gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 09:01:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: more comment cleanup

Browse Source
This commit is contained in:
Julian Sommer
2026-06-12 12:38:31 +10:00
parent 53e8c3f60f
commit 50b5495d26
320 changed files with 3282 additions and 8676 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/eksgw/client.go
+5 -9
View File
@@ -1,8 +1,6 @@
// Package eksgw is the on-VM SigV4 client EKS control-plane helpers use to
// reach the AWS gateway, instead of dialing core NATS. Both eks-gateway-publish
// (bootstrap + state relay) and eks-token-webhook (TokenReview relay) sign with
// the system (Predastore) credentials for service "eks" and POST over HTTPS;
// the gateway authorizes via IAM and relays to the host-side NATS/STS/KV.
// Package eksgw is the on-VM SigV4 client used by EKS helpers (eks-gateway-publish,
// eks-token-webhook) to POST over HTTPS to the AWS gateway instead of NATS.
// Signs with Predastore credentials for service "eks".
package eksgw
import (
@@ -73,10 +71,8 @@ func New(baseURL, caPath, accessKey, secretKey, region string) (*Client, error)
}, nil
}
// Post SigV4-signs and sends a single POST of body to path (e.g.
// "/clusters/alpha/token-review"). It returns the response body on a 2xx, or an
// error (including the gateway status + body) otherwise. No retry — callers that
// need it wrap Post in their own loop.
// Post SigV4-signs and sends body to path, returning the response body on 2xx.
// Errors include the gateway status and body. No retry; callers wrap as needed.
func (c *Client) Post(path string, body []byte) ([]byte, error) {
req, err := http.NewRequest(http.MethodPost, c.baseURL+path, bytes.NewReader(body))
if err != nil {
internal/fipsboot/fipsboot.go
+3 -9
View File
@@ -1,12 +1,6 @@
// Package fipsboot enforces FIPS 140-3 mode at process startup.
//
// Blank-import this package from every binary's main:
//
// import _ "github.com/mulgadc/spinifex/internal/fipsboot"
//
// Builds with GOFIPS140=v1.0.0 link the Go Cryptographic Module v1.0.0
// (NIST CMVP cert #5247). At runtime FIPS mode can still be turned off
// via GODEBUG=fips140=off; init() panics in that case to close the hole.
// Package fipsboot enforces FIPS 140-3 at startup via blank import in every binary.
// Requires GOFIPS140=v1.0.0 at build time; panics if GODEBUG=fips140=off disables
// the runtime FIPS mode.
package fipsboot
import (
internal/tlsconfig/curves.go
+5 -26
View File
@@ -1,34 +1,13 @@
// Package tlsconfig pins the cluster-wide TLS 1.3 curve preference list.
//
// The list is hardcoded because mixed classification on a single cluster is
// not supported and the certified FIPS build is itself the per-deployment
// choice. A typo in a config string would silently downgrade where a missing
// constant cannot.
// Hardcoded to prevent silent downgrade via misconfiguration; the FIPS build
// is the per-deployment choice.
package tlsconfig
import "crypto/tls"
// Curves is the fixed TLS 1.3 curve allowlist:
//
// - X25519MLKEM768 — Cat 3 ML-KEM hybrid, wins under Go 1.26 stdlib filter
// - SecP384r1MLKEM1024 — Cat 5 ML-KEM hybrid, fallback for peers that prefer it
// - X25519 — classical TLS 1.3 fallback for SDKs without ML-KEM (aws CLI v2 today)
// - CurveP384 — classical TLS 1.3 fallback (Java/.NET enterprise SDKs)
//
// PQ-capable peers (browsers since 2024, Go aws-sdk v2 with Go 1.24+) always
// negotiate the hybrid first; classical fallback only triggers for peers that
// cannot do ML-KEM. Those connections still get TLS 1.3 + AES-256-GCM but
// remain HNDL-exposed — accepted trade-off until AWS CLI v2 / bundled
// OpenSSL ships ML-KEM (currently lagging upstream OpenSSL 3.5).
//
// Weak/niche entries from the Go stdlib default (SecP256r1MLKEM768, P-256,
// P-521) are intentionally excluded so they cannot be advertised.
//
// Go 1.26 stdlib treats tls.Config.CurvePreferences as an allowlist filter
// applied to the stdlib default order, not as a user-supplied order.
//
// Callers must not mutate this slice. Assign it directly to
// tls.Config.CurvePreferences; do not append to it.
// Curves is the fixed TLS 1.3 curve allowlist: two ML-KEM hybrids (PQ-capable
// peers always negotiate these) followed by two classical fallbacks for SDKs
// without ML-KEM. Weak Go defaults (SecP256r1MLKEM768, P-256, P-521) excluded.
var Curves = []tls.CurveID{
tls.X25519MLKEM768,
tls.SecP384r1MLKEM1024,
internal/tlsconfig/curves_test.go
+5 -9
View File
@@ -46,11 +46,9 @@ func TestCurves_ExcludesWeakPrimitives(t *testing.T) {
}
}
// TestIntegration_NegotiatesPQHybrid spins up an httptest TLS server
// configured with Curves + TLS 1.3 minimum, dials it, and asserts a PQ
// hybrid is negotiated at TLS 1.3. Per Go 1.26 stdlib filter behavior the
// winner is X25519MLKEM768; the assertion checks for any approved hybrid
// to remain stable across stdlib reorderings of our 4 entries.
// TestIntegration_NegotiatesPQHybrid verifies that a Curves-configured server
// negotiates a PQ hybrid at TLS 1.3; checks for any approved hybrid to stay
// stable across Go stdlib reorderings.
func TestIntegration_NegotiatesPQHybrid(t *testing.T) {
cert := selfSignedCert(t)
srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
@@ -95,10 +93,8 @@ func TestIntegration_NegotiatesPQHybrid(t *testing.T) {
}
}
// TestIntegration_TLS12ClientRejected confirms MinVersion: tls.VersionTLS13
// is enforced — a TLS 1.2-pinned client gets a handshake error rather than
// silently downgrading. This is the negative test that closes the HNDL gap
// on external surfaces.
// TestIntegration_TLS12ClientRejected confirms MinVersion TLS 1.3 is enforced:
// a TLS 1.2-pinned client must get a handshake error, not a silent downgrade.
func TestIntegration_TLS12ClientRejected(t *testing.T) {
cert := selfSignedCert(t)
srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
spinifex/admin/admin.go
+18 -76
View File
@@ -44,13 +44,12 @@ type ConfigSettings struct {
LogDir string
ConfigDir string
// Add more fields as needed
Node string
Az string
Port string
BindIP string
// AdvertiseIP is the off-host dial target rendered into the local node's
// [nodes.X].advertise field. Empty → callers fall back to BindIP.
// AdvertiseIP is the off-host dial target rendered into [nodes.X].advertise.
// Empty → callers fall back to BindIP.
AdvertiseIP string
// Cluster settings
@@ -82,10 +81,8 @@ type ConfigSettings struct {
PoolPrefixLen int // Subnet prefix length (default 24)
PoolDNSServers []string // DNS servers for VM DHCP (auto-detected from host)
// OperatorEmail is the address collected at install time (TUI, SPINIFEX_EMAIL,
// or `spx admin init --email`). Written under [operator] in spinifex.toml so
// reset-dev-env.sh can preserve it across wipes. Empty means no operator
// identity was supplied.
// OperatorEmail is the address collected at install time. Written under [operator]
// in spinifex.toml so it survives wipes. Empty means no identity was supplied.
OperatorEmail string
// Other nodes in the cluster (for config source of truth)
@@ -104,9 +101,8 @@ type ConfigSettings struct {
// Sets gpu_passthrough = true under [nodes.<node>.daemon].
GPUPassthrough bool
// IPSecEnabled toggles cluster-wide OVN native IPsec on intra-AZ Geneve
// (default true). Surfaced under [network] in spinifex.toml so the value
// is visible/auditable; the daemon reads it via cluster config.
// IPSecEnabled toggles cluster-wide OVN native IPsec on intra-AZ Geneve.
// Written under [network] in spinifex.toml; daemon reads it via cluster config.
IPSecEnabled bool
}
@@ -133,22 +129,19 @@ func GenerateConfigFiles(configs []ConfigFile, configSettings ConfigSettings) er
return nil
}
// generateConfigFile creates a configuration file from a template
// GenerateConfigFile creates a configuration file from a template.
func GenerateConfigFile(configPath string, configTemplate string, configSettings ConfigSettings) error {
// Parse the embedded template
tmpl, err := template.New("config").Parse(configTemplate)
if err != nil {
return fmt.Errorf("failed to parse template: %w", err)
}
// Create file with secure permissions
f, err := os.OpenFile(configPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return fmt.Errorf("failed to create config file: %w", err)
}
defer f.Close()
// Execute template
if err := tmpl.Execute(f, configSettings); err != nil {
return fmt.Errorf("failed to execute template: %w", err)
}
@@ -157,7 +150,6 @@ func GenerateConfigFile(configPath string, configTemplate string, configSettings
}
func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string) (caCertPath string) {
// Certificate paths
caCertPath = filepath.Join(configDir, "ca.pem")
caKeyPath := filepath.Join(configDir, "ca.key")
serverCertPath := filepath.Join(configDir, "server.pem")
@@ -171,7 +163,6 @@ func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string) (
if needsGeneration {
fmt.Println("\n🔐 Generating Certificate Authority and SSL certificates...")
// Step 1: Generate CA certificate
if err := GenerateCACert(caCertPath, caKeyPath); err != nil {
fmt.Fprintf(os.Stderr, "Error generating CA certificate: %v\n", err)
os.Exit(1)
@@ -180,7 +171,6 @@ func GenerateCertificatesIfNeeded(configDir string, force bool, bindIP string) (
fmt.Printf(" CA Certificate: %s\n", caCertPath)
fmt.Printf(" CA Key: %s\n", caKeyPath)
// Step 2: Generate server certificate signed by CA (with bind IP in SANs)
if err := GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, bindIP); err != nil {
fmt.Fprintf(os.Stderr, "Error generating server certificate: %v\n", err)
os.Exit(1)
@@ -211,17 +201,14 @@ func GenerateServerCertOnly(configDir string, bindIP string) error {
serverCertPath := filepath.Join(configDir, "server.pem")
serverKeyPath := filepath.Join(configDir, "server.key")
// Verify CA files exist
if !FileExists(caCertPath) || !FileExists(caKeyPath) {
return fmt.Errorf("CA files not found in %s", configDir)
}
// Generate server cert signed by CA with this node's bind IP
return GenerateSignedCert(serverCertPath, serverKeyPath, caCertPath, caKeyPath, bindIP)
}
func CreateServiceDirectories(spxRoot string) {
// Create additional directories
dirs := []string{
filepath.Join(spxRoot, "images"),
filepath.Join(spxRoot, "amis"),
@@ -238,7 +225,6 @@ func CreateServiceDirectories(spxRoot string) {
fmt.Println("\n📁 Creating directory structure...")
for _, dir := range dirs {
// Check if directory exists
if _, err := os.Stat(dir); os.IsNotExist(err) {
if err := os.MkdirAll(dir, 0750); err != nil {
fmt.Fprintf(os.Stderr, "Warning: Could not create %s: %v\n", dir, err)
@@ -248,16 +234,12 @@ func CreateServiceDirectories(spxRoot string) {
fmt.Printf("✅ Directory structure created in %s\n", spxRoot)
}
// Helper functions
func FileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
// ChownRecursive changes ownership of a path and all its contents to the
// specified username. Used after init to hand production directories back
// to the service user when init runs as root via sudo.
// ChownRecursive changes ownership of path and its contents to username.
// Best-effort: errors are logged but do not halt the operation.
func ChownRecursive(path, username string) {
u, err := user.Lookup(username)
@@ -306,7 +288,6 @@ func ChownRecursive(path, username string) {
// SetServiceOwnership sets per-service ownership on data/config directories
// and shared config files to root:spinifex with correct modes.
// Keep in sync with setup.sh create_directories() and plan doc section 2.
func SetServiceOwnership() {
grp, err := user.LookupGroup("spinifex")
if err != nil {
@@ -379,12 +360,11 @@ func SetServiceOwnership() {
}
}
// updateAWSINIFile updates or creates an AWS INI file section with given key-value pairs
// UpdateAWSINIFile updates or creates an AWS INI file section with the given key-value pairs.
func UpdateAWSINIFile(path, section string, values map[string]string) error {
var cfg *ini.File
var err error
// Load existing file or create new one
if FileExists(path) {
cfg, err = ini.Load(path)
if err != nil {
@@ -394,25 +374,21 @@ func UpdateAWSINIFile(path, section string, values map[string]string) error {
cfg = ini.Empty()
}
// Get or create section
sec, err := cfg.NewSection(section)
if err != nil {
// Section already exists, get it
// Section already exists, get it.
sec, err = cfg.GetSection(section)
if err != nil {
return fmt.Errorf("failed to get section: %w", err)
}
}
// Set key-value pairs
for key, value := range values {
sec.Key(key).SetValue(value)
}
// Write atomically with 0600. ini.SaveTo uses os.Create (0666 & ~umask,
// typically 0644), which leaves ~/.aws/credentials world-readable. Render
// to a sibling temp file (os.CreateTemp is 0600) and rename into place so
// the secrets are never exposed, even briefly.
// Write atomically: ini.SaveTo uses os.Create (world-readable); render to a
// sibling temp file (0600) and rename to avoid briefly exposing secrets.
dir := filepath.Dir(path)
tmp, err := os.CreateTemp(dir, ".aws-ini-*.tmp")
if err != nil {
@@ -433,8 +409,7 @@ func UpdateAWSINIFile(path, section string, values map[string]string) error {
return nil
}
// generateAWSAccessKey generates an AWS-style access key
// Format: AKIA + 16 random uppercase alphanumeric characters
// GenerateAWSAccessKey generates an AWS-style access key (AKIA + 16 random alphanumeric chars).
func GenerateAWSAccessKey() (string, error) {
const prefix = "AKIA"
const charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
@@ -452,8 +427,7 @@ func GenerateAWSAccessKey() (string, error) {
return prefix + string(result), nil
}
// generateAWSSecretKey generates an AWS-style secret key
// 40 character base64-encoded string
// GenerateAWSSecretKey generates a 40-character base64-encoded AWS-style secret key.
func GenerateAWSSecretKey() (string, error) {
bytes := make([]byte, 30) // 30 bytes = 40 chars in base64
if _, err := rand.Read(bytes); err != nil {
@@ -488,15 +462,13 @@ func GenerateNATSToken() (string, error) {
return "nats_" + base64.URLEncoding.EncodeToString(bytes)[:32], nil
}
// GenerateCACert generates a Certificate Authority certificate and key
// GenerateCACert generates a Certificate Authority certificate and key.
func GenerateCACert(caCertPath, caKeyPath string) error {
// Generate CA private key
caPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
return fmt.Errorf("failed to generate CA private key: %w", err)
}
// Create CA certificate template
notBefore := time.Now()
notAfter := notBefore.Add(3650 * 24 * time.Hour) // 10 years
@@ -519,13 +491,11 @@ func GenerateCACert(caCertPath, caKeyPath string) error {
MaxPathLen: 1,
}
// Self-sign the CA certificate
caDerBytes, err := x509.CreateCertificate(rand.Reader, &caTemplate, &caTemplate, &caPrivateKey.PublicKey, caPrivateKey)
if err != nil {
return fmt.Errorf("failed to create CA certificate: %w", err)
}
// Write CA certificate to file
caCertOut, err := os.Create(caCertPath)
if err != nil {
return fmt.Errorf("failed to create CA cert file: %w", err)
@@ -536,7 +506,6 @@ func GenerateCACert(caCertPath, caKeyPath string) error {
return fmt.Errorf("failed to write CA cert: %w", err)
}
// Write CA private key to file
caKeyOut, err := os.OpenFile(caKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return fmt.Errorf("failed to create CA key file: %w", err)
@@ -752,15 +721,13 @@ func GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath string,
return nil
}
// generateSelfSignedCert generates a self-signed SSL certificate (legacy, kept for compatibility)
// GenerateSelfSignedCert generates a self-signed SSL certificate (legacy, kept for compatibility).
func GenerateSelfSignedCert(certPath, keyPath string) error {
// Generate private key
privateKey, err := rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
return fmt.Errorf("failed to generate private key: %w", err)
}
// Create certificate template
notBefore := time.Now()
notAfter := notBefore.Add(3650 * 24 * time.Hour) // 10 years
@@ -784,13 +751,11 @@ func GenerateSelfSignedCert(certPath, keyPath string) error {
IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},
}
// Create certificate
derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey)
if err != nil {
return fmt.Errorf("failed to create certificate: %w", err)
}
// Write certificate to file
certOut, err := os.Create(certPath)
if err != nil {
return fmt.Errorf("failed to create cert file: %w", err)
@@ -801,7 +766,6 @@ func GenerateSelfSignedCert(certPath, keyPath string) error {
return fmt.Errorf("failed to write cert: %w", err)
}
// Write private key to file
keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return fmt.Errorf("failed to create key file: %w", err)
@@ -821,11 +785,6 @@ func GenerateSelfSignedCert(certPath, keyPath string) error {
}
// SetupAWSCredentials updates ~/.aws/credentials and ~/.aws/config.
// bindIP is the IP the AWS gateway listens on. If empty or "0.0.0.0", the
// operator's local ~/.aws/config endpoint_url falls back to "localhost" (this
// runs on the same box as the gateway). wanIP is reserved for a future
// --operator-endpoint flag that will let remote operators point their CLI at
// the host's WAN IP; today it is accepted but unused.
// When running under sudo, writes to SUDO_USER's home instead of root's.
func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP, wanIP string) error {
_ = wanIP
@@ -834,8 +793,6 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP, wanIP s
return err
}
// When running under sudo, write to the invoking user's home directory
// so the operator can use AWS_PROFILE=spinifex without sudo.
sudoUser := os.Getenv("SUDO_USER")
if os.Getuid() == 0 && sudoUser != "" {
if u, err := user.Lookup(sudoUser); err == nil {
@@ -851,21 +808,8 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP, wanIP s
credPath := filepath.Join(awsDir, "credentials")
configPath := filepath.Join(awsDir, "config")
// Determine profile name
//profileName := "default"
// Use Spinifex as the default profile
profileName := "spinifex"
if FileExists(credPath) {
// Check if default profile already exists
cfg, err := ini.Load(credPath)
if err == nil && cfg.HasSection("default") {
profileName = "spinifex"
}
}
// Update credentials file
if err := UpdateAWSINIFile(credPath, profileName, map[string]string{
"aws_access_key_id": accessKey,
"aws_secret_access_key": secretKey,
@@ -873,7 +817,6 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP, wanIP s
return err
}
// Update config file
configSection := profileName
if profileName != "default" {
configSection = "profile " + profileName
@@ -961,9 +904,8 @@ func ParsePredastoreNodeIDFromConfig(tomlContent string, ip string) int {
return FindNodeIDByIP(cfg.DB, ip)
}
// SetMIGProfile idempotently writes mig_profile = "<profile>" for the given
// node into spinifex.toml, preserving all other content and comments.
// An empty profile clears the setting (writes mig_profile = "").
// SetMIGProfile idempotently writes mig_profile = "<profile>" for the given node
// into spinifex.toml. An empty profile clears the setting.
func SetMIGProfile(tomlPath, node, profile string) error {
raw, err := os.ReadFile(tomlPath)
if err != nil {
spinifex/admin/dial.go
+4 -13
View File
@@ -2,13 +2,8 @@ package admin
import "net"
// DialTarget returns a host:port suitable for an in-process client to dial,
// given a configured listen address. Inputs of the form "0.0.0.0:N" or "[::]:N"
// are rewritten to "127.0.0.1:N" or "[::1]:N" — a service listening on the
// wildcard is reachable on loopback, and loopback is the TLS SAN that the
// daemon's own cert always covers. Loopback, specific IPs, and bare hostnames
// pass through unchanged. Malformed input (no host:port) passes through so
// callers still see the original value in error messages.
// DialTarget returns a host:port for in-process dialing. Wildcard listeners
// ("0.0.0.0:N", "[::]:N") are rewritten to loopback; specific IPs pass through.
func DialTarget(listenAddr string) string {
host, port, err := net.SplitHostPort(listenAddr)
if err != nil {
@@ -23,12 +18,8 @@ func DialTarget(listenAddr string) string {
return listenAddr
}
// AdvertiseHost picks the host that off-host clients should dial for a service
// listening on listenAddr. If the listen host is a wildcard (0.0.0.0, ::, or
// empty), the caller-supplied advertiseIP is returned; otherwise the listen
// host itself is returned. Port is intentionally dropped — callers append the
// port appropriate for the protocol (some services listen and advertise on
// different ports).
// AdvertiseHost returns the host off-host clients should dial. Wildcard listeners
// return advertiseIP; specific listen hosts return themselves (port is dropped).
func AdvertiseHost(listenAddr, advertiseIP string) string {
host, _, err := net.SplitHostPort(listenAddr)
if err != nil {
spinifex/admin/email.go
+2 -6
View File
@@ -26,12 +26,8 @@ func ValidateEmail(addr string) error {
return nil
}
// ReadOperatorEmail extracts the [operator].email scalar from a spinifex.toml
// file, or returns "" if the file doesn't exist, can't be read, or has no
// [operator] section. Used by `spx admin init --force` (and reset flows) to
// preserve a previously-captured email across re-inits when --email isn't
// supplied on the new invocation. Minimal text scan — no TOML parser, since
// we only need one scalar from one section.
// ReadOperatorEmail extracts the [operator].email scalar from spinifex.toml,
// returning "" on any error or missing section. Uses a text scan, not a full TOML parser.
func ReadOperatorEmail(tomlPath string) string {
f, err := os.Open(tomlPath)
if err != nil {
spinifex/admin/images_remove.go
+11 -33
View File
@@ -46,10 +46,8 @@ func (d Dependents) Empty() bool {
return len(d.Snapshots) == 0 && len(d.Volumes) == 0 && len(d.AMIs) == 0
}
// RemovePreview captures everything the CLI needs to render the confirmation
// prompt and decide whether to proceed: AMI metadata (or zero values when the
// config is missing/corrupt), backing-storage byte counts, and the dependent
// resource set. PreviewRemoveSystemImage performs no deletions.
// RemovePreview captures AMI metadata, byte counts, and dependents for the
// CLI confirmation prompt. PreviewRemoveSystemImage performs no deletions.
type RemovePreview struct {
ImageID string
Name string
@@ -74,11 +72,8 @@ func SnapPrefix(imageID string) string {
return "snap-" + imageID
}
// PreviewRemoveSystemImage walks the object store and gathers everything the
// caller needs to decide whether to remove an admin-imported AMI. It performs
// no mutations. Returned errors indicate unexpected I/O failures only —
// AMI-not-found, corrupt config, and account-owned AMIs are reflected in the
// preview fields, not as errors.
// PreviewRemoveSystemImage gathers AMI metadata and dependents without mutating
// any state. Not-found, corrupt config, and account-owned AMI are preview fields.
func PreviewRemoveSystemImage(store objectstore.ObjectStore, bucket, imageID string) (*RemovePreview, error) {
if !strings.HasPrefix(imageID, "ami-") {
return nil, errors.New(awserrors.ErrorInvalidAMIIDMalformed)
@@ -125,16 +120,8 @@ func PreviewRemoveSystemImage(store objectstore.ObjectStore, bucket, imageID str
return preview, nil
}
// FindAMIDependents walks the bucket to find every resource that transitively
// depends on the given admin-imported AMI:
//
// 1. snap-*/metadata.json whose VolumeID == imageID → derivedSnaps
// (these are CopyImage-created snapshots that point back at the system AMI)
// 2. vol-*/config.json whose VolumeMetadata.SnapshotID ∈ {snap-ami-<id>} ∪ derivedSnaps
// 3. ami-*/config.json whose AMIMetadata.SnapshotID ∈ derivedSnaps
//
// The walk terminates one hop deep. CopyImage of an account AMI uses its own
// VolumeID, not the system AMI's, so further recursion is not required today.
// FindAMIDependents returns snapshots, volumes, and AMIs that depend on the
// given system AMI. Walk terminates one hop deep.
func FindAMIDependents(store objectstore.ObjectStore, bucket, imageID string) (Dependents, error) {
var deps Dependents
@@ -220,16 +207,9 @@ func FindAMIDependents(store objectstore.ObjectStore, bucket, imageID string) (D
return deps, nil
}
// RemoveSystemImage deletes an admin-imported AMI and its backing storage
// after re-validating the preconditions:
//
// 1. ami-<id>/config.json exists and parses (skipped under --force = salvage)
// 2. AMI is system-owned, not account-owned (--force bypasses)
// 3. No volumes / snapshots / AMIs depend on it (--force bypasses)
//
// Deletion order is intentional: the config.json is deleted FIRST so the AMI
// disappears from DescribeImages immediately, preventing new RunInstances
// from racing the block cleanup.
// RemoveSystemImage deletes an admin-imported AMI after re-validating that it
// is system-owned and has no dependents (bypassed by --force). config.json is
// deleted first so the AMI vanishes from DescribeImages before block cleanup.
func RemoveSystemImage(store objectstore.ObjectStore, bucket string, opts RemoveImageOpts) (*RemoveImageResult, error) {
if !strings.HasPrefix(opts.ImageID, "ami-") {
return nil, errors.New(awserrors.ErrorInvalidAMIIDMalformed)
@@ -344,10 +324,8 @@ func readAMIConfig(store objectstore.ObjectStore, bucket, imageID string) (viper
return state.VolumeConfig.AMIMetadata, nil
}
// errCorruptVolumeConfig lets the dependency walk distinguish a vol-*/config.json
// that can't be parsed (recovery is operator-owned, walk continues) from a
// transient transport error (walk must fail closed — silently dropping a
// dependent could lead to deleting blocks underneath a live volume).
// errCorruptVolumeConfig distinguishes an unparse-able config (walk continues)
// from a transport error (walk fails closed to prevent deleting live blocks).
var errCorruptVolumeConfig = errors.New("corrupt volume config")
// readVolumeConfig reads vol-<id>/config.json into VolumeConfig.
spinifex/admin/ipsec.go
+9 -39
View File
@@ -33,33 +33,16 @@ const (
// strongSwan config consumes them as-is.
var oidExtKeyUsageIPSecIKE = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 17}
// IPSecCertPaths returns the canonical paths for the per-node IPsec peer
// credential pair under configDir. Both paths live in
// {configDir}/ipsec/. The CA trust anchor is the existing cluster CA in
// configDir/ca.pem; ovs-monitor-ipsec is pointed at that path directly.
// IPSecCertPaths returns the canonical cert/key paths for the per-node IPsec peer
// credential pair under {configDir}/ipsec/.
func IPSecCertPaths(configDir string) (certPath, keyPath string) {
dir := filepath.Join(configDir, ipsecDirName)
return filepath.Join(dir, ipsecCertFileName), filepath.Join(dir, ipsecKeyFileName)
}
// GenerateIPSecPeerCert issues an IKEv2 peer certificate for this node,
// signed by the existing cluster CA. The cert carries the IPSec-IKE
// extended key usage so strongSwan accepts it as a peer credential.
//
// hostname is the node's primary hostname (used as Common Name + DNS SAN).
// nodeIP is the node's primary cluster IP (added as an IP SAN). Additional
// SANs discovered from the local machine's interfaces are not added here —
// IPSec peer identity should be the explicit cluster identity only, not
// every interface address, to keep peer-identity matching strict.
//
// Cross-cutting invariant: this dnsName SAN must equal the OVS chassis-id
// (Open_vSwitch.external_ids:system-id) on every node, because
// ovs-monitor-ipsec generates strongSwan configs with leftid=@<chassis-id>
// / rightid=@<chassis-id> and charon validates those names against the
// peer cert's dnsName SAN. setup-ovn.sh pins system-id to `hostname -s`
// for exactly this reason — see the comment in setup-ovn.sh near
// `system-id.conf`. Changing the cert identity here without updating
// setup-ovn.sh (or vice versa) breaks IKE with AUTHENTICATION_FAILED.
// GenerateIPSecPeerCert issues an IKEv2 peer certificate for this node signed
// by the existing cluster CA. The cert carries the IPSec-IKE EKU so strongSwan
// accepts it. The dnsName SAN must equal the OVS chassis-id (system-id).
func GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, hostname, nodeIP string) error {
if hostname == "" {
return fmt.Errorf("ipsec peer cert: hostname required")
@@ -127,14 +110,8 @@ func GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, hostname, nodeIP st
return fmt.Errorf("ipsec peer cert: write key: %w", err)
}
// Symlink the cluster CA into charon's default cacerts scan dir.
// ovs-monitor-ipsec writes per-conn `cacert=` references, but
// strongSwan 6.0 only loads CAs declared in standalone `ca` sections
// or files placed in /etc/ipsec.d/cacerts at startup. Without this,
// charon authenticates the local side OK but rejects every inbound
// peer cert as `no trusted RSA public key found for '<peer>'` and
// no SAs negotiate. The dir is on charon's default AppArmor allowlist
// so no profile change is needed (unlike /etc/spinifex/).
// strongSwan 6.0 only loads CAs placed in /etc/ipsec.d/cacerts at startup;
// per-conn cacert= references are insufficient. Install the cluster CA there.
if err := installCAIntoCharonTrustStore(caCertPath); err != nil {
return fmt.Errorf("ipsec peer cert: install CA into charon trust store: %w", err)
}
@@ -166,15 +143,8 @@ func installCAIntoCharonTrustStore(caCertPath string) error {
return fmt.Errorf("rename %s -> %s: %w", staging, link, err)
}
// strongSwan scans /etc/ipsec.d/cacerts only at charon startup. If
// openvswitch-ipsec.service started before this install (the common
// case — the unit is enabled at provision time), charon is running
// with zero CAs loaded and every IKE_AUTH ends in `no trusted RSA
// public key found for '<peer>'`. Trigger a stroke rereadcacerts so
// the symlink we just placed is loaded without bouncing charon.
// Silent failure is expected when charon isn't running yet (admin
// init can run before setup-ovn.sh enables the unit); a later charon
// start will scan cacerts naturally.
// Trigger rereadcacerts so charon picks up the symlink without restart.
// Silent failure is expected when charon isn't running yet.
if err := charonRereadCAs(); err != nil {
slog.Debug("ipsec: rereadcacerts skipped", "err", err)
}
spinifex/admin/ipsec_test.go
+3 -9
View File
@@ -127,12 +127,8 @@ func TestIPSecCertPaths(t *testing.T) {
assert.Equal(t, "/etc/spinifex/ipsec/peer.key", keyPath)
}
// TestInstallCAIntoCharonTrustStore_TriggersRereadCAs locks the fix for the
// race where openvswitch-ipsec started before admin init's CA install: every
// IKE_AUTH failed with `no trusted RSA public key found for '<peer>'` because
// charon had scanned cacerts before the symlink was placed. The trust-store
// installer must trigger `ipsec rereadcacerts` so an already-running charon
// picks up the CA without a restart.
// TestInstallCAIntoCharonTrustStore_TriggersRereadCAs ensures the installer
// calls `ipsec rereadcacerts` so an already-running charon picks up the CA.
func TestInstallCAIntoCharonTrustStore_TriggersRereadCAs(t *testing.T) {
origTrustDir := charonCATrustDir
charonCATrustDir = t.TempDir()
@@ -160,9 +156,7 @@ func TestInstallCAIntoCharonTrustStore_TriggersRereadCAs(t *testing.T) {
}
// TestInstallCAIntoCharonTrustStore_SwallowsRereadError verifies that a failed
// rereadcacerts (e.g. charon not running, ipsec binary missing on a fresh
// install) does not break admin init. The CA symlink must still be in place so
// a later charon start picks it up via the normal cacerts scan.
// rereadcacerts (charon not running) doesn't abort admin init.
func TestInstallCAIntoCharonTrustStore_SwallowsRereadError(t *testing.T) {
origTrustDir := charonCATrustDir
charonCATrustDir = t.TempDir()
spinifex/admin/network_detect.go
+2 -8
View File
@@ -54,7 +54,6 @@ func DetectNetwork() (*DetectedNetwork, error) {
return nil, fmt.Errorf("could not parse default route: %s", defaultRoute)
}
// Get all routes to find interfaces and their subnets
allRoutes, err := exec.Command("ip", "-4", "route", "show", "scope", "link").Output()
if err != nil {
return nil, fmt.Errorf("failed to get routes: %w", err)
@@ -149,7 +148,6 @@ func SuggestPoolRange(wan *DetectedInterface) (start, end string) {
return "", ""
}
// Calculate broadcast address
ip := ipNet.IP.To4()
mask := ipNet.Mask
broadcast := make(net.IP, 4)
@@ -182,13 +180,9 @@ func SuggestPoolRange(wan *DetectedInterface) (start, end string) {
return startIP.String(), endIP.String()
}
// isVirtualInterface returns true for interfaces that shouldn't be
// considered as physical NICs. The defaultRouteDev parameter is the
// interface carrying the default route — it's always considered physical
// even if its name matches a virtual prefix (e.g., br-wan).
// isVirtualInterface returns true for interfaces that shouldn't be considered
// physical NICs. defaultRouteDev is always treated as physical.
func isVirtualInterface(name string, defaultRouteDev string) bool {
// The default route interface is always considered physical — it's the
// WAN uplink, even if it's a bridge (br-wan, br-external, etc.).
if defaultRouteDev != "" && name == defaultRouteDev {
return false
}
spinifex/awsec2query/parser.go
+4 -29
View File
@@ -13,27 +13,6 @@ import (
// Callers should map this to AWS's MalformedQueryString error code.
var ErrSliceTooLarge = errors.New("list parameter exceeds maximum entries")
/*
Sample args
args["Action"] = "RunInstance"
args["ImageId"] = "ami-123456789"
args["MinCount"] = "1"
args["MaxCount"] = "1"
args["InstanceType"] = "t2.micro"
args["KeyName"] = "my-key-pair"
args["SecurityGroup.1"] = "sg-12345678"
args["SecurityGroup.2"] = "sg-87654321"
args["TagSpecification.1.ResourceType"] = "instance"
args["TagSpecification.1.Tag.1.Key"] = "Name"
args["TagSpecification.1.Tag.1.Value"] = "MyInstance"
args["TagSpecification.2.ResourceType"] = "volume"
args["TagSpecification.2.Tag.1.Key"] = "Environment"
args["TagSpecification.2.Tag.1.Value"] = "Production"
*/
func QueryParamsToStruct(params map[string]string, out any) error {
v := reflect.ValueOf(out)
@@ -67,10 +46,8 @@ func setStructFields(v reflect.Value, params map[string]string, prefix string) e
// Check for locationName tag (AWS SDK uses this for query parameter names)
locationName := fieldType.Tag.Get("locationName")
// Try the field name, locationName, and title-cased locationName.
// AWS query params use title case (e.g. "ResourceId") but some SDK
// structs use camelCase locationName (e.g. "resourceId" in DeleteTagsInput
// vs "ResourceId" in CreateTagsInput).
// Try field name, locationName, and title-cased locationName.
// AWS query params use title case but some SDK structs use camelCase locationName.
queryKeys := []string{prefix + fieldName}
if locationName != "" && locationName != fieldName {
queryKeys = append(queryKeys, prefix+locationName)
@@ -153,13 +130,12 @@ func setFieldValue(field reflect.Value, value string) error {
elem := field.Type().Elem()
// Special case: []byte
if elem.Kind() == reflect.Uint8 {
// First try base64 (what the AWS API expects for PublicKeyMaterial).
// Try base64 first (AWS API expectation); fall back to raw bytes.
trimmed := strings.TrimSpace(value)
if decoded, err := base64.StdEncoding.DecodeString(trimmed); err == nil {
field.SetBytes(decoded)
return nil
}
// Some clients may send raw text. Fall back to raw bytes.
field.SetBytes([]byte(value))
return nil
}
@@ -195,8 +171,7 @@ func setFieldValue(field reflect.Value, value string) error {
const maxSliceLen = 1024
func setSliceField(field reflect.Value, params map[string]string, prefix string) error {
// Find all indexed items for this slice.
// Supports both EC2-style (Prefix.N) and IAM/ELBv2-style (Prefix.member.N) formats.
// Collect indexed items; supports EC2-style (Prefix.N) and IAM/ELBv2-style (Prefix.member.N).
indices := make(map[int]bool)
useMember := false
for key := range params {
spinifex/awserrors/awserrors.go
+4 -11
View File
@@ -221,9 +221,7 @@ var (
ErrorInvalidOptionConflict = "InvalidOption.Conflict"
ErrorInvalidPaginationToken = "InvalidPaginationToken"
ErrorInvalidParameter = "InvalidParameter"
// ACM (AWS JSON 1.1). Suffix-less; the JSON error envelope appends
// "Exception" on the wire (InvalidArnException), matching aws-sdk-go acm
// ErrCode* values. Not-found reuses the shared ErrorResourceNotFound.
// ACM error: the JSON envelope appends "Exception" on the wire; not-found reuses ErrorResourceNotFound.
ErrorACMInvalidArn = "InvalidArn"
ErrorInvalidParameterCombination = "InvalidParameterCombination"
ErrorInvalidParameterDependency = "InvalidParameterDependency"
@@ -372,9 +370,7 @@ var (
ErrorResourceCountExceeded = "ResourceCountExceeded"
ErrorResourceCountLimitExceeded = "ResourceCountLimitExceeded"
ErrorResourceLimitExceeded = "ResourceLimitExceeded"
// ErrorResourceNotFound ("ResourceNotFound") is the ACM not-found code. EKS
// must use ErrorEKSResourceNotFound ("ResourceNotFoundException", below) —
// the restjson code the aws eks client expects. Do not cross-wire the two.
// ErrorResourceNotFound is the ACM not-found code. EKS must use ErrorEKSResourceNotFound — do not cross-wire.
ErrorResourceNotFound = "ResourceNotFound"
ErrorEKSResourceInUse = "ResourceInUseException"
ErrorEKSResourceNotFound = "ResourceNotFoundException"
@@ -495,11 +491,8 @@ func ValidErrorCode(code string) string {
return ErrorServerInternal
}
// IsErrorCode reports whether err carries the named AWS error code. Service
// handlers return these codes as plain strings (errors.New(code)), commonly
// wrapped with %w by orchestration callers, so the match is a substring test
// against err.Error() rather than equality. Use it to classify a delete error
// as idempotent (already-gone) or retryable (still-in-use) in teardown paths.
// IsErrorCode reports whether err carries the named AWS error code.
// Uses substring matching because handlers return codes as plain strings, sometimes wrapped with %w.
func IsErrorCode(err error, code string) bool {
return err != nil && strings.Contains(err.Error(), code)
}
spinifex/awserrors/awserrors_test.go
+2 -5
View File
@@ -2,11 +2,8 @@ package awserrors
import "testing"
// TestErrorLookup_Structure asserts invariants of the ErrorLookup map without
// duplicating every entry. Catches wholesale-deletion accidents, invalid HTTP
// codes, empty messages, and regressions of SDK-surfaced codes — the failure
// modes that actually matter — without locking in a 1:1 mirror that has to be
// updated in lockstep with every new error code.
// TestErrorLookup_Structure asserts ErrorLookup invariants: minimum size, valid HTTP codes, non-empty messages.
// Avoids a 1:1 mirror that would need updating with every new error code.
func TestErrorLookup_Structure(t *testing.T) {
if len(ErrorLookup) < 400 {
t.Fatalf("ErrorLookup unexpectedly small: %d entries", len(ErrorLookup))
spinifex/config/config.go
+15 -38
View File
@@ -33,12 +33,8 @@ type ExternalPool struct {
DNSServers []string `mapstructure:"dns_servers"` // DNS servers for VM DHCP (auto-detected from host; fallback: 8.8.8.8, 1.1.1.1)
Region string `mapstructure:"region"` // Scope to region (optional — empty means any region)
AZ string `mapstructure:"az"` // Scope to AZ (optional — empty means any AZ in region)
// GwLrpRangeStart/End reserve a sub-range of the LAN for OVN gateway LRP IPs
// in centralized NAT mode (veth). Each VPC consumes one IP from this
// range so its gateway router port can ARP with a sender IP on the LAN
// subnet — link-local 169.254.0.1/30 makes upstream routers reject ARP per
// RFC 826. Must NOT overlap [RangeStart, RangeEnd] or IPAM and vpcd will
// fight over the same IP.
// GwLrpRangeStart/End reserve a sub-range for OVN gateway LRP IPs in centralized NAT mode.
// Must NOT overlap [RangeStart, RangeEnd] — link-local 169.254/16 is rejected by upstream routers.
GwLrpRangeStart string `mapstructure:"gw_lrp_range_start"`
GwLrpRangeEnd string `mapstructure:"gw_lrp_range_end"`
}
@@ -47,15 +43,12 @@ type ExternalPool struct {
type NetworkConfig struct {
ExternalMode string `mapstructure:"external_mode"` // "pool" or "" (disabled)
ExternalPools []ExternalPool `mapstructure:"external_pools"` // One or more IP pools
// IPSecEnabled toggles OVN native IPsec on every node for intra-AZ Geneve
// (AES-256-GCM via strongSwan). Default true. Operators can flip off for
// trusted single-rack lab topologies; production edge stays on.
// IPSecEnabled toggles OVN native IPsec (AES-256-GCM) on every node. Default true; disable only for trusted lab topologies.
IPSecEnabled bool `mapstructure:"ipsec_enabled"`
}
// BootstrapConfig holds the default VPC infrastructure IDs written by admin init.
// vpcd reads this on startup to ensure OVN topology exists for the bootstrap VPC,
// covering the case where admin init ran before services were started.
// vpcd reads this on startup to ensure OVN topology exists for the bootstrap VPC.
type BootstrapConfig struct {
AccountID string `mapstructure:"account_id"`
VpcId string `mapstructure:"vpc_id"`
@@ -70,8 +63,7 @@ type Config struct {
// Node config
Node string `json:"Node" mapstructure:"node"`
Host string `json:"Host" mapstructure:"host"` // Unique hostname or IP of this node
// AdvertiseIP is the off-host dial target for this node. Empty → callers
// fall back to Host for backward compat with pre-siv-8 cluster configs.
// AdvertiseIP is the off-host dial target. Empty falls back to Host for backward compat.
AdvertiseIP string `json:"AdvertiseIP" mapstructure:"advertise"`
Region string `json:"Region" mapstructure:"region"`
AZ string `json:"AZ" mapstructure:"az"`
@@ -102,12 +94,8 @@ type AWSGWConfig struct {
type ViperblockConfig struct {
ShardWAL *bool `json:"ShardWAL" mapstructure:"shardwal"` // Enable sharded WAL (default false when nil)
// EncryptionKeyFile is the path to the shared 32-byte AES-256 master key
// for viperblock at-rest encryption (chunks, WAL records, VBState/snapshot
// metadata). When empty, volumes are written in cleartext (legacy mode).
// When set, every VB constructed on this node — daemons and AWS handlers
// alike — must load it via masterkey.LoadShared so existing volumes can
// still be opened.
// EncryptionKeyFile is the path to the shared 32-byte AES-256 master key for viperblock at-rest encryption.
// Empty means cleartext. When set, all VB instances must load it via masterkey.LoadShared.
EncryptionKeyFile string `json:"EncryptionKeyFile" mapstructure:"encryption_key_file"`
}
@@ -139,11 +127,9 @@ type GPUModelOverride struct {
Manufacturer string `json:"Manufacturer" mapstructure:"manufacturer"`
Name string `json:"Name" mapstructure:"name"`
MemoryMiB int64 `json:"MemoryMiB" mapstructure:"memory_mib"`
// XVGAOff forces x-vga=off for this GPU in QEMU passthrough, overriding the
// default (on for consumer GPUs, off for known datacenter cards).
// XVGAOff forces x-vga=off in QEMU passthrough, overriding the per-GPU default.
XVGAOff bool `json:"XVGAOff" mapstructure:"xvga_off"`
// MIGProfile overrides the daemon-level MIGProfile for this specific GPU.
// Takes the same format as DaemonConfig.MIGProfile (e.g. "1g.10gb").
// MIGProfile overrides the daemon-level MIGProfile for this GPU (same format, e.g. "1g.10gb").
MIGProfile string `json:"MIGProfile" mapstructure:"mig_profile"`
}
@@ -156,9 +142,7 @@ type DaemonConfig struct {
MgmtBridge string `json:"MgmtBridge" mapstructure:"mgmt_bridge"` // Linux bridge for system instance control plane (default "br-mgmt")
GPUPassthrough bool `json:"GPUPassthrough" mapstructure:"gpu_passthrough"` // Enable VFIO GPU passthrough for g5.* instance types
GPUModelOverrides []GPUModelOverride `json:"GPUModelOverrides" mapstructure:"gpu_model_overrides"` // Dev/test GPU mappings not in the production model list
// MIGProfile enables NVIDIA MIG partitioning on all eligible GPUs on this node.
// Set to a profile name such as "1g.10gb" to activate; empty disables MIG.
// Individual GPUs can override this via GPUModelOverrides[].MIGProfile.
// MIGProfile enables NVIDIA MIG on all eligible GPUs (e.g. "1g.10gb"); empty disables. Per-GPU override via GPUModelOverrides[].MIGProfile.
MIGProfile string `json:"MIGProfile" mapstructure:"mig_profile"`
}
@@ -180,8 +164,7 @@ type NATSSub struct {
Subject string `json:"Subject" mapstructure:"subject"`
}
// NodeBaseDir returns the BaseDir for the current node, or "" if the config
// is nil, the node name is unset, or the node is not found in the Nodes map.
// NodeBaseDir returns the BaseDir for the current node, or "" if config is nil, node is unset, or not found.
func (cc *ClusterConfig) NodeBaseDir() string {
if cc == nil || cc.Node == "" {
slog.Warn("NodeBaseDir: no config or node name set, using global PID path")
@@ -201,8 +184,7 @@ func (cc *ClusterConfig) NodeBaseDir() string {
// AllServices is the default service list when Services is empty (backward compat).
var AllServices = []string{"nats", "predastore", "viperblock", "daemon", "awsgw", "vpcd", "ui"}
// HasService reports whether the node runs the named service.
// An empty Services list means all services (backward compat).
// HasService reports whether the node runs the named service (empty list means all services).
func (c Config) HasService(name string) bool {
services := c.Services
if len(services) == 0 {
@@ -225,8 +207,7 @@ func LoadConfig(configPath string) (*ClusterConfig, error) {
viper.SetEnvPrefix("SPINIFEX")
viper.AutomaticEnv()
// network.ipsec_enabled defaults to true (intra-AZ Geneve encrypted via OVN
// native IPsec). Operators must explicitly set false to disable.
// Default ipsec_enabled to true; operators must explicitly set false to disable.
viper.SetDefault("network.ipsec_enabled", true)
// Try to load config file if it exists
@@ -251,9 +232,7 @@ func LoadConfig(configPath string) (*ClusterConfig, error) {
return nil, fmt.Errorf("error unmarshaling config: %w", err)
}
// Normalize the local node's bind address: 0.0.0.0 means "listen on all
// interfaces" but is not a valid connect address. Only rewrite for the
// local node — remote nodes use real IPs that must not be changed.
// Rewrite 0.0.0.0 in Predastore.Host to 127.0.0.1 for the local node only (not a valid connect address).
if local, ok := config.Nodes[config.Node]; ok {
if strings.HasPrefix(local.Predastore.Host, "0.0.0.0") {
local.Predastore.Host = strings.Replace(local.Predastore.Host, "0.0.0.0", "127.0.0.1", 1)
@@ -268,9 +247,7 @@ func LoadConfig(configPath string) (*ClusterConfig, error) {
return &config, nil
}
// validateClusterConfig rejects legacy upstream-DHCP config and validates
// external pool ranges. The upstream-DHCP-client model was replaced by static
// WAN-pool allocation; legacy keys must be removed from the operator's TOML.
// validateClusterConfig rejects legacy DHCP config keys and validates external pool ranges.
func validateClusterConfig(cc *ClusterConfig) error {
if viper.IsSet("network.external_dhcp") {
return fmt.Errorf("config: [network] external_dhcp is no longer supported; remove the key (static WAN-pool allocation only)")
spinifex/daemon/daemon.go
+128 -403
View File
File diff suppressed because it is too large Load Diff
spinifex/daemon/daemon_handlers_eni.go
+6 -10
View File
@@ -15,12 +15,9 @@ import (
"github.com/nats-io/nats.go"
)
// handleAttachNetworkInterface composes the existing KV-side AttachENI
// with the live QMP pipeline vm.Manager.HotPlugENI. The KV update happens
// first so a daemon crash before the QMP pipeline leaves the ENI record
// in attaching state for the Sprint 3d reconciler to converge. On QMP
// failure the KV record is rolled back to available with the error
// reason persisted in LastAttachError.
// handleAttachNetworkInterface updates KV first (crash-safe attaching state),
// then runs the QMP hot-plug pipeline. On QMP failure the KV record rolls
// back to available with the error persisted in LastAttachError.
func (d *Daemon) handleAttachNetworkInterface(msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) {
slog.Info("Attaching ENI to instance", "instanceId", command.ID)
@@ -99,10 +96,9 @@ func (d *Daemon) handleAttachNetworkInterface(msg *nats.Msg, command types.EC2In
})
}
// handleDetachNetworkInterface reverses the attach pipeline. The KV
// record is marked detaching first so a daemon crash mid-flight leaves
// state for the reconciler; on QMP success the record returns to
// available.
// handleDetachNetworkInterface marks the KV record as detaching first
// (crash-safe), runs the QMP hot-unplug pipeline, then returns the
// record to available on success.
func (d *Daemon) handleDetachNetworkInterface(msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) {
slog.Info("Detaching ENI from instance", "instanceId", command.ID)
spinifex/daemon/daemon_handlers_iam_profile.go
+12 -20
View File
@@ -11,10 +11,9 @@ import (
"github.com/nats-io/nats.go"
)
// handleAssociateIamInstanceProfile services the per-instance Associate path
// dispatched from handleEC2Events. The gateway has already resolved the
// profile reference and enforced iam:PassRole; instance ownership has been
// verified by checkInstanceOwnership before this point.
// handleAssociateIamInstanceProfile services the per-instance Associate path.
// Gateway pre-validates the profile reference and iam:PassRole; ownership is
// checked by checkInstanceOwnership before dispatch.
func (d *Daemon) handleAssociateIamInstanceProfile(msg *nats.Msg, command types.EC2InstanceCommand, instance *vm.VM) {
result, err := d.instanceService.AssociateIamInstanceProfile(instance, command)
if err != nil {
@@ -24,13 +23,9 @@ func (d *Daemon) handleAssociateIamInstanceProfile(msg *nats.Msg, command types.
respondWithJSON(msg, result)
}
// handleIamProfileDisassociate services the ec2.IamProfileAssociation.disassociate
// fan-out subject. Every daemon responds: the owner with the populated
// association after mutating vm.VM, non-owners (or daemons whose VM has a
// different account) with JSON null so the gateway's expectedNodes collector
// can exit before the timeout. Errors short-circuit the fan-out — only the
// owner can produce a meaningful error (e.g. persistence failure) since
// non-owners NoOp.
// handleIamProfileDisassociate services the disassociate fan-out. The owning
// daemon responds with the mutated association; non-owners respond with JSON
// null so the gateway's expectedNodes collector can exit early.
func (d *Daemon) handleIamProfileDisassociate(msg *nats.Msg) {
accountID := utils.AccountIDFromMsg(msg)
input := &ec2.DisassociateIamInstanceProfileInput{}
@@ -49,10 +44,9 @@ func (d *Daemon) handleIamProfileDisassociate(msg *nats.Msg) {
respondWithJSON(msg, result)
}
// handleIamProfileReplace services the ec2.IamProfileAssociation.replace
// fan-out subject. Same response contract as handleIamProfileDisassociate:
// every daemon always responds (JSON null on non-owners) so the gateway
// collector exits early when the cluster is healthy.
// handleIamProfileReplace services the replace fan-out. Same contract as
// handleIamProfileDisassociate: every daemon responds (JSON null on
// non-owners) so the gateway collector can exit early.
func (d *Daemon) handleIamProfileReplace(msg *nats.Msg) {
accountID := utils.AccountIDFromMsg(msg)
input := &ec2.ReplaceIamInstanceProfileAssociationInput{}
@@ -71,11 +65,9 @@ func (d *Daemon) handleIamProfileReplace(msg *nats.Msg) {
respondWithJSON(msg, result)
}
// handleIamProfileDescribe services the ec2.IamProfileAssociation.describe
// fan-out subject. Empty IamInstanceProfileAssociations slice is a valid
// response (no matches on this daemon) and counts toward the gateway's
// expectedNodes early-exit collector. The gateway concatenates per-daemon
// slices.
// handleIamProfileDescribe services the describe fan-out. An empty slice is a
// valid response (no matches on this daemon) and counts toward the gateway's
// expectedNodes collector; the gateway concatenates per-daemon slices.
func (d *Daemon) handleIamProfileDescribe(msg *nats.Msg) {
accountID := utils.AccountIDFromMsg(msg)
input := &ec2.DescribeIamInstanceProfileAssociationsInput{}
spinifex/daemon/daemon_system_dispatch.go
+8 -23
View File
@@ -31,18 +31,9 @@ type systemInstanceTerminateEnvelope struct {
Error string `json:"error,omitempty"`
}
// handleSystemLaunchInstance is the NATS subscriber for
// system.LaunchInstance.{type}[.{nodeID}] requests. The owning daemon (the
// one whose queue-group subscription wins, or the one whose nodeID-targeted
// subscription is hit) runs LaunchSystemInstance locally and the resulting
// VM stays bound to this node — see handleSystemTerminateInstance for the
// matching teardown path.
//
// The launch runs in its own goroutine so the subscription's delivery goroutine
// returns immediately: a multi-minute VM boot must not head-of-line block the
// host-pinned launch subject, or concurrent launches to the same node fail with
// 'no responders' even though the daemon is up (267.4). The NATS request/reply
// contract is preserved — the goroutine responds with the launch result.
// handleSystemLaunchInstance is the NATS subscriber for system.LaunchInstance.
// The launch runs in its own goroutine so a multi-minute VM boot cannot
// head-of-line block concurrent launches to the same node.
func (d *Daemon) handleSystemLaunchInstance(msg *nats.Msg) {
d.systemDispatchWg.Go(func() {
defer func() {
@@ -84,12 +75,9 @@ func (d *Daemon) serveSystemLaunchInstance(msg *nats.Msg) {
respondWithSystemLaunchOutput(msg, output)
}
// LaunchSystemInstanceOnNode launches a system VM on a specific Spinifex host.
// An empty nodeID or the local node runs the launch in-process (matching
// LaunchSystemInstance); any other node is reached over the node-targeted
// system.LaunchInstance.{type}.{nodeID} subject, where the receiving daemon
// runs the launch locally and binds the per-instance terminate subscription
// (handleSystemLaunchInstance). The VM stays bound to the node that runs it.
// LaunchSystemInstanceOnNode launches a system VM on a specific host.
// An empty nodeID or the local node runs in-process; any other node is
// reached via system.LaunchInstance.{type}.{nodeID} — the VM stays on that node.
func (d *Daemon) LaunchSystemInstanceOnNode(nodeID string, input *handlers_elbv2.SystemInstanceInput) (*handlers_elbv2.SystemInstanceOutput, error) {
if nodeID == "" || nodeID == d.node {
return d.LaunchSystemInstance(input)
@@ -156,11 +144,8 @@ func (d *Daemon) subscribeSystemTerminateLocked(instanceID string) error {
}
// handleSystemTerminateInstance is the NATS subscriber for
// system.TerminateInstance.{instanceID}. Only the daemon that owns the VM
// has a subscription on this subject — other daemons never see the request.
//
// Like the launch path, teardown runs in its own goroutine so a slow terminate
// never head-of-line blocks the responder (267.4).
// system.TerminateInstance.{instanceID}. Only the owning daemon subscribes.
// Teardown runs in its own goroutine to avoid head-of-line blocking.
func (d *Daemon) handleSystemTerminateInstance(msg *nats.Msg) {
d.systemDispatchWg.Go(func() {
defer func() {
spinifex/daemon/eks_adapter.go
+1 -7
View File
@@ -9,13 +9,7 @@ import (
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
)
// daemonEKSSubnetResolver adapts the daemon's VPCServiceImpl to the
// handlers_eks subnetVPCResolver-equivalent. handlers/eks declares the
// interface privately; the daemon needs an exported shape only because the
// constructor wires it via EKSServiceDeps.SubnetResolver = ...
//
// The translation is trivial: VPCService.GetSubnet returns a SubnetRecord
// with VpcId, which is the only field the EKS launcher needs.
// daemonEKSSubnetResolver adapts VPCServiceImpl to the EKS SubnetVPCResolver.
type daemonEKSSubnetResolver struct {
d *Daemon
}
spinifex/daemon/eks_deps.go
+6 -34
View File
@@ -11,17 +11,8 @@ import (
"log/slog"
)
// buildEKSServiceDeps assembles the EKSServiceDeps the EKS service needs to
// run CreateCluster/DescribeCluster/ListClusters/DeleteCluster end-to-end.
// All collaborators must already be initialized; callers invoke this from
// Daemon.startCluster after VPC, ELBv2, Instance, and Image services are
// ready.
//
// MasterKey is loaded best-effort from the config dir (the directory holding
// spinifex.toml — same place admin init writes master.key, and the same
// resolution daemon TLS + predastore config use). A missing key disables OIDC
// envelope encryption, which in turn fails CreateCluster at the
// depsReadyForOrchestration gate — the daemon logs and continues.
// buildEKSServiceDeps assembles the EKSServiceDeps. All collaborators must
// already be initialised. MasterKey is loaded best-effort from the config dir.
func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps {
masterKey, err := handlers_iam.LoadMasterKey(filepath.Join(filepath.Dir(d.configPath), "master.key"))
if err != nil {
@@ -30,10 +21,6 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps {
masterKey = nil
}
// CA PEM the K3s server VM uses to verify the AWS gateway's HTTPS cert when
// the eks-gateway-publish helper POSTs its bootstrap envelopes + state
// reports. The same CA signs the AWSGW server cert and the NATS server cert,
// so it is read from the configured NATS CA path.
gatewayCA := ""
if d.config.NATS.CACert != "" {
if caBytes, readErr := os.ReadFile(d.config.NATS.CACert); readErr == nil {
@@ -68,22 +55,9 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps {
}
}
// resolveGatewayHost is the single source of truth for the off-host-reachable
// address LB VMs, EKS OIDC issuers, and the EKS NATS URL all dial. startCluster
// and buildEKSServiceDeps both call it so the OIDC issuer host can never
// diverge from the host the lb-agent actually reaches (M7). Precedence:
//
// 1. br-mgmt present + AWSGW on a dedicated non-loopback IP distinct from
// AdvertiseIP (multi-node: AWSGW on a mgmt-only IP, VPC path can't reach
// it) → the AWSGW bind IP. startCluster additionally adds a bootcmd host
// route via br-mgmt for this case only.
// 2. AdvertiseIP set → AdvertiseIP (single-node, or multi-node where AWSGW
// binds the advertised IP; VMs reach it via VPC → external).
// 3. br-mgmt present + AWSGW on 0.0.0.0, no AdvertiseIP → the br-mgmt IP.
// 4. DevNetworking shim → 10.0.2.2.
// 5. AWSGW bound to a specific IP (no br-mgmt, no advertise) → that IP.
//
// Returns "" if no reachable host can be derived.
// resolveGatewayHost returns the single AWSGW host used by LB VMs, EKS OIDC
// issuers, and the EKS NATS URL. Precedence: mgmt-dedicated AWSGW IP →
// AdvertiseIP → br-mgmt IP → DevNetworking → AWSGW bind IP. Returns "".
func (d *Daemon) resolveGatewayHost() string {
awsgwBindIP := ""
if d.config.AWSGW.Host != "" {
@@ -109,9 +83,7 @@ func (d *Daemon) resolveGatewayHost() string {
return ""
}
// resolveGatewayBaseURL is the HTTPS AWSGW endpoint EKS OIDC issuers are built
// from. Returns "" if no reachable host can be derived, in which case
// CreateCluster fails at depsReadyForOrchestration.
// resolveGatewayBaseURL returns the HTTPS AWSGW base URL for EKS OIDC.
func (d *Daemon) resolveGatewayBaseURL() string {
host := d.resolveGatewayHost()
if host == "" {
spinifex/daemon/eks_worker_launch.go
+4 -10
View File
@@ -15,12 +15,8 @@ import (
// Compile-time check that Daemon satisfies the EKS worker-launch surface.
var _ handlers_eks.WorkerLauncher = (*Daemon)(nil)
// RunWorkerInstance launches a nodegroup worker through the normal customer
// RunInstances path: customer-owned, primary ENI auto-created in input.SubnetId
// with input.SecurityGroupIds, no ManagedBy tag, no mgmt NIC — so the worker is
// visible in DescribeInstances and reclaimed by TerminateInstances like any
// other customer EC2. The EKS service hands us raw cloud-config YAML in
// input.UserData; RunInstances expects it base64-encoded, so wrap it here.
// RunWorkerInstance launches a nodegroup worker via the normal RunInstances path.
// UserData is base64-encoded before forwarding; the worker is customer-visible.
func (d *Daemon) RunWorkerInstance(input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
if d.instanceService == nil {
return nil, errors.New("eks worker: instance service not initialized")
@@ -34,10 +30,8 @@ func (d *Daemon) RunWorkerInstance(input *ec2.RunInstancesInput, accountID strin
return d.instanceService.RunInstances(input, accountID)
}
// TerminateWorkerInstances terminates each nodegroup worker via the same path
// the EC2 NATS terminate handler uses. A worker no longer tracked by the local
// vmMgr is treated as already-gone (idempotent success) so a retried
// DeleteNodegroup / scale-down does not wedge on instances that already drained.
// TerminateWorkerInstances terminates nodegroup workers. Already-gone instances
// are treated as success so retried DeleteNodegroup calls are idempotent.
func (d *Daemon) TerminateWorkerInstances(instanceIDs []string, accountID string) error {
if d.instanceService == nil {
return errors.New("eks worker: instance service not initialized")
spinifex/daemon/local_state.go
+9 -31
View File
@@ -12,9 +12,7 @@ import (
)
const (
// LocalStateSchemaVersion is the on-disk schema version for instance-state.json.
// Bump on any breaking change to the LocalState shape; daemon refuses to start
// on an unknown version rather than silently losing data.
// LocalStateSchemaVersion guards on-disk compatibility; bump on breaking changes.
LocalStateSchemaVersion = 1
// DefaultLocalStateDir is the default directory under DataDir for local state files.
@@ -34,11 +32,8 @@ type LocalState struct {
VMS map[string]*vm.VM `json:"vms"`
}
// LocalStatePath returns the absolute path to the per-node instance state file
// rooted at dataDir. Empty dataDir falls back to DefaultDataDir
// (/var/lib/spinifex), the platform-shared root. The state directory sits at
// the platform root so it lives next to other shared state (predastore,
// viperblock data dirs) rather than nested under the daemon's own subdir.
// LocalStatePath returns the absolute path to the per-node instance state file.
// Empty dataDir falls back to DefaultDataDir.
func LocalStatePath(dataDir string) string {
if dataDir == "" {
dataDir = DefaultDataDir
@@ -46,11 +41,8 @@ func LocalStatePath(dataDir string) string {
return filepath.Join(dataDir, DefaultLocalStateDir, LocalStateFileName)
}
// MarshalLocalState produces the JSON wire form of vms wrapped with the schema
// version. Callers that already hold a stable view of vms (e.g. inside a
// vm.Manager.View callback) can marshal under the lock and pass the bytes to
// WriteLocalStateBytes — that avoids racing json.Marshal against concurrent
// VM-field mutations.
// MarshalLocalState produces the JSON wire form of vms with schema version.
// Call inside a vm.Manager.View callback to avoid races on VM fields.
func MarshalLocalState(vms map[string]*vm.VM) ([]byte, error) {
state := LocalState{
SchemaVersion: LocalStateSchemaVersion,
@@ -63,14 +55,8 @@ func MarshalLocalState(vms map[string]*vm.VM) ([]byte, error) {
return data, nil
}
// WriteLocalState atomically writes the instance state to path. vms must be a
// snapshot owned by the caller (e.g. from vm.Manager.SnapshotMap) — this
// function does not lock. Convenience wrapper around MarshalLocalState +
// WriteLocalStateBytes for callers that own an exclusive snapshot.
//
// Atomicity: marshal → write to <path>.tmp → fsync → rename. The rename is
// atomic on POSIX so a concurrent reader sees either the old file or the new
// one, never a half-written file.
// WriteLocalState atomically writes instance state via marshal → tmp → fsync →
// rename. vms must be a caller-owned snapshot; this function does not lock.
func WriteLocalState(path string, vms map[string]*vm.VM) error {
data, err := MarshalLocalState(vms)
if err != nil {
@@ -80,8 +66,6 @@ func WriteLocalState(path string, vms map[string]*vm.VM) error {
}
// WriteLocalStateBytes atomically writes pre-marshalled state JSON to path.
// Used by hot paths that marshal under a short-lived lock and then commit to
// disk lock-free.
func WriteLocalStateBytes(path string, data []byte) error {
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0o750); err != nil {
@@ -119,14 +103,8 @@ func WriteLocalStateBytes(path string, data []byte) error {
return nil
}
// ReadLocalState reads the per-node instance state from path.
//
// Returns (nil, nil) if the file does not exist — fresh-install signal,
// caller should start with an empty instance map.
//
// Returns an error on:
// - JSON parse failure (corruption — caller refuses start)
// - Unknown SchemaVersion (caller refuses start; safer than silent migration)
// ReadLocalState reads instance state from path. Returns (nil, nil) if the
// file does not exist (fresh install). Errors on JSON failure or unknown schema.
func ReadLocalState(path string) (*LocalState, error) {
data, err := os.ReadFile(path)
if err != nil {
spinifex/daemon/ownership.go
+6 -14
View File
@@ -8,14 +8,11 @@ import (
"github.com/nats-io/nats.go"
)
// checkInstanceOwnership verifies the caller owns the instance.
// Returns true if access is allowed, false if denied (error already sent).
// Instances with an empty AccountID (legacy/migration data) are only
// visible to root (GlobalAccountID).
// checkInstanceOwnership verifies the caller owns the instance. Returns true if
// allowed; false after sending an error. Empty ownerAccountID is root-only.
func checkInstanceOwnership(msg *nats.Msg, instanceID, ownerAccountID string) bool {
callerAccountID := utils.AccountIDFromMsg(msg)
// Untenanted instance: only root can access.
if ownerAccountID == "" {
if callerAccountID != utils.GlobalAccountID {
slog.Warn("Untenanted instance access denied (not root)",
@@ -26,7 +23,6 @@ func checkInstanceOwnership(msg *nats.Msg, instanceID, ownerAccountID string) bo
return true
}
// Normal ownership check
if callerAccountID != ownerAccountID {
slog.Warn("Account does not own instance",
"instanceId", instanceID, "callerAccount", callerAccountID, "ownerAccount", ownerAccountID)
@@ -36,9 +32,8 @@ func checkInstanceOwnership(msg *nats.Msg, instanceID, ownerAccountID string) bo
return true
}
// isInstanceVisible checks if the caller can see this instance (for Describe operations).
// Instances with an empty AccountID (legacy/migration data) are only
// visible to root (GlobalAccountID).
// isInstanceVisible reports whether the caller may see this instance.
// Empty ownerAccountID is root-only.
func isInstanceVisible(callerAccountID, ownerAccountID string) bool {
if ownerAccountID == "" {
return callerAccountID == utils.GlobalAccountID
@@ -46,11 +41,8 @@ func isInstanceVisible(callerAccountID, ownerAccountID string) bool {
return callerAccountID == ownerAccountID
}
// volumeVisibleTo reports whether callerAccountID may operate on a volume with
// the given tenantID. Volumes with an empty tenantID (legacy/migration
// data) are root-only — without this, the short-circuit
// `tenantID != "" && tenantID != caller` matches every caller and lets any
// tenant attach a legacy/migration volume by ID alone.
// volumeVisibleTo reports whether callerAccountID may access a volume.
// Empty tenantID is root-only to prevent untenanted volumes from leaking.
func volumeVisibleTo(tenantID, callerAccountID string) bool {
if tenantID == "" {
return callerAccountID == utils.GlobalAccountID
spinifex/daemon/peer_health.go
+8 -24
View File
@@ -11,10 +11,8 @@ import (
"github.com/mulgadc/spinifex/spinifex/config"
)
// Peer-health probe cadence. Worst-case detection of total partition is
// peerProbeInterval + (peers × peerProbeTimeout); on a 3-node cluster with
// the current settings that lands ~6s after the partition installs — well
// inside the 30s budget DDIL Scenario C asserts on.
// Peer-health probe cadence. Worst-case partition detection on a 3-node
// cluster is ~6s — within the 30s DDIL Scenario C budget.
const (
peerProbeInterval = 2 * time.Second
peerProbeTimeout = 2 * time.Second
@@ -49,14 +47,8 @@ func (d *Daemon) peerNodes() []config.Config {
return out
}
// monitorPeerReachability polls every peer's /health endpoint over the
// cluster network and updates peersReachable. Runs until d.ctx is cancelled.
// No-op on single-node clusters: peersReachable is pinned true in NewDaemon
// and there is nothing to probe.
//
// Probes are issued serially per tick because (a) typical cluster size is 2–3
// peers, (b) any peer responding is enough to declare reachability, and
// (c) serial dialling caps in-flight HTTPS connections during steady state.
// monitorPeerReachability polls every peer's /health and updates peersReachable.
// No-op on single-node. Probes are serial: any one responding suffices.
func (d *Daemon) monitorPeerReachability() {
peers := d.peerNodes()
if len(peers) == 0 {
@@ -99,23 +91,15 @@ func (d *Daemon) probePeersOnce(client *http.Client, peers []config.Config) {
if prev != reachable {
slog.Info("peer reachability changed", "reachable", reachable, "peers", len(peers))
if reachable {
// Scenario C heal edge: local NATS client stayed connected
// to its local server throughout the partition so
// onNATSReconnect never fires. The peer-probe flip is the
// only signal that the cluster is back. Goroutine keeps
// the probe ticker non-blocking; reconcileOnHeal coalesces
// with any concurrent NATS-reconnect path.
// DDIL Scenario C: NATS stayed connected, so this flip is
// the only heal signal. Goroutine keeps the ticker non-blocking.
go d.reconcileOnHeal("peer-probe-heal")
}
}
}
// peerDaemonPort returns the TCP port to dial on peer p. The toml template
// only renders [nodes.<self>.daemon] for the local node — remote-node blocks
// carry just `host = "<ip>"` — so a peer's own DaemonConfig is typically
// empty. Clusters are symmetric, so this falls back to the local daemon's
// bound port. Returns "" only when neither the peer nor self has a parseable
// host:port.
// peerDaemonPort returns the TCP port to dial on peer p. Falls back to the
// local daemon's port since remote-node config blocks typically omit it.
func (d *Daemon) peerDaemonPort(p config.Config) string {
if p.Daemon.Host != "" {
if _, port, err := net.SplitHostPort(p.Daemon.Host); err == nil && port != "" {
spinifex/daemon/reconcile.go
+5 -27
View File
@@ -4,19 +4,8 @@ import (
"log/slog"
)
// reconcileOnHeal runs the post-heal resync. Fires from two edges:
// - onNATSReconnect, when the local NATS client reattaches.
// - probePeersOnce, when peer reachability flips false→true (the only
// signal in Scenario C where the NATS client stayed connected to its
// local server throughout the partition).
//
// Both edges may fire near-simultaneously on the same heal. reconciling
// coalesces them: the second caller observes the flag set and returns,
// keeping the work to one round-trip.
//
// The caller is expected to invoke this in a goroutine — both call sites
// run on latency-sensitive callbacks (NATS client thread, peer-probe
// ticker) and the reconcile path performs KV I/O.
// reconcileOnHeal runs the post-heal resync (WriteState to KV). Coalesces
// concurrent calls via reconciling flag. Must be called in a goroutine.
func (d *Daemon) reconcileOnHeal(reason string) {
if d.jsManager == nil {
return
@@ -29,25 +18,14 @@ func (d *Daemon) reconcileOnHeal(reason string) {
slog.Info("reconcileOnHeal: starting", "reason", reason, "node", d.node)
// Push fresh local running-instance state. This re-establishes our
// row in the cluster-wide instance-state bucket regardless of what
// it held mid-partition (stale, missing, or clobbered by a peer's
// stream snapshot restore).
if err := d.WriteState(); err != nil {
slog.Warn("reconcileOnHeal: WriteState failed", "reason", reason, "error", err)
return
}
// NOTE: an earlier revision called d.ensureDefaultVPCInfrastructure()
// here. DDIL Scenario C regressed because the very first peer-probe
// tick on multi-node startup flips peersReachable false→true and
// triggers this path concurrently with startCluster's own
// ensureDefaultVPCInfrastructure call. The Describe→Create→Attach
// sequence is not cluster-wide singleton, so multiple daemons +
// double calls per daemon could race and produce duplicate IGW
// attachments, corrupting default-VPC routing. Heal-time bootstrap
// re-fire needs leader election or a debounce against the
// startup-time path before re-introducing.
// ensureDefaultVPCInfrastructure is intentionally omitted: the peer-probe
// heal edge and startCluster race without singleton protection and can
// produce duplicate IGW attachments. Re-introduce only with leader election.
slog.Info("reconcileOnHeal: complete", "reason", reason, "revision", d.Revision())
}
spinifex/daemon/resource.go
+27 -84
View File
@@ -17,31 +17,19 @@ import (
// cannot be satisfied.
var errInsufficientCapacity = errors.New("insufficient capacity to satisfy MinCount")
// hostReserve is the host CPU and RAM held back from guest scheduling for the
// always-on infrastructure tier only — the daemon and co-located fixed services
// (NATS, predastore, viperblock, vpcd, awsgw, ui). It deliberately does NOT
// cover per-instance consumers such as the nbdkit process backing a guest's
// volumes: those scale with instance count and disk I/O, so folding them into a
// fixed reserve makes it unsizable. Per-instance cost is charged to the instance
// and bounded by the live-memory admission gate (see liveMemGate). Tunable via
// SPINIFEX_RESERVED_VCPU / SPINIFEX_RESERVED_MEM_GB until a `capacity` command
// lifts it into config.
// hostReserve holds CPU and RAM reserved for the daemon and co-located fixed
// services. Per-instance costs (nbdkit) are charged separately. Tunable via
// SPINIFEX_RESERVED_VCPU / SPINIFEX_RESERVED_MEM_GB.
type hostReserve struct {
vCPU int
memGB float64
}
// defaultHostReserve covers the fixed infrastructure tier (predastore +
// viperblock storage VMs under load, plus the daemon/NATS/awsgw/vpcd/ui stack).
// Size it from the measured fixed-tier footprint of the target host via
// SPINIFEX_RESERVED_MEM_GB; the default is a conservative floor, not a ceiling.
// defaultHostReserve is the conservative floor for the fixed infrastructure tier.
var defaultHostReserve = hostReserve{vCPU: 2, memGB: 2.0}
// resolveHostReserve returns defaultHostReserve, with vCPU/memGB overridden
// when SPINIFEX_RESERVED_VCPU / SPINIFEX_RESERVED_MEM_GB are set to valid
// non-negative values. Invalid values are logged and ignored — keeps a
// typo'd env from silently widening the reserve. Intended as a stop-gap
// until the planned operator-tunable [capacity] config lands.
// resolveHostReserve returns defaultHostReserve overridden by
// SPINIFEX_RESERVED_VCPU / SPINIFEX_RESERVED_MEM_GB when valid.
func resolveHostReserve(getenv func(string) string) hostReserve {
r := defaultHostReserve
if v := getenv("SPINIFEX_RESERVED_VCPU"); v != "" {
@@ -61,13 +49,8 @@ func resolveHostReserve(getenv func(string) string) hostReserve {
return r
}
// resolveHostVCPU returns the detected core count, overridden by
// SPINIFEX_HOST_VCPU when set to a valid positive integer. The override is an
// escape hatch: it pins the schedulable core count for hosts where
// /proc/cpuinfo topology detection misreports (some VMs/containers expose no
// "core id", or fewer physical cores than the host can actually schedule), and
// lets tests decouple from runner topology. Invalid values are logged and
// ignored so a typo can't silently shrink capacity below the reserve.
// resolveHostVCPU returns detected core count, overridden by SPINIFEX_HOST_VCPU
// when valid. Override is an escape hatch for hosts with misreported topology.
func resolveHostVCPU(getenv func(string) string, detected int) int {
if v := getenv("SPINIFEX_HOST_VCPU"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
@@ -78,19 +61,11 @@ func resolveHostVCPU(getenv func(string) string, detected int) int {
return detected
}
// minHostMemHeadroomGB is the minimum schedulable memory we require above
// the reserve, so a host that just meets the reserve still has a small
// amount left to launch the smallest guest type.
// minHostMemHeadroomGB is the minimum schedulable memory above the reserve.
const minHostMemHeadroomGB = 0.5
// applyHostReserve validates that the host meets the minimum size for the
// given reserve and returns the reserve to apply. Pure function — no locks
// or side effects. Exists as a helper for unit-testability of the
// validation bounds.
//
// vCPU and memory are checked separately so the returned error names the
// failing dimension, letting alerting/log filters distinguish a CPU
// shortfall from a memory shortfall.
// applyHostReserve validates host capacity against the reserve and returns the
// reserve to apply. Checks CPU and memory separately for clear error messages.
func applyHostReserve(host hostReserve, totalVCPU int, totalMemGB float64) (vcpu int, mem float64, err error) {
if totalVCPU <= host.vCPU {
return 0, 0, fmt.Errorf(
@@ -107,15 +82,8 @@ func applyHostReserve(host hostReserve, totalVCPU int, totalMemGB float64) (vcpu
return host.vCPU, host.memGB, nil
}
// canAllocateCount returns how many instances of the given type can fit
// in the remaining capacity, capped at maxCount. Pure aside from a single
// slog.Error when remaining capacity is negative — that condition would
// otherwise be silently clamped to zero, hiding a misconfigured reserve
// or allocation accounting drift.
//
// availGPU is the number of free GPUs in the pool; requiresGPU indicates
// that this instance type needs one. When requiresGPU is true and availGPU
// is zero, the result is always 0.
// canAllocateCount returns how many instances of the given type fit in remaining
// capacity, capped at maxCount. Logs an error if capacity is negative.
func canAllocateCount(availVCPU, allocVCPU int, availMem, allocMem float64,
vCPUs int64, memMiB int64, maxCount int,
availGPU int, requiresGPU bool) int {
@@ -150,11 +118,8 @@ func canAllocateCount(availVCPU, allocVCPU int, availMem, allocMem float64,
return max(result, 0)
}
// liveMemCount clamps an accounting-derived launch count n by live host memory:
// how many guests of memGB fit in (availMemGB − reservedMemGB) right now, never
// more than n. Pure function — the live read and instance-type lookup happen in
// liveMemGate. A negative headroom (host already below the reserve) yields 0, so
// admission refuses rather than admitting into OOM territory.
// liveMemCount clamps n by how many guests of memGB fit in live headroom
// (availMemGB − reservedMemGB). Negative headroom yields 0.
func liveMemCount(n int, availMemGB, reservedMemGB, memGB float64) int {
if memGB <= 0 {
return n
@@ -163,10 +128,8 @@ func liveMemCount(n int, availMemGB, reservedMemGB, memGB float64) int {
return max(0, min(n, byLive))
}
// liveMemAdmissionEnabled reports whether the live-memory admission gate is on.
// Default on; SPINIFEX_ADMISSION_LIVE_MEM set to a false-y value ("0", "false",
// "off", "no") disables it, falling back to pure accounting — a rollback hatch
// if the live gate proves too conservative on a busy host.
// liveMemAdmissionEnabled reports whether the live-memory gate is enabled.
// Set SPINIFEX_ADMISSION_LIVE_MEM=0/false/off/no to disable.
func liveMemAdmissionEnabled(getenv func(string) string) bool {
switch strings.ToLower(strings.TrimSpace(getenv("SPINIFEX_ADMISSION_LIVE_MEM"))) {
case "0", "false", "off", "no":
@@ -176,13 +139,8 @@ func liveMemAdmissionEnabled(getenv func(string) string) bool {
}
}
// readMemAvailableGB returns the host's current /proc/meminfo MemAvailable in
// GB. ok=false on non-Linux hosts or any read/parse failure, which disables the
// live gate for that call (fail open). MemAvailable is the kernel's own estimate
// of memory available for new workloads without swapping, already netting
// reclaimable page cache — the figure admission should gate on, and one that
// already reflects co-located storage VMs, nbdkit, and the daemon stack that the
// static -m accounting never sees.
// readMemAvailableGB returns /proc/meminfo MemAvailable in GB. Returns ok=false
// on non-Linux or read failure (fail open — accounting gate still applies).
func readMemAvailableGB() (float64, bool) {
if runtime.GOOS != "linux" {
return 0, false
@@ -200,9 +158,7 @@ func readMemAvailableGB() (float64, bool) {
return float64(kb) / (1024 * 1024), true
}
// parseMemAvailableKB extracts the MemAvailable value (in kB) from /proc/meminfo
// contents. Returns ok=false when the field is absent or unparseable. Pure —
// split out for unit-testability without touching the filesystem.
// parseMemAvailableKB extracts MemAvailable (kB) from /proc/meminfo. Pure.
func parseMemAvailableKB(data []byte) (int64, bool) {
for line := range strings.SplitSeq(string(data), "\n") {
key, val, ok := strings.Cut(line, ":")
@@ -222,29 +178,20 @@ func parseMemAvailableKB(data []byte) (int64, bool) {
return 0, false
}
// nbdkit runs one process per attached volume (spinifex/nbd). Measured on a live
// host: a main volume's nbdkit climbs to ~0.75 GB under guest I/O (128 MiB cache
// + plugin working set + in-flight buffers); aux volumes (-cloudinit, -efi; 0
// cache) stay flat ~32-68 MB. Charged per instance so a busy nbdkit can't
// overcommit memory the static -m budget never saw. Env-tunable; RG-6.
// Default per-volume nbdkit memory charges: main ~0.75 GB under I/O, aux ~96 MiB.
// Env-tunable via SPINIFEX_NBDKIT_{MAIN,AUX}_MIB.
const (
defaultNbdkitMainMiB = 768
defaultNbdkitAuxMiB = 96
)
// defaultMainVolumes / defaultAuxVolumes is the volume layout every instance
// gets: one main root volume plus -cloudinit and -efi aux volumes. Block-device
// mappings add main volumes the static charge can't see at admission; the live
// MemAvailable gate (RG-8) is the backstop for those. RG-6.
// Default volume layout: one main root volume + two aux volumes (cloud-init, efi).
const (
defaultMainVolumes = 1
defaultAuxVolumes = 2
)
// resolveNbdkitCharge returns the per-volume nbdkit memory charge in MiB, with
// main/aux overridable via SPINIFEX_NBDKIT_MAIN_MIB / SPINIFEX_NBDKIT_AUX_MIB so
// an operator can retune from their own nbdkit RSS characterisation without a
// rebuild. Invalid or negative values are logged and ignored.
// resolveNbdkitCharge returns per-volume nbdkit charges, overridable via env.
func resolveNbdkitCharge(getenv func(string) string) (mainMiB, auxMiB int) {
mainMiB, auxMiB = defaultNbdkitMainMiB, defaultNbdkitAuxMiB
if v := getenv("SPINIFEX_NBDKIT_MAIN_MIB"); v != "" {
@@ -264,17 +211,13 @@ func resolveNbdkitCharge(getenv func(string) string) (mainMiB, auxMiB int) {
return mainMiB, auxMiB
}
// nbdkitChargeMiB is the per-instance nbdkit charge — one process per volume over
// the default layout (mainVols main + auxVols aux). Pure; RG-6.
// nbdkitChargeMiB returns the total nbdkit memory charge for mainVols + auxVols.
func nbdkitChargeMiB(mainVols, auxVols, mainMiB, auxMiB int) int64 {
return int64(mainVols*mainMiB + auxVols*auxMiB)
}
// resourceStatsForType computes the InstanceTypeCap for a single instance type
// given the remaining host resources. Pure function — no locks or side effects.
// Callers are responsible for alarming on negative remainVCPU/remainMem
// (see canAllocateCount); resourceStatsForType silently clamps to zero
// because it's invoked in a per-type loop and would otherwise log N times.
// resourceStatsForType computes the InstanceTypeCap for a single type given
// remaining resources. Clamps negative capacity to zero without logging.
func resourceStatsForType(remainVCPU int, remainMem float64, it *ec2.InstanceTypeInfo) types.InstanceTypeCap {
vCPUs := instanceTypeVCPUs(it)
memGB := float64(instanceTypeMemoryMiB(it)) / 1024.0
spinifex/daemon/state.go
+2 -6
View File
@@ -8,12 +8,8 @@ import (
)
// TransitionState validates and applies a state transition on the given instance.
// It sets instance.Status, persists via WriteState, and logs.
// On validation failure, the instance status is unchanged and a wrapped
// vm.ErrInvalidTransition is returned so callers can map it to the AWS
// IncorrectInstanceState error code via errors.Is.
// If WriteState fails, the in-memory status retains the new value (the VM has
// physically changed state regardless) and an error is returned.
// Returns vm.ErrInvalidTransition on bad transitions; WriteState failure is also
// returned but in-memory status is kept since the VM has already changed state.
func (d *Daemon) TransitionState(instance *vm.VM, target vm.InstanceState) error {
var (
current vm.InstanceState
spinifex/daemon/vm_adapters.go
+14 -44
View File
@@ -18,14 +18,8 @@ import (
"github.com/nats-io/nats.go"
)
// Thin adapters that satisfy the vm.Manager collaborator interfaces. The
// bodies delegate to existing daemon methods or services; the manager
// receives them via vm.Deps so it can drive lifecycle without importing the
// daemon package.
// stateStoreAdapter satisfies vm.StateStore by delegating to the cluster
// JetStream manager. It also tolerates a nil JetStreamManager during early
// boot so manager construction can happen before initJetStream returns.
// stateStoreAdapter satisfies vm.StateStore by delegating to JetStreamManager.
// Tolerates nil JetStreamManager during early boot.
type stateStoreAdapter struct {
js *JetStreamManager
}
@@ -69,11 +63,7 @@ func (a *stateStoreAdapter) ListTerminatedInstances() ([]*vm.VM, error) {
}
// volumeMounterAdapter satisfies vm.VolumeMounter by routing ebs.mount /
// ebs.unmount NATS requests, mirroring Daemon.MountVolumes and
// Daemon.unmountInstanceVolumes. The mount path mutates
// instance.EBSRequests.Requests[k].NBDURI; unmount also flips boot/data
// volumes back to "available" via VolumeStateUpdater so the AWS API view
// stays consistent.
// ebs.unmount NATS requests. Unmount also updates volume state via VolumeStateUpdater.
type volumeMounterAdapter struct {
nc *nats.Conn
node string
@@ -205,11 +195,7 @@ func (a *volumeMounterAdapter) MountOne(req *types.EBSRequest) error {
return nil
}
// UnmountOne sends ebs.unmount for a single request. Best-effort: errors
// are logged and tolerated. Shared by AttachVolume rollback and the
// DetachVolume ebs.unmount step where the caller cannot meaningfully act
// on a failed unmount. Internal callers that need to surface rollback
// failures use unmountOne directly.
// UnmountOne sends ebs.unmount; errors are logged and swallowed.
func (a *volumeMounterAdapter) UnmountOne(req types.EBSRequest) {
if err := a.unmountOne(req); err != nil {
slog.Error("UnmountOne failed", "volume", req.Name, "err", err)
@@ -218,8 +204,7 @@ func (a *volumeMounterAdapter) UnmountOne(req types.EBSRequest) {
slog.Info("UnmountOne: volume unmounted successfully", "volume", req.Name)
}
// unmountOne sends ebs.unmount and returns any error so callers (e.g. the
// Mount partial-failure rollback) can wrap it into a richer failure.
// unmountOne sends ebs.unmount and returns any error.
func (a *volumeMounterAdapter) unmountOne(req types.EBSRequest) error {
payload, err := json.Marshal(req)
if err != nil {
@@ -242,10 +227,8 @@ func (a *volumeMounterAdapter) unmountOne(req types.EBSRequest) error {
return nil
}
// instanceTypeResolverAdapter satisfies vm.InstanceTypeResolver. It looks up
// the EC2 InstanceTypeInfo on the daemon's ResourceManager and projects the
// QEMU-relevant numbers (vCPU, memory MiB, architecture) into the
// SDK-agnostic vm.InstanceTypeSpec.
// instanceTypeResolverAdapter satisfies vm.InstanceTypeResolver by projecting
// EC2 instance type info into the SDK-agnostic vm.InstanceTypeSpec.
type instanceTypeResolverAdapter struct {
rm *ResourceManager
}
@@ -272,9 +255,7 @@ func (a *instanceTypeResolverAdapter) Resolve(name string) (vm.InstanceTypeSpec,
}, true
}
// resourceControllerAdapter satisfies vm.ResourceController. Both Allocate
// and Deallocate look up the EC2 InstanceTypeInfo by name before delegating
// to the existing capacity-keyed resource manager methods.
// resourceControllerAdapter satisfies vm.ResourceController.
type resourceControllerAdapter struct {
rm *ResourceManager
}
@@ -309,13 +290,8 @@ func (a *resourceControllerAdapter) CanAllocate(instanceType string, count int)
return a.rm.canAllocate(it, count)
}
// onInstanceRecoveringHook returns the daemon's OnInstanceRecovering
// callback. Restore fires it once per instance about to be relaunched so
// concurrent terminate commands can land on this node before launch
// completes — without it, ec2.cmd.<id> would only be subscribed by
// onInstanceUpHook after launch success, leaving a window where the
// instance is reachable by DescribeInstances (in StatePending) but not
// by EC2 commands.
// onInstanceRecoveringHook subscribes ec2.cmd.<id> early during Restore so
// terminate commands can land before the relaunch completes.
func (d *Daemon) onInstanceRecoveringHook() func(*vm.VM) {
return func(instance *vm.VM) {
d.mu.Lock()
@@ -334,11 +310,8 @@ func (d *Daemon) onInstanceRecoveringHook() func(*vm.VM) {
}
}
// consumeCleanShutdownMarker returns the daemon's
// ConsumeCleanShutdownMarker callback. Returns true (and deletes the
// marker) when a clean shutdown was recorded for this node on the
// previous run; returns false otherwise so Restore takes the cautious
// "validate stale PIDs" path.
// consumeCleanShutdownMarker returns the ConsumeCleanShutdownMarker callback.
// Returns true and deletes the marker when a clean shutdown was recorded.
func (d *Daemon) consumeCleanShutdownMarker() func() bool {
return func() bool {
if d.jsManager == nil {
@@ -354,11 +327,8 @@ func (d *Daemon) consumeCleanShutdownMarker() func() bool {
}
}
// onInstanceUpHook returns the daemon's OnInstanceUp callback. Subscribing
// per-instance NATS topics is the only side-effect; the manager fires this
// synchronously after a successful Pending→Running transition. Returns the
// first subscribe error so the manager (specifically reconnectInstance) can
// roll back QMP rather than persist a half-reachable instance to KV.
// onInstanceUpHook subscribes per-instance NATS topics after Pending→Running.
// Returns the first subscribe error so the manager can roll back QMP.
func (d *Daemon) onInstanceUpHook() func(*vm.VM) error {
return func(instance *vm.VM) error {
d.mu.Lock()
spinifex/filterutil/filterutil.go
+7 -15
View File
@@ -8,11 +8,8 @@ import (
"github.com/aws/aws-sdk-go/service/ec2"
)
// ParseFilters converts AWS SDK filter types into a map[string][]string for
// easier matching. validNames defines the set of accepted filter names.
// Filter names starting with "tag:" are always accepted. Returns an
// InvalidParameterValue-style error if an unsupported filter name is
// encountered.
// ParseFilters converts AWS SDK filter types to map[string][]string.
// tag: prefixed names are always accepted; other names must be in validNames or an error is returned.
func ParseFilters(filters []*ec2.Filter, validNames map[string]bool) (map[string][]string, error) {
if len(filters) == 0 {
return nil, nil
@@ -54,10 +51,8 @@ func MatchesAny(filterValues []string, value string) bool {
return false
}
// MatchesTags checks whether a resource's tags satisfy all tag:Key filters
// present in the filter map. Each tag:Key filter uses OR logic across its
// values — the resource must have the tag and its value must match at least
// one filter value (with wildcard support).
// MatchesTags checks whether a resource's tags satisfy all tag:Key filters in the map.
// Each tag:Key filter uses OR logic across its values, with wildcard support.
func MatchesTags(filters map[string][]string, tags map[string]string) bool {
for name, values := range filters {
if !strings.HasPrefix(name, "tag:") {
@@ -89,9 +84,8 @@ func EC2TagsToMap(tags []*ec2.Tag) map[string]string {
return m
}
// MatchWildcard matches value against a pattern where * matches zero or more
// characters. Case-sensitive — callers needing case-insensitive matching
// (e.g. IAM policy evaluation) should lower-case both inputs before calling.
// MatchWildcard matches value against a pattern where * matches zero or more characters.
// Case-sensitive; callers needing case-insensitive matching should lower-case both inputs.
func MatchWildcard(pattern, value string) bool {
if pattern == "*" {
return true
@@ -103,7 +97,6 @@ func MatchWildcard(pattern, value string) bool {
parts := strings.Split(pattern, "*")
last := len(parts) - 1
// Check that value starts with the first part and ends with the last.
if !strings.HasPrefix(value, parts[0]) {
return false
}
@@ -111,8 +104,7 @@ func MatchWildcard(pattern, value string) bool {
return false
}
// Trim prefix and suffix from the search region so middle parts
// cannot consume characters reserved for the anchored ends.
// Trim anchored ends before scanning middle parts.
remaining := value[len(parts[0]):]
if len(remaining) < len(parts[last]) {
return false
spinifex/formation/formation.go
+3 -4
View File
@@ -99,10 +99,9 @@ type SharedCredentials struct {
AdminSecretKey string `json:"admin_secret_key,omitempty"`
}
// FormationServer is a lightweight HTTP server that coordinates cluster formation.
// Nodes register themselves via POST /formation/join. Once the expected number of
// nodes have joined, the done channel is closed and full cluster data (credentials,
// CA, node list) becomes available via GET /formation/status.
// FormationServer is a lightweight HTTPS server that coordinates cluster formation.
// Nodes register via POST /formation/join; full cluster data is available via
// GET /formation/status once the expected count is reached.
type FormationServer struct {
mu sync.RWMutex
expected int
spinifex/formation/helpers.go
+4 -8
View File
@@ -16,11 +16,9 @@ func hasService(services []string, name string) bool {
return slices.Contains(services, name)
}
// BuildClusterRoutes returns a sorted list of "IP:4248" NATS cluster routes
// for nodes that run the "nats" service. If ClusterIP is set it is used,
// otherwise BindIP. Nodes are sorted by name for deterministic ordering.
// BuildClusterRoutes returns sorted "IP:4248" NATS cluster routes for nodes
// running the "nats" service. ClusterIP is preferred over BindIP.
func BuildClusterRoutes(nodes map[string]NodeInfo) []string {
// Filter to nodes that run NATS
natsNodes := make(map[string]NodeInfo)
for k, n := range nodes {
if hasService(n.Services, "nats") {
@@ -39,11 +37,9 @@ func BuildClusterRoutes(nodes map[string]NodeInfo) []string {
return routes
}
// BuildPredastoreNodes returns a sorted list of PredastoreNodeConfig with
// 1-based IDs for nodes that run the "predastore" service.
// Nodes are sorted by name for deterministic ordering.
// BuildPredastoreNodes returns sorted PredastoreNodeConfig (1-based IDs) for
// nodes running the "predastore" service.
func BuildPredastoreNodes(nodes map[string]NodeInfo) []admin.PredastoreNodeConfig {
// Filter to nodes that run Predastore
predaNodes := make(map[string]NodeInfo)
for k, n := range nodes {
if hasService(n.Services, "predastore") {
spinifex/gpu/bind.go
+2 -3
View File
@@ -94,9 +94,8 @@ func unbindVFIO(sysfsRoot, addr, originalDriver string) error {
return nil
}
// writeSysfs writes value to a sysfs attribute path.
// Uses O_WRONLY|O_CREATE|O_TRUNC so it works against both real sysfs
// (file exists, kernel handles the write) and test temp dirs (file created).
// writeSysfs writes value to a sysfs attribute path. Works against both real
// sysfs and test temp dirs (file created if absent).
func writeSysfs(path, value string) error {
return os.WriteFile(path, []byte(value), 0o600)
}
spinifex/gpu/manager.go
+21 -44
View File
@@ -50,19 +50,10 @@ func (m *Manager) AddMIGGPU(dev GPUDevice) {
m.freeMIGGPUs = append(m.freeMIGGPUs, dev)
}
// Claim associates the first available GPU entry with instanceID and returns
// the claimed device. For MIG entries the NVIDIA driver manages isolation via
// mdev; no VFIO bind is performed. For whole-GPU entries the device and all
// IOMMU group members are bound to vfio-pci as before.
//
// profileName is the nvidia-smi MIG profile name (e.g. "1g.10gb") for MIG
// requests, or "" for whole-GPU passthrough. When a matching pre-carved slice
// exists it is claimed immediately. When none exists and a free MIG GPU is
// available, MIG instances are created on-demand (the lock is released during
// the nvidia-smi calls).
//
// The returned *MIGInstance is non-nil only for MIG entries; callers use its
// presence to distinguish attachment modes.
// Claim associates the first available GPU with instanceID. profileName selects
// a MIG profile (e.g. "1g.10gb") or "" for whole-GPU passthrough. A pre-carved
// slice is claimed immediately; otherwise a free MIG GPU is carved on-demand.
// The returned *MIGInstance is non-nil only for MIG entries.
func (m *Manager) Claim(instanceID, profileName string) (*GPUDevice, *MIGInstance, error) {
m.mu.Lock()
@@ -217,14 +208,10 @@ func (m *Manager) Claim(instanceID, profileName string) (*GPUDevice, *MIGInstanc
return &entry.Device, nil, nil
}
// Release returns the GPU held by instanceID to the available pool.
// For devices that were already bound to vfio-pci before the daemon started
// (e.g. by spx-test-gpu), the device is left vfio-pci bound — QEMU has
// already released the group fd on exit, so the next Claim works immediately.
// For devices that were bound by Claim itself, Release unbinds vfio-pci and
// rebinds to the original driver; if that fails the entry is marked unavailable.
// For MIG slices created dynamically by Claim (tracked in committedProfiles),
// the last release on a GPU destroys all slices and returns the GPU to freeMIGGPUs.
// Release returns the GPU held by instanceID to the pool. Pre-bound devices
// (already vfio-pci before the daemon) are left bound; Claim-bound devices are
// unbound and rebind to their original driver (failure marks them unavailable).
// For dynamically carved MIG GPUs, the last release destroys all slices.
func (m *Manager) Release(instanceID string) error {
m.mu.Lock()
defer m.mu.Unlock()
@@ -295,9 +282,8 @@ func (m *Manager) Release(instanceID string) error {
return firstErr
}
// tryDestroyIdleMIGGPULocked destroys all MIG instances on pciAddr if none are
// currently claimed, then returns the GPU to freeMIGGPUs. Must be called with
// m.mu held; releases and reacquires it around the nvidia-smi call.
// tryDestroyIdleMIGGPULocked destroys MIG instances on pciAddr when none are
// claimed and returns the GPU to freeMIGGPUs. Must be called with m.mu held.
func (m *Manager) tryDestroyIdleMIGGPULocked(pciAddr string) {
for _, e := range m.pool {
if e.Device.PCIAddress == pciAddr && e.MIGInstance != nil && e.InstanceID != "" {
@@ -347,9 +333,7 @@ func (m *Manager) tryDestroyIdleMIGGPULocked(pciAddr string) {
}
// MarkFailed marks a whole-GPU entry as permanently unavailable after a
// VFIO/QEMU launch error. The entry is skipped by Claim until the daemon
// restarts. Call this after releasing a GPU whose QEMU crashed during startup
// so the same broken device is not immediately re-assigned to the next instance.
// VFIO/QEMU launch error. Claim skips it until the daemon restarts.
func (m *Manager) MarkFailed(pciAddress string) {
m.mu.Lock()
defer m.mu.Unlock()
@@ -364,9 +348,8 @@ func (m *Manager) MarkFailed(pciAddress string) {
}
}
// MarkMIGFailed marks a specific MIG slice as permanently unavailable after a
// QEMU launch error. Only the slice identified by mdevPath is affected; other
// slices on the same physical GPU remain available.
// MarkMIGFailed marks a MIG slice unavailable after a QEMU launch error.
// Other slices on the same GPU remain available.
func (m *Manager) MarkMIGFailed(mdevPath string) {
m.mu.Lock()
defer m.mu.Unlock()
@@ -381,9 +364,7 @@ func (m *Manager) MarkMIGFailed(mdevPath string) {
}
}
// AddMIGInstances appends pool entries for each MIG slice on device. Called by
// the daemon after CreateInstances or ListInstances populates the slice set for
// a MIG-enabled GPU.
// AddMIGInstances appends pool entries for each MIG slice on device.
func (m *Manager) AddMIGInstances(device GPUDevice, instances []MIGInstance) {
m.mu.Lock()
defer m.mu.Unlock()
@@ -429,8 +410,7 @@ func (m *Manager) TotalCount() int {
return len(m.pool)
}
// PoolEntry is a read-only snapshot of a single pool slot, safe to inspect
// without holding the manager lock. Returned by Snapshot.
// PoolEntry is a read-only snapshot of a single pool slot, returned by Snapshot.
type PoolEntry struct {
Device GPUDevice
MIGInstance *MIGInstance // nil for whole-GPU entries
@@ -438,9 +418,8 @@ type PoolEntry struct {
Available bool
}
// Snapshot returns a point-in-time copy of every pool slot plus any free
// MIG GPUs (not yet committed to a profile). The caller receives owned copies
// and may inspect them without further synchronisation.
// Snapshot returns a point-in-time copy of every pool slot including free MIG GPUs.
// The caller receives owned copies and may inspect them without synchronisation.
func (m *Manager) Snapshot() []PoolEntry {
m.mu.Lock()
defer m.mu.Unlock()
@@ -465,9 +444,8 @@ func (m *Manager) Snapshot() []PoolEntry {
return out
}
// ReclaimByAddress marks the GPU at addr as claimed by instanceID without
// performing sysfs writes. Used on daemon restart to re-register GPUs that
// are still bound to vfio-pci from a previous run.
// ReclaimByAddress marks the GPU at addr as claimed without sysfs writes.
// Used on daemon restart for GPUs still bound to vfio-pci from a previous run.
func (m *Manager) ReclaimByAddress(addr, instanceID string) error {
m.mu.Lock()
defer m.mu.Unlock()
@@ -513,9 +491,8 @@ func (m *Manager) ReclaimByAddress(addr, instanceID string) error {
return fmt.Errorf("GPU %s not found in pool", addr)
}
// ReclaimByMdev marks the MIG slice at mdevPath as claimed by instanceID
// without performing sysfs writes. Used on daemon restart to re-register slices
// whose VMs are still running.
// ReclaimByMdev marks the MIG slice at mdevPath as claimed without sysfs writes.
// Used on daemon restart for slices whose VMs are still running.
func (m *Manager) ReclaimByMdev(mdevPath, instanceID string) error {
m.mu.Lock()
defer m.mu.Unlock()
spinifex/gpu/mig.go
+3 -5
View File
@@ -166,11 +166,9 @@ func createComputeInstance(pciAddr string, giID int) (int, error) {
return parseCreatedCIID(string(out))
}
// enrichMdevPaths walks basePath (/sys/bus/mdev/devices) and matches UUIDs to
// instances by reading each entry's symlink target (which contains the parent
// GPU's PCI address) and the GI ID from the gpu_instance_id sysfs attribute.
// basePath is parameterised so tests can inject a temp directory with fake
// symlinks without touching real sysfs.
// enrichMdevPaths matches mdev UUIDs to instances via symlink target (parent GPU
// PCI address) and gpu_instance_id sysfs attribute. basePath is parameterised
// so tests can inject a temp dir without touching real sysfs.
func enrichMdevPaths(basePath, pciAddr string, instances []MIGInstance) error {
entries, err := os.ReadDir(basePath)
if err != nil {
spinifex/gpu/reset.go
+3 -12
View File
@@ -9,18 +9,9 @@ import (
"strings"
)
// resetPCI issues a PCI Function-Level Reset for addr via sysfs. It is a no-op
// for non-AMD devices: NVIDIA and Intel GPUs handle the unbind/rebind cycle
// without requiring an explicit reset.
//
// AMD Instinct (CDNA) data-centre GPUs expose a reset sysfs node and support
// FLR correctly — writing "1" to it clears the hardware state left behind by
// QEMU, allowing the next Claim to bind a clean device. Consumer AMD GPUs
// (Polaris/Vega/Navi) have a firmware reset bug that FLR does not fully fix;
// those architectures are not the target here.
//
// Returns nil when the reset attribute is absent so callers treat this as
// best-effort.
// resetPCI issues a PCI Function-Level Reset for AMD devices via sysfs.
// No-op for NVIDIA/Intel which handle unbind/rebind without an explicit reset.
// Returns nil when the sysfs reset attribute is absent (best-effort).
func resetPCI(sysfsRoot, addr string) error {
vendorBytes, _ := os.ReadFile(filepath.Join(sysfsRoot, "bus/pci/devices", addr, "vendor"))
if strings.TrimSpace(string(vendorBytes)) != "0x1002" {
spinifex/handlers/acm/service.go
+1 -5
View File
@@ -2,11 +2,7 @@ package handlers_acm
import "github.com/aws/aws-sdk-go/service/acm"
// ACMService is the minimal AWS Certificate Manager-alike control plane: BYO
// certificate import and lifecycle. It covers only what ELBv2 HTTPS listeners
// need to resolve a Certificates[].CertificateArn to PEM material. Request
// flows (RequestCertificate + DNS/email validation), renewal, export, and
// tagging are intentionally out of scope for now.
// ACMService covers BYO certificate import and lifecycle for ELBv2 HTTPS listeners.
type ACMService interface {
ImportCertificate(input *acm.ImportCertificateInput, accountID string) (*acm.ImportCertificateOutput, error)
DescribeCertificate(input *acm.DescribeCertificateInput, accountID string) (*acm.DescribeCertificateOutput, error)
spinifex/handlers/acm/service_nats.go
+1 -2
View File
@@ -10,8 +10,7 @@ import (
const defaultTimeout = 30 * time.Second
// NATSACMService is the gateway-side client that forwards ACM operations to
// the daemon over NATS request/response.
// NATSACMService forwards ACM operations to the daemon over NATS request/response.
type NATSACMService struct {
natsConn *nats.Conn
}
spinifex/handlers/acm/store.go
+3 -6
View File
@@ -20,10 +20,8 @@ const (
KeyPrefixCert = "cert."
)
// CertRecord is the stored representation of an imported certificate. PEM
// material (leaf, chain, private key) is held alongside the parsed metadata
// surfaced by DescribeCertificate. AccountID scopes ownership so list/describe
// never cross account boundaries.
// CertRecord is the stored representation of an imported certificate.
// AccountID scopes ownership so list/describe never cross account boundaries.
type CertRecord struct {
CertificateArn string `json:"certificate_arn"`
AccountID string `json:"account_id"`
@@ -62,8 +60,7 @@ func NewStore(nc *nats.Conn) (*Store, error) {
return &Store{kv: kv}, nil
}
// certKey derives the KV key from a certificate ARN. The ARN's certificate ID
// (the UUID after "certificate/") is the stable per-record key.
// certKey derives the KV key from a certificate ARN using the UUID after "certificate/".
func certKey(certArn string) string {
id := certArn
if i := strings.LastIndex(certArn, "/"); i >= 0 {
spinifex/handlers/ec2/eip/service_impl.go
+9 -16
View File
@@ -412,15 +412,13 @@ func addressMatchesFilters(record *EIPRecord, filters map[string][]string) bool
return filterutil.MatchesTags(filters, record.Tags)
}
// DescribeAddressesAttribute returns per-EIP attributes (e.g. domain-name).
// Spinifex doesn't support reverse-DNS PTR records, so PtrRecord is left nil.
// Unlike DescribeAddresses, this returns an empty list (not an error) when
// requested AllocationIds are not found. This matches real AWS behavior.
// DescribeAddressesAttribute returns per-EIP attributes. PtrRecord is always nil
// (no reverse-DNS support). Returns an empty list, not an error, for missing IDs.
func (s *EIPServiceImpl) DescribeAddressesAttribute(input *ec2.DescribeAddressesAttributeInput, accountID string) (*ec2.DescribeAddressesAttributeOutput, error) {
var addresses []*ec2.AddressAttribute
if len(input.AllocationIds) > 0 {
// Direct lookups — O(n) on requested IDs rather than scanning all EIPs.
// Direct lookups by requested IDs.
for _, id := range input.AllocationIds {
if id == nil {
continue
@@ -570,10 +568,9 @@ func (s *EIPServiceImpl) findByAssociationID(accountID, associationID string) (*
return nil, "", 0, errors.New(awserrors.ErrorInvalidAssociationIDNotFound)
}
// AssociatedPublicIPForInstance returns the public IP of the Elastic IP
// associated with instanceID, if any. The daemon uses it to re-announce a
// post-launch EIP's dnat_and_snat on VM relaunch, where the instance's own
// PublicIP field is unset (only auto-assigned public IPs populate it).
// AssociatedPublicIPForInstance returns the public IP of the EIP associated with
// instanceID, if any. Used by the daemon to re-announce dnat_and_snat on relaunch
// when the instance's own PublicIP field is unset (EIP-assigned vs auto-assigned).
func (s *EIPServiceImpl) AssociatedPublicIPForInstance(accountID, instanceID string) (string, bool) {
if instanceID == "" {
return "", false
@@ -602,13 +599,9 @@ func (s *EIPServiceImpl) AssociatedPublicIPForInstance(accountID, instanceID str
return "", false
}
// publishNATEvent publishes a NAT lifecycle event to NATS for vpcd consumption.
// This is fire-and-forget; errors are logged but do not fail the API response.
//
// PortName must match the OVN logical switch port name ("port-<eni-id>") because
// vpcd sets NAT.LogicalPort to this value in distributed NAT mode (direct
// bridge). A mismatch creates a dnat_and_snat row pointing at a nonexistent
// port, and OVN never programs the DNAT flow.
// publishNATEvent publishes a NAT lifecycle event to NATS for vpcd (fire-and-forget).
// PortName must use topology.Port(eniID) to match the OVN logical switch port name;
// a mismatch causes OVN to never program the DNAT flow.
func (s *EIPServiceImpl) publishNATEvent(topic, vpcID, externalIP, logicalIP, eniID, mac string) {
utils.PublishEvent(s.natsConn, topic, natEvent{
VpcId: vpcID,
spinifex/handlers/ec2/eip/service_impl_test.go
+6 -11
View File
@@ -82,8 +82,8 @@ func TestEIP_Release(t *testing.T) {
}, testAccountID)
require.NoError(t, err)
// Round-robin: re-allocating does NOT hand back the just-released IP
// (siv-246); the cursor advances, so the new EIP differs.
// Round-robin: re-allocating does NOT hand back the just-released IP;
// the cursor advances, so the new EIP differs.
out2, err := svc.AllocateAddress(&ec2.AllocateAddressInput{}, testAccountID)
require.NoError(t, err)
assert.NotEqual(t, allocatedIP, *out2.PublicIp, "released EIP must not be reused immediately")
@@ -558,12 +558,9 @@ func TestEIP_DescribeAddresses_FilterByTag(t *testing.T) {
assert.Equal(t, *out.AllocationId, *desc.Addresses[0].AllocationId)
}
// TestEIP_PublishNATEvent_PortNameHasPortPrefix is a regression test for a bug
// where EIPServiceImpl.publishNATEvent sent the raw ENI id as PortName. vpcd
// writes PortName into NAT.LogicalPort in distributed NAT mode (direct bridge),
// so a mismatch with the OVN logical switch port name ("port-<eni>") produces
// a dnat_and_snat row pointing at a nonexistent port — OVN never programs the
// flow and the EIP black-holes.
// TestEIP_PublishNATEvent_PortNameHasPortPrefix verifies that publishNATEvent
// uses topology.Port(eniID) as PortName. A raw ENI id mismatch creates a
// dnat_and_snat row pointing at a nonexistent OVN port, black-holing the EIP.
func TestEIP_PublishNATEvent_PortNameHasPortPrefix(t *testing.T) {
svc, _ := setupTestEIP(t)
@@ -580,9 +577,7 @@ func TestEIP_PublishNATEvent_PortNameHasPortPrefix(t *testing.T) {
var got natEvent
require.NoError(t, json.Unmarshal(msg.Data, &got))
assert.Equal(t, "port-"+eniID, got.PortName,
"PortName must match the OVN logical switch port name (port-<eni>); "+
"otherwise vpcd's distributed NAT creates a dnat_and_snat rule "+
"pointing at a nonexistent logical port and the flow is never programmed")
"PortName must be port-<eni> to match the OVN logical switch port name")
assert.Equal(t, "10.0.0.5", got.LogicalIP)
assert.Equal(t, "198.51.100.10", got.ExternalIP)
}
spinifex/handlers/ec2/eip/types.go
+2 -4
View File
@@ -16,10 +16,8 @@ type EIPRecord struct {
ENIId string `json:"eni_id,omitempty"`
InstanceId string `json:"instance_id,omitempty"`
PrivateIp string `json:"private_ip,omitempty"`
// MacAddress is the associated ENI's MAC, captured at associate-time so
// the vpcd reconcile can re-apply the distributed-shape dnat_and_snat
// (external_mac/logical_port) after a host reboot without re-querying the
// ENI. Empty while the EIP is unassociated.
// MacAddress is the ENI's MAC captured at associate-time so vpcd can
// re-apply dnat_and_snat after a host reboot. Empty when unassociated.
MacAddress string `json:"mac_address,omitempty"`
VpcId string `json:"vpc_id,omitempty"`
State string `json:"state"` // "allocated", "associated"
spinifex/handlers/ec2/igw/service_impl.go
+5 -13
View File
@@ -37,12 +37,9 @@ type IGWRecord struct {
CreatedAt time.Time `json:"created_at"`
}
// GatePublisher recomputes the per-subnet egress gate/ungate decision for
// every subnet in a VPC. Wired by the daemon after IGW + RouteTable services
// are constructed so AttachInternetGateway / DetachInternetGateway can fan
// out gate decisions immediately instead of waiting up to 5 minutes for the
// reconciler's drift tick to install OVN DROP policies on unrouted subnets.
// Nil-safe: when unset the handler falls back to publish-only behaviour.
// GatePublisher recomputes per-subnet egress gate/ungate decisions for a VPC.
// Wired by the daemon so IGW attach/detach triggers immediate OVN policy updates
// rather than waiting for the reconciler's drift tick.
type GatePublisher interface {
PublishGateDecisionsForVPC(accountID, vpcID, destCidr string)
}
@@ -332,13 +329,8 @@ func (s *IGWServiceImpl) AttachInternetGateway(input *ec2.AttachInternetGatewayI
}
}
// Gate fan-out intentionally skipped on attach. The default-VPC
// bootstrap path attaches the IGW and immediately publishes a
// 0.0.0.0/0 → IGW CreateRoute event; the two race across distinct
// NATS subjects (gate vs ungate) and ungate-then-gate leaves the
// subnet DROPped. Subnets without a default route stay protected by
// the reconciler's drift pass (loadSubnetDropGates). Detach is
// race-free because no follow-up CreateRoute fires.
// Gate fan-out is intentionally skipped on attach to avoid a race with
// the bootstrap CreateRoute path. Detach triggers gate fan-out directly.
slog.Info("AttachInternetGateway completed", "internetGatewayId", igwID, "vpcId", vpcID, "accountID", accountID)
spinifex/handlers/ec2/igw/service_impl_test.go
+2 -5
View File
@@ -699,11 +699,8 @@ func TestDeleteInternetGateway_PublishesNoEvent(t *testing.T) {
}
}
// TestAttachInternetGateway_NoGateFanOut documents that attach intentionally
// skips the gate fan-out to avoid racing the default-VPC bootstrap path,
// which attaches the IGW and immediately adds a 0.0.0.0/0 → IGW route. The
// two events would race across the gate/ungate NATS subjects and leave the
// default subnet DROPped when ungate landed first.
// TestAttachInternetGateway_NoGateFanOut verifies that IGW attach skips gate
// fan-out to avoid racing the bootstrap CreateRoute path.
func TestAttachInternetGateway_NoGateFanOut(t *testing.T) {
svc, _ := setupTestIGWService(t)
pub := &fakeGatePublisher{}
spinifex/handlers/ec2/image/service_impl.go
+10 -18
View File
@@ -744,10 +744,9 @@ func (s *ImageServiceImpl) putSnapshotMetadata(snapshotID, volumeID string, volu
return handlers_ec2_snapshot.WriteSnapshotConfig(s.store, s.bucketName, snapshotID, &cfg)
}
// CopyImage clones an AMI same-region, metadata-only: the new snapshot shares
// the source's VolumeID and a fresh config.json points at it.
// Source visibility is checked before the O(n) name-uniqueness scan so typos
// and cross-account sources fast-fail without a full AMI listing.
// CopyImage clones an AMI same-region, metadata-only: the new snapshot shares the
// source's VolumeID and a fresh config.json points at it. Source visibility is checked
// before the name-uniqueness scan so cross-account sources fast-fail.
func (s *ImageServiceImpl) CopyImage(input *ec2.CopyImageInput, accountID string) (*ec2.CopyImageOutput, error) {
if input == nil || input.Name == nil || *input.Name == "" ||
input.SourceImageId == nil || *input.SourceImageId == "" {
@@ -779,12 +778,8 @@ func (s *ImageServiceImpl) CopyImage(input *ec2.CopyImageInput, accountID string
}
srcSnap, err := handlers_ec2_snapshot.ReadSnapshotConfig(s.store, s.bucketName, srcMeta.SnapshotID)
if err != nil {
// Bundled system AMIs (admin-imported via `spx admin images import`)
// store blocks under ami-xxx/ with no standalone snap-xxx/metadata.json
// — the SnapshotID field is a viperblock-internal snap reference. To
// keep copy-image AWS-compatible (copy of a public/shared AMI must
// succeed), synthesize a minimal snap view from the AMI itself: the
// bundled prefix IS the volume, so VolumeID = sourceImageID.
// Bundled system AMIs have no standalone snap-xxx/metadata.json; synthesize
// a minimal snap view using VolumeID = sourceImageID so CopyImage succeeds.
if objectstore.IsNoSuchKeyError(err) && srcMeta.ImageOwnerAlias != "" && !utils.IsAccountID(srcMeta.ImageOwnerAlias) {
srcSnap = &handlers_ec2_snapshot.SnapshotConfig{
SnapshotID: srcMeta.SnapshotID,
@@ -943,10 +938,9 @@ func (s *ImageServiceImpl) clusterEncryptionEnabled() bool {
return mkey != nil
}
// synthesizeRootBlockDeviceMapping returns /dev/sda1 with size+snapshot from
// AMIMetadata, or nil for non-EBS AMIs. encrypted reflects the cluster-level
// encryption-at-rest posture (master key configured) since AMI metadata does
// not carry a per-image encryption flag of its own.
// synthesizeRootBlockDeviceMapping returns /dev/sda1 with size+snapshot from AMIMetadata,
// or nil for non-EBS AMIs. encrypted reflects the cluster-level encryption posture
// (master key configured); AMI metadata carries no per-image encryption flag.
func synthesizeRootBlockDeviceMapping(meta viperblock.AMIMetadata, encrypted bool) []*ec2.BlockDeviceMapping {
if meta.RootDeviceType != "ebs" {
return nil
@@ -1108,10 +1102,8 @@ func (s *ImageServiceImpl) DeregisterImage(input *ec2.DeregisterImageInput, acco
return &ec2.DeregisterImageOutput{}, nil
}
// ModifyImageAttribute writes a modifiable AMI attribute. Gateway normalises
// into Attribute+Value; only description is writable. Ownership is checked
// before the attribute switch so cross-account callers always see
// UnauthorizedOperation.
// ModifyImageAttribute writes a modifiable AMI attribute; only description is writable.
// Ownership is checked first so cross-account callers always see UnauthorizedOperation.
func (s *ImageServiceImpl) ModifyImageAttribute(input *ec2.ModifyImageAttributeInput, accountID string) (*ec2.ModifyImageAttributeOutput, error) {
if input == nil || input.ImageId == nil || *input.ImageId == "" ||
input.Attribute == nil || *input.Attribute == "" {
spinifex/handlers/ec2/image/service_impl_test.go
+8 -17
View File
@@ -200,10 +200,8 @@ func TestDescribeImages_AfterCreate(t *testing.T) {
assert.Equal(t, testAccountID, *result.Images[0].OwnerId)
}
// TestDescribeImages_BootModeProjection asserts that AMIMetadata.BootMode
// round-trips through DescribeImages. Empty BootMode (legacy AMIs registered
// before the field existed) must pass through as an empty string, not get
// synthesized to a default.
// TestDescribeImages_BootModeProjection asserts that AMIMetadata.BootMode round-trips
// through DescribeImages; empty BootMode on legacy AMIs must pass through unchanged.
func TestDescribeImages_BootModeProjection(t *testing.T) {
svc, store := setupTestImageService(t)
@@ -871,10 +869,8 @@ func createTestAMIConfigFull(t *testing.T, store *objectstore.MemoryObjectStore,
require.NoError(t, err)
}
// TestDescribeImages_FilterBy verifies single-attribute filters. Each subtest
// builds its own AMI fixtures and asserts which IDs the filter selects.
// Multi-filter, wildcard, error-path, and no-filter cases live in their own
// dedicated tests below.
// TestDescribeImages_FilterBy verifies single-attribute filters; each subtest builds
// its own AMI fixtures. Multi-filter, wildcard, and error-path cases have their own tests.
func TestDescribeImages_FilterBy(t *testing.T) {
tests := []struct {
name string
@@ -1542,11 +1538,8 @@ func putTestAMIConfigWithSnapshot(t *testing.T, store *objectstore.MemoryObjectS
require.NoError(t, err)
}
// seedCopyableAMI writes a matching (snapshot, AMI) pair so CopyImage can
// complete end-to-end. Returns the AMI metadata that was persisted (callers
// can customise fields before seeding by tweaking the returned VolumeID /
// tags via preceding calls, but the helper takes the simple path for the
// happy case).
// seedCopyableAMI writes a matching (snapshot, AMI) pair so CopyImage can complete
// end-to-end.
func seedCopyableAMI(t *testing.T, store *objectstore.MemoryObjectStore, imageID, name, owner, snapshotID, volumeID string, sizeGiB int64) {
t.Helper()
cfg := handlers_ec2_snapshot.SnapshotConfig{
@@ -1704,10 +1697,8 @@ func TestCopyImage_SystemAMICopiedIntoCallerAccount(t *testing.T) {
assert.Equal(t, "spinifex", srcMeta.ImageOwnerAlias)
}
// Bundled system AMIs (admin-imported) have no standalone snap-xxx/metadata.json
// — their SnapshotID refers to a viperblock-internal snap, and blocks live under
// ami-xxx/. CopyImage must still succeed (AWS parity: copy of a public AMI works),
// falling back to synthesizing a snap view where VolumeID = sourceImageID.
// Bundled system AMIs have no standalone snap-xxx/metadata.json; CopyImage must
// still succeed by synthesizing a snap view where VolumeID = sourceImageID.
func TestCopyImage_BundledSystemAMINoStandaloneSnap(t *testing.T) {
svc, store := setupTestImageService(t)
spinifex/handlers/ec2/instance/service.go
+8 -25
View File
@@ -43,19 +43,9 @@ type TerminateStoppedInstanceOutput struct {
InstanceID string `json:"instanceId"`
}
// ResourceCapacityProvider exposes per-node instance-type availability used by
// DescribeInstanceTypes. Implemented by daemon.ResourceManager.
//
// GetAvailableInstanceTypeInfos reports types gated by free schedulable
// capacity (one entry per free slot when showCapacity is true). It is used
// when callers ask "what could you launch right now?" — typically with the
// capacity=true filter for aggregated cluster-wide accounting.
//
// GetSupportedInstanceTypeInfos reports every type this node is configured
// to run, regardless of current allocation. This is the AWS-compatible
// answer for plain DescribeInstanceTypes and is what tooling like the
// Terraform AWS provider expects when it performs a metadata lookup after
// RunInstances.
// ResourceCapacityProvider exposes per-node instance-type availability.
// GetAvailableInstanceTypeInfos gates on free capacity; GetSupportedInstanceTypeInfos
// returns all configured types regardless of allocation.
type ResourceCapacityProvider interface {
GetAvailableInstanceTypeInfos(showCapacity bool) []*ec2.InstanceTypeInfo
GetSupportedInstanceTypeInfos() []*ec2.InstanceTypeInfo
@@ -129,10 +119,8 @@ type SubnetInfo struct {
MapPublicIpOnLaunch bool
}
// ENIInfo carries the subset of ENI metadata RunInstances needs to attach a
// pre-created ENI as the primary interface. Translated from
// handlers/ec2/vpc.ENIRecord by the daemon adapter to avoid a cyclic
// instance↔vpc import.
// ENIInfo carries the ENI metadata RunInstances needs for a pre-created primary
// interface. Translated from handlers/ec2/vpc.ENIRecord to avoid a cyclic import.
type ENIInfo struct {
NetworkInterfaceID string
SubnetID string
@@ -143,14 +131,9 @@ type ENIInfo struct {
SecurityGroupIDs []string
}
// ENICreator covers the VPC/ENI operations RunInstances performs while
// auto-attaching a primary interface. Implemented via an adapter over
// handlers/ec2/vpc.VPCServiceImpl on the daemon. DetachENI lives here (not on
// ENIDeleter) because the launch-time NAT-rollback path needs to flip the ENI
// to "available" before ENIDeleter.DeleteNetworkInterface, which rejects
// in-use ENIs. GetENI supports the pre-created-ENI launch path where the
// caller passes NetworkInterfaces[0].NetworkInterfaceId and RunInstances
// attaches the existing ENI instead of creating a new one.
// ENICreator covers the VPC/ENI operations RunInstances uses to auto-attach a
// primary interface. DetachENI is here (not ENIDeleter) so the NAT-rollback
// path can flip the ENI to "available" before deletion.
type ENICreator interface {
GetDefaultSubnet(accountID string) (*SubnetInfo, error)
GetSubnet(accountID, subnetID string) (*SubnetInfo, error)
spinifex/handlers/ec2/instance/service_impl.go
+129 -307
View File
@@ -89,10 +89,9 @@ instance-id: {{.InstanceID}}
local-hostname: {{.Hostname}}
`
// cloudInitNetworkConfigWildcard enables DHCP on all NICs via wildcard match.
// Used when there's no dual-NIC setup (non-VPC or VPC without DEV_NETWORKING).
// The "e*" glob matches both traditional names (eth0, eth1 — Alpine/older
// kernels) and predictable names (ens3, enp0s3 — systemd-based distros).
// cloudInitNetworkConfigWildcard enables DHCP on all NICs via "e*" glob,
// covering both traditional (eth0) and predictable (ens3) names.
// Used when there is no dual-NIC setup (non-VPC or no DEV_NETWORKING).
const cloudInitNetworkConfigWildcard = `network:
version: 2
ethernets:
@@ -103,19 +102,16 @@ const cloudInitNetworkConfigWildcard = `network:
dhcp-identifier: mac
`
// cloudInitNetworkConfigDisabled tells cloud-init to do no network rendering
// at all. Used on RHEL-family guests where Spinifex writes NM keyfiles via
// user-data write_files; without this stub cloud-init's RHEL network module
// falls back to autodetect-DHCP and drops a second keyfile next to ours.
// cloudInitNetworkConfigDisabled suppresses cloud-init network rendering.
// Used on RHEL guests where NM keyfiles are written via write_files;
// without this stub cloud-init would drop a competing autoconnect keyfile.
const cloudInitNetworkConfigDisabled = `network:
config: disabled
`
// selectNetworkConfigForFamily returns the cloud-init network-config payload
// for the given distro family. RHEL-family guests get an explicit
// "config: disabled" stub because their NM keyfiles are written via user-data
// write_files; shipping a netplan config alongside would let cloud-init's RHEL
// network module drop a competing cloud-init-enpXsY.nmconnection.
// selectNetworkConfigForFamily returns the cloud-init network-config for the distro.
// RHEL gets "config: disabled" to prevent cloud-init from dropping a competing
// NM keyfile alongside the ones written via user-data write_files.
func selectNetworkConfigForFamily(distroFamily, eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string) string {
if distroFamily == "rhel" {
return cloudInitNetworkConfigDisabled
@@ -128,26 +124,8 @@ func selectNetworkConfigForFamily(distroFamily, eniMAC, devMAC, mgmtMAC, mgmtIP
}
// generateNetworkConfig produces the cloud-init network-config for the instance.
//
// Per-interface config is generated when eniMAC is present (VPC NIC). This allows
// adding the mgmt NIC with a static IP and optionally the dev NIC with route
// suppression. Without per-interface config, the wildcard fallback does DHCP on
// all NICs — which won't work for the mgmt NIC (no DHCP server on br-mgmt).
//
// extraENIMACs configures additional VPC NICs for multi-subnet system VMs
// (e.g. multi-AZ ALB VMs). Each extra MAC produces a DHCP ethernet block named
// vpc1, vpc2, ... so each interface pulls its address from the subnet it lives in.
//
// The dev NIC still gets an IP via DHCP (needed for hostfwd port forwarding)
// but dhcp4-overrides prevents it from installing routes or DNS.
//
// The primary VPC NIC (vpc0) also carries an on-link route to the IMDS metadata
// IP (169.254.169.254): under the L2 datapath the metadata localport lives on
// the guest's subnet switch and answers ARP directly, so the guest must treat
// the address as on-link rather than steering it via the default gateway. The
// route is reached via the primary ENI, matching AWS; it is not delivered via
// DHCP option 121 (which would force the default route into option 121 per
// RFC 3442) and the default gateway stays option 3.
// Emits per-interface blocks when eniMAC is set; falls back to wildcard DHCP
// when eniMAC is empty.
func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string, emitIMDSRoute bool) string {
if eniMAC == "" {
return cloudInitNetworkConfigWildcard
@@ -161,10 +139,8 @@ func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs
dhcp4: true
dhcp-identifier: mac
`, eniMAC)
// The gateway-less IMDS on-link route renders only under netplan. cloud-init's
// eni renderer (Alpine) crashes on a route with no gateway, taking the whole
// network-config down with it, so eni-rendered guests get this route via
// buildAlpineCloudInit instead (emitIMDSRoute=false).
// Alpine's eni renderer crashes on gateway-less routes; omit here and
// deliver via buildAlpineCloudInit instead (emitIMDSRoute=false).
if emitIMDSRoute {
cfg += fmt.Sprintf(` routes:
- to: %s/32
@@ -203,8 +179,7 @@ func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs
addresses:
- "%s/24"
`, mgmtMAC, mgmtIP)
// Route for multi-node LB mgmt traffic is delivered to the LB microVM
// via the fw_cfg netcfg blob, not here.
// LB mgmt route is delivered via fw_cfg netcfg, not here.
}
return cfg
@@ -217,46 +192,27 @@ type CloudInitData struct {
UserDataCloudConfig string
UserDataScript string
CACertPEM string
// DistroFamily selects per-distro branches in the cloud-init template.
// "debian" | "rhel" | "alpine". Empty falls through to debian rendering
// (legacy AMIs registered before the field existed).
// DistroFamily selects per-distro rendering: "debian" | "rhel" | "alpine".
// Empty falls through to debian (legacy AMIs without this field).
DistroFamily string
// SudoGroup is the OS group cloud-init adds the user to for passwordless
// sudo. "sudo" on debian/ubuntu/alpine, "wheel" on RHEL-family.
// SudoGroup is "sudo" on debian/ubuntu/alpine, "wheel" on RHEL-family.
SudoGroup string
// RHELWriteFiles is the pre-indented YAML block of NM keyfile entries
// (one per NIC) appended under the merged write_files: key. Empty on
// non-RHEL guests; the network-config ISO file carries the YAML netplan
// equivalent instead.
// RHELWriteFiles is the pre-indented YAML NM keyfile block (one per NIC).
// Empty on non-RHEL guests; those use the network-config ISO instead.
RHELWriteFiles string
// RHELRunCmd is the pre-indented YAML block of runcmd entries (restorecon
// + nmcli reload/up) needed to load and bring up the keyfiles. Empty on
// non-RHEL guests.
// RHELRunCmd is the pre-indented runcmd YAML (restorecon + nmcli reload/up).
// Empty on non-RHEL guests.
RHELRunCmd string
// AlpineWriteFiles / AlpineRunCmd carry the IMDS on-link route delivery for
// Alpine guests (a persistent /etc/local.d script + OpenRC `local` enable),
// since the eni renderer cannot emit the gateway-less route in the
// network-config. Empty on non-Alpine guests and on non-VPC instances.
// AlpineWriteFiles / AlpineRunCmd deliver the IMDS on-link route via a
// local.d script (eni renderer can't emit gateway-less routes).
// Empty on non-Alpine or non-VPC instances.
AlpineWriteFiles string
AlpineRunCmd string
}
// buildRHELCloudInit produces the cloud-init write_files (NM keyfiles) and
// runcmd (restorecon + nmcli) blocks needed to bring up networking on a
// RHEL-family guest. Returns empty strings when eniMAC is empty (wildcard /
// non-VPC path) so NM's default autoconnect handles the interface.
//
// Mirrors generateNetworkConfig's per-interface structure: vpc0 (DHCP + on-link
// IMDS route) + optional vpc1..N for extra ENIs, dev0 (DHCP with route/DNS
// suppression), mgmt0 (static). Owning the keyfile bytes directly avoids relying on cloud-init's
// v2-to-NM renderer round-tripping dhcp-identifier: mac → dhcp-client-id=mac,
// which OVN's MAC-keyed DHCP requires.
//
// File mode 0600 root:root is load-bearing — NM ignores keyfiles otherwise.
// restorecon defers to host SELinux policy (no-op on non-SELinux systems);
// nmcli reload forces NM to rescan system-connections after write_files
// completes in cloud-init's config stage, and nmcli up brings each interface
// online without waiting on autoconnect heuristics.
// runcmd blocks for RHEL networking. Returns empty strings when eniMAC is empty.
// File mode 0600 is required — NM ignores world-readable keyfiles.
func buildRHELCloudInit(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string) (writeFiles, runCmd string) {
if eniMAC == "" {
return "", ""
@@ -309,15 +265,12 @@ func appendRHELDHCPKeyfile(b *strings.Builder, name, mac string, suppressDefault
b.WriteString(" method=auto\n")
b.WriteString(" dhcp-client-id=mac\n")
if suppressDefaults {
// Equivalent to netplan dhcp4-overrides {use-routes: false, use-dns: false}:
// dev NIC gets an IP for hostfwd but must not install a default route or DNS.
// Dev NIC gets DHCP IP for hostfwd but must not install routes or DNS.
b.WriteString(" never-default=true\n")
b.WriteString(" ignore-auto-dns=true\n")
}
if imdsRoute {
// On-link route to the IMDS metadata IP so the guest ARPs 169.254.169.254
// directly on this NIC; the per-subnet localport answers. No next hop =
// link scope. Matches the netplan path in generateNetworkConfig.
// On-link IMDS route so guest ARPs 169.254.169.254 on this NIC (link scope).
fmt.Fprintf(b, " route1=%s/32\n", handlers_imds.MetaDataServerIP)
}
b.WriteString(" [ipv6]\n")
@@ -341,18 +294,9 @@ func appendRHELStaticKeyfile(b *strings.Builder, name, mac, ip string) {
b.WriteString(" method=disabled\n")
}
// buildAlpineCloudInit produces the cloud-init write_files + runcmd blocks that
// deliver the IMDS on-link route on Alpine guests. Returns empty strings when
// eniMAC is empty (wildcard / non-VPC path).
//
// Alpine renders networking via cloud-init's eni backend, which crashes on a
// gateway-less route (it assumes every route has a next hop), so the route is
// omitted from the netplan network-config (see generateNetworkConfig) and added
// here instead. A persistent /etc/local.d script keeps the on-link route present
// across reboots — runcmd runs once per instance, so the OpenRC `local` service
// re-applies it on every boot. cloud-init's eni renderer renames the primary
// VPC NIC to vpc0 via a 70-persistent-net.rules udev rule, so the device name is
// stable. `ip route replace` is idempotent.
// buildAlpineCloudInit produces the write_files + runcmd blocks that install the
// IMDS on-link route on Alpine guests. Alpine's eni renderer crashes on
// gateway-less routes, so the route is delivered via a persistent local.d script.
func buildAlpineCloudInit(eniMAC string) (writeFiles, runCmd string) {
if eniMAC == "" {
return "", ""
@@ -364,10 +308,8 @@ func buildAlpineCloudInit(eniMAC string) (writeFiles, runCmd string) {
wf.WriteString(" permissions: '0755'\n")
wf.WriteString(" content: |\n")
wf.WriteString(" #!/bin/sh\n")
// Resolve the primary VPC NIC via its default route, not by name: the
// persistent-net udev rename to vpc0 doesn't apply on the Alpine AMI, so the
// kernel name is eth0 at first boot. The mgmt NIC (eth1) carries no default
// route, so this always picks the VPC egress NIC — the one IMDS rides.
// Resolve via default route: udev rename to vpc0 hasn't run yet at first boot;
// the mgmt NIC has no default route so this always selects the VPC egress NIC.
wf.WriteString(" dev=$(ip route show default | awk '{print $5; exit}')\n")
fmt.Fprintf(&wf, " [ -n \"$dev\" ] && ip route replace %s/32 dev \"$dev\" scope link\n", handlers_imds.MetaDataServerIP)
@@ -403,13 +345,9 @@ type volumeParams struct {
deleteOnTermination bool
}
// floorVolumeSizeToAMI ensures requested is at least the AMI's snapshot size.
// Cloud images embed a fixed-size partition table; truncating below it (e.g.
// a 10 GiB Rocky image requested at the 4 GiB default) hangs the guest in
// dracut waiting on a root UUID that lives past the cut-off. AWS rejects
// under-sized requests with InvalidParameterValue — spinifex silently rounds
// up since we don't surface that error code today and the user-visible
// failure (dracut hang) is worse than a slightly bigger volume.
// floorVolumeSizeToAMI ensures the requested size is at least the AMI's snapshot
// size. Under-sizing hangs the guest in dracut; we round up silently rather than
// returning InvalidParameterValue, since a slightly larger volume is less harmful.
func floorVolumeSizeToAMI(loader AMIMetaLoader, imageID string, requested int) int {
if loader == nil || !strings.HasPrefix(imageID, "ami-") {
return requested
@@ -592,14 +530,9 @@ func (s *InstanceServiceImpl) RunInstance(input *ec2.RunInstancesInput) (*vm.VM,
return instance, ec2Instance, nil
}
// PrepareRunInstances validates RunInstances input, allocates per-node capacity,
// creates per-instance VM + ec2.Instance metadata, auto-resolves the subnet,
// creates the primary ENI, and auto-assigns a public IP when the subnet has
// MapPublicIpOnLaunch. It does NOT touch vmMgr, WriteState, or NATS
// subscriptions — those are daemon concerns so the daemon can respond to AWS
// between Prepare and Launch (preserving the original respond-then-launch
// timing). Returns the reservation, the prepared VMs, and the instance-type
// info so the caller can deallocate on failure.
// PrepareRunInstances validates input, allocates capacity, creates VM metadata,
// auto-creates the primary ENI, and auto-assigns a public IP when needed.
// Does NOT touch vmMgr or NATS — callers insert VMs then call LaunchRunInstances.
func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, []*vm.VM, *ec2.InstanceTypeInfo, error) {
if accountID == "" {
return nil, nil, nil, errors.New(awserrors.ErrorServerInternal)
@@ -627,8 +560,7 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput,
slog.Error("PrepareRunInstances: AMI not found", "imageId", *input.ImageId, "err", err)
return nil, nil, nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound)
}
// Caller must own the AMI or it must carry a non-account-ID owner alias
// (e.g. "self", "spinifex", "") indicating a system/legacy AMI.
// Caller must own the AMI or the owner alias must be non-account-ID (e.g. "self", "spinifex", "").
amiOwner := amiMeta.ImageOwnerAlias
if amiOwner != "" && amiOwner != accountID && utils.IsAccountID(amiOwner) {
slog.Warn("PrepareRunInstances: AMI not owned by caller", "imageId", *input.ImageId, "amiOwner", amiOwner, "accountID", accountID)
@@ -658,9 +590,8 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput,
launchCount := allocatableCount
slog.Info("PrepareRunInstances: count determined", "min", minCount, "max", maxCount, "launching", launchCount)
// Allocate is atomic per call (check + commit under a single write lock),
// so this loop can never overcommit. Under contention it may allocate
// fewer than allocatableCount; the < minCount branch below rolls back.
// Each Allocate is atomic; under contention we may get fewer than
// allocatableCount — the < minCount branch below rolls back.
var allocatedCount int
for i := 0; i < launchCount; i++ {
if err := s.resourceMgr.Allocate(instanceType); err != nil {
@@ -692,8 +623,7 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput,
}
instance.BootMode = amiMeta.BootMode
// Terraform with associate_public_ip_address sends subnet/SG via
// NetworkInterfaces[0]; lift to top-level so the rest works uniformly.
// Terraform may pass subnet/SG via NetworkInterfaces[0]; lift to top-level.
if (input.SubnetId == nil || *input.SubnetId == "") &&
len(input.NetworkInterfaces) > 0 && input.NetworkInterfaces[0] != nil {
nic := input.NetworkInterfaces[0]
@@ -705,11 +635,8 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput,
}
}
// Pre-created ENI path: when the caller passes
// NetworkInterfaces[0].NetworkInterfaceId, attach that existing ENI
// instead of auto-creating one. AWS-parity behaviour; consumed by the
// EKS cluster launcher which needs the ENI to live under the customer
// account before the K3s server VM is created.
// Pre-created ENI: attach the existing ENI instead of auto-creating one
// (AWS parity; used by EKS launcher to pre-create the ENI in customer VPC).
preExistingENIID := ""
if len(input.NetworkInterfaces) > 0 && input.NetworkInterfaces[0] != nil {
preExistingENIID = aws.StringValue(input.NetworkInterfaces[0].NetworkInterfaceId)
@@ -760,9 +687,8 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput,
instance.ENIMac = *eni.MacAddress
if _, attachErr := s.eniCreator.AttachENI(accountID, instance.ENIId, instance.ID, 0); attachErr != nil {
// Without the attachment, RegisterTargets (TargetType=instance)
// resolves ENIs via attachment.instance-id, finds none, and
// silently drops the target; terminate leaks the auto-EIP.
// Without the attachment RegisterTargets silently drops the target;
// aborting prevents a leak of the auto-assigned EIP.
slog.Error("PrepareRunInstances: AttachENI failed — aborting launch",
"eniId", instance.ENIId, "instanceId", instance.ID, "err", attachErr)
if _, delErr := s.eniDeleter.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{
@@ -813,12 +739,8 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput,
az := s.config.AZ
publicIP, poolName, allocErr := s.ipAllocator.AllocateIP(region, az, handlers_ec2_vpc.PurposeENIPublic, "", *eni.NetworkInterfaceId, instance.ID)
if allocErr != nil {
// No qemu-hostfwd fallback exists: an instance on a
// MapPublicIpOnLaunch subnet with no public IP is
// unreachable, so fail the launch instead of booting a
// running-but-unreachable instance. The ENI is attached
// with a primary IP — detach before delete (in-use ENIs
// reject deletion).
// Fail rather than boot an unreachable instance;
// detach before delete since in-use ENIs reject deletion.
slog.Error("PrepareRunInstances: public IP allocation failed — aborting launch",
"instanceId", instance.ID, "eniId", *eni.NetworkInterfaceId, "err", allocErr)
if detErr := s.eniCreator.DetachENI(accountID, *eni.NetworkInterfaceId); detErr != nil {
@@ -842,7 +764,7 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput,
if natErr := utils.AddNAT(s.natsConn, *eni.VpcId, publicIP, *eni.PrivateIpAddress, portName, *eni.MacAddress); natErr != nil {
slog.Error("PrepareRunInstances: vpc.add-nat failed — rolling back public IP to avoid surfacing an unreachable address",
"instanceId", instance.ID, "publicIp", publicIP, "pool", poolName, "err", natErr)
// Timeout may have committed the rule after our window; neutralise before releasing the IP.
// Neutralise before releasing in case timeout committed the rule.
utils.PublishNATEvent(s.natsConn, "vpc.delete-nat", *eni.VpcId, publicIP, *eni.PrivateIpAddress, portName, *eni.MacAddress)
s.rollbackAutoAssignedPublicIP(accountID, instance.ID, *eni.NetworkInterfaceId, publicIP, poolName)
lastRunErr = natErr
@@ -897,12 +819,9 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput,
return reservation, instances, instanceType, nil
}
// attachPreCreatedENI looks up an existing ENI in accountID, verifies it is
// not already in-use, attaches it as device-0 to instance, and populates the
// VM + ec2.Instance with the ENI's network identity (private IP, MAC, subnet,
// VPC, security groups). Returns an error without mutating any state if the
// lookup or attach fails. Public-IP auto-assignment is skipped on the
// pre-created path because the caller is expected to manage that out-of-band.
// attachPreCreatedENI verifies the ENI is available, attaches it as device-0,
// and populates the VM + ec2.Instance. Public-IP auto-assignment is skipped;
// the caller manages that out-of-band.
func (s *InstanceServiceImpl) attachPreCreatedENI(accountID, eniID string, instance *vm.VM, ec2Instance *ec2.Instance) error {
eniInfo, err := s.eniCreator.GetENI(accountID, eniID)
if err != nil {
@@ -953,15 +872,12 @@ func (s *InstanceServiceImpl) attachPreCreatedENI(accountID, eniID string, insta
return nil
}
// LaunchRunInstances takes the VMs created by PrepareRunInstances (already
// inserted into vmMgr by the caller) and runs the heavyweight launch loop:
// volume preparation, GPU claim, vmMgr.Run. Failures mark the VM as failed
// and deallocate resources but do not abort the loop — partial success
// matches AWS behaviour where some instances in a reservation may fail.
// LaunchRunInstances runs the heavyweight launch loop for VMs already inserted
// into vmMgr: volume prep, GPU claim, vmMgr.Run. Partial failures are tolerated.
func (s *InstanceServiceImpl) LaunchRunInstances(instances []*vm.VM, input *ec2.RunInstancesInput, instanceType *ec2.InstanceTypeInfo) {
var successCount int
for _, instance := range instances {
// Skip if a concurrent request terminated the instance during prepare.
// Skip if a concurrent terminate raced with prepare.
status := s.vmMgr.Status(instance)
if status != vm.StatePending && status != vm.StateProvisioning {
slog.Info("LaunchRunInstances: instance state changed during provisioning, skipping launch",
@@ -969,8 +885,7 @@ func (s *InstanceServiceImpl) LaunchRunInstances(instances []*vm.VM, input *ec2.
continue
}
// Pre-compute dev MAC so cloud-init can generate per-interface netplan
// that suppresses the default route on the dev/hostfwd NIC.
// Pre-compute dev MAC for per-interface cloud-init netplan (route suppression).
if s.config.Daemon.DevNetworking && instance.ENIId != "" {
instance.DevMAC = vm.GenerateDevMAC(instance.ID)
}
@@ -1028,10 +943,8 @@ func (s *InstanceServiceImpl) LaunchRunInstances(instances []*vm.VM, input *ec2.
slog.Info("LaunchRunInstances: completed", "requested", len(instances), "launched", successCount)
}
// RunInstances satisfies the InstanceService interface for non-daemon callers
// (tests, mocks). The daemon's NATS handler bypasses this method and calls
// PrepareRunInstances + LaunchRunInstances directly so it can respond to AWS
// between the two phases, preserving the original respond-then-launch timing.
// RunInstances is for non-daemon callers (tests). The daemon calls
// PrepareRunInstances + LaunchRunInstances directly to respond before launching.
func (s *InstanceServiceImpl) RunInstances(input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error) {
reservation, instances, instanceType, err := s.PrepareRunInstances(input, accountID)
if err != nil {
@@ -1044,9 +957,7 @@ func (s *InstanceServiceImpl) RunInstances(input *ec2.RunInstancesInput, account
return reservation, nil
}
// RebootInstance handles an ec2.cmd reboot for an instance already running on
// this node. Returns nil on success; the dispatching daemon handler maps the
// returned error to a NATS error response and responds with `{}` on nil.
// RebootInstance handles an ec2.cmd reboot for a running instance on this node.
func (s *InstanceServiceImpl) RebootInstance(instance *vm.VM, command spxtypes.EC2InstanceCommand) error {
slog.Info("RebootInstance: rebooting instance", "id", command.ID)
@@ -1068,16 +979,12 @@ func (s *InstanceServiceImpl) RebootInstance(instance *vm.VM, command spxtypes.E
return nil
}
// StartInstance handles an ec2.cmd start for an instance that is stopped on
// this node (local vmMgr state, not the cross-node shared-KV path). Returns
// nil on success; the dispatching daemon handler responds with the running
// status payload.
// StartInstance handles an ec2.cmd start for a locally stopped instance.
func (s *InstanceServiceImpl) StartInstance(instance *vm.VM, command spxtypes.EC2InstanceCommand) error {
slog.Info("StartInstance: starting instance", "id", command.ID)
status := s.vmMgr.Status(instance)
// StateError is startable: a crash/recovery-failed instance can be manually
// recovered. The manager transitions Error->Pending before relaunch.
// StateError is startable so a crashed instance can be manually recovered.
if status != vm.StateStopped && status != vm.StateError {
slog.Error("StartInstance: instance not in a startable state", "instanceId", command.ID, "status", status)
return errors.New(awserrors.ErrorIncorrectInstanceState)
@@ -1091,9 +998,8 @@ func (s *InstanceServiceImpl) StartInstance(instance *vm.VM, command spxtypes.EC
}
}
// Clear stop attribute before launch so WriteState inside the manager
// persists the correct attributes. Without this, a daemon restart after
// a stop→start cycle would see StopInstance=true and skip reconnecting QEMU.
// Clear stop attribute before launch; a stale StopInstance=true would
// cause the daemon to skip QEMU reconnect on restart.
s.vmMgr.UpdateState(instance.ID, func(v *vm.VM) { v.Attributes = command.Attributes })
if err := s.vmMgr.Start(instance.ID); err != nil {
@@ -1117,10 +1023,8 @@ func (s *InstanceServiceImpl) StartInstance(instance *vm.VM, command spxtypes.EC
return nil
}
// StopOrTerminateInstance handles an ec2.cmd stop or terminate for an
// instance running on this node. Validates the transition synchronously,
// stamps the command attributes onto the VM, then dispatches the actual
// Stop/Terminate in a background goroutine so the daemon can ack immediately.
// StopOrTerminateInstance handles an ec2.cmd stop or terminate. Validates the
// transition synchronously, then dispatches Stop/Terminate in a goroutine.
func (s *InstanceServiceImpl) StopOrTerminateInstance(instance *vm.VM, command spxtypes.EC2InstanceCommand) error {
isTerminate := command.Attributes.TerminateInstance
action := "Stopping"
@@ -1132,11 +1036,8 @@ func (s *InstanceServiceImpl) StopOrTerminateInstance(instance *vm.VM, command s
slog.Info("StopOrTerminateInstance: "+action, "id", command.ID)
// Fold the termination-protection check, idempotency check, transition
// validation, and attribute stamp into one lock acquisition so a racing
// ModifyInstanceAttribute can't clear protection between the gates. The
// async lifecycle goroutine below persists on its own state transitions,
// so no synchronous persist is needed here.
// Single lock acquisition covers protection check, idempotency, transition
// validation, and attribute stamp to prevent races with ModifyInstanceAttribute.
var (
protected, raced, stateMismatch bool
currentState vm.InstanceState
@@ -1156,10 +1057,9 @@ func (s *InstanceServiceImpl) StopOrTerminateInstance(instance *vm.VM, command s
return
}
v.Attributes = command.Attributes
// Auto-disassociate IAM instance profile on terminate (matches AWS).
// Stop/Start preserves the binding; only terminate clears it. Done
// under the same lock as the state transition so DescribeInstances
// never observes a terminated instance still advertising a profile.
// Auto-disassociate IAM profile on terminate (AWS parity; stop/start preserves it).
// Done under the same lock so DescribeInstances never sees a terminated
// instance advertising a profile.
if isTerminate && v.IamInstanceProfileArn != "" {
slog.Info("IAM instance profile auto-disassociated",
"instance_id", v.ID,
@@ -1181,13 +1081,12 @@ func (s *InstanceServiceImpl) StopOrTerminateInstance(instance *vm.VM, command s
return errors.New(awserrors.ErrorOperationNotPermitted)
}
if raced {
// Idempotent: a concurrent terminate goroutine is already cleaning up.
// Idempotent: a concurrent terminate is already in progress.
slog.Info("StopOrTerminateInstance: instance already shutting down, terminate is idempotent", "instanceId", instance.ID)
return nil
}
if stateMismatch {
// Surface IncorrectInstanceState synchronously so the AWS SDK sees
// 400 instead of a stale 200. The async path re-validates.
// Surface IncorrectInstanceState synchronously (AWS SDK expects 400).
slog.Warn("StopOrTerminateInstance: instance in incorrect state for "+strings.ToLower(action),
"instanceId", instance.ID, "currentState", string(currentState))
return errors.New(awserrors.ErrorIncorrectInstanceState)
@@ -1213,14 +1112,9 @@ func (s *InstanceServiceImpl) StopOrTerminateInstance(instance *vm.VM, command s
return nil
}
// AssociateIamInstanceProfile attaches an instance profile to a running
// instance. The gateway has already resolved the profile reference to a
// canonical ARN and enforced iam:PassRole; this method validates that the
// instance has no existing profile (returns IamInstanceProfileAlreadyAssociated
// otherwise) and atomically writes the ARN + a freshly generated
// iip-assoc-… ID to vm.VM. AssociationId is never reused — every Associate
// produces a new one. InstanceProfile.Id is left nil; the gateway enriches it
// from its IAMService cache before returning to the caller.
// AssociateIamInstanceProfile attaches an instance profile to a running instance.
// Validates no existing profile, then atomically writes the ARN + new association ID.
// InstanceProfile.Id is left nil; the gateway enriches it from IAMService.
func (s *InstanceServiceImpl) AssociateIamInstanceProfile(instance *vm.VM, command spxtypes.EC2InstanceCommand) (*ec2.IamInstanceProfileAssociation, error) {
if command.IamProfileAssociationData == nil || command.IamProfileAssociationData.InstanceProfileArn == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -1261,13 +1155,8 @@ func (s *InstanceServiceImpl) AssociateIamInstanceProfile(instance *vm.VM, comma
}, nil
}
// DisassociateIamProfileAssociation finds the VM carrying the requested
// AssociationId in this daemon's vmMgr and clears both profile fields
// atomically. Returns nil when no local VM owns the ID, or when the caller's
// account doesn't own the matching VM, so the gateway fan-out collector can
// advance past this daemon's NoOp response without waiting on the deadline.
// The accountID scope mirrors checkInstanceOwnership semantics applied
// per-instance on ec2.cmd.<id>.
// DisassociateIamProfileAssociation clears the profile association on the local VM.
// Returns nil when no local VM owns the ID so the gateway fan-out can skip this node.
func (s *InstanceServiceImpl) DisassociateIamProfileAssociation(input *ec2.DisassociateIamInstanceProfileInput, accountID string) (*ec2.IamInstanceProfileAssociation, error) {
if input == nil || input.AssociationId == nil || *input.AssociationId == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -1281,9 +1170,7 @@ func (s *InstanceServiceImpl) DisassociateIamProfileAssociation(input *ec2.Disas
var clearedArn string
timestamp := time.Now().UTC()
_, err := s.vmMgr.UpdateAndPersist(owner, func(v *vm.VM) bool {
// Re-check under the manager lock: another fan-out (or a concurrent
// terminate) could have cleared or replaced the association between
// our snapshot read and this mutation.
// Re-check under lock: a concurrent terminate may have cleared the association.
if v.IamInstanceProfileAssociationId != associationID {
return false
}
@@ -1297,8 +1184,7 @@ func (s *InstanceServiceImpl) DisassociateIamProfileAssociation(input *ec2.Disas
return nil, errors.New(awserrors.ErrorServerInternal)
}
if clearedArn == "" {
// Lost a race with terminate / another disassociate. NoOp so the
// gateway collector treats it as "not this daemon's instance".
// Lost a race; NoOp so the gateway collector skips this node.
return nil, nil
}
@@ -1313,12 +1199,9 @@ func (s *InstanceServiceImpl) DisassociateIamProfileAssociation(input *ec2.Disas
}, nil
}
// ReplaceIamProfileAssociation finds the VM carrying the requested
// AssociationId and swaps it for a new (ARN, AssociationId) pair atomically.
// The new AssociationId is always freshly generated — AWS never reuses IDs
// across replace. Returns nil when no local VM owns the old ID or when the
// caller's account doesn't own the matching VM. The gateway has already
// resolved IamInstanceProfile.Arn to a canonical ARN before publishing.
// ReplaceIamProfileAssociation atomically swaps the profile ARN and AssociationId.
// Returns nil when no local VM owns the old ID. The gateway has already resolved
// the ARN to canonical form.
func (s *InstanceServiceImpl) ReplaceIamProfileAssociation(input *ec2.ReplaceIamInstanceProfileAssociationInput, accountID string) (*ec2.IamInstanceProfileAssociation, error) {
if input == nil || input.AssociationId == nil || *input.AssociationId == "" ||
input.IamInstanceProfile == nil || aws.StringValue(input.IamInstanceProfile.Arn) == "" {
@@ -1363,12 +1246,9 @@ func (s *InstanceServiceImpl) ReplaceIamProfileAssociation(input *ec2.ReplaceIam
}, nil
}
// DescribeIamProfileAssociations walks this daemon's vmMgr and returns every
// live association visible to the caller's account matching the supplied
// filters (empty filters match all). Empty result is a valid response: the
// gateway aggregates per-daemon slices, so missing daemons just yield fewer
// rows. Filter names other than "instance-id" / "state" surface as
// InvalidParameterValue, short-circuiting the fan-out at the gateway.
// DescribeIamProfileAssociations returns live associations on this node visible to
// the caller. Supports "instance-id" and "state" filters; unknown filters return
// InvalidParameterValue.
func (s *InstanceServiceImpl) DescribeIamProfileAssociations(input *ec2.DescribeIamInstanceProfileAssociationsInput, accountID string) (*ec2.DescribeIamInstanceProfileAssociationsOutput, error) {
assocFilter := stringPtrSliceToSet(input.AssociationIds)
instFilter := make(map[string]bool)
@@ -1409,8 +1289,7 @@ func (s *InstanceServiceImpl) DescribeIamProfileAssociations(input *ec2.Describe
if len(instFilter) > 0 && !instFilter[v.ID] {
return
}
// v1 only emits "associated" — there are no async transitions, so any
// state filter that excludes "associated" yields an empty result.
// Only "associated" state exists; other state filters yield empty results.
const liveState = ec2.IamInstanceProfileAssociationStateAssociated
if len(stateFilter) > 0 && !stateFilter[liveState] {
return
@@ -1425,12 +1304,8 @@ func (s *InstanceServiceImpl) DescribeIamProfileAssociations(input *ec2.Describe
return out, nil
}
// findInstanceByAssociationID is the lock-protected lookup used by the
// fan-out mutators. It snapshots the matching instance ID under the manager
// lock and returns (id, true) only when the caller's account owns the VM,
// so cross-tenant Disassociate / Replace cannot reach into another account's
// instance. The actual mutation re-validates inside UpdateAndPersist to close
// the read/mutate race window.
// findInstanceByAssociationID returns the VM ID for the given AssociationId
// if the caller's account owns it. Mutations re-validate under UpdateAndPersist.
func (s *InstanceServiceImpl) findInstanceByAssociationID(associationID, accountID string) (string, bool) {
var owner string
s.vmMgr.ForEach(func(v *vm.VM) {
@@ -1558,10 +1433,8 @@ func (s *InstanceServiceImpl) newViperblock(volumeName string, size int, volumeC
return vb, err
}
// restoreSlogDefault re-installs the daemon's Info-level slog handler after
// viperblock.New mutates the global slog default via its SetDebug method
// (see viperblock.go SetDebug — it calls slog.SetDefault with LevelError,
// silencing every Info/Warn in the entire process).
// restoreSlogDefault re-installs the Info-level slog handler after viperblock.New
// resets it to LevelError via SetDebug, silencing all Info/Warn globally.
func restoreSlogDefault() {
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
Level: slog.LevelInfo,
@@ -1583,12 +1456,8 @@ func (s *InstanceServiceImpl) prepareRootVolume(input *ec2.RunInstancesInput, im
return errors.New(awserrors.ErrorServerInternal)
}
// Load the state from the remote backend. Only os.ErrNotExist means
// "volume doesn't exist, clone from AMI" — every other error (HMAC
// integrity failure, key mismatch, transient backend 5xx, JSON parse
// failure) must abort. Cloning from AMI on a tamper/mismatch would
// overwrite the legitimate volume's config.json with AMI base-map
// state and lose access to the live data.
// Only os.ErrNotExist means "clone from AMI"; any other error must abort.
// Treating a tamper/mismatch as missing would overwrite live volume state.
_, err = vb.LoadStateRequest("")
if err != nil && !errors.Is(err, os.ErrNotExist) {
slog.Error("Failed to load root volume state from backend",
@@ -1673,11 +1542,9 @@ func (s *InstanceServiceImpl) cloneAMIToVolume(input *ec2.RunInstancesInput, siz
return nil
}
// prepareEFIVolume creates the per-VM EFI variable store. The volume is
// sized to match the firmware's VARS template (QEMU pflash refuses any other
// size) and seeded with the template bytes so EFI variables — BootOrder,
// BootNext, secure-boot state — survive across reboots. arch is the AMI
// architecture string ("x86_64" | "arm64").
// prepareEFIVolume creates the per-VM EFI variable store, sized exactly to the
// firmware VARS template (pflash requires byte-exact size) and seeded from it.
// arch is "x86_64" | "arm64".
func (s *InstanceServiceImpl) prepareEFIVolume(imageId string, volumeConfig viperblock.VolumeConfig, instance *vm.VM, arch string) error {
codePath, varsTemplate, varsSize, err := vm.FirmwarePaths(arch)
if err != nil {
@@ -1698,10 +1565,8 @@ func (s *InstanceServiceImpl) prepareEFIVolume(imageId string, volumeConfig vipe
efiVolumeName := fmt.Sprintf("%s-efi", imageId)
efiVolumeConfig := volumeConfig
efiVolumeConfig.VolumeMetadata.VolumeID = efiVolumeName
// Zero SizeGiB so viperblock's LoadState doesn't reconcile the EFI
// volume's persisted byte-exact size up to the parent root volume's
// GiB-rounded size — QEMU pflash rejects any VARS volume larger than
// the firmware's expected variable region.
// Zero SizeGiB prevents viperblock from rounding the EFI volume size
// up to GiB boundaries; pflash rejects any size beyond the VARS region.
efiVolumeConfig.VolumeMetadata.SizeGiB = 0
efiVb, err := s.newViperblock(efiVolumeName, int(varsSize), efiVolumeConfig)
@@ -1716,11 +1581,8 @@ func (s *InstanceServiceImpl) prepareEFIVolume(imageId string, volumeConfig vipe
return errors.New(awserrors.ErrorServerInternal)
}
// Distinguish "not found" (seed from template) from any other backend
// failure (transient S3 5xx, JSON unmarshal, etc.). Treating a transient
// failure as "not found" would silently re-seed a volume on every retry,
// clobbering guest-set BootOrder. Backends signal missing-object by
// wrapping os.ErrNotExist (see viperblock.classifyStateLoad).
// Only os.ErrNotExist means "seed from template"; other errors must abort.
// Treating a transient failure as missing would clobber guest-set BootOrder.
_, loadErr := efiVb.LoadStateRequest("")
if loadErr != nil && !errors.Is(loadErr, os.ErrNotExist) {
slog.Error("Failed to load EFI volume state from backend", "name", efiVolumeName, "err", loadErr)
@@ -1755,9 +1617,7 @@ func (s *InstanceServiceImpl) prepareEFIVolume(imageId string, volumeConfig vipe
}
}
// Close is the durability boundary after WriteAt+Flush; if it fails the
// VARS volume may be partially written and QEMU pflash will refuse to
// start. Fail the launch loudly rather than enqueueing a corrupt volume.
// Close is the durability boundary; a partial VARS write causes pflash to refuse launch.
if err := efiVb.Close(); err != nil {
slog.Error("Failed to close EFI Viperblock store", "err", err)
return errors.New(awserrors.ErrorServerInternal)
@@ -1777,10 +1637,8 @@ func (s *InstanceServiceImpl) prepareEFIVolume(imageId string, volumeConfig vipe
return nil
}
// instanceArchitecture pulls the AWS architecture string off an
// ec2.InstanceTypeInfo. Returns "" when the spec is nil or malformed; the
// caller's firmware probe surfaces that as a clear error rather than a
// silent BIOS fallback.
// instanceArchitecture returns the AWS architecture string from the instance type.
// Returns "" on nil/malformed spec; the caller's firmware probe surfaces the error.
func instanceArchitecture(it *ec2.InstanceTypeInfo) string {
if it == nil || it.ProcessorInfo == nil || len(it.ProcessorInfo.SupportedArchitectures) == 0 || it.ProcessorInfo.SupportedArchitectures[0] == nil {
return ""
@@ -1788,9 +1646,8 @@ func instanceArchitecture(it *ec2.InstanceTypeInfo) string {
return *it.ProcessorInfo.SupportedArchitectures[0]
}
// prepareCloudInitVolume creates cloud-init ISO with SSH keys and user data.
// rootVolumeId is the per-instance root volume ID (not the AMI ID), ensuring
// each instance gets its own cloud-init volume with fresh SSH keys and metadata.
// prepareCloudInitVolume creates the cloud-init ISO with SSH keys and user data.
// Uses rootVolumeId (not AMI ID) so each instance gets its own volume.
func (s *InstanceServiceImpl) prepareCloudInitVolume(input *ec2.RunInstancesInput, rootVolumeId string, volumeConfig viperblock.VolumeConfig, instance *vm.VM) error {
slog.Info("Creating cloud-init volume")
@@ -1875,9 +1732,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
// Generate instance metadata
hostname := generateHostname(instance.ID)
// Retrieve SSH pubkey from S3 — required for instance access.
// Password authentication is not supported; instances without a key
// pair have no remote access method.
// Retrieve SSH pubkey from S3 — password auth is not supported.
keyName := ""
if input.KeyName != nil {
keyName = *input.KeyName
@@ -1906,9 +1761,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
}
}
// Read CA certificate for injection into guest cloud-init.
// Derive the config directory: BaseDir (e.g. ~/spinifex/spinifex/) sits one
// level below the data root; the CA cert is at <data-root>/config/ca.pem.
// Read CA cert from <data-root>/config/ca.pem and inject into guest cloud-init.
var caCertPEM string
dataRoot := filepath.Dir(strings.TrimSuffix(s.config.BaseDir, "/"))
caCertPath := filepath.Join(dataRoot, "config", "ca.pem")
@@ -1929,9 +1782,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
slog.Error("failed to read CA cert for guest cloud-init injection", "path", caCertPath, "error", err)
}
// Re-read AMI metadata for DistroFamily rather than persisting it on
// vm.VM — cloud-init renders once at first launch and the ISO is sealed,
// so no consumer needs DistroFamily after this point.
// Re-read AMI metadata for DistroFamily; not persisted on VM since the ISO is sealed once.
var distroFamily string
if s.amiLoader != nil && input.ImageId != nil && *input.ImageId != "" {
amiMeta, err := s.amiLoader.GetAMIConfig(*input.ImageId)
@@ -1988,11 +1839,8 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
}
}
// Alpine's eni renderer can't emit the gateway-less IMDS route, so deliver it
// out-of-band via a local.d script. Only when the caller didn't supply its own
// #cloud-config: that payload owns the write_files/runcmd keys (e.g. the EKS
// k3s server VM, which bakes the same route into its own block), and a second
// write_files key here would collide — last key wins, silently dropping a block.
// Inject Alpine IMDS route only when no #cloud-config user-data is present;
// a caller-supplied payload already owns write_files/runcmd and a second key collides.
if distroFamily == "alpine" && userData.UserDataCloudConfig == "" {
userData.AlpineWriteFiles, userData.AlpineRunCmd = buildAlpineCloudInit(instance.ENIMac)
}
@@ -2032,11 +1880,8 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
return errors.New(awserrors.ErrorServerInternal)
}
// Add network-config: per-interface when VPC+dev (suppresses dev default route),
// wildcard DHCP otherwise. Extra ENI MACs produce additional DHCP NICs for
// multi-subnet system VMs (multi-AZ ALBs). RHEL-family guests get a
// "config: disabled" stub here so cloud-init doesn't drop a second NM
// keyfile alongside the ones we wrote via user-data write_files.
// Add network-config: per-interface for VPC+dev, wildcard DHCP otherwise.
// RHEL gets "config: disabled" to prevent a competing NM keyfile.
networkConfig := selectNetworkConfigForFamily(distroFamily, instance.ENIMac, instance.DevMAC, instance.MgmtMAC, instance.MgmtIP, extraMACs)
err = writer.AddFile(strings.NewReader(networkConfig), "network-config")
if err != nil {
@@ -2129,9 +1974,8 @@ var DescribeInstancesValidFilters = map[string]bool{
}
// DescribeInstanceStatusValidFilters lists the filter names accepted by
// DescribeInstanceStatus. event.*, instance-status.* and system-status.*
// are rejected: Mulga's static-payload health model has no events and a
// single value per status field, so those filters can't usefully narrow.
// DescribeInstanceStatus. event.* and status.* filters are excluded as
// Mulga's health model has a single static value per status field.
var DescribeInstanceStatusValidFilters = map[string]bool{
"availability-zone": true,
"instance-state-code": true,
@@ -2235,21 +2079,15 @@ func (s *InstanceServiceImpl) DescribeInstances(input *ec2.DescribeInstancesInpu
reservationMap := make(map[string]*ec2.Reservation)
// Iterate under the manager lock — VM fields (Status, Instance, Reservation,
// PublicIP, PlacementGroupName) are mutated through manager-locked
// Inspect/UpdateState elsewhere, so reading them lock-free would race.
// Iterate under the manager lock to avoid races on Status, Instance, PublicIP, etc.
s.vmMgr.View(func(vms map[string]*vm.VM) {
for _, instance := range vms {
if !IsInstanceVisible(accountID, instance.AccountID) {
continue
}
// Platform-managed system VMs (LB HAProxy, EKS control plane) carry a
// ManagedBy tag and must stay out of customer EC2 listings, the same
// way they're filtered from the UI Nodes page. LB VMs are owned by the
// system account so IsInstanceVisible already hides them, but the EKS
// control-plane VM is owned by the customer account (its ENI lives in
// the customer VPC), so guard on ManagedBy here. Root/operator callers
// still see system instances.
// Platform-managed VMs (LB, EKS control plane) are hidden from customer
// listings. LBs are system-account-owned; EKS control-plane VMs are
// customer-account-owned so ManagedBy guards them explicitly.
if instance.ManagedBy != "" && accountID != utils.GlobalAccountID {
continue
}
@@ -2316,12 +2154,9 @@ func (s *InstanceServiceImpl) DescribeInstances(input *ec2.DescribeInstancesInpu
return &ec2.DescribeInstancesOutput{Reservations: reservations}, nil
}
// DescribeInstanceTypes returns instance types this node supports. Without
// the "capacity" filter the response lists every supported type once,
// matching AWS semantics — callers (e.g. the Terraform AWS provider) expect
// a stable answer that does not depend on current allocation. With
// `capacity=true` each type expands to one entry per free schedulable slot
// so callers can aggregate cluster-wide remaining capacity.
// DescribeInstanceTypes returns supported instance types. With filter
// capacity=true, expands each type to one entry per free slot for cluster-wide
// capacity aggregation.
func (s *InstanceServiceImpl) DescribeInstanceTypes(input *ec2.DescribeInstanceTypesInput, _ string) (*ec2.DescribeInstanceTypesOutput, error) {
slog.Info("Processing DescribeInstanceTypes request from this node")
@@ -2443,11 +2278,8 @@ func (s *InstanceServiceImpl) describeInstancesFromKV(input *ec2.DescribeInstanc
return &ec2.DescribeInstancesOutput{Reservations: reservations}, nil
}
// ModifyInstanceAttribute applies a single attribute change. InstanceType and
// UserData require the instance to be stopped. DisableApiTermination works on
// both running and stopped instances. SourceDestCheck is a networking concept
// that does not apply to bare-metal VMs; it is accepted as a no-op on any
// instance state so Terraform and the AWS CLI do not error out.
// ModifyInstanceAttribute applies a single attribute change. SourceDestCheck
// is a no-op on bare-metal VMs. InstanceType/UserData require a stopped instance.
func (s *InstanceServiceImpl) ModifyInstanceAttribute(input *ec2.ModifyInstanceAttributeInput, accountID string) (*ec2.ModifyInstanceAttributeOutput, error) {
if input.InstanceId == nil || *input.InstanceId == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -2567,10 +2399,8 @@ func (s *InstanceServiceImpl) ModifyInstanceAttribute(input *ec2.ModifyInstanceA
return &ec2.ModifyInstanceAttributeOutput{}, nil
}
// StoppedInstanceNode returns the LastNode field stored in the shared KV for a
// stopped instance. Returns "" when the store is unavailable, the instance is
// not found, or the entry has no LastNode set. Used by the daemon's ec2.start
// handler to route start requests back to the original node when possible.
// StoppedInstanceNode returns the node that last hosted the stopped instance,
// used to route start requests back to the original node when possible.
func (s *InstanceServiceImpl) StoppedInstanceNode(instanceID string) string {
if s.stoppedStore == nil {
return ""
@@ -2686,10 +2516,8 @@ func (s *InstanceServiceImpl) StartStoppedInstance(input *StartStoppedInstanceIn
return &StartStoppedInstanceOutput{Status: "running", InstanceID: instance.ID}, nil
}
// TerminateStoppedInstance picks up a stopped instance from shared KV, deletes
// its volumes, releases its public IP and ENI, writes it to the terminated
// bucket, then removes it from the stopped bucket. No QEMU shutdown or unmount
// is needed — the instance was already drained during stop.
// TerminateStoppedInstance terminates a stopped instance: deletes its volumes,
// releases its public IP and ENI, and moves it to the terminated KV bucket.
func (s *InstanceServiceImpl) TerminateStoppedInstance(input *TerminateStoppedInstanceInput, accountID string) (*TerminateStoppedInstanceOutput, error) {
if input.InstanceID == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -2791,14 +2619,8 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(instance *vm.VM, instanceID
_ = instanceID
}
// rollbackAutoAssignedPublicIP unwinds an auto-assign that proceeded past
// ENI create + attach + IPAM allocation + ENI update but failed at the
// vpc.add-nat barrier. Order matters: clear the ENI record first so any
// concurrent DescribeNetworkInterfaces stops surfacing the unreachable IP,
// then release the IPAM lease so the pool slot can be reused, then detach
// and delete the ENI itself — the failing instance never enters vmMgr, so
// terminateCleanup will not run and the ENI would otherwise leak. Detach
// must precede DeleteNetworkInterface; the latter rejects in-use ENIs.
// rollbackAutoAssignedPublicIP unwinds a failed auto-assign: clears the ENI
// public IP record, releases the IPAM lease, then detaches and deletes the ENI.
func (s *InstanceServiceImpl) rollbackAutoAssignedPublicIP(accountID, instanceID, eniID, publicIP, poolName string) {
if s.eniCreator != nil {
if err := s.eniCreator.UpdateENIPublicIP(accountID, eniID, "", ""); err != nil {
spinifex/handlers/ec2/instance/service_impl_test.go
+17 -38
View File
@@ -547,10 +547,8 @@ func TestCloudInitMetaTemplateRendering(t *testing.T) {
assert.Contains(t, rendered, "local-hostname:")
}
// TestCloudInitVolumeNamePerInstance verifies that AMI-based launches produce
// unique root volume IDs, which in turn produce unique cloud-init volume names.
// This prevents the bug where a cached cloud-init ISO (keyed by AMI) would
// serve stale SSH keys or hostnames to subsequent instances.
// TestCloudInitVolumeNamePerInstance verifies that each launch produces a unique
// root volume ID so cloud-init ISOs are not shared across instances.
func TestRunInstance_NoImageId(t *testing.T) {
instanceTypes := map[string]*ec2.InstanceTypeInfo{
"t3.micro": {InstanceType: aws.String("t3.micro")},
@@ -857,7 +855,7 @@ func TestCloudInitVolumeNamePerInstance(t *testing.T) {
}
}
// --- 1b-pre Describe* coverage (siv-22 parts) ------------------------------
// --- Describe* coverage -----------------------------------------------------
type fakeResourceCapacityProvider struct {
types []*ec2.InstanceTypeInfo
@@ -2238,7 +2236,7 @@ func TestStartStoppedInstance_GPUClaimFailureRollsBack(t *testing.T) {
assert.Empty(t, store.deletedStopped, "stopped-KV entry must remain on rollback")
}
// --- PrepareRunInstances / ec2.cmd dispatch tests (1b-pre phase 2d) ---------
// --- PrepareRunInstances / ec2.cmd dispatch tests ---------------------------
type fakeAMILoader struct {
byID map[string]viperblock.AMIMetadata
@@ -2727,12 +2725,9 @@ func TestPrepareRunInstances_PublicIPAutoAssigned(t *testing.T) {
}
}
// TestPrepareRunInstances_NATFailureRollsBackPublicIP regresses the silent
// corruption where vpc.add-nat NACKs (or vpcd is offline) but the launch
// returned an ec2.Instance carrying the unreachable IP, the ENI record was
// updated with that IP, and the IPAM pool entry stayed allocated. With the
// fix in place, NAT failure must drop the instance from the response, clear
// the ENI public IP, release the IPAM lease, and deallocate capacity.
// TestPrepareRunInstances_NATFailureRollsBackPublicIP verifies that a vpc.add-nat
// failure drops the instance, clears the ENI public IP, releases the IPAM
// lease, and deallocates capacity.
func TestPrepareRunInstances_NATFailureRollsBackPublicIP(t *testing.T) {
eni := &fakeENICreator{
subnet: &SubnetInfo{SubnetID: "subnet-1", VpcID: "vpc-1", MapPublicIpOnLaunch: true},
@@ -2824,12 +2819,9 @@ func TestPrepareRunInstances_NATFailureRollsBackPublicIP(t *testing.T) {
}
}
// TestPrepareRunInstances_PublicIPAllocFailureAbortsLaunch regresses the
// silent failure where a failed public-IP allocation (pool exhausted) was
// logged at WARN and ignored, so the instance booted running with no public
// IP. There is no qemu-hostfwd fallback, so such an instance is unreachable.
// The fix must fail the launch: detach + delete the auto-created ENI,
// deallocate capacity, drop the instance, and surface InsufficientAddressCapacity.
// TestPrepareRunInstances_PublicIPAllocFailureAbortsLaunch verifies that a
// failed public-IP allocation aborts the launch: detaches and deletes the ENI,
// deallocates capacity, and returns InsufficientAddressCapacity.
func TestPrepareRunInstances_PublicIPAllocFailureAbortsLaunch(t *testing.T) {
eni := &fakeENICreator{
subnet: &SubnetInfo{SubnetID: "subnet-1", VpcID: "vpc-1", MapPublicIpOnLaunch: true},
@@ -2896,16 +2888,9 @@ func TestPrepareRunInstances_ENICreateFailureDeallocates(t *testing.T) {
require.Len(t, prov.deallocated, 1, "ENI failure must trigger deallocate")
}
// TestPrepareRunInstances_ENIAttachFailureRollsBack regresses the silent
// corruption where AttachENI failure was logged-and-ignored, leaving the
// auto-created ENI unattached in vpcService. Later RegisterTargets with
// TargetType=instance enumerates ENIs by attachment.instance-id, finds
// none, and drops the target — the customer's ALB has 0 healthy targets
// with no log line connecting the two events. The dangling ENI also leaks
// its auto-assigned EIP on terminate because DeleteNetworkInterface
// cannot resolve the instance link. With the fix, AttachENI failure must
// delete the auto-created ENI, deallocate capacity, and drop the instance
// from the reservation.
// TestPrepareRunInstances_ENIAttachFailureRollsBack verifies that an AttachENI
// failure deletes the auto-created ENI, deallocates capacity, and drops the
// instance from the reservation.
func TestPrepareRunInstances_ENIAttachFailureRollsBack(t *testing.T) {
eni := &fakeENICreator{
attachErr: errors.New("vpc attach refused"),
@@ -3038,10 +3023,8 @@ func TestStartInstance_AllocateFails(t *testing.T) {
assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, err.Error())
}
// TestStartInstance_ErrorStateStartable verifies a crash/recovery-failed
// instance (StateError) is accepted past the startability guard. Reaching the
// resource Allocate step (here forced to fail) proves the guard did not reject
// it with IncorrectInstanceState — manual recovery of errored instances works.
// TestStartInstance_ErrorStateStartable verifies a StateError instance passes
// the startability guard and reaches resource allocation.
func TestStartInstance_ErrorStateStartable(t *testing.T) {
id := "i-err"
mgr := mgrWith(map[string]*vm.VM{
@@ -3068,12 +3051,8 @@ func TestRebootInstance_NotFound(t *testing.T) {
assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
}
// TestStartInstance_NotFound verifies that when vmMgr.Start cannot find the
// instance (e.g. it was terminated between the daemon's pre-dispatch lookup
// and the manager call), the service returns InvalidInstanceID.NotFound
// rather than collapsing to a generic Server.InternalError. The AWS SDK
// retries 500s, so the prior behaviour produced duplicate Start attempts
// before the client surfaced the failure.
// TestStartInstance_NotFound verifies that a missing instance returns
// InvalidInstanceID.NotFound rather than a generic internal error.
func TestStartInstance_NotFound(t *testing.T) {
id := "i-missing"
mgr := mgrWith(nil)
spinifex/handlers/ec2/instance/service_nats.go
+3 -8
View File
@@ -10,14 +10,9 @@ import (
"github.com/nats-io/nats.go"
)
// NATSInstanceService handles instance operations via NATS messaging.
//
// Note: DescribeInstances and DescribeInstanceTypes subscriptions are
// fan-out (no queue group); a single NATSRequest only captures one
// responder. Cluster-wide aggregation is the gateway's job (see
// spinifex/gateway/ec2/instance). The methods below are kept for symmetry
// with the InstanceService interface and intentionally fail rather than
// silently returning a one-node view.
// NATSInstanceService handles instance operations via NATS. Fan-out methods
// (DescribeInstances, DescribeInstanceTypes) intentionally error — use the
// gateway's scatter-gather helper for cluster-wide results.
type NATSInstanceService struct {
natsConn *nats.Conn
}
spinifex/handlers/ec2/key/service_impl.go
+2 -4
View File
@@ -367,10 +367,8 @@ func (s *KeyServiceImpl) ValidateKeyPairExists(accountID, keyName string) error
return err
}
// GetPublicKeyMaterial returns the stored OpenSSH public key line, trimmed to a
// single clean line. NoSuchKey maps to ErrorInvalidKeyPairNotFound; every other
// fault is returned as-is so the caller can distinguish absence (404) from a
// backend error (500).
// GetPublicKeyMaterial returns the stored OpenSSH public key line, trimmed to a single
// line. NoSuchKey maps to ErrorInvalidKeyPairNotFound; other errors are returned as-is.
func (s *KeyServiceImpl) GetPublicKeyMaterial(accountID, keyName string) (string, error) {
keyPath := fmt.Sprintf("keys/%s/%s", accountID, keyName)
result, err := s.store.GetObject(&s3.GetObjectInput{
spinifex/handlers/ec2/placementgroup/service_impl.go
+1 -3
View File
@@ -317,9 +317,7 @@ func (s *PlacementGroupServiceImpl) UpdatePlacementGroupRecord(accountID, groupN
const maxCASRetries = 5
// ReserveSpreadNodes atomically reserves node slots for a spread placement group launch.
// It reads the group record, filters eligible nodes (excluding already-occupied ones),
// selects nodes, writes placeholder entries, and returns the selected nodes.
// Uses CAS with retries following the IPAM pattern.
// Filters occupied nodes, selects up to MaxCount, writes placeholders via CAS with retries.
func (s *PlacementGroupServiceImpl) ReserveSpreadNodes(input *ReserveSpreadNodesInput, accountID string) (*ReserveSpreadNodesOutput, error) {
if input.GroupName == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
spinifex/handlers/ec2/placementgroup/service_impl_test.go
+3 -6
View File
@@ -562,12 +562,9 @@ func TestRemoveInstance_Success(t *testing.T) {
assert.Equal(t, []string{"i-bbb"}, record.NodeInstances["node-b"])
}
// TestRemoveInstance_NodeNotFound / _GroupNotFound pin the idempotency
// contract in service_impl.go:453-497 — RemoveInstance must succeed when the
// node or group is missing, since callers (instance Terminate) may race with
// group deletion or have already moved the instance off the node. Production
// marks both paths with //nolint:nilerr // intentional; without these tests a
// future cleanup could remove the annotations and silently flip the contract.
// TestRemoveInstance_NodeNotFound / _GroupNotFound pin the idempotency contract:
// RemoveInstance must succeed when the node or group is missing (callers may race
// with group deletion or have already moved the instance off the node).
func TestRemoveInstance_NodeNotFound(t *testing.T) {
svc := setupTestService(t)
spinifex/handlers/ec2/routetable/service_impl.go
+30 -64
View File
@@ -128,15 +128,9 @@ func (s *RouteTableServiceImpl) getVPCCidr(accountID, vpcID string) (string, err
return vpcRecord.CidrBlock, nil
}
// mainRouteTable returns the VPC's main route table or (nil, nil) if none
// exists. Used to enumerate effective routes for subnets that have no
// explicit RT association (AWS implicit-main semantics).
//
// Deterministic across duplicate IsMain=true records: see preferMain. A
// duplicate is a bootstrap-race symptom — the canonical fix lives in
// VPCServiceImpl.createMainRouteTable, but the picker stays defensive so a
// stale dup (e.g. cross-AZ restore, manual KV edit) can't break gate
// recomputation by selecting the orphan non-deterministically.
// mainRouteTable returns the VPC's main route table, or (nil, nil) if none exists.
// Applies AWS implicit-main semantics; deterministic across duplicate IsMain=true
// records via preferMain.
func (s *RouteTableServiceImpl) mainRouteTable(accountID, vpcID string) (*RouteTableRecord, error) {
rts, err := s.allRouteTablesForVPC(accountID, vpcID)
if err != nil {
@@ -205,12 +199,9 @@ func (s *RouteTableServiceImpl) subnetsImplicitlyOnMainRT(accountID, vpcID strin
return implicit, nil
}
// effectiveRouteTable returns the route table whose routes apply to subnetID
// per AWS semantics: explicit non-main association if one exists, otherwise
// the VPC's main RT. Returns (nil, nil) when neither is present.
//
// Main-RT selection is deterministic across IsMain=true duplicates via
// preferMain — see mainRouteTable.
// effectiveRouteTable returns the route table whose routes apply to subnetID per AWS
// semantics: explicit non-main association if present, otherwise the VPC's main RT.
// Returns (nil, nil) when neither is present. Main-RT ties broken via preferMain.
func (s *RouteTableServiceImpl) effectiveRouteTable(accountID, vpcID, subnetID string) (*RouteTableRecord, error) {
rts, err := s.allRouteTablesForVPC(accountID, vpcID)
if err != nil {
@@ -230,12 +221,9 @@ func (s *RouteTableServiceImpl) effectiveRouteTable(accountID, vpcID, subnetID s
return main, nil
}
// preferMain reports whether candidate should replace current as the chosen
// main RT. Ordered tiebreakers when two IsMain=true records survive for one
// VPC: (1) more routes — user-added 0.0.0.0/0 etc. flag the "real" RT vs an
// orphan that only carries the implicit local route; (2) oldest CreatedAt —
// the original wins over a late dup; (3) smaller RouteTableId — total order
// so the same input always produces the same answer across nodes.
// preferMain reports whether candidate should replace current as the chosen main RT.
// Tiebreakers: (1) more routes, (2) oldest CreatedAt, (3) smaller RouteTableId —
// giving a deterministic total order across duplicate IsMain=true records.
func preferMain(current, candidate *RouteTableRecord) bool {
if current == nil {
return true
@@ -249,10 +237,8 @@ func preferMain(current, candidate *RouteTableRecord) bool {
return candidate.RouteTableId < current.RouteTableId
}
// subnetHasDefaultEgress reports whether subnetID's effective RT carries a
// destCidr route pointing at any internet-bound target (IGW or NAT GW). If
// false, the subnet must be gated with a DROP policy because the VPC LR's
// router-wide default static route would otherwise let packets egress.
// subnetHasDefaultEgress reports whether subnetID's effective RT carries a destCidr
// route pointing at an IGW or NAT GW; false means the subnet must be gated with a DROP policy.
func (s *RouteTableServiceImpl) subnetHasDefaultEgress(accountID, vpcID, subnetID, destCidr string) (bool, error) {
rt, err := s.effectiveRouteTable(accountID, vpcID, subnetID)
if err != nil {
@@ -275,12 +261,9 @@ func (s *RouteTableServiceImpl) subnetHasDefaultEgress(accountID, vpcID, subnetI
return false, nil
}
// publishSubnetEgressGateDecision recomputes the gate/ungate decision for one
// subnet+destCidr pair and emits either vpc.gate-subnet-egress (when the
// effective RT lacks an internet-bound route) or vpc.ungate-subnet-egress
// (when one is present). Called after every per-subnet IGW/NATGW event so
// the drop policy state always matches the AWS RT view. Side-effect only;
// errors are logged.
// publishSubnetEgressGateDecision emits vpc.gate-subnet-egress or vpc.ungate-subnet-egress
// for one subnet+destCidr based on whether the effective RT has an internet-bound route.
// Side-effect only; errors are logged.
func (s *RouteTableServiceImpl) publishSubnetEgressGateDecision(accountID, vpcID, subnetID, destCidr string) {
if s.natsConn == nil || subnetID == "" || vpcID == "" {
return
@@ -320,11 +303,9 @@ func (s *RouteTableServiceImpl) publishSubnetEgressGateDecision(accountID, vpcID
"topic", topic, "vpcId", vpcID, "subnetId", subnetID, "destinationCidr", destCidr)
}
// PublishGateDecisionsForVPC recomputes gate/ungate for every subnet in
// vpcID against destCidr by enumerating all subnets in the VPC and resolving
// each one's effective RT. Called from the IGW handler on Attach / Detach so
// per-subnet OVN DROP policies are reinstated immediately instead of waiting
// for the reconciler's drift tick. destCidr defaults to 0.0.0.0/0.
// PublishGateDecisionsForVPC recomputes gate/ungate for every subnet in vpcID against
// destCidr, resolving each subnet's effective RT. Called on IGW Attach/Detach.
// destCidr defaults to 0.0.0.0/0.
func (s *RouteTableServiceImpl) PublishGateDecisionsForVPC(accountID, vpcID, destCidr string) {
if s.natsConn == nil || vpcID == "" {
return
@@ -909,10 +890,8 @@ func (s *RouteTableServiceImpl) AssociateRouteTable(input *ec2.AssociateRouteTab
return nil, err
}
// Subnet leaves implicit main-RT membership: tear down per-subnet rules
// that came from the main RT's routes before the explicit RT's rules
// take effect. Skip if the explicit RT IS the main RT (can't happen via
// public API but DescribeRouteTables permits the query).
// Subnet leaves implicit main-RT membership: tear down main RT's per-subnet rules
// before the explicit RT's rules take effect.
if !record.IsMain {
if mainRT, err := s.mainRouteTable(accountID, record.VpcId); err != nil {
slog.Warn("AssociateRouteTable: main RT lookup failed", "vpcId", record.VpcId, "err", err)
@@ -995,10 +974,7 @@ func (s *RouteTableServiceImpl) DisassociateRouteTable(input *ec2.DisassociateRo
s.publishNatGatewayEventsForAssociation(accountID, "vpc.delete-nat-gateway", &record, departingSubnetID)
s.publishIGWRouteEventsForAssociation(accountID, "vpc.delete-igw-route", &record, departingSubnetID)
// Subnet falls back to implicit main-RT membership: re-install
// per-subnet rules for the main RT's routes (no-op if departing
// table WAS the main, which can't happen since DisassociateMain
// is rejected above).
// Subnet falls back to implicit main-RT membership: re-install main RT's per-subnet rules.
if mainRT, err := s.mainRouteTable(accountID, record.VpcId); err != nil {
slog.Warn("DisassociateRouteTable: main RT lookup failed", "vpcId", record.VpcId, "err", err)
} else if mainRT != nil && mainRT.RouteTableId != record.RouteTableId {
@@ -1247,10 +1223,8 @@ func rtbMatchesFilters(record *RouteTableRecord, filters map[string][]string) bo
return filterutil.MatchesTags(filters, record.Tags)
}
// publishNatGatewayEvents publishes vpc.add-nat-gateway events for each subnet
// associated with this route table, so vpcd creates the SNAT rules. Called by
// CreateRoute when a NAT GW route is added to a table that may already have
// subnet associations.
// publishNatGatewayEvents publishes vpc.add-nat-gateway for each associated subnet
// so vpcd creates the SNAT rules.
func (s *RouteTableServiceImpl) publishNatGatewayEvents(accountID string, record *RouteTableRecord, vpcID, natgwID, publicIp, destCidr string) {
if s.natsConn == nil {
return
@@ -1324,10 +1298,8 @@ func (s *RouteTableServiceImpl) publishNatGatewayDeleteEvents(accountID string,
}
}
// publishNatGatewayEventsForAssociation emits one NAT GW SNAT event per NAT GW
// route on the table, scoped to a single subnet. Called when a subnet joins or
// leaves a route table that already has NAT GW routes so OVN SNAT state tracks
// association lifecycle (terraform creates the route first, then associates).
// publishNatGatewayEventsForAssociation emits one NAT GW SNAT event per NAT GW route
// on the table, scoped to a single subnet joining or leaving the table.
func (s *RouteTableServiceImpl) publishNatGatewayEventsForAssociation(accountID, topic string, record *RouteTableRecord, subnetID string) {
if s.natsConn == nil || subnetID == "" {
return
@@ -1398,10 +1370,8 @@ func (s *RouteTableServiceImpl) publishNatGatewayEventForSubnet(accountID, topic
slog.Info("NAT GW event published", "topic", topic, "subnetCidr", subnet.CidrBlock, "publicIp", publicIp, "subnetId", subnetID, "destinationCidr", destCidr)
}
// publishIGWRouteEvents emits one vpc.{add,delete}-igw-route event per subnet
// associated with the route table. Called by CreateRoute/DeleteRoute when an
// IGW route is added or removed against a table that may already carry
// subnet associations.
// publishIGWRouteEvents emits one vpc.{add,delete}-igw-route event per associated
// subnet when an IGW route is added or removed from a table.
func (s *RouteTableServiceImpl) publishIGWRouteEvents(accountID, topic string, record *RouteTableRecord, vpcID, igwID, destCidr string) {
if s.natsConn == nil {
return
@@ -1430,10 +1400,8 @@ func (s *RouteTableServiceImpl) publishIGWRouteEvents(accountID, topic string, r
}
}
// publishIGWRouteEventsForAssociation emits one IGW route event per IGW route
// on the table, scoped to a single subnet. Called when a subnet joins or leaves
// a route table that already has IGW routes so OVN policy state tracks
// association lifecycle (terraform creates the route first, then associates).
// publishIGWRouteEventsForAssociation emits one IGW route event per IGW route on the
// table, scoped to a single subnet joining or leaving the table.
func (s *RouteTableServiceImpl) publishIGWRouteEventsForAssociation(accountID, topic string, record *RouteTableRecord, subnetID string) {
if s.natsConn == nil || subnetID == "" {
return
@@ -1446,10 +1414,8 @@ func (s *RouteTableServiceImpl) publishIGWRouteEventsForAssociation(accountID, t
}
}
// publishIGWRouteEventForSubnet publishes a single vpc.{add,delete}-igw-route
// event. Side-effect only — logs and swallows errors so a publish failure
// doesn't fail the caller's API response. The subscriber (network/subscribers)
// resolves the gateway nexthop and installs the OVN Logical_Router_Policy.
// publishIGWRouteEventForSubnet publishes a single vpc.{add,delete}-igw-route event.
// Side-effect only — logs and swallows errors so a publish failure doesn't fail the caller.
func (s *RouteTableServiceImpl) publishIGWRouteEventForSubnet(topic, subnetID, vpcID, igwID, destCidr string) {
if s.natsConn == nil {
return
spinifex/handlers/ec2/routetable/service_impl_test.go
+17 -38
View File
@@ -491,10 +491,8 @@ func receiveNatGWEvent(t *testing.T, sub *nats.Subscription, wantCidr string) (n
return evt.NatGatewayId, evt.PublicIp
}
// TestAssociateRouteTable_PublishesNatGatewayEvent covers the terraform flow:
// CreateRouteTable → CreateRoute(NAT GW) → AssociateRouteTable(subnet). CreateRoute
// runs against an empty-association table so no event fires there; Associate must
// emit the event so vpcd installs the SNAT rule for the joining subnet.
// TestAssociateRouteTable_PublishesNatGatewayEvent covers the terraform flow where
// AssociateRouteTable must emit a NAT GW event so vpcd installs the SNAT rule.
func TestAssociateRouteTable_PublishesNatGatewayEvent(t *testing.T) {
svc := setupTestService(t)
sub, err := svc.natsConn.SubscribeSync("vpc.add-nat-gateway")
@@ -577,10 +575,8 @@ func TestDisassociateRouteTable_PublishesNatGatewayDeleteEvent(t *testing.T) {
assert.Equal(t, "nat-test1", natgwID)
}
// TestDeleteRoute_NATGW_PublishesDeleteForAssociatedSubnets covers DeleteRoute
// on a NATGW route: every subnet associated with the table receives a
// vpc.delete-nat-gateway event so the subscriber tears down both the SNAT rule
// and the per-subnet egress policy (mirror of the IGW DeleteRoute path).
// TestDeleteRoute_NATGW_PublishesDeleteForAssociatedSubnets verifies that every
// associated subnet receives a vpc.delete-nat-gateway event when a NATGW route is removed.
func TestDeleteRoute_NATGW_PublishesDeleteForAssociatedSubnets(t *testing.T) {
svc := setupTestService(t)
addSub, err := svc.natsConn.SubscribeSync("vpc.add-nat-gateway")
@@ -707,10 +703,8 @@ func TestCreateRoute_IGW_PublishesAddIGWRouteForAssociatedSubnets(t *testing.T)
assert.Equal(t, "igw-test1", igwID)
}
// TestAssociateRouteTable_PublishesAddIGWRouteForExistingRoute covers the
// terraform ordering: CreateRoute(IGW) → AssociateRouteTable. CreateRoute runs
// against an empty-association table so no event fires; Associate must emit
// the event so the subscriber installs the policy for the joining subnet.
// TestAssociateRouteTable_PublishesAddIGWRouteForExistingRoute covers the terraform
// ordering: AssociateRouteTable must emit an IGW route event for the joining subnet.
func TestAssociateRouteTable_PublishesAddIGWRouteForExistingRoute(t *testing.T) {
svc := setupTestService(t)
sub, err := svc.natsConn.SubscribeSync("vpc.add-igw-route")
@@ -825,10 +819,8 @@ func drainIGWSubnets(t *testing.T, sub *nats.Subscription, count int) map[string
return got
}
// TestCreateRoute_IGW_OnMainRT_FansOutToImplicitSubnets covers the primary
// publish-gap fixed in this change: CreateRoute(IGW) on a VPC's main RT must
// emit one vpc.add-igw-route event per subnet that has no explicit non-main
// RT association (AWS implicit-main semantics).
// TestCreateRoute_IGW_OnMainRT_FansOutToImplicitSubnets verifies CreateRoute(IGW) on
// the main RT emits one vpc.add-igw-route per implicit-main subnet.
func TestCreateRoute_IGW_OnMainRT_FansOutToImplicitSubnets(t *testing.T) {
svc := setupTestService(t)
sub, err := svc.natsConn.SubscribeSync("vpc.add-igw-route")
@@ -882,10 +874,8 @@ func TestCreateRoute_IGW_OnMainRT_SkipsExplicitlyAssociatedSubnets(t *testing.T)
assert.False(t, got["subnet-priv1"], "explicitly-associated subnet must NOT receive main-RT event")
}
// TestAssociateRouteTable_RemovesMainRTRoutesForJoiningSubnet covers the
// symmetric fix: when a subnet leaves implicit main-RT membership for an
// explicit non-main RT, the main RT's per-subnet rules must be torn down so
// state matches AWS effective-route semantics.
// TestAssociateRouteTable_RemovesMainRTRoutesForJoiningSubnet verifies that the main
// RT's per-subnet rules are torn down when a subnet joins an explicit non-main RT.
func TestAssociateRouteTable_RemovesMainRTRoutesForJoiningSubnet(t *testing.T) {
svc := setupTestService(t)
delSub, err := svc.natsConn.SubscribeSync("vpc.delete-igw-route")
@@ -913,13 +903,8 @@ func TestAssociateRouteTable_RemovesMainRTRoutesForJoiningSubnet(t *testing.T) {
assert.Len(t, got, 1)
}
// TestDisassociateRouteTable_RestoresMainRTRoutesForDepartingSubnet covers
// the symmetric add-back: when a subnet leaves an explicit RT and falls back
// to implicit main-RT membership, the main RT's per-subnet rules must be
// re-installed.
// receiveGateEvent reads one message from sub and returns the (subnet,
// destCidr) it carried. Used for both vpc.gate-subnet-egress and
// vpc.ungate-subnet-egress since the payload shape is identical.
// receiveGateEvent reads one gate/ungate message from sub and returns the
// (subnet, destCidr) it carried.
func receiveGateEvent(t *testing.T, sub *nats.Subscription) (subnetID, destCidr string) {
t.Helper()
msg, err := sub.NextMsg(time.Second)
@@ -951,10 +936,8 @@ func drainGateSubnets(t *testing.T, sub *nats.Subscription, limit int) map[strin
return got
}
// TestCreateRoute_IGW_PublishesUngateForAssociatedSubnets: when a subnet is
// already associated to a route table and an IGW default route is added,
// the gate decision flips to "egress allowed" so the subscriber must remove
// any previously-installed drop policy.
// TestCreateRoute_IGW_PublishesUngateForAssociatedSubnets: adding an IGW default route
// to a table must flip associated subnets to "egress allowed" (ungate).
func TestCreateRoute_IGW_PublishesUngateForAssociatedSubnets(t *testing.T) {
svc := setupTestService(t)
ungate, err := svc.natsConn.SubscribeSync("vpc.ungate-subnet-egress")
@@ -1269,13 +1252,9 @@ func TestPublishGateDecisionsForVPC_UngatesSubnetWithIGWRoute(t *testing.T) {
assert.Len(t, gateGot, 1)
}
// TestEffectiveRouteTable_DeterministicWithDuplicateMain seeds two
// IsMain=true RTs for the same VPC (the bootstrap-race symptom) and asserts
// effectiveRouteTable picks the one with more routes — the "real" RT carries
// any user-added 0.0.0.0/0 entry, while the orphan only has the implicit
// local route. Non-deterministic selection produced the run 26699045536
// guest-SSH timeout: CreateRoute's post-write gate recompute happened to
// resolve to the orphan and published vpc.gate-subnet-egress.
// TestEffectiveRouteTable_DeterministicWithDuplicateMain seeds two IsMain=true RTs
// for the same VPC and asserts effectiveRouteTable deterministically picks the one
// with more routes (the "real" RT) over the orphan with only the implicit local route.
func TestEffectiveRouteTable_DeterministicWithDuplicateMain(t *testing.T) {
svc := setupTestService(t)
now := time.Now()
spinifex/handlers/ec2/volume/service_impl.go
+12 -26
View File
@@ -785,10 +785,9 @@ type volumeModificationResult struct {
err error
}
// fetchVolumeModificationsByIDs reads each requested volume's config in
// parallel, returning a result slice positionally aligned with volumeIDs.
// Cross-tenant volumes surface as InvalidVolume.NotFound, mirroring
// DescribeVolumes' silent tenant scoping.
// fetchVolumeModificationsByIDs reads each requested volume's config in parallel,
// returning results positionally aligned with volumeIDs. Cross-tenant volumes surface
// as InvalidVolume.NotFound.
func (s *VolumeServiceImpl) fetchVolumeModificationsByIDs(volumeIDs []*string, accountID string) []volumeModificationResult {
results := make([]volumeModificationResult, len(volumeIDs))
var wg sync.WaitGroup
@@ -980,10 +979,8 @@ func (s *VolumeServiceImpl) GetVolumeConfig(volumeID string) (*viperblock.Volume
return cfg, err
}
// getVolumeConfigAndEncryption reads config.json and surfaces both the
// VolumeConfig and the authoritative VBState.EncryptionEnabled flag. Wrapper-
// only blobs (pre-VBState volumes or freshly-seeded test fixtures) report
// encryptionEnabled=false — they predate encryption-at-rest by construction.
// getVolumeConfigAndEncryption reads config.json and returns the VolumeConfig plus the
// VBState.EncryptionEnabled flag. Pre-VBState blobs report encryptionEnabled=false.
func (s *VolumeServiceImpl) getVolumeConfigAndEncryption(volumeID string) (*viperblock.VolumeConfig, bool, error) {
configKey := volumeID + "/config.json"
@@ -1043,16 +1040,9 @@ func (s *VolumeServiceImpl) putVolumeConfig(volumeID string, cfg *viperblock.Vol
return nil
}
// mergeVolumeConfig reads existing config.json from S3 and merges the new
// VolumeConfig into it, preserving full VBState when present. If no existing
// VBState is found, it returns a plain volumeConfigWrapper.
//
// Refuses to merge an encrypted-at-rest VBState: the persisted blob carries a
// trailing 16-byte AES-GCM tag bound to the JSON bytes, and re-marshaling
// here without the master key would strip the tag and brick the volume.
// Spinifex does not currently hold the viperblock master key, so the safe
// behavior is to fail loud rather than silently corrupt the on-disk state.
// Track full encryption-aware merge under a separate bead.
// mergeVolumeConfig reads existing config.json from S3 and merges the new VolumeConfig
// into it, preserving full VBState when present. Refuses to merge encrypted VBState —
// re-marshaling without the master key would corrupt the on-disk AES-GCM tag.
func (s *VolumeServiceImpl) mergeVolumeConfig(configKey string, cfg *viperblock.VolumeConfig) ([]byte, error) {
getResult, err := s.store.GetObject(&s3.GetObjectInput{
Bucket: aws.String(s.bucketName),
@@ -1072,10 +1062,8 @@ func (s *VolumeServiceImpl) mergeVolumeConfig(configKey string, cfg *viperblock.
return nil, fmt.Errorf("failed to read existing config: %w", err)
}
// Decoder (not Unmarshal) so a trailing 16-byte AES-GCM tag does not
// produce a parse error and route the body to the wrapper-only path —
// that would silently overwrite an encrypted config.json with a tag-less
// blob and brick the volume.
// Use Decoder (not Unmarshal): a trailing AES-GCM tag must not cause a parse error
// that would silently route to the wrapper-only path and brick the volume.
var state viperblock.VBState
if decodeErr := json.NewDecoder(bytes.NewReader(body)).Decode(&state); decodeErr != nil || state.BlockSize == 0 {
// Not a full VBState (new volume or wrapper-only) -- write wrapper
@@ -1180,10 +1168,8 @@ func (s *VolumeServiceImpl) ModifyVolume(input *ec2.ModifyVolumeInput, accountID
}
targetIOPS := int64(volMeta.IOPS)
// Persist the modification record alongside the volume metadata so
// DescribeVolumesModifications can read it back. spinifex applies
// modifications synchronously, so the persisted state is always
// completed/100/EndTime==StartTime.
// Persist the modification record so DescribeVolumesModifications can read it back.
// Modifications are synchronous, so state is always completed/100.
now := time.Now()
cfg.Modification = &viperblock.VolumeModification{
VolumeID: volumeID,
spinifex/handlers/ec2/volume/service_impl_test.go
+1 -4
View File
@@ -963,10 +963,7 @@ func TestCreateVolume_StampsAccountID(t *testing.T) {
store := objectstore.NewMemoryObjectStore()
svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
// CreateVolume will fail at viperblock layer (no real S3), but the
// VolumeConfig should have TenantID set if we can inspect it.
// We test via the validation path — CreateVolume with invalid size
// should NOT fail because of accountID.
// Test via the validation path — CreateVolume should not fail because of accountID.
_, err := svc.CreateVolume(&ec2.CreateVolumeInput{
Size: aws.Int64(1),
AvailabilityZone: aws.String("ap-southeast-2a"),
spinifex/handlers/ec2/vpc/eni.go
+8 -21
View File
@@ -575,10 +575,8 @@ func (s *VPCServiceImpl) UpdateENI(accountID, eniId string, fn func(*ENIRecord))
return nil
}
// FindENIByAttachment scans the per-account ENI bucket for a record whose
// AttachmentId matches attachmentId. Returns NotFound semantics when no
// match exists. Used by DetachNetworkInterface, whose AWS input identifies
// the attachment by id rather than by ENI id.
// FindENIByAttachment scans the ENI bucket for the record with the given
// AttachmentId. Used by DetachNetworkInterface which identifies by attachment ID.
func (s *VPCServiceImpl) FindENIByAttachment(accountID, attachmentId string) (ENIRecord, error) {
keys, err := s.eniKV.Keys()
if err != nil && !errors.Is(err, nats.ErrNoKeysFound) {
@@ -721,10 +719,8 @@ func (s *VPCServiceImpl) publishPortEvent(topic, eniId, subnetId, vpcId, private
})
}
// requestPortEvent sends a port lifecycle event via request-reply with the
// shared SG-event timeout. Used for vpc.create-port so vpcd OVSDB failures
// (which leave the LSP outside any SG port group) surface to the caller
// instead of being swallowed.
// requestPortEvent sends a port lifecycle event via request-reply so vpcd
// OVSDB failures surface to the caller rather than being swallowed.
func (s *VPCServiceImpl) requestPortEvent(topic, eniId, subnetId, vpcId, privateIP, macAddr string, sgIds []string) error {
return utils.RequestEvent(s.natsConn, topic, portEventPayload{
NetworkInterfaceId: eniId,
@@ -737,9 +733,7 @@ func (s *VPCServiceImpl) requestPortEvent(topic, eniId, subnetId, vpcId, private
}
// requestUpdatePortSGsEvent sends a vpc.update-port-sgs event to vpcd via
// request-reply so OVN port-group reconciliation errors surface to the caller
// instead of being swallowed. The payload is declarative — vpcd reads its
// libovsdb cache to compute the diff.
// request-reply. Errors surface to the caller; vpcd computes the OVN diff.
func (s *VPCServiceImpl) requestUpdatePortSGsEvent(eniId, privateIP string, sgIds []string) error {
return utils.RequestEvent(s.natsConn, "vpc.update-port-sgs", struct {
NetworkInterfaceId string `json:"network_interface_id"`
@@ -768,16 +762,9 @@ func (s *VPCServiceImpl) publishNATEvent(topic, vpcId, externalIP, logicalIP, po
})
}
// validateSGAttachment is the boundary validator shared by RunInstances (via
// CreateNetworkInterface), CreateNetworkInterface itself, and
// ModifyNetworkInterfaceAttribute. It runs *before* any KV write so failures
// don't leave half-state.
//
// AWS contract reproduced here:
// - InvalidGroup.NotFound when an SG ID is unknown to the account.
// - InvalidParameterValue when an SG's VPC differs from the resolved VPC.
// - SecurityGroupsPerInterfaceLimitExceeded when >5 SGs (AWS default).
// - MissingParameter when the list is empty (every ENI must have ≥1 SG).
// validateSGAttachment validates SG IDs before any KV write. Returns
// InvalidGroup.NotFound, InvalidParameterValue, SecurityGroupsPerInterfaceLimitExceeded,
// or MissingParameter per AWS contract.
const sgPerInterfaceLimit = 5
func (s *VPCServiceImpl) validateSGAttachment(accountID string, sgIds []string, vpcId string) error {
spinifex/handlers/ec2/vpc/external_ipam.go
+4 -8
View File
@@ -25,10 +25,8 @@ const (
KVBucketExternalIPAMVersion = external.KVBucketStaticPoolVersion
)
// ExternalPoolConfig is the admin-defined pool from spinifex.toml.
// Duplicated against network/external.ExternalPoolConfig — the duplicate
// stays until ExternalIPAM moves into network/external (deferred L5
// cleanup tracked separately).
// ExternalPoolConfig is the admin-defined IP pool from spinifex.toml.
// Duplicated from network/external.ExternalPoolConfig pending consolidation.
type ExternalPoolConfig struct {
Name string
Source string // "static" (default) or "dhcp"
@@ -47,10 +45,8 @@ type ExternalPoolConfig struct {
GwLrpRangeEnd string
}
// ExternalIPAM is the AWS-facing entry point for external IP allocation.
// State + CAS logic lives in external.StaticPoolAllocator for static
// pools and dhcp.DHCPPoolAllocator (via vpcd RPC) for dhcp-sourced
// pools; ExternalIPAM dispatches by pool name.
// ExternalIPAM is the AWS-facing entry point for external IP allocation,
// dispatching to StaticPoolAllocator or dhcp.DHCPPoolAllocator per pool name.
type ExternalIPAM struct {
kv nats.KeyValue
pools []ExternalPoolConfig
spinifex/handlers/ec2/vpc/external_ipam_test.go
+2 -2
View File
@@ -100,8 +100,8 @@ func TestExternalIPAM_Release(t *testing.T) {
err = ipam.ReleaseIP("wan", "192.168.1.151")
require.NoError(t, err)
// Round-robin: a released IP is NOT reused immediately (siv-246). Allocation
// resumes past the cursor, so the next IP is .153, not the freed .151.
// Round-robin: a released IP is NOT reused immediately. Allocation resumes
// past the cursor, so the next IP is .153, not the freed .151.
ip3, _, err := ipam.AllocateIP("", "", PurposeENIPublic, "", "eni-3", "i-3")
require.NoError(t, err)
assert.Equal(t, "192.168.1.153", ip3)
spinifex/handlers/ec2/vpc/invariants_test.go
+5 -13
View File
@@ -12,15 +12,9 @@ import (
// IMDS-datapath invariant.
// TestI4_ENIIndexBucketShape asserts the spinifex-network-eni-by-vpc-ip reverse
// index keeps its contract: keys parse as "vpcID/ip" with a valid IP, values
// carry a non-empty eni_id, and no mutable/denormalised fields creep back into
// the value. The index is identity-only (eni_id + the immutable account_id) by
// design — every mutable field (IPs, MAC, instance ID, profile ARN) is read
// live off the ENIRecord + instance record at request time. A denormalised
// field here resurrects the staleness class the design deliberately removed:
// an IMDS handler trusting a stale profile ARN would mint credentials for the
// wrong role.
// TestI4_ENIIndexBucketShape asserts the eni-by-vpc-ip reverse index keeps its
// contract: keys parse as "vpcID/ip", values carry only immutable identity
// fields (eni_id + account_id), and no mutable/denormalised fields are present.
func TestI4_ENIIndexBucketShape(t *testing.T) {
_, nc := setupTestVPCServiceWithNC(t)
kv := openTestENIByIPBucket(t, nc)
@@ -30,10 +24,8 @@ func TestI4_ENIIndexBucketShape(t *testing.T) {
require.NoError(t, idx.Put("vpc-aaaaaaaa", "10.0.1.5", "eni-aaa", "111122223333"))
require.NoError(t, idx.Put("vpc-bbbbbbbb", "10.0.2.9", "eni-bbb", "444455556666"))
// allowedFields is the closed set the value JSON may contain. Adding a field
// to eniByIPValue without updating this set fails the test by design — a
// reviewer must justify any new field as immutable identity, not cached
// mutable state.
// allowedFields is the closed set the value JSON may contain; any new field
// must be immutable identity, not cached mutable state.
allowedFields := map[string]struct{}{"eni_id": {}, "account_id": {}}
keys, err := kv.Keys()
spinifex/handlers/ec2/vpc/ipam.go
+2 -6
View File
@@ -55,12 +55,8 @@ func NewIPAMWithKV(kv nats.KeyValue) *IPAM {
return &IPAM{kv: kv}
}
// AllocateIP allocates an IP address from the given subnet.
// It reserves the first 4 and last IP per AWS convention:
// .0=network, .1=gateway, .2=DNS, .3=reserved, .255=broadcast (for /24).
// Uses CAS for conflict-free allocation across nodes. purpose is one of the
// Purpose* constants in purpose.go; ownerID is the ENI / instance / EIP
// resource ID that owns this allocation.
// AllocateIP allocates an IP from the subnet, reserving the first 4 and last
// addresses per AWS convention. Uses CAS for conflict-free multi-node allocation.
func (m *IPAM) AllocateIP(subnetId, cidrBlock, purpose, ownerID string) (string, error) {
for attempt := range 5 {
record, revision, err := m.getRecord(subnetId)
spinifex/handlers/ec2/vpc/security_group.go
+27 -60
View File
@@ -27,12 +27,9 @@ var sgIDRegex = regexp.MustCompile(`^sg-[0-9a-f]{17}$`)
// re-implementing the format check.
var SGRuleIDRegex = regexp.MustCompile(`^sgr-[0-9a-f]{17}$`)
// validateSGRule rejects values that could break out of an OVN ACL match-expression token.
// CidrIp must be IPv4 and round-trip to canonical form (so "10.0.0.5/8" with host bits set is
// rejected, as is anything containing operators/whitespace that net.ParseCIDR would not accept).
// IPv6 is rejected because the ACL builder in vpcd/acl.go is IPv4-only — accepting an IPv6 CIDR
// would persist a rule that OVN can never program.
// SourceSG must match the spinifex SG-ID format. At least one source must be specified.
// validateSGRule rejects CidrIp values that are non-canonical or IPv6 (OVN ACL
// builder is IPv4-only), and SourceSG values not matching the sg-ID format.
// At least one source must be specified.
func validateSGRule(r SGRule) error {
if r.CidrIp == "" && r.SourceSG == "" {
return errors.New("rule must specify CidrIp or SourceSG")
@@ -71,11 +68,8 @@ const (
)
// SGRule represents a single ingress or egress rule in a security group.
//
// RuleId is the persistent AWS-style identifier returned by
// DescribeSecurityGroupRules. It is assigned at insert by Authorize* and by
// the v1→v2 migration for pre-existing rules; sgRuleKey deliberately excludes
// it so duplicate-content rules are still rejected regardless of ID.
// RuleId is assigned on Authorize and excluded from sgRuleKey so
// duplicate-content rules are rejected regardless of ID.
type SGRule struct {
RuleId string `json:"rule_id"` // sgr-<17 hex>; assigned on Authorize, backfilled by migration
IpProtocol string `json:"ip_protocol"` // "tcp", "udp", "icmp", "-1" (all)
@@ -262,11 +256,9 @@ func (s *VPCServiceImpl) DeleteSecurityGroup(input *ec2.DeleteSecurityGroupInput
return &ec2.DeleteSecurityGroupOutput{}, nil
}
// validateSGRuleReferences ensures every SourceSG named in `rules` resolves to
// an SG in the caller's account that lives in the same VPC as the rule-owning
// SG. Cross-VPC and missing references both return InvalidGroup.NotFound,
// matching AWS (peering not supported; AWS uses the same code for "doesn't
// exist" and "exists in another VPC").
// validateSGRuleReferences returns InvalidGroup.NotFound if any SourceSG in
// rules is missing or belongs to a different VPC (cross-VPC refs not
// supported; AWS uses the same error for both cases).
func (s *VPCServiceImpl) validateSGRuleReferences(accountID, ownerVpcId string, rules []SGRule) error {
for _, r := range rules {
if r.SourceSG == "" {
@@ -492,10 +484,8 @@ var describeSecurityGroupRulesValidFilters = map[string]bool{
}
// DescribeSecurityGroupRules returns a flat list of SecurityGroupRule objects
// across every security group in the caller's account, optionally narrowed by
// SecurityGroupRuleIds or filters (group-id, security-group-rule-id, tag:*,
// tag-key). MaxResults and NextToken are accepted but ignored — matches the
// existing Describe* precedent in spinifex.
// for the caller's account, optionally narrowed by SecurityGroupRuleIds or
// filters. MaxResults and NextToken are accepted but ignored.
func (s *VPCServiceImpl) DescribeSecurityGroupRules(input *ec2.DescribeSecurityGroupRulesInput, accountID string) (*ec2.DescribeSecurityGroupRulesOutput, error) {
requested := make(map[string]bool)
if input != nil {
@@ -579,9 +569,8 @@ func (s *VPCServiceImpl) DescribeSecurityGroupRules(input *ec2.DescribeSecurityG
}
// sgRuleMatchesFilters applies the DescribeSecurityGroupRules filter set to a
// single rule within its parent record. Rule-level tags are not yet persisted,
// so tag:* and tag-key filters always fail to match (consistent with returning
// the empty set for tag queries until per-rule tagging lands).
// single rule. Rule-level tags are not yet persisted, so tag:* and tag-key
// filters always return false.
func sgRuleMatchesFilters(record *SecurityGroupRecord, rule SGRule, filters map[string][]string) bool {
for name, values := range filters {
if strings.HasPrefix(name, "tag:") {
@@ -601,11 +590,8 @@ func sgRuleMatchesFilters(record *SecurityGroupRecord, rule SGRule, filters map[
// rule.Tags is not yet populated; tag-key excludes every rule.
return false
default:
// Unreachable today because ParseFilters rejects anything not in
// describeSecurityGroupRulesValidFilters or starting with "tag:".
// If a future change adds a name to that map without adding a case
// here, this Error gives a Sentry breadcrumb instead of silently
// returning an empty result set for a documented filter.
// Unreachable: ParseFilters rejects names not in the valid set.
// Logs an error if a future map entry lacks a matching case.
slog.Error("sgRuleMatchesFilters: filter accepted by ParseFilters but no case", "filter", name)
return false
}
@@ -614,10 +600,8 @@ func sgRuleMatchesFilters(record *SecurityGroupRecord, rule SGRule, filters map[
}
// sgRuleToSecurityGroupRule flattens a stored SGRule into the AWS API shape.
// accountID supplies GroupOwnerId and (for SG-source rules) the
// ReferencedGroupInfo.UserId. VpcId is derived from the parent record —
// spinifex enforces same-VPC SG references at security_group.go:validateSGRuleReferences,
// so the referenced SG's VpcId always equals the parent's.
// accountID supplies GroupOwnerId and ReferencedGroupInfo.UserId; VpcId is
// derived from the parent record (same-VPC references are enforced on write).
func sgRuleToSecurityGroupRule(record *SecurityGroupRecord, rule SGRule, isEgress bool, accountID string) *ec2.SecurityGroupRule {
out := &ec2.SecurityGroupRule{
SecurityGroupRuleId: aws.String(rule.RuleId),
@@ -913,11 +897,8 @@ func (s *VPCServiceImpl) sgRecordToEC2(record *SecurityGroupRecord, accountID st
}
// sgParseMode selects how ipPermissionsToSGRules handles IPv6 sources.
// Spinifex stores IPv4-only rules (the OVN ACL builder in vpcd/acl.go cannot
// program IPv6), so Ipv6Ranges are unrepresentable. Authorize must reject
// them; Revoke must tolerate them silently because the Terraform AWS provider
// auto-revokes the default ::/0 IPv6 egress on every aws_security_group it
// creates and would otherwise abort every apply that touches an SG.
// Authorize rejects IPv6; Revoke silently skips it (Terraform auto-revokes
// ::/0 IPv6 egress and must not abort on no-op).
type sgParseMode int
const (
@@ -925,15 +906,9 @@ const (
sgParseRevoke
)
// ipPermissionsToSGRules converts AWS IpPermission slice to SGRule slice, validating
// every tenant-supplied CidrIp/SourceSG. This is the only path that constructs SGRule
// from external input — validating here makes it impossible for a future handler to
// bypass the check.
//
// Permissions whose only source is Ipv6Ranges contribute no rules. In
// sgParseAuthorize they cause an error; in sgParseRevoke they are skipped.
// Mixed v4+v6 permissions keep their v4/SG entries and silently drop the v6
// part on revoke (no stored rule can match v6 anyway).
// ipPermissionsToSGRules converts AWS IpPermission slice to SGRule slice,
// validating every tenant-supplied CidrIp/SourceSG. IPv6-only permissions
// error in Authorize mode and are silently skipped in Revoke mode.
func ipPermissionsToSGRules(perms []*ec2.IpPermission, mode sgParseMode) ([]SGRule, error) {
var rules []SGRule
for _, perm := range perms {
@@ -1049,11 +1024,9 @@ func sgRulesToIpPermissions(rules []SGRule) []*ec2.IpPermission {
return result
}
// assertRulesPresent returns InvalidPermission.NotFound on the first revoke
// rule that does not match an existing rule. AWS rejects revoke of a
// non-existent rule rather than treating it as a no-op; matching that
// behaviour lets Terraform / SDK consumers distinguish "already gone" from
// "operator typo".
// assertRulesPresent returns InvalidPermission.NotFound if any rule in
// toRevoke is absent from existing, matching AWS's non-idempotent revoke
// behavior.
func assertRulesPresent(existing, toRevoke []SGRule) error {
if len(toRevoke) == 0 {
return nil
@@ -1091,23 +1064,17 @@ func sgRuleKey(r SGRule) string {
return fmt.Sprintf("%s:%d:%d:%s:%s", r.IpProtocol, r.FromPort, r.ToPort, r.CidrIp, r.SourceSG)
}
// vpcdSGEventTimeout bounds the synchronous handler↔vpcd round-trip for SG
// events (vpc.create-sg, vpc.delete-sg, vpc.update-sg, vpc.update-port-sgs).
// On timeout the caller surfaces the error to the API client; the periodic
// reconciler is the safety net that converges any drift afterwards.
// vpcdSGEventTimeout bounds the synchronous vpcd round-trip for SG events.
const vpcdSGEventTimeout = 5 * time.Second
// requestSGEvent sends a security group lifecycle event to vpcd via
// request-reply and waits for vpcd's success/error response. Used by the
// public SG API surface so vpcd-side ACL/port-group failures are visible to
// the caller instead of swallowed.
// request-reply and surfaces vpcd-side failures to the API caller.
func (s *VPCServiceImpl) requestSGEvent(topic string, evt SGEvent) error {
return utils.RequestEvent(s.natsConn, topic, evt, vpcdSGEventTimeout)
}
// createDefaultSecurityGroupInternal provisions the per-VPC default SG with
// AWS-equivalent rules: allow all inbound from self, allow all outbound to
// 0.0.0.0/0. Bypasses the public-API "default" reserved-name guard. Used by
// AWS-equivalent rules, bypassing the public-API reserved-name guard. Used by
// CreateVpc and EnsureDefaultVPC.
func (s *VPCServiceImpl) createDefaultSecurityGroupInternal(accountID, vpcId string) (string, error) {
groupId := utils.GenerateResourceID("sg")
spinifex/handlers/ec2/vpc/security_group_test.go
+27 -53
View File
@@ -294,10 +294,9 @@ func TestRevokeSecurityGroupIngress_NotFound(t *testing.T) {
assert.Error(t, err)
}
// TestRevokeSecurityGroupIngress_RuleNotFound: revoking a rule that doesn't
// exist in the SG must return InvalidPermission.NotFound, matching AWS.
// Terraform / SDK consumers branch on this code to distinguish "already
// revoked, idempotent" from "operator typo".
// TestRevokeSecurityGroupIngress_RuleNotFound asserts that revoking a missing
// rule returns InvalidPermission.NotFound, matching AWS non-idempotent
// revoke behavior.
func TestRevokeSecurityGroupIngress_RuleNotFound(t *testing.T) {
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
@@ -347,11 +346,9 @@ func TestRevokeSecurityGroupEgress_NotFound(t *testing.T) {
assert.Error(t, err)
}
// TestRevokeSecurityGroupEgress_IPv6OnlyIsNoOp guards against regression of
// the Terraform AWS provider's auto-revoke of the default ::/0 IPv6 egress
// rule. Spinifex stores no IPv6 rules, so the call must succeed as a no-op
// instead of returning InvalidParameterValue (which surfaces as bare
// "UnknownError" to terraform and aborts every aws_security_group apply).
// TestRevokeSecurityGroupEgress_IPv6OnlyIsNoOp asserts that revoking a
// IPv6-only permission succeeds as a no-op; Terraform auto-revokes ::/0 IPv6
// egress and must not abort when spinifex stores no IPv6 rules.
func TestRevokeSecurityGroupEgress_IPv6OnlyIsNoOp(t *testing.T) {
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
@@ -390,9 +387,8 @@ func TestRevokeSecurityGroupIngress_IPv6OnlyIsNoOp(t *testing.T) {
require.True(t, aws.BoolValue(out.Return))
}
// TestAuthorizeSecurityGroupEgress_IPv6Rejected guards the inverse: authorize
// with IPv6 sources must still fail. Spinifex cannot enforce IPv6 — accepting
// the call would persist a rule OVN can never program.
// TestAuthorizeSecurityGroupEgress_IPv6Rejected asserts that authorizing
// IPv6 sources is rejected; spinifex cannot program IPv6 OVN ACLs.
func TestAuthorizeSecurityGroupEgress_IPv6Rejected(t *testing.T) {
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
@@ -484,9 +480,8 @@ func TestIpPermissionsToSGRules_SourceSG(t *testing.T) {
assert.Equal(t, srcSG, rules[0].SourceSG)
}
// TestIpPermissionsToSGRules_IPv6OnlyAuthorize rejects an authorize call whose
// only source is Ipv6Ranges. Spinifex's OVN ACL builder is IPv4-only — an IPv6
// rule would persist as state OVN cannot program.
// TestIpPermissionsToSGRules_IPv6OnlyAuthorize rejects authorize calls whose
// only source is Ipv6Ranges (OVN ACL builder is IPv4-only).
func TestIpPermissionsToSGRules_IPv6OnlyAuthorize(t *testing.T) {
allProto := "-1"
perms := []*ec2.IpPermission{
@@ -502,9 +497,8 @@ func TestIpPermissionsToSGRules_IPv6OnlyAuthorize(t *testing.T) {
}
// TestIpPermissionsToSGRules_IPv6OnlyRevoke tolerates IPv6-only permissions on
// revoke (returns zero rules, no error). The Terraform AWS provider sends
// exactly this shape when it auto-revokes the default ::/0 IPv6 egress rule
// after every aws_security_group create.
// revoke (returns zero rules, no error), matching Terraform's auto-revoke of
// the default ::/0 IPv6 egress rule.
func TestIpPermissionsToSGRules_IPv6OnlyRevoke(t *testing.T) {
allProto := "-1"
perms := []*ec2.IpPermission{
@@ -520,9 +514,7 @@ func TestIpPermissionsToSGRules_IPv6OnlyRevoke(t *testing.T) {
}
// TestIpPermissionsToSGRules_MixedV4V6Revoke keeps the v4 rule and drops the
// v6 part on a mixed-source permission. Mixed perms are uncommon from real
// tooling but worth covering — IPv6 must never silently corrupt the IPv4
// parse output.
// v6 part on a mixed-source permission on revoke.
func TestIpPermissionsToSGRules_MixedV4V6Revoke(t *testing.T) {
proto := "tcp"
perms := []*ec2.IpPermission{
@@ -957,11 +949,8 @@ func TestDescribeSecurityGroups_FilterNoResults(t *testing.T) {
// --- Phase 3: default SG lifecycle, dependency checks, quotas ---
// findDefaultSGInVPC returns the default-SG ID a CreateVpc call provisions.
// Walks the SG KV bucket and filters by IsDefault=true so a regression in
// the discriminator field (rather than the GroupName="default" convention)
// is caught — production code (FindDefaultSGForVPC, DeleteVpc cascade,
// DeleteSecurityGroup default guard) keys off IsDefault, not GroupName.
// findDefaultSGInVPC returns the default-SG ID for a VPC by filtering on
// IsDefault=true (production code keys off IsDefault, not GroupName).
func findDefaultSGInVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) string {
t.Helper()
keys, err := svc.sgKV.Keys()
@@ -1290,10 +1279,8 @@ func TestAuthorizeSecurityGroupEgress_RuleLimit(t *testing.T) {
func TestCreateSecurityGroup_VpcdError_RecordPersists(t *testing.T) {
svc, _ := setupTestVPCServiceWithFailingVpcd(t, "forced-create-sg-error")
// Bootstrap a VPC with a working stub by swapping in a success responder
// for vpc.create-sg only during VPC creation. We do that by short-circuit:
// CreateVpc here will fail because it also publishes vpc.create-sg.
// Instead seed a VPC record directly so we can isolate the SG path.
// CreateVpc also publishes vpc.create-sg and would fail; seed the VPC
// record directly to isolate the SG path.
seedAvailableVPC(t, svc, "vpc-fail000000000")
_, err := svc.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{
@@ -1397,11 +1384,8 @@ func TestRevokeSecurityGroupIngress_VpcdError_Propagated(t *testing.T) {
assert.Contains(t, err.Error(), "forced-revoke-error")
}
// TestAuthorizeSecurityGroupEgress_VpcdError_Propagated and the matching
// Revoke variant lock Phase 7's sync contract on the egress path. Without
// these, an OVN port-group failure during egress rule provisioning would be
// silently swallowed — the KV record would gain the rule but OVN would never
// enforce it.
// TestAuthorizeSecurityGroupEgress_VpcdError_Propagated asserts that vpcd
// failures on the egress path are surfaced to the caller.
func TestAuthorizeSecurityGroupEgress_VpcdError_Propagated(t *testing.T) {
svc, nc := setupTestVPCServiceWithNC(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
@@ -1444,9 +1428,8 @@ func TestRevokeSecurityGroupEgress_VpcdError_Propagated(t *testing.T) {
assert.Contains(t, err.Error(), "forced-egress-revoke-error")
}
// TestFindDefaultSGForVPC_NoMatch: an account with SGs in vpcA must return ""
// when asked for the default SG of vpcB. The "" return — not an error —
// signals "no default exists; create one" to RunInstances' fallback path.
// TestFindDefaultSGForVPC_NoMatch asserts that a lookup for vpcB's default SG
// returns "" (not an error) when only vpcA has SGs.
func TestFindDefaultSGForVPC_NoMatch(t *testing.T) {
svc := setupTestVPCService(t)
vpcA := createTestVPC(t, svc, "10.0.0.0/16")
@@ -1457,10 +1440,8 @@ func TestFindDefaultSGForVPC_NoMatch(t *testing.T) {
assert.Equal(t, "", got, "no default SG → empty string, not error")
}
// TestFindDefaultSGForVPC_SkipsMalformedRecord: a corrupt SG entry under the
// account prefix must be skipped, not crash the scan. Without this guard a
// single bad write to the SG bucket would brick the default-SG lookup
// cluster-wide.
// TestFindDefaultSGForVPC_SkipsMalformedRecord asserts that a corrupt SG entry
// is skipped rather than aborting the scan.
func TestFindDefaultSGForVPC_SkipsMalformedRecord(t *testing.T) {
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
@@ -1517,9 +1498,7 @@ func TestDeleteSecurityGroup_FailsClosedOnCorruptENI(t *testing.T) {
// --- Helpers ---
// seedAvailableVPC writes a VPC record directly to KV in "available" state,
// bypassing CreateVpc (which itself depends on a working vpcd stub). Used to
// isolate non-CreateVpc paths in failing-vpcd tests.
// seedAvailableVPC writes a VPC record directly to KV, bypassing CreateVpc.
func seedAvailableVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) {
t.Helper()
rec := VPCRecord{VpcId: vpcID, CidrBlock: "10.0.0.0/16", State: "available"}
@@ -1529,9 +1508,7 @@ func seedAvailableVPC(t *testing.T, svc *VPCServiceImpl, vpcID string) {
require.NoError(t, err)
}
// failingDeleteResponder layers a failing handler for vpc.delete-sg on top of
// the default success stubs (NATS picks the responder that subscribes to the
// queue group; here neither uses a queue, so we drain the success sub first).
// failingDeleteResponder overrides the vpc.delete-sg stub to return a failure.
func failingDeleteResponder(t *testing.T, nc *nats.Conn, msg string) {
t.Helper()
testutil.OverrideVpcdStubResponse(nc, "vpc.delete-sg", []byte(`{"success":false,"error":"`+msg+`"}`))
@@ -1762,11 +1739,8 @@ func TestAuthorizeSecurityGroupIngress_AssignsRuleIDs(t *testing.T) {
}
}
// TestAuthorizeSecurityGroupIngress_RejectsContentDuplicate verifies the
// key-based dedup still catches duplicate-content rules now that RuleId is in
// the struct. The old slices.Contains check would have missed this because
// freshly-generated IDs make two otherwise-identical rules unequal under Go
// struct equality.
// TestAuthorizeSecurityGroupIngress_RejectsContentDuplicate verifies key-based
// dedup catches duplicate-content rules regardless of generated RuleId values.
func TestAuthorizeSecurityGroupIngress_RejectsContentDuplicate(t *testing.T) {
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.0.0.0/16")
spinifex/handlers/ec2/vpc/service_impl.go
+7 -19
View File
@@ -35,11 +35,8 @@ const (
KVBucketVNICounterVersion = 1
)
// VPCRecord represents a stored VPC. AZ stamps the local availability zone
// at create time; the network/reconcile package filters its intent set by
// matching `vpc.AZ == localAZ` (with the legacy rule that empty matches
// every AZ for pre-Phase-2.2 records). New records always populate AZ from
// the daemon's config.AZ.
// VPCRecord represents a stored VPC. AZ is stamped at create time; empty AZ
// is a legacy value that matches every AZ in the reconciler.
type VPCRecord struct {
VpcId string `json:"vpc_id"`
CidrBlock string `json:"cidr_block"`
@@ -994,10 +991,8 @@ type BootstrapIDs struct {
SubnetId string
}
// EnsureDefaultVPC creates a default VPC and subnet if none exists for the given account.
// This matches AWS behavior where a default VPC is present on account creation.
// Safe to call multiple times — no-ops if a default VPC already exists.
// Returns the default VPC info (whether newly created or pre-existing).
// EnsureDefaultVPC creates a default VPC and subnet if none exists for the
// account. Safe to call multiple times — no-ops if already present.
func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string, bootstrap ...BootstrapIDs) (*DefaultVPCInfo, error) {
if s.vpcKV == nil {
return nil, nil // No persistence, skip
@@ -1136,16 +1131,9 @@ func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string, bootstrap ...Bootstr
}, nil
}
// createMainRouteTable writes a main route table record directly to the route table KV bucket.
// This avoids a circular import (routetable package imports vpc package for validation).
//
// Idempotency: when concurrent daemons race EnsureDefaultVPC for the same
// bootstrap-pinned VPC, both would otherwise mint a fresh random rtb-ID
// and Put it, leaving the KV with two IsMain=true records for one VPC.
// effectiveRouteTable then resolves the subnet's RT non-deterministically,
// and CreateRoute's post-write gate recompute can publish
// vpc.gate-subnet-egress against the orphan (route-less) main RT, installing
// a Priority-1100 DROP that kills egress on the default subnet.
// createMainRouteTable writes a main route table record directly to the route
// table KV bucket. Avoids a circular import; idempotent via findMainRouteTableID
// so concurrent EnsureDefaultVPC calls do not mint duplicate main RTs.
func (s *VPCServiceImpl) createMainRouteTable(accountID, vpcID, vpcCidr string) error {
if existing, err := s.findMainRouteTableID(accountID, vpcID); err != nil {
return fmt.Errorf("check existing main route table: %w", err)
spinifex/handlers/ec2/vpc/service_impl_test.go
+6 -14
View File
@@ -33,10 +33,8 @@ func setupTestVPCService(t *testing.T) *VPCServiceImpl {
}
// setupTestVPCServiceWithFailingVpcd creates a VPC service whose vpcd stub
// always returns success=false. Used by Phase 7 propagation tests to assert
// vpcd-side errors surface to the API caller. The default-VPC bootstrap is
// not needed here — these tests CreateVpc themselves once they have working
// stubs swapped in.
// always returns success=false. Used to assert vpcd-side errors surface to the
// API caller.
func setupTestVPCServiceWithFailingVpcd(t *testing.T, errMsg string) (*VPCServiceImpl, *nats.Conn) {
t.Helper()
_, nc, _ := testutil.StartTestJetStream(t)
@@ -597,11 +595,8 @@ func TestEnsureDefaultVPC_SkipsWhenDefaultExists(t *testing.T) {
}
// TestCreateMainRouteTable_Idempotent asserts a second createMainRouteTable
// call for the same VPC is a no-op. Without this guard, concurrent multi-node
// EnsureDefaultVPC calls (admin account, bootstrap-pinned VpcId) each Put a
// random rtb-ID with IsMain=true, and the resulting duplicate causes
// CreateRoute's post-write gate recompute to resolve effective RT to the
// orphan (route-less) record and publish vpc.gate-subnet-egress.
// call for the same VPC is a no-op. Concurrent calls otherwise create duplicate
// IsMain=true records, corrupting route-table resolution.
func TestCreateMainRouteTable_Idempotent(t *testing.T) {
svc := setupTestVPCService(t)
vpcID := createTestVPC(t, svc, "10.99.0.0/16") // CreateVpc auto-calls createMainRouteTable
@@ -649,11 +644,8 @@ func countMainRouteTablesForVPC(t *testing.T, svc *VPCServiceImpl, vpcID string)
}
// TestEnsureDefaultVPC_NoVpcdResponder simulates the daemon-startup race where
// EnsureDefaultVPC runs before vpcd has subscribed to vpc.create-sg. Without
// the fix, the synchronous SG round-trip errors out and EnsureDefaultVPC
// returns early, leaving the VPC without a default subnet or main route table.
// After the fix the SG step is best-effort, so subnet + RTB still land in KV
// and vpcd's reconcile-sgs loop converges the OVN port group asynchronously.
// EnsureDefaultVPC runs before vpcd has subscribed. The SG step is best-effort,
// so subnet and RTB must still land in KV when vpcd is absent.
func TestEnsureDefaultVPC_NoVpcdResponder(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
svc, err := NewVPCServiceImplWithNATS(nil, nc)
spinifex/handlers/ec2/vpc/service_nats_test.go
+2 -5
View File
@@ -207,11 +207,8 @@ func TestNATSVPCService_Timeout(t *testing.T) {
_, nc := setupTestVPCServiceWithNC(t)
client := &NATSVPCService{natsConn: nc}
// Use a very short timeout to make the test fast
// We can't easily override the 30s timeout in the production code,
// so instead test with a topic that has no subscriber.
// The standard NATSVPCService uses 30s which is too long for tests.
// Just verify the constructor works.
// Production code uses a 30s timeout; just verify the constructor works
// without exercising the full round-trip.
_ = client
_ = time.Second // reference time package
}
spinifex/handlers/eks/access_entry.go
+18 -27
View File
@@ -12,10 +12,9 @@ import (
"github.com/nats-io/nats.go"
)
// Access-entry types (Q9). v1 supports STANDARD principals only; the node
// types (EC2_LINUX/EC2_WINDOWS/FARGATE_LINUX) are accepted by AWS but the
// nodegroup join path (Sprint 6d) handles node identity, so they are rejected
// here until that lands.
// Access-entry types. v1 supports STANDARD principals only; node types
// (EC2_LINUX/EC2_WINDOWS/FARGATE_LINUX) are rejected until the nodegroup
// join path wires them.
const (
AccessEntryTypeStandard = "STANDARD"
@@ -23,9 +22,8 @@ const (
accessScopeNamespace = "namespace"
)
// Supported AWS-managed access policies (Q9). The map value is the K8s
// ClusterRole the policy maps onto; the token webhook + RBAC bindings consume
// it. Any policy ARN outside this set is rejected with InvalidParameterValue.
// supportedAccessPolicies maps each supported AWS-managed policy ARN to the
// K8s ClusterRole it grants. Policy ARNs outside this set are rejected.
var supportedAccessPolicies = map[string]string{
"arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy": "cluster-admin",
"arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminPolicy": "admin",
@@ -33,15 +31,13 @@ var supportedAccessPolicies = map[string]string{
"arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy": "view",
}
// ErrAccessEntryNotFound is returned by GetAccessEntryRecord / the CAS helper
// when no entry exists for the principal. Callers translate to the AWS shape
// (ResourceNotFoundException) at the service boundary.
// ErrAccessEntryNotFound is returned when no entry exists for the principal.
// Callers translate it to ResourceNotFoundException at the service boundary.
var ErrAccessEntryNotFound = errors.New("eks: access entry not found")
// AccessEntryARN composes the access-entry ARN. AWS uses a per-entry UUID; this
// build keys the discriminator off the principal-ARN hash instead so the ARN is
// deterministic (no RNG) and one-per-principal — clients address access entries
// by principalArn, not by this ARN, so the divergence is informational only.
// AccessEntryARN composes the access-entry ARN, keying the discriminator off
// the principal-ARN hash for determinism. Clients address entries by
// principalArn, not this ARN, so the divergence from AWS's UUID is informational.
func AccessEntryARN(region, accountID, cluster, principalARN string) string {
return fmt.Sprintf("arn:aws:eks:%s:%s:access-entry/%s/%s",
region, accountID, cluster, PrincipalARNHash(principalARN))
@@ -85,8 +81,7 @@ func GetAccessEntryRecord(kv nats.KeyValue, cluster, principalARN string) (*Acce
return &rec, nil
}
// ListAccessEntryRecords returns every access entry under a cluster, sorted by
// principal ARN for stable output.
// ListAccessEntryRecords returns all access entries under a cluster, sorted by principal ARN.
func ListAccessEntryRecords(kv nats.KeyValue, cluster string) ([]*AccessEntryRecord, error) {
if cluster == "" {
return nil, errors.New("eks: ListAccessEntryRecords empty cluster")
@@ -121,8 +116,7 @@ func ListAccessEntryRecords(kv nats.KeyValue, cluster string) ([]*AccessEntryRec
return out, nil
}
// DeleteAccessEntryRecord removes one entry. Returns ErrAccessEntryNotFound if
// it did not exist so DeleteAccessEntry can surface ResourceNotFoundException.
// DeleteAccessEntryRecord removes one entry; returns ErrAccessEntryNotFound if absent.
func DeleteAccessEntryRecord(kv nats.KeyValue, cluster, principalARN string) error {
key := AccessEntryKey(cluster, principalARN)
if _, err := kv.Get(key); err != nil {
@@ -137,8 +131,8 @@ func DeleteAccessEntryRecord(kv nats.KeyValue, cluster, principalARN string) err
return nil
}
// casUpdateAccessEntry does a revision-checked read-modify-write. mutate returns
// true if it changed a field. Returns ErrAccessEntryNotFound if absent.
// casUpdateAccessEntry does a revision-checked read-modify-write.
// mutate returns true when a field changed. Returns ErrAccessEntryNotFound if absent.
func casUpdateAccessEntry(kv nats.KeyValue, cluster, principalARN string, mutate func(*AccessEntryRecord) bool) (*AccessEntryRecord, error) {
key := AccessEntryKey(cluster, principalARN)
for range maxClusterStateCASRetries {
@@ -180,9 +174,8 @@ func sortAccessEntries(recs []*AccessEntryRecord) {
}
}
// validateAccessScope checks an AccessScope per AWS rules: type ∈
// {cluster, namespace}; namespace scope requires at least one namespace and
// cluster scope must not carry any.
// validateAccessScope validates an AccessScope: type must be "cluster" or
// "namespace"; namespace scope requires ≥1 namespace, cluster scope must have none.
func validateAccessScope(scope *eks.AccessScope) (AccessScope, error) {
if scope == nil || aws.StringValue(scope.Type) == "" {
return AccessScope{}, errors.New("eks: accessScope.type is required")
@@ -237,14 +230,12 @@ func associatedPolicyToAWS(p AssociatedAccessPolicy) *eks.AssociatedAccessPolicy
}
}
// defaultKubernetesUsername mirrors AWS: when the caller omits username, EKS
// derives it from the principal ARN.
// defaultKubernetesUsername mirrors AWS behaviour: username defaults to principalARN when omitted.
func defaultKubernetesUsername(principalARN string) string {
return principalARN
}
// newAccessEntryRecord builds a record, defaulting the username to the
// principal ARN when unset (AWS behaviour). now stamps Created/Modified.
// newAccessEntryRecord builds a record; defaults username to principalARN when unset.
func newAccessEntryRecord(region, accountID, cluster, principalARN, username string, groups []string, entryType string, tags map[string]string, now time.Time) *AccessEntryRecord {
if username == "" {
username = defaultKubernetesUsername(principalARN)
spinifex/handlers/eks/addon_catalog.go
+8 -15
View File
@@ -2,26 +2,20 @@ package handlers_eks
import "slices"
// AddonSpec describes one Spinifex-supported managed add-on. The catalog is a
// static in-binary registry (air-gapped; there is no internet add-on catalog),
// so DescribeAddonVersions and CreateAddon validate against these entries.
// AddonSpec describes one Spinifex-supported managed add-on in the static
// in-binary catalog. DescribeAddonVersions and CreateAddon validate against it.
type AddonSpec struct {
// Name is the AWS add-on name (e.g. "aws-load-balancer-controller").
Name string
// Versions lists the supported versions newest-first; [0] is the default.
Versions []string
// DefaultVersion is the version CreateAddon applies when none is requested.
// Always equal to Versions[0].
// Versions lists supported versions newest-first; [0] is the default.
Versions []string
DefaultVersion string
// RequiresIRSA reports whether the add-on needs an IAM-role-for-service-
// account binding to operate (surfaced as requiresIamPermissions).
// RequiresIRSA indicates the add-on needs an IAM role for service-account binding.
RequiresIRSA bool
// Description is a short human-readable summary.
Description string
Description string
}
// addonCatalog is the bundled set of supported add-ons. Keep newest version
// first in each Versions slice; newAddonSpec enforces DefaultVersion = [0].
// addonCatalog is the bundled add-on registry. Keep newest version first per slice.
var addonCatalog = buildAddonCatalog(
newAddonSpec("aws-load-balancer-controller", true,
"Provisions ELBv2 load balancers for Kubernetes Service/Ingress resources.",
@@ -31,8 +25,7 @@ var addonCatalog = buildAddonCatalog(
"1.11.1"),
)
// newAddonSpec builds a spec, pinning DefaultVersion to the first (newest)
// version. Panics on an empty version list — the catalog is build-time data.
// newAddonSpec builds a spec with DefaultVersion = versions[0]. Panics on empty versions.
func newAddonSpec(name string, requiresIRSA bool, description string, versions ...string) AddonSpec {
if len(versions) == 0 {
panic("eks: addon spec " + name + " has no versions")
spinifex/handlers/eks/addon_installer.go
+12 -23
View File
@@ -8,27 +8,19 @@ import (
"github.com/nats-io/nats.go"
)
// AddonInstaller delivers (and removes) a managed add-on's Kubernetes manifests
// to a cluster. It is deliberately transport-agnostic: the add-on state machine
// depends only on this narrow contract so the VM-side delivery transport (which
// drops manifests into the K3s server auto-deploy dir) can drop in without
// touching the op layer.
//
// The reachability constraint (cluster_reconciler.go:73-79) means delivery must
// be VM-side, not a daemon apiserver apply — a private cluster's apiserver is
// not host-reachable.
// AddonInstaller delivers and removes a managed add-on's Kubernetes manifests.
// Transport-agnostic: delivery must be VM-side because a private cluster's
// apiserver is not host-reachable.
type AddonInstaller interface {
// Install renders and delivers the add-on described by rec. It does not flip
// the record to ACTIVE — that transition is gated on the cluster's state
// report confirming the manifest applied and pods are ready.
// Install stages the add-on manifest. Does not transition to ACTIVE — that
// is gated on the cluster's state report confirming pods are ready.
Install(accountID, cluster string, rec *AddonRecord) error
// Uninstall removes the add-on's delivered manifests from the cluster.
// Uninstall removes the add-on's manifests from the cluster.
Uninstall(accountID, cluster, addon string) error
}
// stagedManifest is the artifact the VM-side delivery transport consumes. It
// names the bundled add-on + version and carries the operator-supplied config
// so the VM can render the final Kubernetes objects from the baked manifests.
// stagedManifest is consumed by the VM-side delivery transport: add-on name,
// version, and operator-supplied config for rendering the Kubernetes objects.
type stagedManifest struct {
AddonName string `json:"addonName"`
AddonVersion string `json:"addonVersion"`
@@ -36,19 +28,16 @@ type stagedManifest struct {
ConfigurationValues string `json:"configurationValues,omitempty"`
}
// stagingInstaller renders the bundled manifest descriptor for an add-on and
// stages it in the per-account KV at AddonManifestKey. The VM-side delivery
// slice (separate bead) pulls staged manifests and applies them via the K3s
// auto-deploy dir; until then an installed add-on sits CREATING — honest, not a
// false ACTIVE.
// stagingInstaller stages the manifest descriptor in KV at AddonManifestKey.
// The VM-side delivery slice applies it via the K3s auto-deploy dir; until then
// the add-on sits CREATING.
type stagingInstaller struct {
nc *nats.Conn
}
var _ AddonInstaller = (*stagingInstaller)(nil)
// newStagingInstaller builds the default installer bound to the daemon's NATS
// connection.
// newStagingInstaller returns a stagingInstaller bound to the daemon NATS connection.
func newStagingInstaller(nc *nats.Conn) *stagingInstaller {
return &stagingInstaller{nc: nc}
}
spinifex/handlers/eks/addon_state.go
+12 -20
View File
@@ -10,8 +10,7 @@ import (
"github.com/nats-io/nats.go"
)
// AddonStatus mirrors the AWS EKS addon.status enum verbatim so clients match
// against the values without translation.
// AddonStatus mirrors the AWS EKS addon.status enum verbatim.
type AddonStatus string
const (
@@ -23,9 +22,8 @@ const (
AddonStatusCreateFailed AddonStatus = "CREATE_FAILED"
)
// AddonRecord is the persisted-state envelope for a managed add-on bound to a
// cluster. It tracks the AWS-visible lifecycle status plus the IRSA role and
// opaque configuration the installer needs.
// AddonRecord is the persisted state for a managed add-on: lifecycle status,
// IRSA role, and opaque configuration the installer needs.
type AddonRecord struct {
AddonName string `json:"addonName"`
AddonVersion string `json:"addonVersion"`
@@ -39,13 +37,11 @@ type AddonRecord struct {
ModifiedAt time.Time `json:"modifiedAt"`
}
// ErrAddonNotFound is returned by GetAddonRecord / the CAS helper when no
// record exists for the add-on. Callers translate to the AWS shape
// (ResourceNotFoundException) at the service boundary.
// ErrAddonNotFound is returned when no record exists for the add-on.
// Callers translate it to ResourceNotFoundException at the service boundary.
var ErrAddonNotFound = errors.New("eks: addon not found")
// AddonARN composes the deterministic ARN for a managed add-on (one per
// add-on name per cluster).
// AddonARN composes the deterministic ARN for a managed add-on.
func AddonARN(region, accountID, cluster, addon string) string {
return fmt.Sprintf("arn:aws:eks:%s:%s:addon/%s/%s", region, accountID, cluster, addon)
}
@@ -88,10 +84,8 @@ func GetAddonRecord(kv nats.KeyValue, cluster, addon string) (*AddonRecord, erro
return &rec, nil
}
// ListAddonRecords returns every add-on record under a cluster, sorted by add-on
// name for stable output. The staged-manifest sub-keys
// (clusters/{c}/addons/{addon}/manifest) are skipped — only the record keys
// (one path segment under the prefix) are returned.
// ListAddonRecords returns all add-on records under a cluster, sorted by name.
// Staged-manifest sub-keys (one extra path segment) are skipped.
func ListAddonRecords(kv nats.KeyValue, cluster string) ([]*AddonRecord, error) {
if cluster == "" {
return nil, errors.New("eks: ListAddonRecords empty cluster")
@@ -109,8 +103,7 @@ func ListAddonRecords(kv nats.KeyValue, cluster string) ([]*AddonRecord, error)
if !strings.HasPrefix(k, prefix) {
continue
}
// Skip sub-keys (e.g. the staged manifest); a record key is exactly one
// segment under the prefix.
// Skip sub-keys (e.g. staged manifest); record keys are one segment under the prefix.
if strings.Contains(strings.TrimPrefix(k, prefix), "/") {
continue
}
@@ -131,8 +124,7 @@ func ListAddonRecords(kv nats.KeyValue, cluster string) ([]*AddonRecord, error)
return out, nil
}
// DeleteAddonRecord removes one record (and its staged manifest, if any).
// Returns ErrAddonNotFound if the record did not exist.
// DeleteAddonRecord removes one record and its staged manifest. Returns ErrAddonNotFound if absent.
func DeleteAddonRecord(kv nats.KeyValue, cluster, addon string) error {
key := AddonKey(cluster, addon)
if _, err := kv.Get(key); err != nil {
@@ -152,8 +144,8 @@ func DeleteAddonRecord(kv nats.KeyValue, cluster, addon string) error {
return nil
}
// casUpdateAddon does a revision-checked read-modify-write. mutate returns true
// if it changed a field. Returns ErrAddonNotFound if absent.
// casUpdateAddon does a revision-checked read-modify-write.
// mutate returns true when a field changed. Returns ErrAddonNotFound if absent.
func casUpdateAddon(kv nats.KeyValue, cluster, addon string, mutate func(*AddonRecord) bool) (*AddonRecord, error) {
key := AddonKey(cluster, addon)
for range maxClusterStateCASRetries {
spinifex/handlers/eks/addons.go
+8 -16
View File
@@ -32,9 +32,7 @@ func (s *EKSServiceImpl) ListAddons(input *eks.ListAddonsInput, accountID string
return &eks.ListAddonsOutput{Addons: aws.StringSlice(names)}, nil
}
// DescribeAddonVersions returns the bundled add-on catalog, optionally filtered
// to a single add-on by name. It is cluster-independent (the catalog is static)
// so it does not touch the KV store.
// DescribeAddonVersions returns the static add-on catalog, optionally filtered by name.
func (s *EKSServiceImpl) DescribeAddonVersions(input *eks.DescribeAddonVersionsInput, _ string) (*eks.DescribeAddonVersionsOutput, error) {
filter := ""
if input != nil {
@@ -51,9 +49,8 @@ func (s *EKSServiceImpl) DescribeAddonVersions(input *eks.DescribeAddonVersionsI
return &eks.DescribeAddonVersionsOutput{Addons: out}, nil
}
// CreateAddon validates the add-on against the bundled catalog, persists a
// CREATING record, and stages it for delivery. The record reaches ACTIVE only
// once the cluster's state report confirms delivery (separate slice).
// CreateAddon validates, persists a CREATING record, and stages it for delivery.
// Transitions to ACTIVE once the cluster state report confirms delivery.
func (s *EKSServiceImpl) CreateAddon(input *eks.CreateAddonInput, accountID string) (*eks.CreateAddonOutput, error) {
if input == nil {
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
@@ -125,8 +122,7 @@ func (s *EKSServiceImpl) DescribeAddon(input *eks.DescribeAddonInput, accountID
return &eks.DescribeAddonOutput{Addon: addonRecordToAWS(cluster, rec)}, nil
}
// UpdateAddon CASes the version/config/role onto the record, marks it UPDATING,
// and re-stages it for delivery.
// UpdateAddon CASes new version/config/role onto the record, marks it UPDATING, and re-stages it.
func (s *EKSServiceImpl) UpdateAddon(input *eks.UpdateAddonInput, accountID string) (*eks.UpdateAddonOutput, error) {
if input == nil {
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
@@ -177,8 +173,7 @@ func (s *EKSServiceImpl) UpdateAddon(input *eks.UpdateAddonInput, accountID stri
}}, nil
}
// DeleteAddon marks the record DELETING, removes the staged manifest, and
// deletes the record.
// DeleteAddon marks the record DELETING, removes the staged manifest, then deletes the record.
func (s *EKSServiceImpl) DeleteAddon(input *eks.DeleteAddonInput, accountID string) (*eks.DeleteAddonOutput, error) {
if input == nil {
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
@@ -239,8 +234,7 @@ func addonRecordToAWS(cluster string, rec *AddonRecord) *eks.Addon {
return out
}
// addonSpecToAWS converts a catalog spec to the SDK AddonInfo shape, expanding
// each supported version into an AddonVersionInfo.
// addonSpecToAWS converts a catalog spec to the SDK AddonInfo shape.
func addonSpecToAWS(spec AddonSpec) *eks.AddonInfo {
versions := make([]*eks.AddonVersionInfo, 0, len(spec.Versions))
for _, v := range spec.Versions {
@@ -255,8 +249,7 @@ func addonSpecToAWS(spec AddonSpec) *eks.AddonInfo {
}
}
// addonInstaller returns the wired installer, defaulting to the staging
// installer bound to the daemon NATS connection when none was injected.
// addonInstaller returns the injected installer or the default stagingInstaller.
func (s *EKSServiceImpl) addonInstaller() AddonInstaller {
if s.deps.AddonInstaller != nil {
return s.deps.AddonInstaller
@@ -264,8 +257,7 @@ func (s *EKSServiceImpl) addonInstaller() AddonInstaller {
return newStagingInstaller(s.deps.NATSConn)
}
// markAddonFailed best-effort flips a record to CREATE_FAILED with the reason
// after an installer error so a failed install is not left looking healthy.
// markAddonFailed best-effort flips a record to CREATE_FAILED with the error reason.
func (s *EKSServiceImpl) markAddonFailed(acctKV nats.KeyValue, cluster, addon string, cause error) {
now := time.Now().UTC()
if _, err := casUpdateAddon(acctKV, cluster, addon, func(r *AddonRecord) bool {
spinifex/handlers/eks/awsgw_endpoint.go
+2 -7
View File
@@ -7,16 +7,11 @@ import (
"strings"
)
// ClusterOIDCIssuer returns the public OIDC issuer URL for one cluster. This
// is the URL kube-apiserver advertises as service-account-issuer and the URL
// that AWS STS / external IRSA validators fetch /.well-known/openid-configuration
// against. Format per plan §10.6:
// ClusterOIDCIssuer returns the public OIDC issuer URL:
//
// {gatewayBaseURL}/oidc/eks/{region}/{accountID}/{clusterName}
//
// gatewayBaseURL is the awsgw HTTPS endpoint (e.g. "https://10.0.0.1:9999"),
// without a trailing /oidc. Validates that region/accountID/clusterName are
// non-empty and URL-safe.
// gatewayBaseURL is the awsgw HTTPS endpoint without a trailing slash.
func ClusterOIDCIssuer(gatewayBaseURL, region, accountID, clusterName string) (string, error) {
if gatewayBaseURL == "" {
return "", errors.New("eks: ClusterOIDCIssuer empty gatewayBaseURL")
spinifex/handlers/eks/claim.go
+5 -9
View File
@@ -8,15 +8,11 @@ import (
)
// claimKey atomically creates key with payload. owned is true when this caller
// created it (won the claim). On an existing key it returns owned=false with the
// current value + revision so the caller can decide reclaim-vs-conflict; a key
// that vanished between the Create and the Get returns owned=false, existing=nil
// so the caller can retry the claim.
//
// This is the shared idempotency primitive for create handlers: the first
// caller to Create a resource's identity key wins and does the launching work;
// a duplicate (SDK retry, gateway re-publish that spawned a second handler)
// loses and never launches, so concurrent creates cannot double-launch + leak.
// wins the CAS-create. On an existing key it returns owned=false with the
// current value + revision; a key that vanished between Create and Get returns
// owned=false, existing=nil so the caller can retry.
// This is the shared idempotency primitive: the first caller wins and launches;
// duplicates (SDK retries, re-published handlers) lose and never launch.
func claimKey(kv nats.KeyValue, key string, payload []byte) (owned bool, existing []byte, rev uint64, err error) {
if _, cerr := kv.Create(key, payload); cerr == nil {
return true, nil, 0, nil
spinifex/handlers/eks/cluster_reconciler.go
+34 -59
View File
@@ -18,44 +18,36 @@ const (
defaultReconcileInterval = 30 * time.Second
defaultHealthzTimeout = 5 * time.Second
// defaultCreateTimeout bounds how long a cluster may sit in CREATING before
// the reconciler gives up and marks it FAILED. Measured from meta.CreatedAt
// (absolute), so a VM that never boots reaches a terminal state even across
// daemon restarts. Plan Risk #1.
// being marked FAILED. Measured from meta.CreatedAt so a VM that never boots
// reaches a terminal state even across daemon restarts.
defaultCreateTimeout = 15 * time.Minute
// defaultStateStaleAfter bounds how old the control plane's last NATS state
// report may be before health is treated as unknown/failed. The CP publishes
// every ~30s; 3x that tolerates a missed tick without flapping.
// defaultStateStaleAfter is the maximum age of a CP state report before
// health is treated as unknown. CP publishes every ~30s; 3× tolerates a missed tick.
defaultStateStaleAfter = 90 * time.Second
)
// natsSubscriber is the minimum surface the reconciler needs to consume the
// control-plane state report; *nats.Conn satisfies it, tests stub it.
// natsSubscriber is the minimal subscribe surface the reconciler needs; *nats.Conn satisfies it.
type natsSubscriber interface {
Subscribe(subject string, cb nats.MsgHandler) (*nats.Subscription, error)
}
// ErrReconcilerLeaseLost is returned by Run when the leader lease was lost
// mid-loop. Callers can re-Acquire and re-Run if they want to keep driving
// reconcile from the same node.
// ErrReconcilerLeaseLost is returned by Run when the leader lease is lost mid-loop.
var ErrReconcilerLeaseLost = errors.New("eks: reconciler lease lost")
// ErrReconcilerClusterDeleting is returned by Run when meta.Status is
// observed as DELETING; the DeleteCluster service-impl path is responsible
// for teardown, so the reconciler exits without further work.
// ErrReconcilerClusterDeleting is returned by Run when meta.Status is DELETING;
// DeleteCluster owns teardown so the reconciler exits.
var ErrReconcilerClusterDeleting = errors.New("eks: cluster deleting")
// ErrReconcilerClusterFailed is returned by Run when meta.Status is FAILED.
var ErrReconcilerClusterFailed = errors.New("eks: cluster failed")
// HTTPDoer is the minimum http.Client surface the reconciler needs for
// /healthz probes; tests stub it without a TLS server.
// HTTPDoer is the minimal http.Client surface needed for /healthz probes.
type HTTPDoer interface {
Do(req *http.Request) (*http.Response, error)
}
// ClusterReconciler drives one cluster's CREATING → ACTIVE → DELETING
// lifecycle under a leader lease. One instance per (accountID, clusterName);
// the leader lease arbitrates which spinifex node actually drives reconcile.
// ClusterReconciler drives one cluster's CREATING → ACTIVE → DELETING lifecycle
// under a leader lease. One instance per (accountID, clusterName).
type ClusterReconciler struct {
leaderKV nats.KeyValue
acctKV nats.KeyValue
@@ -70,17 +62,15 @@ type ClusterReconciler struct {
createTimeout time.Duration
httpClient HTTPDoer
// State-report health source. When stateSub is non-nil the reconciler gates
// on the CP's NATS self-report (latest) rather than the HTTP /healthz probe —
// the apiserver is bound to the VPC node-ip and unreachable from the host.
// When stateSub is non-nil, health is gated on the CP's NATS self-report
// rather than the HTTP /healthz probe (apiserver is VPC-only, host-unreachable).
stateSub natsSubscriber
stateSubject string
stateStaleAfter time.Duration
latest atomic.Pointer[ServerStateReport]
}
// ReconcilerOption tunes a ClusterReconciler. Tests use the With* helpers to
// shrink tickers and inject a stub HTTP client.
// ReconcilerOption tunes a ClusterReconciler.
type ReconcilerOption func(*ClusterReconciler)
// WithReconcileInterval overrides the default reconcile loop period.
@@ -88,8 +78,7 @@ func WithReconcileInterval(d time.Duration) ReconcilerOption {
return func(r *ClusterReconciler) { r.interval = d }
}
// WithLeaseRefresh overrides the default leader-lease refresh period. Should
// be less than KVBucketEKSLeaderTTL.
// WithLeaseRefresh overrides the leader-lease refresh period (should be < KVBucketEKSLeaderTTL).
func WithLeaseRefresh(d time.Duration) ReconcilerOption {
return func(r *ClusterReconciler) { r.leaseRefresh = d }
}
@@ -99,8 +88,7 @@ func WithHealthzTimeout(d time.Duration) ReconcilerOption {
return func(r *ClusterReconciler) { r.healthTimeout = d }
}
// WithCreateTimeout overrides how long a cluster may remain CREATING before the
// reconciler marks it FAILED.
// WithCreateTimeout overrides the CREATING→FAILED timeout.
func WithCreateTimeout(d time.Duration) ReconcilerOption {
return func(r *ClusterReconciler) { r.createTimeout = d }
}
@@ -110,9 +98,8 @@ func WithHTTPClient(c HTTPDoer) ReconcilerOption {
return func(r *ClusterReconciler) { r.httpClient = c }
}
// WithStateSource configures the reconciler to gate health on the control
// plane's NATS state report (subject) consumed via sub, instead of the HTTP
// /healthz probe. The subscription is opened in Run and closed when it returns.
// WithStateSource gates health on the CP's NATS state report instead of an
// HTTP /healthz probe. The subscription is opened in Run and closed on return.
func WithStateSource(sub natsSubscriber, subject string) ReconcilerOption {
return func(r *ClusterReconciler) {
r.stateSub = sub
@@ -120,15 +107,13 @@ func WithStateSource(sub natsSubscriber, subject string) ReconcilerOption {
}
}
// WithStateStaleAfter overrides how old the last state report may be before
// health is treated as failing.
// WithStateStaleAfter overrides the maximum age of a state report before health is treated as failing.
func WithStateStaleAfter(d time.Duration) ReconcilerOption {
return func(r *ClusterReconciler) { r.stateStaleAfter = d }
}
// NewClusterReconciler constructs a ClusterReconciler. healthURL is the
// fully-qualified probe target (e.g. https://nlb-dns:443/healthz) or empty
// to skip /healthz probing (CREATING transitions on bootstrap state alone).
// NewClusterReconciler constructs a ClusterReconciler. healthURL is the probe
// target or empty to skip /healthz probing (bootstrap state alone drives transitions).
func NewClusterReconciler(leaderKV, acctKV nats.KeyValue, accountID, clusterName, holderID, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, error) {
if leaderKV == nil {
return nil, errors.New("eks: NewClusterReconciler nil leaderKV")
@@ -177,9 +162,8 @@ func reconcilerLeaderKey(accountID, clusterName string) string {
}
// AcquireLease attempts a CAS Create on the leader bucket. Returns
// (release, true) if this caller is now the leader; (nil, false) otherwise.
// The release closure performs a best-effort kv.Delete; the bucket's 60s
// TTL reaps stale leases when release does not run (crashed leader).
// (release, true) on success; (nil, false) otherwise. The bucket TTL reaps
// stale leases when release does not run.
func (r *ClusterReconciler) AcquireLease() (func(), bool) {
key := reconcilerLeaderKey(r.accountID, r.clusterName)
if _, err := r.leaderKV.Create(key, []byte(r.holderID)); err != nil {
@@ -196,8 +180,7 @@ func (r *ClusterReconciler) AcquireLease() (func(), bool) {
}, true
}
// RefreshLease re-asserts ownership via a CAS update on the leader key.
// Returns false if the key was taken by another holder or the CAS conflicts.
// RefreshLease re-asserts ownership via a CAS update. Returns false if another holder won.
func (r *ClusterReconciler) RefreshLease() bool {
key := reconcilerLeaderKey(r.accountID, r.clusterName)
entry, err := r.leaderKV.Get(key)
@@ -218,9 +201,8 @@ func (r *ClusterReconciler) RefreshLease() bool {
return true
}
// Run drives reconcile until ctx is cancelled, the lease is lost, or the
// cluster reaches a terminal state. Caller is expected to AcquireLease first;
// Run does not re-acquire.
// Run drives reconcile until ctx is cancelled, the lease is lost, or the cluster
// reaches a terminal state. Caller must AcquireLease first.
func (r *ClusterReconciler) Run(ctx context.Context) error {
if r.stateSub != nil && r.stateSubject != "" {
sub, err := r.stateSub.Subscribe(r.stateSubject, func(m *nats.Msg) {
@@ -327,10 +309,8 @@ func (r *ClusterReconciler) reconcileOnce(ctx context.Context) error {
return nil
}
// observe returns the current health issue ("" when healthy) and the reported
// node count. With a NATS state source it reads the control plane's latest
// self-report (rejecting a stale or unhealthy one); otherwise it falls back to
// the HTTP /healthz probe (node count is then 0, the probe carries no count).
// observe returns the health issue ("" = healthy) and node count from the CP's
// latest NATS self-report, or from the HTTP /healthz probe when no NATS source is set.
func (r *ClusterReconciler) observe(ctx context.Context) (issue string, nodeCount int) {
if r.stateSub != nil && r.stateSubject != "" {
report := r.latest.Load()
@@ -352,11 +332,9 @@ func (r *ClusterReconciler) observe(ctx context.Context) (issue string, nodeCoun
return "", 0
}
// failIfCreateTimedOut transitions a still-CREATING cluster to FAILED once it
// has exceeded createTimeout since meta.CreatedAt, and returns
// ErrReconcilerClusterFailed so Run exits the loop. While still within the
// window it returns nil so the next tick retries. A zero or absent CreatedAt
// disables the deadline (no false-positive fail on a malformed record).
// failIfCreateTimedOut marks the cluster FAILED once createTimeout has elapsed
// since meta.CreatedAt. Returns nil while still within the window so the next
// tick retries. A zero CreatedAt disables the deadline.
func (r *ClusterReconciler) failIfCreateTimedOut(meta *ClusterMeta, reason string) error {
if r.createTimeout <= 0 || meta.CreatedAt.IsZero() {
return nil
@@ -376,12 +354,9 @@ func (r *ClusterReconciler) failIfCreateTimedOut(meta *ClusterMeta, reason strin
return ErrReconcilerClusterFailed
}
// bootstrapReady returns true once the four NATS bootstrap KV artifacts are
// present: node token, admin kubeconfig, the OIDC JWKS verified-marker, and the
// CA on meta. The JWKS gate is the verified-marker (written only after the VM's
// published JWKS passes the controller cross-check), NOT the pre-seeded
// OIDCJWKSKey — so a cluster never transitions to ACTIVE on an unverified or
// mismatched signing key.
// bootstrapReady returns true once the four bootstrap KV artifacts are present:
// node token, admin kubeconfig, OIDC JWKS verified-marker, and CA on meta.
// Uses the verified-marker (not OIDCJWKSKey) so ACTIVE always implies a verified key.
func (r *ClusterReconciler) bootstrapReady(meta *ClusterMeta) (bool, string) {
if meta.CertificateAuthorityB64 == "" {
return false, "CA absent on meta"
spinifex/handlers/eks/cluster_state.go
+39 -86
View File
@@ -10,9 +10,7 @@ import (
"github.com/nats-io/nats.go"
)
// ClusterStatus is the persisted lifecycle state for a cluster. Values match
// the AWS EKS Cluster.status string set so DescribeCluster can pass them
// through verbatim.
// ClusterStatus is the persisted cluster lifecycle state; values match the AWS EKS status enum.
type ClusterStatus string
const (
@@ -22,26 +20,19 @@ const (
ClusterStatusFailed ClusterStatus = "FAILED"
)
// ClusterVpcConfig is the persisted subset of eks.VpcConfigResponse that the
// reconciler + DescribeCluster need at-rest.
// ClusterVpcConfig is the persisted subset of eks.VpcConfigResponse.
type ClusterVpcConfig struct {
SubnetIds []string `json:"subnetIds"`
SecurityGroupIds []string `json:"securityGroupIds,omitempty"`
VpcId string `json:"vpcId,omitempty"`
// EndpointPublicAccess / EndpointPrivateAccess control apiserver endpoint
// exposure (AWS resourcesVpcConfig parity). Public ⇒ internet-facing cluster
// NLB (external-pool front-end IP, LAN-reachable); private ⇒ internal NLB
// (VPC-only). PublicAccessCidrs records the allowed source ranges for the
// public endpoint (stored for parity + Describe; ingress enforcement is a
// follow-up).
// EndpointPublicAccess/EndpointPrivateAccess mirror AWS resourcesVpcConfig.
// PublicAccessCidrs stores allowed source ranges for the public endpoint.
EndpointPublicAccess bool `json:"endpointPublicAccess"`
EndpointPrivateAccess bool `json:"endpointPrivateAccess"`
PublicAccessCidrs []string `json:"publicAccessCidrs,omitempty"`
}
// ClusterMeta is the persisted control-plane record for one cluster. The
// blob lives at ClusterMetaKey(name) inside the per-account KV bucket and is
// the source of truth for DescribeCluster.
// ClusterMeta is the persisted control-plane record at ClusterMetaKey(name); source of truth for DescribeCluster.
type ClusterMeta struct {
Name string `json:"name"`
Arn string `json:"arn"`
@@ -50,9 +41,8 @@ type ClusterMeta struct {
Version string `json:"version"`
RoleArn string `json:"roleArn"`
Endpoint string `json:"endpoint,omitempty"`
// EndpointIP is the host/LAN-reachable front-end IP the cluster NLB exposes
// (public-access ⇒ external-pool IP; private ⇒ VPC IP). The apiserver serving
// cert SANs this IP, so kubectl reaches Endpoint with TLS verification on.
// EndpointIP is the NLB front-end IP (external-pool for public, VPC IP for private).
// The apiserver serving cert SANs this IP for TLS verification.
EndpointIP string `json:"endpointIp,omitempty"`
OIDCIssuer string `json:"oidcIssuer,omitempty"`
CertificateAuthorityB64 string `json:"certificateAuthorityB64,omitempty"`
@@ -60,52 +50,35 @@ type ClusterMeta struct {
ControlPlaneInstanceID string `json:"controlPlaneInstanceId,omitempty"`
ControlPlaneENIID string `json:"controlPlaneEniId,omitempty"`
ControlPlaneENIIP string `json:"controlPlaneEniIp,omitempty"`
// ControlPlaneMgmtIP is the control-plane VM's br-mgmt NIC address. Until
// authoritative DNS (Eclipso/Route53) lands, the host-side reconciler cannot
// resolve or route to the VPC-internal NLB DNS endpoint, so it probes
// /healthz on this host-reachable address instead.
// ControlPlaneMgmtIP is the CP VM's br-mgmt NIC address, used as the /healthz
// probe target until authoritative in-VPC DNS is available.
ControlPlaneMgmtIP string `json:"controlPlaneMgmtIp,omitempty"`
// ControlPlaneNodes lists the placed control-plane server VMs. HA spread
// holds one entry per distinct host; the single-CP path holds one. [0] is the
// primary the NLB target + egress SNAT wire to until per-node NLB
// registration (231.7.3). The scalar ControlPlane* fields above mirror [0]
// for readers that predate this field (reconciler, teardown) and for clusters
// persisted before HA spread existed (empty ControlPlaneNodes).
// ControlPlaneNodes lists placed CP server VMs; [0] is the primary for NLB
// and egress. Scalar ControlPlane* fields mirror [0] for back-compat.
ControlPlaneNodes []ControlPlaneNode `json:"controlPlaneNodes,omitempty"`
// ControlPlaneSpreadGroup is the spread placement-group name reserving the CP
// hosts; "" for the single-CP fallback. DeleteCluster releases + deletes it.
// ControlPlaneSpreadGroup is the spread placement-group name; "" for single-CP.
ControlPlaneSpreadGroup string `json:"controlPlaneSpreadGroup,omitempty"`
// EgressEIPAllocationID / EgressEIPPublicIP track the hidden pool address
// SNAT'd to the control-plane VM for egress-only internet (image pulls).
// Released + the snat removed on DeleteCluster.
// EgressEIPAllocationID / EgressEIPPublicIP track the hidden-pool SNAT address
// for CP VM egress (image pulls). Released on DeleteCluster.
EgressEIPAllocationID string `json:"egressEipAllocationId,omitempty"`
EgressEIPPublicIP string `json:"egressEipPublicIp,omitempty"`
NLBArn string `json:"nlbArn,omitempty"`
NLBTargetGroupArn string `json:"nlbTargetGroupArn,omitempty"`
CreatedAt time.Time `json:"createdAt"`
// HealthIssue is the last /healthz probe failure reason for an ACTIVE
// cluster; empty means healthy. DescribeCluster surfaces a non-empty value
// as a ClusterHealth issue so a dead control plane is visible behind the
// still-ACTIVE status. LastHealthProbe stamps when the health state changed.
// HealthIssue is the last health failure reason ("" = healthy).
// DescribeCluster surfaces it as a ClusterHealth issue.
HealthIssue string `json:"healthIssue,omitempty"`
// No omitempty/omitzero: stdlib encoding/json ignores omitzero (json/v2
// only) and never treats a time.Time struct as empty, so this field always
// serializes. The tag states that honestly rather than implying it elides.
// No omitempty: stdlib encoding/json never treats a time.Time as empty.
LastHealthProbe time.Time `json:"lastHealthProbe"`
// NodeCount is the node total from the control plane's last NATS state
// report (server + joined agents). Surfaces nodegroup join/scale progress
// without the host needing apiserver reachability.
// NodeCount is the node total from the CP's last NATS state report.
NodeCount int `json:"nodeCount,omitempty"`
// BuiltinIngress records whether the cluster opted into K3s' bundled
// traefik + servicelb (dev / interim in-VPC app exposure), derived from the
// managed-ingress tag at CreateCluster. Default false = AWS parity (built-ins
// disabled; ingress via the AWS Load Balancer Controller).
// BuiltinIngress is true when the cluster opted into K3s' bundled traefik + servicelb.
// Default false = AWS parity (ingress via the AWS Load Balancer Controller).
BuiltinIngress bool `json:"builtinIngress,omitempty"`
}
// ControlPlaneNode identifies one placed control-plane server VM and the host
// it landed on. NodeID is the Spinifex host — distinct per entry under HA
// spread, empty for a single control plane launched on the local node.
// ControlPlaneNode identifies one placed CP server VM. NodeID is the Spinifex
// host; empty for a single-CP launched on the local node.
type ControlPlaneNode struct {
NodeID string `json:"nodeId,omitempty"`
InstanceID string `json:"instanceId"`
@@ -114,15 +87,13 @@ type ControlPlaneNode struct {
MgmtIP string `json:"mgmtIp,omitempty"`
}
// ErrClusterNotFound is returned by GetClusterMeta / SetClusterStatus /
// DeleteClusterPrefix when the cluster meta key is absent. Callers translate
// to the AWS shape (ResourceNotFoundException) at the service boundary.
// ErrClusterNotFound is returned when the cluster meta key is absent.
// Callers translate it to ResourceNotFoundException at the service boundary.
var ErrClusterNotFound = errors.New("eks: cluster not found")
const maxClusterStateCASRetries = 5
// PutClusterMeta writes the meta record unconditionally. Used at
// CreateCluster time to lay down the initial CREATING record.
// PutClusterMeta writes the meta record unconditionally.
func PutClusterMeta(kv nats.KeyValue, meta *ClusterMeta) error {
if meta == nil {
return errors.New("eks: PutClusterMeta nil meta")
@@ -160,11 +131,9 @@ func GetClusterMeta(kv nats.KeyValue, name string) (*ClusterMeta, error) {
return &meta, nil
}
// casUpdateMeta does a revision-checked read-modify-write of the cluster meta
// record. mutate receives the decoded meta and returns true if it changed a
// field (false short-circuits to a no-op success). Retries on KV CAS conflict
// (concurrent reconciler write) up to maxClusterStateCASRetries. Returns
// ErrClusterNotFound if the meta record was deleted underneath us.
// casUpdateMeta does a revision-checked read-modify-write of the cluster meta.
// mutate returns true when a field changed. Retries on CAS conflict up to
// maxClusterStateCASRetries. Returns ErrClusterNotFound if deleted concurrently.
func casUpdateMeta(kv nats.KeyValue, name string, mutate func(*ClusterMeta) bool) error {
if name == "" {
return errors.New("eks: casUpdateMeta empty name")
@@ -200,8 +169,7 @@ func casUpdateMeta(kv nats.KeyValue, name string, mutate func(*ClusterMeta) bool
return fmt.Errorf("eks: casUpdateMeta %s exhausted CAS retries", name)
}
// SetClusterStatus does a revision-checked update of the meta.Status field.
// Returns ErrClusterNotFound if the meta record was deleted underneath us.
// SetClusterStatus does a CAS update of meta.Status. Returns ErrClusterNotFound if deleted concurrently.
func SetClusterStatus(kv nats.KeyValue, name string, status ClusterStatus) error {
if name == "" {
return errors.New("eks: SetClusterStatus empty name")
@@ -215,11 +183,8 @@ func SetClusterStatus(kv nats.KeyValue, name string, status ClusterStatus) error
})
}
// MarkClusterFailed transitions a cluster to FAILED with a human-readable
// reason, but only from CREATING — a cluster already DELETING/ACTIVE/FAILED is
// left untouched so a late bootstrap error cannot clobber a concurrent delete
// or an already-healthy cluster. Returns ErrClusterNotFound if the meta record
// was deleted underneath us.
// MarkClusterFailed transitions the cluster to FAILED with a reason, but only
// from CREATING — so a late error cannot clobber a concurrent delete or an active cluster.
func MarkClusterFailed(kv nats.KeyValue, name, reason string) error {
if name == "" {
return errors.New("eks: MarkClusterFailed empty name")
@@ -234,10 +199,8 @@ func MarkClusterFailed(kv nats.KeyValue, name, reason string) error {
})
}
// SetClusterCertificateAuthority does a revision-checked update of the
// meta.CertificateAuthorityB64 field. The NATS bootstrap subscriber calls
// this once the K3s server VM publishes its CA on the bootstrap bus. Returns
// ErrClusterNotFound if the meta record was deleted underneath us.
// SetClusterCertificateAuthority does a CAS update of meta.CertificateAuthorityB64.
// Called by the bootstrap subscriber when the K3s VM publishes its CA.
func SetClusterCertificateAuthority(kv nats.KeyValue, name, caB64 string) error {
if name == "" {
return errors.New("eks: SetClusterCertificateAuthority empty name")
@@ -254,12 +217,8 @@ func SetClusterCertificateAuthority(kv nats.KeyValue, name, caB64 string) error
})
}
// SetClusterHealth records the latest /healthz probe outcome for a cluster:
// issue is the failure reason (empty clears it / marks healthy). Writes only
// when the health state changes, stamping LastHealthProbe at the transition so
// a persistently-unhealthy cluster carries the time it first went bad rather
// than churning the KV every probe interval. Returns ErrClusterNotFound if the
// meta record was deleted underneath us.
// SetClusterHealth records the latest health outcome; issue="" means healthy.
// Only writes on a state change, stamping LastHealthProbe at the transition.
func SetClusterHealth(kv nats.KeyValue, name, issue string) error {
if name == "" {
return errors.New("eks: SetClusterHealth empty name")
@@ -274,12 +233,8 @@ func SetClusterHealth(kv nats.KeyValue, name, issue string) error {
})
}
// SetClusterHealthState records the latest health outcome together with the
// node count from the control-plane state report. issue is the failure reason
// (empty clears it / marks healthy). LastHealthProbe is stamped whenever health
// or node count changes; a steady-state healthy cluster with an unchanging node
// count produces no KV write. Returns ErrClusterNotFound if the meta record was
// deleted underneath us.
// SetClusterHealthState records health + node count. LastHealthProbe is stamped
// on any change; no KV write when both are unchanged.
func SetClusterHealthState(kv nats.KeyValue, name, issue string, nodeCount int) error {
if name == "" {
return errors.New("eks: SetClusterHealthState empty name")
@@ -295,10 +250,8 @@ func SetClusterHealthState(kv nats.KeyValue, name, issue string, nodeCount int)
})
}
// DeleteClusterPrefix removes every KV key under clusters/{name}/. Called
// from DeleteCluster after the OIDC private key has been zeroized.
// Returns the first delete error encountered but continues sweeping so
// best-effort cleanup proceeds.
// DeleteClusterPrefix removes every KV key under clusters/{name}/.
// Returns the first error but continues sweeping.
func DeleteClusterPrefix(kv nats.KeyValue, name string) error {
if name == "" {
return errors.New("eks: DeleteClusterPrefix empty name")
spinifex/handlers/eks/k3s_ha_control_plane.go
+37 -77
View File
@@ -18,15 +18,11 @@ import (
"github.com/nats-io/nats.go"
)
// haControlPlaneCount is the control-plane spread width: an odd quorum of three
// so an embedded-etcd control plane tolerates a single host loss. Fewer than
// this many schedulable hosts falls back to a single control-plane VM (today's
// behaviour) instead of failing the create.
// haControlPlaneCount is the HA spread width (odd quorum tolerating one host loss).
// Fewer schedulable hosts fall back to a single CP VM.
const haControlPlaneCount = 3
// controlPlanePlacer is the spread-placement surface the HA control-plane
// orchestrator needs. handlers/ec2/placementgroup.PlacementGroupService (wired
// by the daemon as NewNATSPlacementGroupService) satisfies it.
// controlPlanePlacer is the spread-placement surface for HA CP orchestration.
type controlPlanePlacer interface {
CreatePlacementGroup(*ec2.CreatePlacementGroupInput, string) (*ec2.CreatePlacementGroupOutput, error)
DeletePlacementGroup(*ec2.DeletePlacementGroupInput, string) (*ec2.DeletePlacementGroupOutput, error)
@@ -38,25 +34,17 @@ type controlPlanePlacer interface {
var _ controlPlanePlacer = (handlers_ec2_placementgroup.PlacementGroupService)(nil)
// HostScheduler answers the capacity + placement fan-out questions the HA
// control-plane orchestrator asks. The daemon wires a NATS implementation; unit
// tests fake it so the orchestration logic needs no live cluster.
// HostScheduler answers capacity + placement fan-out questions for HA CP placement.
type HostScheduler interface {
// SchedulableHosts returns the distinct node IDs that can fit at least one
// VM of instanceType.
// SchedulableHosts returns node IDs that can fit at least one VM of instanceType.
SchedulableHosts(instanceType string) []string
// InstanceHosts maps each instance ID to the node currently hosting it.
// Instances absent from the result are not (yet) visible to the fan-out.
// InstanceHosts maps each instance ID to its hosting node; absent entries are not yet visible.
InstanceHosts(instanceIDs []string) map[string]string
}
// placeControlPlane places the cluster's control-plane VM(s). With at least
// haControlPlaneCount schedulable hosts it spreads that many k3s server VMs
// across distinct hosts, all-or-nothing; otherwise it launches a single
// control-plane VM on the local node (today's behaviour). Returns the placed
// nodes ([0] is the primary the NLB + egress wire to until 231.7.3) and the
// spread placement-group name ("" for the single-CP fallback) the caller
// persists for DeleteCluster teardown.
// placeControlPlane places the cluster's CP VMs. With ≥ haControlPlaneCount
// schedulable hosts it spreads servers all-or-nothing; otherwise falls back to
// a single CP. Returns placed nodes ([0] = primary) and spread group name ("" for single-CP).
func (s *EKSServiceImpl) placeControlPlane(accountID, clusterName string, tmpl K3sServerInput) ([]ControlPlaneNode, string, error) {
instanceType := tmpl.InstanceType
if instanceType == "" {
@@ -83,10 +71,7 @@ func (s *EKSServiceImpl) placeControlPlane(accountID, clusterName string, tmpl K
MaxCount: haControlPlaneCount,
}, pgAccount)
if err != nil {
// Could not reserve the full quorum of distinct hosts (capacity raced
// away between the status fan-out and the CAS reservation). Drop the
// group and fall back to a single control plane rather than failing the
// whole create.
// Capacity raced away between fan-out and reservation; fall back to single CP.
slog.Warn("placeControlPlane: spread reservation failed, falling back to single control plane",
"cluster", clusterName, "group", groupName, "err", err)
s.deleteSpreadGroup(groupName, pgAccount)
@@ -106,8 +91,7 @@ func (s *EKSServiceImpl) placeControlPlane(accountID, clusterName string, tmpl K
launched = append(launched, controlPlaneNode(r.node, r.out))
}
// All-or-nothing: any launch failure rolls back every VM that did come up,
// releases the reservation, drops the group, and fails the create.
// All-or-nothing: any launch failure rolls back all VMs, releases the reservation.
if len(launchErrs) > 0 {
slog.Error("placeControlPlane: partial spread launch, rolling back",
"cluster", clusterName, "launched", len(launched), "failed", len(launchErrs))
@@ -115,10 +99,8 @@ func (s *EKSServiceImpl) placeControlPlane(accountID, clusterName string, tmpl K
return nil, "", fmt.Errorf("eks: HA control-plane launch failed: %w", errors.Join(launchErrs...))
}
// Confirm each VM actually landed on its reserved host (acceptance: verified
// via the node-VM fan-out). The node-targeted launch subject already pins
// placement, so an instance not yet visible to the fan-out is tolerated;
// only a definitive wrong-host placement triggers rollback.
// Verify each VM landed on its reserved host. Not-yet-visible instances are
// tolerated; only a definitive wrong-host placement triggers rollback.
if err := s.verifyControlPlaneSpread(launched); err != nil {
slog.Error("placeControlPlane: placement verification failed, rolling back",
"cluster", clusterName, "err", err)
@@ -144,8 +126,7 @@ func (s *EKSServiceImpl) placeControlPlane(accountID, clusterName string, tmpl K
return launched, groupName, nil
}
// launchSingleControlPlane launches one control-plane VM on the local node, the
// pre-HA behaviour. NodeID is empty: the local launch is node-blind.
// launchSingleControlPlane launches one CP VM on the local node (pre-HA fallback).
func (s *EKSServiceImpl) launchSingleControlPlane(tmpl K3sServerInput) ([]ControlPlaneNode, string, error) {
in := tmpl
in.TargetNodeID = ""
@@ -162,13 +143,9 @@ type cpLaunchResult struct {
err error
}
// launchControlPlaneSpread launches the reserved nodes as one etcd quorum:
// nodes[0] is the first server (cluster-init), and its ENI IP is the join
// endpoint the remaining servers register against — so it must come up first.
// nodes[1..] then launch in parallel as join servers (each remote launch blocks
// on a full-AMI boot round trip, so sequential joins would be N×). Results stay
// index-aligned with nodes. If the first server fails, the joins are skipped
// (no endpoint to join) so the caller's all-or-nothing rollback fires.
// launchControlPlaneSpread launches the reserved nodes as one etcd quorum.
// nodes[0] is the cluster-init server whose ENI IP the remaining servers join;
// nodes[1..] launch in parallel. Results are index-aligned with nodes.
func (s *EKSServiceImpl) launchControlPlaneSpread(tmpl K3sServerInput, nodes []string) []cpLaunchResult {
results := make([]cpLaunchResult, len(nodes))
@@ -200,13 +177,11 @@ func (s *EKSServiceImpl) launchControlPlaneSpread(tmpl K3sServerInput, nodes []s
return results
}
// errFirstServerFailed marks join-server results skipped when the cluster-init
// server never came up — the joins have no endpoint to register against.
// errFirstServerFailed marks join results skipped when the cluster-init server failed to launch.
var errFirstServerFailed = errors.New("eks: first control-plane server failed; join servers skipped")
// verifyControlPlaneSpread confirms every launched VM sits on its reserved host
// and that no two share a host. A VM not yet visible to the fan-out is tolerated
// (listing lag — the node-targeted launch subject already guarantees the host).
// verifyControlPlaneSpread confirms every CP VM sits on its reserved host with
// no duplicates. VMs not yet visible to the fan-out are tolerated.
func (s *EKSServiceImpl) verifyControlPlaneSpread(nodes []ControlPlaneNode) error {
ids := make([]string, 0, len(nodes))
for _, n := range nodes {
@@ -233,10 +208,8 @@ func (s *EKSServiceImpl) verifyControlPlaneSpread(nodes []ControlPlaneNode) erro
return nil
}
// rollbackControlPlaneSpread tears down a partial HA placement: terminate every
// VM that launched, release the reservation (clears the empty placeholders —
// the record was never finalized on any rollback path), and drop the group.
// Best-effort: a stranded internal placement group is harmless and re-creatable.
// rollbackControlPlaneSpread terminates launched VMs, releases the reservation,
// and drops the group. Best-effort: a leaked internal group is harmless.
func (s *EKSServiceImpl) rollbackControlPlaneSpread(accountID, groupName, pgAccount string, launched []ControlPlaneNode, reserved []string) {
for _, n := range launched {
if err := TerminateK3sServerVM(s.deps.VPCK3s, s.deps.Instance, accountID, n.InstanceID, n.ENIID); err != nil {
@@ -252,9 +225,7 @@ func (s *EKSServiceImpl) rollbackControlPlaneSpread(accountID, groupName, pgAcco
s.deleteSpreadGroup(groupName, pgAccount)
}
// ensureSpreadGroup creates the per-cluster spread placement group. A duplicate
// (a prior attempt left it behind) is reused — ReserveSpreadNodes filters
// already-occupied hosts, so reserving against a stale group is safe.
// ensureSpreadGroup creates the spread placement group, reusing it if it already exists.
func (s *EKSServiceImpl) ensureSpreadGroup(groupName, pgAccount string) error {
_, err := s.deps.PlacementGroup.CreatePlacementGroup(&ec2.CreatePlacementGroupInput{
GroupName: aws.String(groupName),
@@ -278,9 +249,8 @@ func (s *EKSServiceImpl) deleteSpreadGroup(groupName, pgAccount string) {
}
}
// teardownSpreadGroup removes the cluster's CP instances from the spread group
// and deletes it. Best-effort: a leaked internal group strands nothing billable.
// Runs only for HA clusters (ControlPlaneSpreadGroup set).
// teardownSpreadGroup removes CP instances from the spread group and deletes it.
// No-op for single-CP clusters (ControlPlaneSpreadGroup == "").
func (s *EKSServiceImpl) teardownSpreadGroup(meta *ClusterMeta) {
if meta.ControlPlaneSpreadGroup == "" {
return
@@ -302,9 +272,8 @@ func (s *EKSServiceImpl) teardownSpreadGroup(meta *ClusterMeta) {
s.deleteSpreadGroup(meta.ControlPlaneSpreadGroup, pgAccount)
}
// controlPlaneTeardownNodes returns the control-plane VMs DeleteCluster must
// tear down. It prefers the multi-node list; for clusters persisted before that
// field existed it synthesizes a single entry from the scalar fields.
// controlPlaneTeardownNodes returns CP VMs for DeleteCluster. Prefers
// ControlPlaneNodes; falls back to scalar fields for older persisted clusters.
func controlPlaneTeardownNodes(meta *ClusterMeta) []ControlPlaneNode {
if len(meta.ControlPlaneNodes) > 0 {
return meta.ControlPlaneNodes
@@ -320,10 +289,8 @@ func controlPlaneTeardownNodes(meta *ClusterMeta) []ControlPlaneNode {
}}
}
// haSpreadGroupName namespaces the spread group by account + cluster. The group
// lives under the system account (admin.SystemAccountID) so it never surfaces in
// a customer's DescribePlacementGroups; the account therefore cannot
// disambiguate same-named clusters across tenants, so it is folded into the name.
// haSpreadGroupName namespaces the spread group by account + cluster so same-named
// clusters across tenants don't collide. Lives under the system account.
func haSpreadGroupName(accountID, clusterName string) string {
return "eks-cp-" + accountID + "-" + clusterName
}
@@ -342,27 +309,22 @@ func controlPlaneNode(nodeID string, out *K3sServerOutput) ControlPlaneNode {
var _ HostScheduler = (*natsHostScheduler)(nil)
// hostFanoutTimeout bounds a node status / node VMs fan-out. Sized for a LAN
// round trip to every daemon; CreateCluster is already multi-second so a fixed
// short collection window trades a little latency for reliably seeing every host
// (a missed host would wrongly drop the cluster to single-CP).
// hostFanoutTimeout bounds a node status/VMs fan-out. Short fixed window
// trades a little latency for reliably seeing all hosts before placement.
const hostFanoutTimeout = time.Second
type natsHostScheduler struct {
nc *nats.Conn
}
// NewNATSHostScheduler builds the NATS fan-out HostScheduler the daemon wires
// into EKSServiceDeps.
// NewNATSHostScheduler returns a NATS fan-out HostScheduler for EKSServiceDeps.
func NewNATSHostScheduler(nc *nats.Conn) HostScheduler {
return &natsHostScheduler{nc: nc}
}
func (h *natsHostScheduler) SchedulableHosts(instanceType string) []string {
// The control-plane VM is a system type (sys.*), which node.status omits from
// its per-type capacity list (system types are hidden from customers). A
// system VM still consumes host vCPU/memory like any guest, so size the fit
// against each node's raw schedulable headroom and the type's footprint.
// sys.* types are hidden from customers in node.status but still consume
// vCPU/memory, so fit is checked against raw schedulable headroom.
vcpu, memGB, ok := instancetypes.SpecForSystemType(instanceType)
if !ok {
slog.Warn("SchedulableHosts: unknown system instance type, no schedulable hosts",
@@ -385,9 +347,8 @@ func (h *natsHostScheduler) SchedulableHosts(instanceType string) []string {
return hosts
}
// nodeFitsSystemInstance reports whether a node's schedulable headroom
// (Total - Reserved - Alloc, the same arithmetic node.status documents for guest
// scheduling) admits at least one VM of the given vCPU/memory footprint.
// nodeFitsSystemInstance reports whether a node's headroom (Total - Reserved - Alloc)
// fits at least one VM of the given vCPU/memory footprint.
func nodeFitsSystemInstance(st types.NodeStatusResponse, vcpu int, memGB float64) bool {
remainVCPU := st.TotalVCPU - st.ReservedVCPU - st.AllocVCPU
remainMem := st.TotalMemGB - st.ReservedMemGB - st.AllocMemGB
@@ -414,8 +375,7 @@ func (h *natsHostScheduler) InstanceHosts(instanceIDs []string) map[string]strin
return out
}
// fanout publishes an empty request on subject and invokes handle for every
// reply that arrives within hostFanoutTimeout.
// fanout publishes on subject and calls handle for each reply within hostFanoutTimeout.
func (h *natsHostScheduler) fanout(subject string, handle func([]byte)) {
if h.nc == nil {
return
spinifex/handlers/eks/k3s_server_vm.go
+69 -198
View File
@@ -17,108 +17,73 @@ import (
"github.com/mulgadc/spinifex/spinifex/tags"
)
// imdsServerIP is the link-local IMDS address (mirror of
// handlers/imds.MetaDataServerIP, duplicated to avoid an import cycle:
// imds → sts → eks).
// imdsServerIP is the link-local IMDS address (duplicated here to avoid an imds→sts→eks import cycle).
const imdsServerIP = "169.254.169.254"
// ErrEKSServerAMINotFound is returned by the launcher when no AMI carrying the
// EKS managed-by tag exists in the account. CreateCluster maps it to a precise
// client error: it signals an operator/config gap (the eks-server image was
// never built/imported), not an unrecoverable internal fault.
// ErrEKSServerAMINotFound is returned when no AMI with the EKS managed-by tag
// exists. Signals an operator/config gap (image not built/imported).
var ErrEKSServerAMINotFound = errors.New("eks: eks-server AMI not found")
// k3sVPCProvisioner is the subset of handlers_ec2_vpc.VPCService that the K3s
// server VM launcher needs. Narrow so tests can fake it without implementing
// the full VPC surface.
// k3sVPCProvisioner is the narrow VPC surface the K3s server VM launcher needs.
type k3sVPCProvisioner interface {
CreateNetworkInterface(input *ec2.CreateNetworkInterfaceInput, accountID string) (*ec2.CreateNetworkInterfaceOutput, error)
DeleteNetworkInterface(input *ec2.DeleteNetworkInterfaceInput, accountID string) (*ec2.DeleteNetworkInterfaceOutput, error)
}
// k3sInstanceLauncher is the system-instance launch surface the K3s control-plane
// VM needs. The VM boots from the eks-server AMI (BootAMI) and must get a
// management-bridge NIC so it can reach the daemon's NATS endpoint off its
// tenant VPC subnet — which only the system-instance path provides.
// *daemon.Daemon satisfies this interface.
// k3sInstanceLauncher is the system-instance launch surface for the K3s CP VM.
// The VM boots from the eks-server AMI and needs a mgmt-bridge NIC via the
// system-instance path to reach the daemon's NATS endpoint.
type k3sInstanceLauncher interface {
LaunchSystemInstance(input *sysinstance.SystemInstanceInput) (*sysinstance.SystemInstanceOutput, error)
// LaunchSystemInstanceOnNode places the VM on a specific Spinifex host for
// HA control-plane spread. An empty nodeID (or the local node) launches
// in-process, matching LaunchSystemInstance.
// LaunchSystemInstanceOnNode pins the VM to a specific host for HA spread.
// An empty nodeID launches in-process like LaunchSystemInstance.
LaunchSystemInstanceOnNode(nodeID string, input *sysinstance.SystemInstanceInput) (*sysinstance.SystemInstanceOutput, error)
TerminateSystemInstance(instanceID string) error
}
// k3sAMIResolver is the subset of handlers_ec2_image.ImageService the launcher
// uses to resolve the eks-server AMI name to an AMI ID at launch time.
// k3sAMIResolver is the narrow AMI surface for resolving the eks-server AMI ID.
type k3sAMIResolver interface {
DescribeImages(input *ec2.DescribeImagesInput, accountID string) (*ec2.DescribeImagesOutput, error)
}
const (
// defaultK3sServerInstanceType is the internal system type the k3s
// control-plane VM boots as (2 vCPU / 4 GB / 40 GB root). A sys.* type
// keeps the CP VM out of customer DescribeInstanceTypes and lets the daemon
// register the node-targeted system.LaunchInstance.{type}.{nodeID} subject
// the HA spread path uses. Callers can override via K3sServerInput.InstanceType.
// defaultK3sServerInstanceType is the sys.* type for the CP VM; keeps it out of
// customer DescribeInstanceTypes and enables node-targeted HA launch subjects.
defaultK3sServerInstanceType = "sys.medium"
// k3sOIDCSigningKeyPath is the on-VM path where cloud-init drops the
// per-cluster OIDC private key PEM. K3s reads it via the
// service-account-signing-key-file kube-apiserver arg (signs SA tokens).
// k3sOIDCSigningKeyPath is the on-VM OIDC private key PEM path.
k3sOIDCSigningKeyPath = "/etc/rancher/k3s/oidc-signing-key.pem"
// k3sOIDCPublicKeyPath is where cloud-init drops the matching public key
// PEM. kube-apiserver's service-account-key-file requires a PUBLIC key
// (it rejects a private-key PEM with "data does not contain any valid RSA
// or ECDSA public keys" and crash-loops), so it must be a separate file.
// k3sOIDCPublicKeyPath is the matching public key PEM; kube-apiserver's
// --service-account-key-file requires a PUBLIC key, so it must be separate.
k3sOIDCPublicKeyPath = "/etc/rancher/k3s/oidc-signing-key.pub.pem"
// k3sFirstBootEnvPath is the env file k3s-first-boot.sh sources at boot
// (see scripts/images/eks-server/k3s-first-boot.sh ENVFILE).
// k3sFirstBootEnvPath is the env file k3s-first-boot.sh sources at boot.
k3sFirstBootEnvPath = "/etc/spinifex-eks/first-boot.env"
// agentEnvPath is the env file the k3s-agent OpenRC service sources for its
// K3S_URL/K3S_TOKEN/K3S_NODE_NAME/K3S_NODE_LABEL. Path matches the AGENT_ENVFILE
// in scripts/images/eks-node/eks-node-role.sh and k3s-agent.initd.
// agentEnvPath is the env file the k3s-agent OpenRC service sources for
// K3S_URL/K3S_TOKEN/K3S_NODE_NAME/K3S_NODE_LABEL.
agentEnvPath = "/etc/spinifex-eks/agent.env"
// k3sGatewayCAPath is the on-VM destination for the AWS gateway TLS CA cert
// PEM. The K3s VM uses it to verify the gateway's HTTPS cert when the
// eks-gateway-publish helper POSTs bootstrap envelopes + state reports.
// Path matches k3s-first-boot.sh EKS_GATEWAY_CA.
// k3sGatewayCAPath is the on-VM gateway TLS CA cert PEM path.
k3sGatewayCAPath = "/etc/spinifex-eks/gateway-ca.pem"
// k3sTokenWebhookKubeconfigPath is the apiserver token-webhook config the
// eks-token-webhook service (ordered `before k3s`) writes before the
// apiserver starts. Wired via the authentication-token-webhook-config-file
// apiserver arg so bearer tokens minted by `aws eks get-token` resolve
// through the webhook. Must match the webhook's EKS_WEBHOOK_KUBECONFIG default.
// k3sTokenWebhookKubeconfigPath is the apiserver token-webhook kubeconfig
// written before k3s starts, wired via --authentication-token-webhook-config-file.
k3sTokenWebhookKubeconfigPath = "/etc/spinifex-eks/token-webhook.kubeconfig" //nolint:gosec // file path, not a credential
// k3sConfigPath is the K3s server config file cloud-init writes; K3s
// reads it at startup (overrides the AMI-baked config.yaml.skel).
// k3sConfigPath is the K3s server config file written by cloud-init.
k3sConfigPath = "/etc/rancher/k3s/config.yaml"
// k3sResolvConfPath is the on-VM resolver file. The Alpine eks-server AMI
// runs dhcpcd, whose resolv.conf hook fails to persist the DHCP-supplied
// nameservers ("can't create /etc/resolv.conf: nonexistent directory"), so
// the VM boots with no resolver and containerd cannot resolve registry-1.
// docker.io to pull the system-pod images. cloud-init writes a static
// resolver here; the dhcpcd hook never clobbers it (it errors before it
// would). Reachable via the cluster's egress SNAT.
// k3sResolvConfPath is the on-VM resolver path. The Alpine AMI's dhcpcd hook
// cannot create /etc/resolv.conf, so cloud-init writes a static resolver here.
k3sResolvConfPath = "/etc/resolv.conf"
// k3sResolvConf is the static resolver content. Public anycast resolvers
// reached over the control-plane VM's egress path.
// k3sResolvConf is the static resolver; reached via the cluster's egress SNAT.
k3sResolvConf = "nameserver 1.1.1.1\nnameserver 8.8.8.8"
)
// K3sServerInput is the launcher's input shape. AccountID is the customer
// account; the ENI + VM both live there in v1 (SystemAccount-owned VM is
// deferred behind cross-account-ENI work). Region is carried for future
// region-aware AMI lookups but not consumed today.
// K3sServerInput is the K3s server VM launcher's input shape.
type K3sServerInput struct {
AccountID string
ClusterName string
@@ -126,49 +91,34 @@ type K3sServerInput struct {
SubnetID string
ControlPlaneSGID string
NLBDNS string
// EndpointIP is the reachable cluster NLB front-end IP. When set it is added
// to the apiserver serving-cert SANs (tls-san) so kubectl reaching the
// cluster on this IP validates TLS. Empty for an internal endpoint with no
// front-end IP read back.
// EndpointIP is the NLB front-end IP added to the apiserver cert SANs for TLS.
// Empty for an internal endpoint with no front-end IP.
EndpointIP string
OIDCIssuer string
OIDCPrivateKeyPEM string
OIDCPublicKeyPEM string
// Gateway broker config: the control-plane VM publishes its bootstrap
// envelopes + state reports via SigV4-signed HTTPS POST to the AWS gateway
// (the ELBv2 lb-agent model), not by dialing core NATS. GatewayURL is the
// mgmt-reachable AWSGW endpoint; AccessKey/SecretKey are the system
// (Predastore) SigV4 creds; GatewayCACert is the PEM that signs the gateway
// server cert.
// Gateway broker config: CP VM publishes via SigV4-signed HTTPS POST to AWSGW.
// GatewayURL is the mgmt-reachable endpoint; AccessKey/SecretKey are system
// SigV4 creds; GatewayCACert signs the gateway TLS cert.
GatewayURL string
AccessKey string
SecretKey string
GatewayCACert string
InstanceType string
// TargetNodeID pins the control-plane VM to a specific Spinifex host for HA
// spread across distinct failure domains. Empty launches on the local
// daemon node (the single-CP default).
// TargetNodeID pins the VM to a specific host for HA spread; empty = local node.
TargetNodeID string
// JoinToken is the shared k3s cluster token (server secret). Set on every
// control-plane server so servers 2..N authenticate into the first server's
// embedded-etcd quorum and workers continue to register. Empty preserves
// k3s' per-server auto-generated token (pre-HA single-CP behaviour).
// JoinToken is the shared k3s cluster token so HA servers join the etcd quorum.
// Empty = k3s auto-generated token (single-CP).
JoinToken string
// ServerURL, when set, boots this VM as a JOIN server: it registers with the
// existing control plane at this https://<first-server-ip>:6443 endpoint and
// joins the etcd quorum WITHOUT cluster-init. Empty = the first server, which
// cluster-inits the datastore. A non-empty ServerURL requires JoinToken.
// ServerURL boots this VM as a JOIN server joining the quorum at the given endpoint.
// Empty = first server (cluster-init). Non-empty requires JoinToken.
ServerURL string
// BuiltinIngress keeps K3s' bundled traefik + servicelb enabled (interim
// in-VPC app exposure). When false the config disables them for AWS parity,
// where Service type=LoadBalancer / Ingress are satisfied by the AWS Load
// Balancer Controller instead. The CreateCluster default is ON until that
// controller ships (see managedIngressTagKey).
// BuiltinIngress keeps K3s' bundled traefik + servicelb enabled.
// When false, both are disabled for AWS parity (ingress via the LB Controller).
BuiltinIngress bool
}
// K3sServerOutput carries the identifiers the caller needs to persist into
// ClusterMeta and to register the ENI IP with the cluster NLB target group.
// K3sServerOutput carries identifiers to persist in ClusterMeta and register with the NLB.
type K3sServerOutput struct {
InstanceID string
ENIID string
@@ -176,15 +126,9 @@ type K3sServerOutput struct {
MgmtIP string
}
// LaunchK3sServerVM provisions the K3s control-plane VM for an EKS cluster.
// Sequence: resolve the eks-server AMI, pre-create the customer-account ENI
// in the supplied subnet with the control-plane SG, render cloud-init user
// data (env vars, OIDC PEM, K3s config), then call RunInstances with
// NetworkInterfaces[0].NetworkInterfaceId pointing at the pre-created ENI.
// Returns instance/ENI identity so the caller can register the ENI IP with
// the cluster NLB target group and persist the IDs in ClusterMeta. On
// RunInstances failure the pre-created ENI is deleted best-effort so the
// caller does not leak a customer-account resource.
// LaunchK3sServerVM provisions the K3s CP VM: resolves the AMI, pre-creates the
// ENI, renders cloud-init user-data, then launches via RunInstances. On failure
// the ENI is deleted best-effort to avoid leaking a customer-account resource.
func LaunchK3sServerVM(
vpcSvc k3sVPCProvisioner,
instSvc k3sInstanceLauncher,
@@ -267,10 +211,8 @@ func LaunchK3sServerVM(
}, nil
}
// TerminateK3sServerVM tears down the K3s server VM for a cluster. The
// instance termination cascades ENI detach inside the customer instance path,
// then this helper deletes the ENI explicitly so DeleteCluster does not leak
// it. Missing instance/ENI is a no-op so retries stay safe.
// TerminateK3sServerVM terminates the K3s server VM and deletes the ENI.
// Missing instance/ENI is a no-op for idempotent retries.
func TerminateK3sServerVM(
vpcSvc k3sVPCProvisioner,
instSvc k3sInstanceLauncher,
@@ -281,9 +223,7 @@ func TerminateK3sServerVM(
}
var firstErr error
if instanceID != "" {
// A retried DeleteCluster reaches here after the VM already drained, so
// "instance not found" is the steady-state success case, not a failure —
// treat it as idempotent so teardown can proceed to the ENI/SG/KV sweep.
// "instance not found" on a retry is idempotent success; proceed to the ENI/SG/KV sweep.
if err := instSvc.TerminateSystemInstance(instanceID); err != nil {
if errors.Is(err, sysinstance.ErrSystemInstanceNotFound) {
slog.Debug("TerminateK3sServerVM: instance already gone", "instanceId", instanceID)
@@ -300,15 +240,10 @@ func TerminateK3sServerVM(
switch {
case awserrors.IsErrorCode(err, awserrors.ErrorInvalidNetworkInterfaceIDNotFound),
awserrors.IsErrorCode(err, awserrors.ErrorInvalidNetworkInterfaceNotFound):
// The instance-terminate cascade (or a prior retry) already
// deleted the ENI. Idempotent success — must NOT block the SG +
// KV sweep, or the cluster wedges in DELETING permanently.
// ENI already gone (instance-terminate cascade or prior retry); idempotent success.
slog.Debug("TerminateK3sServerVM: ENI already gone", "eniId", eniID)
default:
// InvalidNetworkInterface.InUse (the VM is still terminating
// async and holds the ENI) and any other error are retryable:
// surface so the cluster stays DELETING and the reconciler
// retries once the instance releases the ENI.
// InUse = VM still terminating async; surface error so the reconciler retries.
slog.Warn("TerminateK3sServerVM: ENI delete failed", "eniId", eniID, "err", err)
if firstErr == nil {
firstErr = fmt.Errorf("delete ENI %s: %w", eniID, err)
@@ -319,12 +254,8 @@ func TerminateK3sServerVM(
return firstErr
}
// lookupEKSServerAMI resolves the EKS control-plane AMI by the
// spinifex:managed-by=eks tag the build pipeline stamps on it, rather than a
// brittle exact name. This survives the planned server+agent → single EKS AMI
// unify untouched (the unified AMI keeps the tag; role is chosen per-instance
// at launch). If more than one AMI carries the tag (e.g. server + agent both
// imported pre-unify), the newest by CreationDate wins.
// lookupEKSServerAMI resolves the EKS CP AMI by the spinifex:managed-by=eks tag
// rather than a brittle exact name. If multiple AMIs match, the newest wins.
func lookupEKSServerAMI(amiSvc k3sAMIResolver, accountID string) (string, error) {
out, err := amiSvc.DescribeImages(&ec2.DescribeImagesInput{
Filters: []*ec2.Filter{
@@ -401,11 +332,8 @@ func validateK3sServerInput(in K3sServerInput) error {
return nil
}
// GenerateK3sClusterToken mints the shared k3s cluster token (256 bits of
// crypto-random, hex-encoded) seeded into every control-plane server so servers
// 2..N join the first server's etcd quorum and workers register. Generated per
// CreateCluster; not persisted (the node-token derived from it is published on
// the bootstrap bus for workers).
// GenerateK3sClusterToken mints the shared k3s cluster token (256 bits, hex-encoded).
// Servers 2..N and workers use it to join; the derived node-token is published on the bootstrap bus.
func GenerateK3sClusterToken() (string, error) {
var b [32]byte
if _, err := rand.Read(b[:]); err != nil {
@@ -414,9 +342,7 @@ func GenerateK3sClusterToken() (string, error) {
return hex.EncodeToString(b[:]), nil
}
// k3sServerJoinURL builds the registration endpoint a join server dials: the
// first server's ENI private IP on the k3s supervisor port. Token-based join is
// trust-on-first-use, so the IP need not be in the apiserver cert SANs.
// k3sServerJoinURL builds the join endpoint (first server's ENI IP on port 6443).
func k3sServerJoinURL(ip string) string {
return "https://" + net.JoinHostPort(ip, "6443")
}
@@ -442,11 +368,7 @@ type userDataFile struct {
func buildK3sUserData(in K3sServerInput) string {
nlbEndpoint := "https://" + net.JoinHostPort(in.NLBDNS, strconv.FormatInt(clusterNLBListenPort, 10))
// Role the eks-node-role first-boot selector reads to enable the
// control-plane services. The first server is "server" (cluster-init +
// bootstrap publisher); servers 2..N are "server-join" (register into the
// existing quorum, no bootstrap re-publish). Nodegroup workers seed "agent"
// through their own path.
// "server" = first CP (cluster-init + bootstrap publisher); "server-join" = HA join server.
role := "server"
if in.ServerURL != "" {
role = "server-join"
@@ -463,42 +385,15 @@ func buildK3sUserData(in K3sServerInput) string {
"EKS_CLUSTER_NAME=" + in.ClusterName,
"EKS_NLB_ENDPOINT=" + nlbEndpoint,
"EKS_OIDC_ISSUER=" + in.OIDCIssuer,
// Gate deferred built-in ingress: k3s.initd stages a traefik .skip marker
// before boot and the state-reporter removes it once the apiserver is
// stable. Set only for clusters that want built-in ingress.
// Deferred ingress: k3s.initd stages a traefik .skip marker; state-reporter removes it once stable.
"EKS_DEFER_TRAEFIK=" + boolFlag(in.BuiltinIngress),
}, "\n")
// cluster-init (first server) selects the embedded etcd datastore (required
// for multi-server HA); servers 2..N instead carry `server: <first>` + the
// shared token and join the quorum without cluster-init.
// etcd-expose-metrics surfaces etcd's own wal_fsync_duration_seconds and
// backend_commit_duration_seconds on 127.0.0.1:2381/metrics so control-plane
// commit latency is measurable directly, not inferred.
//
// anonymous-auth=true: k3s hardens it off by default, which makes the
// apiserver answer 401 to an unauthenticated /healthz. The cluster reconciler
// probes https://<NLB>/healthz anonymously to gate ACTIVE, so it must be
// reachable; the default RBAC binds only the health/version non-resource
// paths to system:unauthenticated, so this exposes nothing else.
// In real EKS neither traefik nor servicelb exists; Service type=LoadBalancer
// and Ingress are reconciled by the AWS Load Balancer Controller. When a
// cluster does not opt into built-in ingress (BuiltinIngress=false) the K3s
// built-ins are disabled outright for parity.
//
// When a cluster DOES want built-in ingress (dev / interim, until the AWS LB
// controller add-on lands) traefik stays ENABLED in config so k3s writes its
// manifest — but its helm-install Job (klipper-helm image pull + chart
// install) hammers the embedded etcd during the fragile bootstrap window,
// which can tip etcd fsync latency past the apiserver / leaderelection
// deadlines and crash the control plane. So traefik is deferred, NOT disabled:
// k3s.initd stages a deploy-controller .skip marker before the apiserver
// starts (EKS_DEFER_TRAEFIK gates it), and the state-reporter removes that
// marker once the apiserver is sustainedly healthy — traefik then installs
// with etcd headroom. Disabling traefik (`disable: traefik`) is wrong for this:
// k3s never writes traefik.yaml, leaving no manifest to un-skip. servicelb
// (klipper) is lazy — it acts only when a LoadBalancer Service exists — so it
// carries no bootstrap cost and stays enabled whenever built-in ingress is on.
// First server uses cluster-init (embedded etcd); join servers set `server: <first>` + token.
// etcd-expose-metrics: surfaces etcd fsync/commit latency on 127.0.0.1:2381/metrics.
// anonymous-auth=true: reconciler probes /healthz unauthenticated to gate ACTIVE; RBAC limits exposure.
// When BuiltinIngress=false, traefik+servicelb are disabled for AWS LB Controller parity.
// When true, traefik is deferred (not disabled) to avoid hammering etcd during bootstrap.
var configLines []string
if in.ServerURL == "" {
configLines = append(configLines, "cluster-init: true")
@@ -510,13 +405,7 @@ func buildK3sUserData(in K3sServerInput) string {
}
configLines = append(configLines,
"etcd-expose-metrics: true",
// Keep user workloads off the control plane (EKS parity — the CP is never
// a worker). Critical here because the server node is schedulable by
// default: a workload landing on the CP pulls its images onto the same
// NBD/viperblock disk the embedded etcd fsyncs to, starving etcd until the
// apiserver crashes and the VM OOMs. k3s' packaged addons tolerate
// CriticalAddonsOnly, so they still run (here, draining to the nodegroup);
// NoExecute also evicts anything already scheduled, not just future pods.
// Prevent user workloads on the CP (EKS parity). k3s packaged addons tolerate CriticalAddonsOnly.
"node-taint:",
" - CriticalAddonsOnly=true:NoExecute",
)
@@ -528,9 +417,7 @@ func buildK3sUserData(in K3sServerInput) string {
)
}
configLines = append(configLines, "tls-san:", " - "+in.NLBDNS)
// The reachable front-end IP must be a cert SAN so kubectl reaching the
// cluster on https://<EndpointIP>:443 validates TLS. k3s accepts IP literals
// in tls-san and classifies them as IP SANs.
// EndpointIP must be a cert SAN for TLS validation via https://<EndpointIP>:443.
if in.EndpointIP != "" {
configLines = append(configLines, " - "+in.EndpointIP)
}
@@ -543,30 +430,21 @@ func buildK3sUserData(in K3sServerInput) string {
" - anonymous-auth=true",
" - authentication-token-webhook-config-file="+k3sTokenWebhookKubeconfigPath,
" - authentication-token-webhook-cache-ttl=5m",
// Pin v1 so the apiserver decodes the webhook's authentication.k8s.io/v1
// TokenReview response; the default v1beta1 rejects the GVK mismatch (401).
// v1: default v1beta1 rejects authentication.k8s.io/v1 TokenReview response (401).
" - authentication-token-webhook-version=v1",
)
k3sConfig := strings.Join(configLines, "\n")
files := []userDataFile{
// first-boot.env carries the system SigV4 secret key, so keep it
// root-only (0600).
// 0600: contains system SigV4 secret key.
{Path: k3sFirstBootEnvPath, Perms: "0600", Body: envBody},
{Path: k3sOIDCSigningKeyPath, Perms: "0600", Body: strings.TrimRight(in.OIDCPrivateKeyPEM, "\n")},
{Path: k3sOIDCPublicKeyPath, Perms: "0644", Body: strings.TrimRight(in.OIDCPublicKeyPEM, "\n")},
{Path: k3sConfigPath, Perms: "0644", Body: k3sConfig},
{Path: k3sGatewayCAPath, Perms: "0644", Body: strings.TrimRight(in.GatewayCACert, "\n")},
// IMDS on-link route. Alpine's cloud-init eni renderer crashes on a
// gateway-less route in network-config, so it's delivered out-of-band:
// a persistent local.d script (re-applied every boot by the OpenRC
// `local` service, enabled via runcmd below) ARPs 169.254.169.254
// directly on the VPC subnet. This block owns the only write_files key,
// so it must carry the route itself — appending a second write_files via
// the generic cloud-init template would collide (last key wins, silently
// dropping the k3s config). The device is resolved via the default route
// (the VPC egress NIC), not a name: the persistent-net rename to vpc0
// doesn't apply on the Alpine AMI, so the kernel name is eth0 at boot.
// IMDS on-link route via persistent local.d script: Alpine cloud-init crashes on a
// gateway-less network-config route, so it's written here instead. Uses default-route
// device (eth0 on Alpine, not vpc0) to ARP 169.254.169.254 directly on the VPC subnet.
{
Path: "/etc/local.d/imds-onlink-route.start",
Perms: "0755",
@@ -579,14 +457,8 @@ func buildK3sUserData(in K3sServerInput) string {
var buf strings.Builder
buf.WriteString("#cloud-config\n")
// Resolver via bootcmd, NOT write_files: on the Alpine AMI /etc/resolv.conf
// is a dangling symlink (its target dir does not exist — which is why the
// dhcpcd hook cannot persist DHCP DNS), and pointing write_files at it makes
// cloud-init follow the dead link, fail, and abort the WHOLE write_files
// block (dropping first-boot.env + the k3s config). bootcmd runs before
// write_files in the init stage and as a shell, so it can drop the symlink
// and write a real file. Containerd needs this to resolve registry-1.docker.
// io for system-pod image pulls; reachable over the cluster egress SNAT.
// bootcmd (not write_files): /etc/resolv.conf is a dangling symlink on Alpine; write_files
// follows it, fails, and aborts the entire block. bootcmd drops the symlink first.
buf.WriteString("bootcmd:\n")
buf.WriteString(" - rm -f " + k3sResolvConfPath + "\n")
fmt.Fprintf(&buf, " - printf '%s\\n' > %s\n",
@@ -604,8 +476,7 @@ func buildK3sUserData(in K3sServerInput) string {
}
}
// Enable + start the OpenRC `local` service so the IMDS route script runs on
// first boot and is re-applied on every subsequent boot.
// Enable OpenRC `local` so the IMDS route script runs on every boot.
buf.WriteString("runcmd:\n")
buf.WriteString(" - [ rc-update, add, local, default ]\n")
buf.WriteString(" - [ rc-service, local, start ]\n")
spinifex/handlers/eks/nats_bootstrap.go
+21 -61
View File
@@ -14,19 +14,13 @@ import (
"github.com/nats-io/nats.go"
)
// errBootstrapPermanent marks a bootstrap persist failure that no amount of
// retrying will fix — a malformed envelope, a non-base64 CA, or a JWKS whose
// kid does not match the controller-generated keypair. Transient failures
// (KV write blips, JWKS-not-yet-propagated) are NOT wrapped with this, so
// persistWithRetry keeps retrying them. A permanent error fails the cluster.
// errBootstrapPermanent marks failures that retrying won't fix (malformed envelope,
// non-base64 CA, JWKS kid mismatch). Transient errors (KV blips) are NOT wrapped.
var errBootstrapPermanent = errors.New("bootstrap permanent error")
const (
// bootstrapPersistAttempts bounds inline retries of a single subject's
// persist op on transient errors. A core-NATS publication is delivered
// once, so a transient KV blip must be ridden out in-handler rather than
// by tearing down the subscription and waiting for a re-publish that
// never comes.
// bootstrapPersistAttempts bounds inline retries on transient errors.
// Core-NATS delivers each publication once, so KV blips must be retried in-handler.
bootstrapPersistAttempts = 3
bootstrapPersistBackoff = 200 * time.Millisecond
)
@@ -62,29 +56,21 @@ func BootstrapSubject(accountID, clusterName, kind string) string {
return fmt.Sprintf("eks.bus.%s.%s.%s", accountID, clusterName, kind)
}
// BootstrapEnvelope is the JSON wire shape every k3s-first-boot.sh
// publication uses. Each subject populates the field matching its kind; the
// rest stay empty. Versioned implicitly by field shape — adding new optional
// fields is backwards-compatible.
// BootstrapEnvelope is the JSON wire shape for k3s-first-boot.sh publications.
// Each subject populates its matching field; unused fields stay empty.
type BootstrapEnvelope struct {
// Token is the K3s bootstrap node-token for the cluster (k3s-bootstrap-token).
Token string `json:"token,omitempty"`
// AdminKubeconfig is the cluster-admin kubeconfig YAML
// (k3s-admin-kubeconfig).
// AdminKubeconfig is the cluster-admin kubeconfig YAML.
AdminKubeconfig string `json:"adminKubeconfig,omitempty"`
// JWKS is the K3s-published RFC 7517 JWKS document. The bootstrap
// subscriber cross-checks it against the controller-generated keypair
// (k3s-oidc-jwks).
// JWKS is the K3s-published JWKS document; cross-checked against the controller-generated keypair.
JWKS string `json:"jwks,omitempty"`
// CertificateAuthority is the base64-encoded K3s server CA PEM
// (k3s-ca). Written onto meta.CertificateAuthorityB64.
// CertificateAuthority is the base64-encoded K3s server CA PEM; written to meta.CertificateAuthorityB64.
CertificateAuthority string `json:"certificateAuthority,omitempty"`
}
// NATSBootstrap collects the four single-shot bootstrap publications from a
// K3s server VM, validates the OIDC JWKS against what the controller wrote
// pre-VM, persists each artifact into the per-cluster KV bucket, and writes
// the CA onto meta.CertificateAuthorityB64. The reconciler observes
// NATSBootstrap collects the four single-shot bootstrap publications from the K3s VM,
// validates the OIDC JWKS, and persists each artifact to KV. Reconciler observes
// completion via the four KV keys + meta.CertificateAuthorityB64.
type NATSBootstrap struct {
nc *nats.Conn
@@ -121,20 +107,10 @@ func NewNATSBootstrap(nc *nats.Conn, kv nats.KeyValue, masterKey []byte, account
}, nil
}
// BootstrapPendingKinds reports which bootstrap subjects still lack their
// persisted artifact for a cluster, so a daemon-restart resume re-subscribes
// only to what is missing. The K3s VM publishes each subject once over core
// NATS and never republishes, so re-waiting on an already-arrived subject
// would deadlock.
//
// JWKS is tracked by its verified-marker (OIDCJWKSVerifiedKey), written by
// persistJWKS only after the cross-check passes — NOT by OIDCJWKSKey, which the
// controller pre-seeds at create time. Until the marker exists the cross-check
// has not positively passed, so resume re-subscribes to JWKS like any other
// pending subject. (If the VM published JWKS during the restart window the
// re-subscribe waits for a republish that never comes and the cluster fails on
// the create timeout — the same one-shot limitation every bootstrap subject
// shares, now applied uniformly so ACTIVE always implies a verified key.)
// BootstrapPendingKinds reports which bootstrap artifacts are still missing so
// a daemon-restart resumes only on pending subjects (K3s publishes each once).
// JWKS is tracked via OIDCJWKSVerifiedKey (set after cross-check), not OIDCJWKSKey
// (pre-seeded by the controller), so ACTIVE always implies a verified keypair.
func BootstrapPendingKinds(kv nats.KeyValue, clusterName string, meta *ClusterMeta) []string {
var pending []string
if _, err := kv.Get(NodeTokenKey(clusterName)); err != nil {
@@ -192,18 +168,9 @@ func (b *NATSBootstrap) Run(ctx context.Context) error {
return b.RunForKinds(ctx, nil)
}
// RunForKinds subscribes only to the requested bootstrap subjects (nil = all
// four) and returns when:
// - every requested subject has arrived and persisted successfully (nil),
// - ctx is cancelled (ctx.Err()),
// - one of the persistence handlers returns an error (first error).
//
// The subset form drives daemon-restart recovery: a cluster that already has
// some artifacts in KV re-subscribes only to the missing ones, since the K3s
// VM published each subject once over core NATS and will not republish.
//
// Each subject is consumed at most once; subsequent publications on the same
// subject are dropped. Subscriptions are always cleaned up before return.
// RunForKinds subscribes to the requested bootstrap subjects (nil = all four) and
// returns when all arrive, ctx is cancelled, or a handler errors. Drives daemon-restart
// recovery by re-subscribing only to missing artifacts. Each subject is consumed at most once.
func (b *NATSBootstrap) RunForKinds(ctx context.Context, kinds []string) error {
subs, err := b.subjectHandlers(kinds)
if err != nil {
@@ -319,10 +286,7 @@ func (b *NATSBootstrap) persistJWKS(data []byte) error {
if err := assertJWKSMatch(existing.Value(), []byte(env.JWKS)); err != nil {
return err
}
// Record the positive cross-check as a distinct marker. The reconciler
// gates ACTIVE on this key — never on OIDCJWKSKey, which the controller
// pre-seeds at create time — so a cluster cannot reach ACTIVE on an
// unverified or mismatched signing key.
// Reconciler gates ACTIVE on this verified-marker, not OIDCJWKSKey (pre-seeded by controller).
if _, err := b.kv.Put(OIDCJWKSVerifiedKey(b.clusterName), []byte("verified")); err != nil {
return fmt.Errorf("kv put %s: %w", OIDCJWKSVerifiedKey(b.clusterName), err)
}
@@ -351,12 +315,8 @@ func unmarshalBootstrapEnvelope(data []byte) (BootstrapEnvelope, error) {
return env, nil
}
// assertJWKSMatch verifies that every key in the incoming JWKS matches a
// controller-generated key by both key id (kid) and key type (kty). A kid
// mismatch means the VM generated its own signing key instead of consuming our
// PEM; a kty mismatch (e.g. an RSA key carrying our EC key's kid) means the
// published key cannot be the one the controller distributed even though the
// kid collides. Either rejects the publication.
// assertJWKSMatch verifies every incoming JWKS key matches the controller-generated
// keypair by kid and kty; a mismatch rejects the publication.
func assertJWKSMatch(existing, incoming []byte) error {
var ex, in oidcJWKS
if err := json.Unmarshal(existing, &ex); err != nil {
spinifex/handlers/eks/nlb.go
+20 -66
View File
@@ -10,9 +10,7 @@ import (
"github.com/mulgadc/spinifex/spinifex/tags"
)
// nlbProvisioner is the subset of handlers_elbv2.ELBv2Service that the
// cluster NLB helpers need. Narrow so tests can fake with hand-rolled
// recorders without implementing the full ELBv2 surface.
// nlbProvisioner is the narrow ELBv2 surface needed by cluster NLB helpers.
type nlbProvisioner interface {
CreateLoadBalancer(input *elbv2.CreateLoadBalancerInput, accountID string) (*elbv2.CreateLoadBalancerOutput, error)
DescribeLoadBalancers(input *elbv2.DescribeLoadBalancersInput, accountID string) (*elbv2.DescribeLoadBalancersOutput, error)
@@ -31,38 +29,28 @@ type nlbProvisioner interface {
SetLoadBalancerIngressCIDRs(lbArn string, cidrs []string, accountID string) error
}
// AWS load-balancer + target-group names are bounded to 32 alphanumeric or
// hyphen characters. EnsureClusterNLB rejects clusters whose derived name
// would exceed this before issuing the create call so callers see a clear
// error instead of a generic ELBv2 InvalidParameterValue.
// maxELBv2NameLen is the AWS limit on LB/TG names (alphanumeric + hyphens).
const maxELBv2NameLen = 32
// k3sAPIServerPort is the in-VM K3s api-server bind port. The NLB target
// group sends traffic to this port on the K3s server VM's ENI.
// k3sAPIServerPort is the K3s apiserver port; NLB target group forwards here.
const k3sAPIServerPort int64 = 6443
// clusterNLBListenPort is the customer-facing port the cluster NLB exposes.
// kubectl + the AWS SDK both expect TLS on 443.
const clusterNLBListenPort int64 = 443
// ClusterNLB is the in-process tuple produced by EnsureClusterNLB. Each ARN
// is persisted into ClusterMeta so the reconciler + DeleteCluster can act on
// the resources without re-resolving by name.
// ClusterNLB is the NLB resource tuple produced by EnsureClusterNLB; ARNs are persisted to ClusterMeta.
type ClusterNLB struct {
LoadBalancerArn string
TargetGroupArn string
ListenerArn string
DNSName string
// FrontendIP is the NLB's reachable front-end address: the external-pool
// public IP for an internet-facing cluster NLB, or the VPC private IP for an
// internal one. Used as the kubeconfig endpoint host + apiserver cert SAN.
// Empty if the backing LB record has no address yet.
// FrontendIP is the public IP (internet-facing) or VPC IP (internal) of the NLB front-end.
// Used as the kubeconfig endpoint host and apiserver cert SAN; empty if not yet provisioned.
FrontendIP string
}
// ClusterNLBName returns the deterministic NLB name for a cluster. Stable so
// DeleteCluster can locate the NLB from clusterName alone after a daemon
// restart that lost the in-memory ARN.
// ClusterNLBName returns the deterministic NLB name for a cluster.
func ClusterNLBName(clusterName string) string {
return "eks-" + clusterName
}
@@ -73,24 +61,9 @@ func ClusterTargetGroupName(clusterName string) string {
return "eks-" + clusterName + "-cp"
}
// EnsureClusterNLB provisions (or returns) the cluster's NLB + target group +
// :443→:6443 TCP listener. Idempotent on clusterName: existing resources are
// reused so DeleteCluster failure-retry and reconciler crash-recovery both
// converge without duplicate creates.
//
// The K3s server VM ENI IP is NOT registered here — Stage 4
// (k3s_server_vm.go) calls RegisterClusterTarget once the VM has its ENI IP.
//
// internetFacing selects the NLB scheme: true ⇒ internet-facing (external-pool
// front-end IP, LAN-reachable, for a public cluster endpoint); false ⇒ internal
// (VPC-only). After ensuring the LB, the reachable FrontendIP is read back from
// the backing LB record so the caller can use it as the endpoint host + cert SAN.
//
// publicAccessCidrs narrows who may reach an internet-facing front-end below the
// scheme default (0.0.0.0/0). It maps the cluster's publicAccessCidrs onto the
// NLB managed-SG ingress; ignored for an internal NLB (its ingress already
// tracks the VPC CIDR) and for the wide-open default, which the LB carries out
// of the box.
// EnsureClusterNLB provisions (or reuses) the cluster NLB, target group, and
// :443→:6443 TCP listener. Idempotent; ENI registration is done separately by
// RegisterClusterTarget. publicAccessCidrs narrows ingress for internet-facing LBs.
func EnsureClusterNLB(nlbp nlbProvisioner, accountID, clusterName string, subnetIDs []string, internetFacing bool, publicAccessCidrs []string) (*ClusterNLB, error) {
if clusterName == "" {
return nil, errors.New("eks: EnsureClusterNLB empty cluster name")
@@ -118,17 +91,12 @@ func EnsureClusterNLB(nlbp nlbProvisioner, accountID, clusterName string, subnet
if err := ensureClusterListener(nlbp, accountID, lbName, out); err != nil {
return nil, err
}
// Narrow the internet-facing front-end to the cluster's publicAccessCidrs
// when they restrict below the wide-open scheme default the LB already
// carries. The override is idempotent, so re-running on retry converges.
if internetFacing && narrowsPublicAccess(publicAccessCidrs) {
if err := nlbp.SetLoadBalancerIngressCIDRs(out.LoadBalancerArn, publicAccessCidrs, accountID); err != nil {
return nil, fmt.Errorf("eks: set NLB ingress CIDRs for %s: %w", lbName, err)
}
}
// Read the front-end IP back from the LB record (DescribeLoadBalancers →
// populated LoadBalancerAddresses). Best-effort: an empty IP is not fatal
// here — the caller falls back to the DNS-name endpoint.
// Best-effort front-end IP read-back; caller falls back to DNS name if empty.
if lb, err := lookupLBByName(nlbp, accountID, lbName); err == nil && lb != nil {
out.FrontendIP = frontendIPFromLB(lb, internetFacing)
} else if err != nil {
@@ -137,10 +105,7 @@ func EnsureClusterNLB(nlbp nlbProvisioner, accountID, clusterName string, subnet
return out, nil
}
// narrowsPublicAccess reports whether publicAccessCidrs restrict the front-end
// below the wide-open default an internet-facing NLB already carries. Empty (no
// override) or the lone 0.0.0.0/0 default are no-ops and skip the extra ELBv2
// round-trip.
// narrowsPublicAccess reports whether publicAccessCidrs restrict below the 0.0.0.0/0 default.
func narrowsPublicAccess(cidrs []string) bool {
if len(cidrs) == 0 {
return false
@@ -151,9 +116,8 @@ func narrowsPublicAccess(cidrs []string) bool {
return true
}
// frontendIPFromLB extracts the reachable front-end address from a described
// load balancer: the public IpAddress for an internet-facing LB, else the VPC
// PrivateIPv4Address. Returns the first non-empty match across AZs.
// frontendIPFromLB returns the public IpAddress (internet-facing) or PrivateIPv4Address (internal)
// from the described LB; first non-empty match across AZs.
func frontendIPFromLB(lb *elbv2.LoadBalancer, internetFacing bool) string {
for _, az := range lb.AvailabilityZones {
for _, addr := range az.LoadBalancerAddresses {
@@ -174,8 +138,7 @@ func frontendIPFromLB(lb *elbv2.LoadBalancer, internetFacing bool) string {
return ""
}
// RegisterClusterTarget attaches one K3s server ENI IP to the cluster TG. Thin
// wrapper over RegisterClusterTargets for callers with a single IP.
// RegisterClusterTarget attaches one ENI IP to the cluster TG.
func RegisterClusterTarget(nlbp nlbProvisioner, accountID, tgArn, eniIP string) error {
if eniIP == "" {
return errors.New("eks: RegisterClusterTarget empty ENI IP")
@@ -183,12 +146,8 @@ func RegisterClusterTarget(nlbp nlbProvisioner, accountID, tgArn, eniIP string)
return RegisterClusterTargets(nlbp, accountID, tgArn, []string{eniIP})
}
// RegisterClusterTargets attaches every control-plane server ENI IP to the
// cluster TG so the NLB forwards :443 traffic to :6443 on each CP VM. With all
// N HA servers registered, the TG health check (:6443) drains a member when its
// node dies and kubectl/apiserver traffic fails over to the survivors. Empty
// IPs are skipped; the ELBv2 RegisterTargets impl dedupes on (id, port), so
// re-invocation (e.g. by the reconciler) is idempotent.
// RegisterClusterTargets attaches all CP ENI IPs to the cluster TG. Empty IPs are
// skipped; RegisterTargets deduplicates on (id, port) so re-invocation is idempotent.
func RegisterClusterTargets(nlbp nlbProvisioner, accountID, tgArn string, eniIPs []string) error {
if tgArn == "" {
return errors.New("eks: RegisterClusterTargets empty TG arn")
@@ -215,9 +174,7 @@ func RegisterClusterTargets(nlbp nlbProvisioner, accountID, tgArn string, eniIPs
return nil
}
// DeregisterClusterTarget removes the K3s server ENI IP from the cluster TG.
// Called by DeleteCluster before VM termination so health checks stop hitting
// a dying target.
// DeregisterClusterTarget removes an ENI IP from the cluster TG before VM termination.
func DeregisterClusterTarget(nlbp nlbProvisioner, accountID, tgArn, eniIP string) error {
if tgArn == "" {
return errors.New("eks: DeregisterClusterTarget empty TG arn")
@@ -237,11 +194,8 @@ func DeregisterClusterTarget(nlbp nlbProvisioner, accountID, tgArn, eniIP string
return nil
}
// DeleteClusterNLB tears down the cluster NLB + TG. DeleteLoadBalancer
// cascades listener cleanup inside the ELBv2 impl, so listeners do not need
// an explicit delete pass. Missing resources are no-ops to keep DeleteCluster
// retries safe. Both sub-sweeps always run; LB delete error takes precedence
// over TG delete error when both fail.
// DeleteClusterNLB tears down the cluster NLB and TG. DeleteLoadBalancer cascades
// listener cleanup. Missing resources are no-ops; LB error takes precedence if both fail.
func DeleteClusterNLB(nlbp nlbProvisioner, accountID, clusterName string) error {
if clusterName == "" {
return errors.New("eks: DeleteClusterNLB empty cluster name")
spinifex/handlers/eks/nodegroup.go
+6 -21
View File
@@ -226,12 +226,7 @@ func (s *EKSServiceImpl) createNodegroup(acctKV nats.KeyValue, input *eks.Create
ModifiedAt: now,
}
// Atomically claim the nodegroup record before any SG provisioning or worker
// launch. This is the idempotency barrier: a duplicate CreateNodegroup (SDK
// retry / gateway re-publish that spawned a second handler) loses the claim
// here and returns without doing any provisioning work, so concurrent creates
// cannot double-launch workers. The L169 read above stays only as a cheap
// fast-path reject.
// Atomically claim the record; concurrent/duplicate CreateNodegroup requests lose here.
owned, err := ClaimNodegroupRecord(acctKV, rec)
if err != nil {
return nil, err
@@ -240,13 +235,8 @@ func (s *EKSServiceImpl) createNodegroup(acctKV nats.KeyValue, input *eks.Create
return nil, errors.New(awserrors.ErrorEKSResourceInUse)
}
// Claim won — the nodegroup is now CREATING. SG provisioning, token decrypt,
// AMI lookup and worker launch are a multi-minute job that must not run in the
// gateway request path (it head-of-line blocks the shared responder and the
// gateway times out and re-publishes — 267.4). Snapshot the CREATING reply
// before the launch, which mutates rec (InstanceIDs/Status), then run the
// provisioning on a background goroutine. Failures surface via the record's
// CREATE_FAILED status, which DeleteNodegroup reclaims.
// Snapshot the CREATING reply before background launch (launch mutates rec).
// Failures surface as CREATE_FAILED, which DeleteNodegroup reclaims.
out := &eks.CreateNodegroupOutput{Nodegroup: nodegroupRecordToAWS(rec)}
s.launchWG.Go(func() {
@@ -325,9 +315,7 @@ func (s *EKSServiceImpl) launchNodegroupInfra(lc nodegroupLaunchCtx) {
return
}
// v1 marks ACTIVE on successful RunInstances (instance-level), not on
// k8s node-Ready — node-Ready gating needs a kubectl probe from the
// reconciler and lands in a follow-up.
// v1: ACTIVE on RunInstances success; node-Ready gating is a future improvement.
rec.Status = eks.NodegroupStatusActive
rec.ModifiedAt = time.Now().UTC()
if err := PutNodegroupRecord(acctKV, rec); err != nil {
@@ -469,9 +457,7 @@ func (s *EKSServiceImpl) updateNodegroupConfig(acctKV nats.KeyValue, input *eks.
}
if err := s.reconcileWorkerCount(acctKV, accountID, rec, meta); err != nil {
// reconcileWorkerCount may have launched some workers before failing; their
// IDs are already on rec. Persist so DeleteNodegroup can reclaim them rather
// than leaking the partial scale-up.
// Persist partial IDs so DeleteNodegroup can reclaim a failed scale-up.
rec.ModifiedAt = time.Now().UTC()
if perr := PutNodegroupRecord(acctKV, rec); perr != nil {
slog.Error("updateNodegroupConfig: persist after reconcile failure",
@@ -533,8 +519,7 @@ func (s *EKSServiceImpl) reconcileWorkerCount(acctKV nats.KeyValue, accountID st
}
func (s *EKSServiceImpl) updateNodegroupVersion(input *eks.UpdateNodegroupVersionInput) (*eks.UpdateNodegroupVersionOutput, error) {
// Worker AMI version upgrades (drain + replace) land in a follow-up; v1
// nodegroups are fixed at their create-time version.
// Worker AMI version upgrades (drain + replace) are not implemented in v1.
if input != nil {
slog.Info("UpdateNodegroupVersion not implemented in v1",
"cluster", aws.StringValue(input.ClusterName), "nodegroup", aws.StringValue(input.NodegroupName))
spinifex/handlers/eks/nodegroup_userdata.go
+4 -18
View File
@@ -17,21 +17,10 @@ type agentUserDataInput struct {
NodeName string
}
// buildAgentUserData renders the cloud-config YAML a nodegroup worker consumes
// at first boot. It mirrors buildK3sUserData: a bootcmd resolver fix, a single
// write_files block (first-boot.env + agent.env + the IMDS on-link route
// script), and a runcmd that enables the OpenRC `local` service. Output is
// unencoded YAML; the caller base64-wraps it for the RunInstances UserData
// field. The eks-node-role first-boot selector reads SPINIFEX_K3S_ROLE=agent
// (and the presence of agent.env) to enable the k3s-agent service, which sources
// agent.env for K3S_URL/K3S_TOKEN.
// buildAgentUserData renders the cloud-config YAML for a nodegroup worker VM.
// Mirrors buildK3sUserData structure; SPINIFEX_K3S_ROLE=agent enables k3s-agent.
func buildAgentUserData(in agentUserDataInput) string {
// K3S_URL targets the control-plane ENI's in-VPC private IP on :6443
// because the cluster NLB DNS endpoint is not resolvable in-VPC yet.
// Workers share the VPC with the server, so this is directly reachable and
// TLS-pinned by the join token's CA hash (SAN/hostname irrelevant for the
// k3s agent join). Switch to the NLB endpoint on :443 once authoritative
// in-VPC DNS is available.
// K3S_URL uses the CP ENI IP directly (NLB DNS not yet resolvable in-VPC).
k3sURL := "https://" + net.JoinHostPort(in.ControlPlaneENIIP, "6443")
nodeLabel := "eks.amazonaws.com/nodegroup=" + in.NodegroupName
@@ -61,10 +50,7 @@ func buildAgentUserData(in agentUserDataInput) string {
var buf strings.Builder
buf.WriteString("#cloud-config\n")
// Resolver fix via bootcmd (not write_files) for the same reason as the
// server path: /etc/resolv.conf is a dangling symlink on the Alpine AMI, so
// write_files would follow the dead link and abort the whole block.
// Containerd needs a working resolver to pull the worker's system images.
// bootcmd (not write_files): /etc/resolv.conf is a dangling symlink on Alpine; see buildK3sUserData.
buf.WriteString("bootcmd:\n")
buf.WriteString(" - rm -f " + k3sResolvConfPath + "\n")
fmt.Fprintf(&buf, " - printf '%s\\n' > %s\n",
spinifex/handlers/eks/oidc_keypair.go
+11 -27
View File
@@ -17,11 +17,8 @@ import (
"github.com/nats-io/nats.go"
)
// PublicKeyPEMFromPrivate parses a PKCS#8 private-key PEM and returns the PKIX
// public-key PEM. kube-apiserver's --service-account-key-file requires a PUBLIC
// key (ParsePublicKeysPEM rejects private-key blocks), while the matching
// --service-account-signing-key-file takes the private key; the K3s server VM
// needs both, seeded to separate files.
// PublicKeyPEMFromPrivate parses a PKCS#8 private-key PEM and returns the PKIX public-key PEM.
// kube-apiserver --service-account-key-file requires a public key; signing uses the private key.
func PublicKeyPEMFromPrivate(privPEM string) (string, error) {
block, _ := pem.Decode([]byte(privPEM))
if block == nil {
@@ -42,9 +39,7 @@ func PublicKeyPEMFromPrivate(privPEM string) (string, error) {
return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: pubDER})), nil
}
// oidcJWK and oidcJWKS are the RFC 7517 wire shapes the per-cluster JWKS
// document uses. handlers_sts decodes the same JSON via its own JWK / JWKS
// types — wire compatibility (not Go type identity) is the contract.
// oidcJWK and oidcJWKS are the RFC 7517 wire shapes for the per-cluster JWKS document.
type oidcJWK struct {
Kty string `json:"kty"`
Kid string `json:"kid"`
@@ -59,16 +54,11 @@ type oidcJWKS struct {
Keys []oidcJWK `json:"keys"`
}
// P-256 coordinate width per SEC1 — JWK requires fixed-length base64url, not
// the variable-length big.Int byte representation.
// p256CoordLen is the P-256 coordinate width; JWK requires fixed-length base64url.
const p256CoordLen = 32
// GenerateClusterOIDCKeypair creates a fresh ECDSA-P256 keypair for the
// cluster, PEM-marshals the private key as PKCS8, envelope-encrypts the PEM
// with masterKey, writes the ciphertext to OIDCSigningKeyKey and the JWKS
// document to OIDCJWKSKey. Returns the plaintext private-key PEM and the JWKS
// bytes so the caller (CreateCluster) can pass them straight to user-data
// without a follow-up KV read + decrypt.
// GenerateClusterOIDCKeypair creates an ECDSA-P256 keypair, encrypts the private key PEM
// into KV, and writes the JWKS document. Returns plaintext PEM + JWKS for immediate use.
func GenerateClusterOIDCKeypair(kv nats.KeyValue, clusterName string, masterKey []byte) (privPEM string, jwksBytes []byte, err error) {
if clusterName == "" {
return "", nil, errors.New("eks: GenerateClusterOIDCKeypair empty cluster name")
@@ -106,9 +96,8 @@ func GenerateClusterOIDCKeypair(kv nats.KeyValue, clusterName string, masterKey
return string(pemBlock), jwksBytes, nil
}
// LoadClusterOIDCPrivateKey reads the encrypted PEM from KV, decrypts with
// masterKey, parses it to *ecdsa.PrivateKey. Returns ErrClusterNotFound when
// the key is absent so DeleteCluster idempotency works.
// LoadClusterOIDCPrivateKey decrypts and parses the cluster OIDC private key from KV.
// Returns ErrClusterNotFound if absent (supports idempotent DeleteCluster).
func LoadClusterOIDCPrivateKey(kv nats.KeyValue, clusterName string, masterKey []byte) (*ecdsa.PrivateKey, error) {
if clusterName == "" {
return nil, errors.New("eks: LoadClusterOIDCPrivateKey empty cluster name")
@@ -145,14 +134,9 @@ func LoadClusterOIDCPrivateKey(kv nats.KeyValue, clusterName string, masterKey [
return priv, nil
}
// ZeroizeClusterOIDCKey overwrites the encrypted private-key blob with zeros
// of the same length, then purges the key. DeleteCluster calls this first so
// a crash after this point leaves no recoverable key material. Purge (not
// Delete) drops the key's entire revision history — a Delete leaves a tombstone
// over prior revisions, so the original encrypted PEM would survive in history
// until age/limit rolls it off. The account bucket is History=1 regardless
// (see store.go), so this is defense-in-depth against a widened history.
// Absent key is a no-op — supports idempotent DeleteCluster retries.
// ZeroizeClusterOIDCKey overwrites the encrypted key blob with zeros then purges it.
// Purge (not Delete) drops all revision history, leaving no recoverable key material.
// Absent key is a no-op for idempotent DeleteCluster retries.
func ZeroizeClusterOIDCKey(kv nats.KeyValue, clusterName string) error {
if clusterName == "" {
return errors.New("eks: ZeroizeClusterOIDCKey empty cluster name")
spinifex/handlers/eks/reconciler_registry.go
+7 -22
View File
@@ -9,15 +9,11 @@ import (
"github.com/nats-io/nats.go"
)
// ErrLeaseHeld signals that a reconciler was not spawned because another node
// holds the leader lease. Spawn treats it as benign: it drops the registry
// entry (rather than recording a phantom holder) so a later Spawn re-attempts
// once the holder's lease TTL lapses.
// ErrLeaseHeld signals the leader lease is held by another node.
// Spawn treats it as benign and retries on the next tick.
var ErrLeaseHeld = errors.New("eks: reconciler lease held by another node")
// ReconcilerRegistry tracks active per-cluster reconciler goroutines so the
// daemon can spawn, stop, and graceful-shutdown them in unison. One entry per
// (accountID, clusterName) key.
// ReconcilerRegistry tracks active per-cluster reconciler goroutines; one entry per (accountID, clusterName).
type ReconcilerRegistry struct {
mu sync.Mutex
holders map[string]*reconcilerHandle
@@ -34,15 +30,10 @@ func NewReconcilerRegistry() *ReconcilerRegistry {
return &ReconcilerRegistry{holders: map[string]*reconcilerHandle{}}
}
// SpawnReconcilerFn is the per-cluster goroutine starter the registry uses.
// Tests inject a fake; production wires a real ClusterReconciler under the
// hood (acquires lease, then loops via Run).
// SpawnReconcilerFn starts a per-cluster reconciler goroutine; injectable for tests.
type SpawnReconcilerFn func(ctx context.Context, accountID, clusterName string) (release func(), err error)
// Spawn launches one reconciler goroutine for (accountID, clusterName). If
// one is already running for that pair, returns nil without spawning a
// duplicate. SpawnFn is responsible for acquiring the leader lease + invoking
// Run; the registry only owns the ctx and the release.
// Spawn launches one reconciler goroutine for (accountID, clusterName), or no-ops if already running.
func (r *ReconcilerRegistry) Spawn(parent context.Context, accountID, clusterName string, spawnFn SpawnReconcilerFn) error {
if accountID == "" || clusterName == "" {
return errors.New("eks: ReconcilerRegistry.Spawn empty ids")
@@ -71,8 +62,6 @@ func (r *ReconcilerRegistry) Spawn(parent context.Context, accountID, clusterNam
cancel()
close(done)
if errors.Is(err, ErrLeaseHeld) {
// Lease held elsewhere — not a failure. The entry is already
// dropped above, so a later Spawn retries after the holder's TTL.
return nil
}
return err
@@ -146,12 +135,8 @@ func registryKey(accountID, clusterName string) string {
return accountID + "/" + clusterName
}
// RunClusterReconciler is the production SpawnReconcilerFn: it constructs a
// ClusterReconciler, acquires the leader lease, then drives Run in a
// goroutine. Returns (release, nil) on lease acquired, or (nil, error) on
// constructor failure. If another node holds the lease, returns
// (nil, ErrLeaseHeld) so the registry drops the entry and re-attempts on a
// later tick instead of recording a phantom holder.
// RunClusterReconciler is the production SpawnReconcilerFn: constructs a ClusterReconciler,
// acquires the lease, and drives Run in a goroutine. Returns ErrLeaseHeld if another node wins.
func RunClusterReconciler(
ctx context.Context,
leaderKV, acctKV nats.KeyValue,
spinifex/handlers/eks/security_groups.go
+14 -59
View File
@@ -11,9 +11,7 @@ import (
"github.com/mulgadc/spinifex/spinifex/tags"
)
// sgProvisioner is the subset of handlers_ec2_vpc.VPCService that the cluster
// SG helpers need. Narrow so tests can fake with hand-rolled recorders without
// implementing the full VPCService surface.
// sgProvisioner is the narrow VPC surface needed by cluster SG helpers.
type sgProvisioner interface {
CreateSecurityGroup(input *ec2.CreateSecurityGroupInput, accountID string) (*ec2.CreateSecurityGroupOutput, error)
DescribeSecurityGroups(input *ec2.DescribeSecurityGroupsInput, accountID string) (*ec2.DescribeSecurityGroupsOutput, error)
@@ -21,16 +19,13 @@ type sgProvisioner interface {
AuthorizeSecurityGroupIngress(input *ec2.AuthorizeSecurityGroupIngressInput, accountID string) (*ec2.AuthorizeSecurityGroupIngressOutput, error)
}
// clusterEKSClusterTagKey is stamped on every cluster-scoped resource so the
// DeleteCluster sweep + the UI hide-from-customer filter can group them.
// clusterEKSClusterTagKey groups all cluster-scoped resources for DeleteCluster sweeps.
const clusterEKSClusterTagKey = "spinifex:eks-cluster"
// clusterEKSRoleTagKey distinguishes the control-plane SG from the nodegroup
// SG when both live in the same VPC.
// clusterEKSRoleTagKey distinguishes the control-plane SG from the nodegroup SG.
const clusterEKSRoleTagKey = "spinifex:eks-role"
// clusterEKSNodegroupTagKey is stamped on worker instances so the nodegroup
// they belong to is recoverable from EC2 tags alone.
// clusterEKSNodegroupTagKey is stamped on worker instances to identify their nodegroup.
const clusterEKSNodegroupTagKey = "spinifex:eks-nodegroup"
const (
@@ -38,9 +33,7 @@ const (
clusterEKSRoleNodegroup = "nodegroup"
)
// ClusterControlPlaneSGName returns the deterministic SG name used for the
// cluster's control-plane SG. Stable so Stage-6 DeleteCluster can locate it
// from clusterName alone.
// ClusterControlPlaneSGName returns the deterministic control-plane SG name for a cluster.
func ClusterControlPlaneSGName(clusterName string) string {
return fmt.Sprintf("eks-cluster-%s-control-plane-sg", clusterName)
}
@@ -51,12 +44,8 @@ func ClusterNodegroupSGName(clusterName string) string {
return fmt.Sprintf("eks-cluster-%s-nodegroup-sg", clusterName)
}
// EnsureClusterSGs creates (or returns existing) control-plane + nodegroup
// security groups in the customer VPC. Both are tagged
// spinifex:managed-by=eks + spinifex:eks-cluster=<name>. Idempotent: an SG
// whose group-name + vpc-id match the expected pair is reused without a
// second create call. The two creates are independent — a failure on the
// nodegroup SG does not unwind the control-plane SG (DeleteCluster cleans up).
// EnsureClusterSGs creates or reuses the control-plane and nodegroup SGs in the customer VPC.
// Idempotent; the two creates are independent (DeleteCluster handles partial teardown).
func EnsureClusterSGs(sgp sgProvisioner, accountID, clusterName, vpcID string) (controlPlaneSGID, nodegroupSGID string, err error) {
if clusterName == "" {
return "", "", errors.New("eks: EnsureClusterSGs empty cluster name")
@@ -84,10 +73,7 @@ func EnsureClusterSGs(sgp sgProvisioner, accountID, clusterName, vpcID string) (
return controlPlaneSGID, nodegroupSGID, nil
}
// DeleteClusterSGs removes both cluster SGs in the customer VPC. Missing SGs
// are no-ops so DeleteCluster can retry safely. Delete errors are logged and
// the sweep continues; the first error is returned so the caller surfaces it
// without halting the broader teardown.
// DeleteClusterSGs removes both cluster SGs. Missing SGs are no-ops; first error is returned.
func DeleteClusterSGs(sgp sgProvisioner, accountID, clusterName, vpcID string) error {
if clusterName == "" {
return errors.New("eks: DeleteClusterSGs empty cluster name")
@@ -148,18 +134,8 @@ func ensureClusterSG(sgp sgProvisioner, accountID, vpcID, clusterName, name, des
return *out.GroupId, nil
}
// EnsureNodegroupSGRules authorizes the ingress rules a nodegroup's workers
// need to join the K3s control plane and form the flannel overlay. All rules
// are group-to-group (UserIdGroupPairs) so they stay correct as instances come
// and go. Idempotent: the InvalidPermission.Duplicate error a re-run triggers
// for an already-authorized rule is treated as success.
//
// Rules:
// - agent → server apiserver/supervisor: 6443/tcp (ngSG → cpSG)
// - flannel VXLAN: 8472/udp both directions between cpSG and ngSG, and
// ngSG ↔ ngSG
// - kubelet: 10250/tcp (cpSG → ngSG)
// - intra-nodegroup all traffic: ngSG self-reference
// EnsureNodegroupSGRules authorizes ingress for flannel VXLAN (8472/udp), apiserver
// (6443/tcp), and kubelet (10250/tcp) between CP and nodegroup SGs. Idempotent.
func EnsureNodegroupSGRules(sgp sgProvisioner, accountID, clusterName, cpSGID, ngSGID string) error {
if cpSGID == "" || ngSGID == "" {
return errors.New("eks: EnsureNodegroupSGRules empty SG id")
@@ -208,13 +184,8 @@ func EnsureNodegroupSGRules(sgp sgProvisioner, accountID, clusterName, cpSGID, n
return nil
}
// EnsureControlPlaneIngress authorizes the ingress the cluster NLB needs to
// reach the K3s apiserver. The NLB's backing LB VM forwards the front-end
// listener to the apiserver on k3sAPIServerPort from inside the VPC, so the
// control-plane SG must admit that port from the VPC CIDR. Public exposure is
// gated separately at the NLB front-end by publicAccessCidrs; this rule only
// governs the in-VPC LB→apiserver hop. Idempotent: the
// InvalidPermission.Duplicate error a re-run triggers is treated as success.
// EnsureControlPlaneIngress admits the NLB→apiserver hop from the VPC CIDR on k3sAPIServerPort.
// Public access is gated separately at the NLB front-end. Idempotent.
func EnsureControlPlaneIngress(sgp sgProvisioner, accountID, cpSGID, vpcCIDR string) error {
if cpSGID == "" {
return errors.New("eks: EnsureControlPlaneIngress empty control-plane SG id")
@@ -242,24 +213,8 @@ func EnsureControlPlaneIngress(sgp sgProvisioner, accountID, cpSGID, vpcCIDR str
return nil
}
// EnsureControlPlaneHAIngress authorizes the server-to-server ingress an HA
// control plane needs so a join server's embedded etcd can peer with the rest
// of the quorum. The control-plane SG is self-referencing here: every rule's
// source and target are the same group, so the rules stay correct as CP VMs are
// launched, lost, and rebuilt. A single cluster-init server never exercises
// these ports, which is why they were absent until multi-server HA: the join
// server registers over the supervisor port (already admitted) but its etcd
// member can never replicate the raft log without the peer port, so the quorum
// silently never forms and the node never reports Ready.
//
// Rules (cpSG → cpSG):
// - etcd client+peer: 2379-2380/tcp — embedded-etcd quorum replication
// - kubelet: 10250/tcp — server-to-server kubelet (metrics, logs, exec)
//
// The supervisor/apiserver port (6443) is already admitted from the VPC CIDR by
// EnsureControlPlaneIngress, which covers the CP peers, so it is not repeated.
// Idempotent: the InvalidPermission.Duplicate error a re-run triggers for an
// already-authorized rule is treated as success.
// EnsureControlPlaneHAIngress authorizes self-referencing CP SG rules for HA etcd peering:
// etcd client+peer (2379-2380/tcp) and kubelet (10250/tcp) within cpSG. Idempotent.
func EnsureControlPlaneHAIngress(sgp sgProvisioner, accountID, cpSGID string) error {
if cpSGID == "" {
return errors.New("eks: EnsureControlPlaneHAIngress empty control-plane SG id")
spinifex/handlers/eks/service.go
+4 -13
View File
@@ -1,19 +1,11 @@
// Package handlers_eks provides the EKS (Amazon Elastic Kubernetes Service)
// control-plane API surface for Spinifex. The interface declared here is the
// stable contract every gateway and daemon caller binds to; the NATS wrapper
// and concrete service implementation both satisfy it.
// Package handlers_eks implements the EKS control-plane API surface for Spinifex.
package handlers_eks
import "github.com/aws/aws-sdk-go/service/eks"
// EKSService is the AWS EKS control-plane contract exposed to spinifex
// gateway and daemon callers. Every method maps 1:1 to an AWS EKS API action
// (PascalCase). Signatures use the aws-sdk-go input/output types verbatim so
// downstream callers can pass straight through from generated SDK code.
// EKSService is the EKS control-plane contract; one method per AWS EKS API action.
type EKSService interface {
// Cluster. CreateCluster also takes the caller's resolved IAM principal ARN
// (from the X-Principal-ARN header) so it can mint the bootstrap
// cluster-creator-admin AccessEntry; "" skips that step.
// Cluster. callerPrincipalARN is the IAM principal for the bootstrap AccessEntry; "" skips it.
CreateCluster(input *eks.CreateClusterInput, accountID, callerPrincipalARN string) (*eks.CreateClusterOutput, error)
DescribeCluster(input *eks.DescribeClusterInput, accountID string) (*eks.DescribeClusterOutput, error)
ListClusters(input *eks.ListClustersInput, accountID string) (*eks.ListClustersOutput, error)
@@ -48,8 +40,7 @@ type EKSService interface {
DescribeAddon(input *eks.DescribeAddonInput, accountID string) (*eks.DescribeAddonOutput, error)
UpdateAddon(input *eks.UpdateAddonInput, accountID string) (*eks.UpdateAddonOutput, error)
// OIDC identity-provider configs (control-plane API only; the anonymous
// OIDC issuer routes live with the awsgw IRSA wiring).
// OIDC identity-provider configs.
AssociateIdentityProviderConfig(input *eks.AssociateIdentityProviderConfigInput, accountID string) (*eks.AssociateIdentityProviderConfigOutput, error)
DescribeIdentityProviderConfig(input *eks.DescribeIdentityProviderConfigInput, accountID string) (*eks.DescribeIdentityProviderConfigOutput, error)
ListIdentityProviderConfigs(input *eks.ListIdentityProviderConfigsInput, accountID string) (*eks.ListIdentityProviderConfigsOutput, error)
spinifex/handlers/eks/service_impl.go
+38 -161
View File
@@ -23,19 +23,13 @@ import (
"github.com/nats-io/nats.go"
)
// SubnetVPCResolver resolves a subnet ID to its parent VPC ID and the VPC's
// CIDR block. Narrow so EKSServiceImpl can stay free of the wider
// handlers/ec2/vpc surface; the daemon adapts VPCServiceImpl onto this
// contract.
// SubnetVPCResolver resolves a subnet to its VPC ID and CIDR.
type SubnetVPCResolver interface {
GetSubnetVPC(accountID, subnetID string) (vpcID string, err error)
GetVPCCIDR(accountID, vpcID string) (cidr string, err error)
}
// EKSServiceDeps wires every external collaborator EKSServiceImpl needs.
// The narrow per-area interfaces (sgProvisioner, nlbProvisioner, etc.) are
// already defined alongside their consumers; the daemon adapts each
// service's concrete type onto the interface.
// EKSServiceDeps wires the external collaborators EKSServiceImpl needs.
type EKSServiceDeps struct {
Config *config.Config
NATSConn *nats.Conn
@@ -44,13 +38,8 @@ type EKSServiceDeps struct {
Region string
HolderID string
// Gateway broker config handed to the K3s server VM so it can publish its
// bootstrap envelopes + state reports via SigV4-signed HTTPS POST to the AWS
// gateway (the ELBv2 lb-agent model) instead of dialing core NATS.
// SystemGatewayURL is the mgmt-reachable AWSGW endpoint (distinct from
// GatewayBaseURL, which is the OIDC issuer host); SystemAccessKey/SecretKey
// are the system (Predastore) SigV4 creds; GatewayCACert is the PEM that
// signs the gateway server cert.
// Gateway broker config for the K3s server VM (SigV4 HTTPS to AWSGW).
// SystemGatewayURL is the mgmt-reachable endpoint; GatewayCACert signs its TLS cert.
SystemGatewayURL string
SystemAccessKey string
SystemSecretKey string
@@ -65,42 +54,28 @@ type EKSServiceDeps struct {
EIP eipProvisioner
Worker WorkerLauncher
// PlacementGroup reserves distinct hosts for HA control-plane spread;
// Scheduler answers the capacity + placement fan-out the spread path needs.
// The daemon wires the NATS implementations.
// PlacementGroup and Scheduler support HA control-plane spread (NATS-backed).
PlacementGroup controlPlanePlacer
Scheduler HostScheduler
// AddonInstaller delivers managed-addon manifests to clusters. Optional:
// when nil the service defaults to the KV staging installer
// (newStagingInstaller) bound to NATSConn.
// AddonInstaller delivers managed-addon manifests; nil defaults to the KV staging installer.
AddonInstaller AddonInstaller
}
// WorkerLauncher is the narrow EC2 surface CreateNodegroup needs to launch and
// terminate nodegroup worker instances. Workers go through the normal
// customer-owned RunInstances path (no ManagedBy tag, no mgmt NIC) so they are
// visible in DescribeInstances and reclaimed by TerminateInstances like any
// other customer EC2. The daemon adapts its concrete instance service onto this.
// Exported so the daemon can assert *Daemon satisfies it with a compile-time
// check.
// WorkerLauncher is the narrow EC2 surface for launching nodegroup worker instances.
// Workers use the customer-owned RunInstances path (no ManagedBy tag, no mgmt NIC).
type WorkerLauncher interface {
RunWorkerInstance(input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error)
TerminateWorkerInstances(instanceIDs []string, accountID string) error
}
// eipProvisioner is the narrow EIP surface CreateCluster needs to give the
// hidden K3s control-plane VM an egress-only public IP (the distributed-NAT
// dev/edge topology has no shared masquerade address). The daemon adapts the
// concrete EIP service onto this.
// eipProvisioner is the narrow EIP surface for allocating a CP VM egress IP.
type eipProvisioner interface {
AllocateAddress(input *ec2.AllocateAddressInput, accountID string) (*ec2.AllocateAddressOutput, error)
ReleaseAddress(input *ec2.ReleaseAddressInput, accountID string) (*ec2.ReleaseAddressOutput, error)
}
// EKSServiceImpl is the daemon-side EKSService. CreateCluster /
// DescribeCluster / ListClusters / DeleteCluster have real bodies; every
// other action is still NotImplemented pending follow-up sprints.
// EKSServiceImpl is the daemon-side EKSService implementation.
type EKSServiceImpl struct {
deps EKSServiceDeps
leaderKV nats.KeyValue
@@ -110,27 +85,18 @@ type EKSServiceImpl struct {
bgCtx context.Context
bgCancel context.CancelFunc
// launchWG tracks in-flight asynchronous create launches (control-plane and
// nodegroup VM provisioning kicked off after the fast CREATING response).
// Not part of request handling — exists so tests can deterministically await
// launch completion (WaitLaunches).
// launchWG tracks in-flight async create launches for test determinism (WaitLaunches).
launchWG sync.WaitGroup
}
var _ EKSService = (*EKSServiceImpl)(nil)
// WaitLaunches blocks until all asynchronous create launches started by this
// service have finished. Intended for tests; production create handlers return
// as soon as the CREATING record is claimed.
// WaitLaunches blocks until all async create launches finish. Intended for tests.
func (s *EKSServiceImpl) WaitLaunches() { s.launchWG.Wait() }
const defaultK8sVersion = "1.32"
// NewEKSServiceImpl wires the Store, leader bucket, reconciler registry, and
// per-cluster background-goroutine context. Validates that MasterKey,
// GatewayBaseURL, Region, and HolderID are set when called from the live
// daemon path; the legacy NewEKSServiceImplWithNATS shim leaves them empty
// and bodies that need them return ServerInternal.
// NewEKSServiceImpl initialises EKSServiceImpl, wiring the leader KV and reconciler registry.
func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) {
if deps.NATSConn == nil {
return nil, errors.New("eks: NewEKSServiceImpl nil NATSConn")
@@ -153,10 +119,7 @@ func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) {
}, nil
}
// NewEKSServiceImplWithNATS is a back-compat shim. Bodies that need
// orchestration deps return ServerInternal when invoked through this shim;
// the daemon-handler routing test calls it to verify NATS subject wiring
// without standing up the full dependency graph.
// NewEKSServiceImplWithNATS is a back-compat shim for tests that only need NATS wiring.
func NewEKSServiceImplWithNATS(cfg *config.Config, nc *nats.Conn) (*EKSServiceImpl, error) {
return NewEKSServiceImpl(EKSServiceDeps{Config: cfg, NATSConn: nc})
}
@@ -173,10 +136,7 @@ func (s *EKSServiceImpl) Shutdown() {
s.registry.StopAll()
}
// SpawnRegisteredReconcilers enumerates every cluster in every per-account
// bucket and registers a reconciler goroutine for each cluster in
// CREATING or ACTIVE state. Called by the daemon on boot so a node restart
// resumes lifecycle reconcile without waiting for the next CreateCluster.
// SpawnRegisteredReconcilers resumes reconciler goroutines for all CREATING/ACTIVE clusters on daemon boot.
func (s *EKSServiceImpl) SpawnRegisteredReconcilers() error {
if !s.depsReadyForOrchestration() {
slog.Debug("SpawnRegisteredReconcilers: deps not ready, skipping")
@@ -210,10 +170,7 @@ func (s *EKSServiceImpl) SpawnRegisteredReconcilers() error {
if meta.Status != ClusterStatusCreating && meta.Status != ClusterStatusActive {
continue
}
// A restart mid-CREATING lost the bootstrap subscriptions; the K3s
// VM publishes its one-shot core-NATS messages once and to nobody.
// Re-subscribe to whichever artifacts are still missing so the
// cluster can still reach ACTIVE instead of hanging in CREATING.
// Re-subscribe to missing artifacts; K3s publishes each one-shot message once.
if meta.Status == ClusterStatusCreating {
if pending := BootstrapPendingKinds(acctKV, cluster, meta); len(pending) > 0 {
slog.Info("SpawnRegisteredReconcilers: resuming bootstrap",
@@ -231,9 +188,7 @@ func (s *EKSServiceImpl) depsReadyForOrchestration() bool {
return len(s.missingOrchestrationDeps()) == 0
}
// requireOrchestrationDeps returns a client-facing ServiceUnavailable (with the
// specific unmet deps logged at ERROR) when the daemon is missing orchestration
// wiring, so the failure is diagnosable instead of a bare ServerInternal.
// requireOrchestrationDeps returns ServiceUnavailable with missing deps logged when orchestration is not wired.
func (s *EKSServiceImpl) requireOrchestrationDeps(op string) error {
missing := s.missingOrchestrationDeps()
if len(missing) == 0 {
@@ -243,9 +198,7 @@ func (s *EKSServiceImpl) requireOrchestrationDeps(op string) error {
return errors.New(awserrors.ErrorServiceUnavailable)
}
// missingOrchestrationDeps names the deps required for cluster orchestration
// that are unset, so a "deps not ready" rejection can log the precise cause
// instead of an opaque ServerInternal whose only hint is a one-time boot WARN.
// missingOrchestrationDeps returns names of unset orchestration deps.
func (s *EKSServiceImpl) missingOrchestrationDeps() []string {
var missing []string
if s.deps.VPCSG == nil {
@@ -295,22 +248,13 @@ func (s *EKSServiceImpl) missingOrchestrationDeps() []string {
// --- Cluster ---
// logCreateErr records the precise cause of a CreateCluster step failure at
// ERROR. The request surface collapses these wrapped errors to an opaque
// ServerInternal ("An internal error has occurred... contact AWS re:Post"), so
// without this the wrapped error never reaches the operator. Returns the error
// wrapped with the stage for the caller to surface.
// logCreateErr logs a CreateCluster step failure at ERROR and returns it wrapped with the stage name.
func logCreateErr(name, accountID, stage string, err error) error {
slog.Error("CreateCluster: "+stage+" failed", "cluster", name, "accountID", accountID, "err", err)
return fmt.Errorf("%s: %w", stage, err)
}
// managedIngressTagKey is the CreateCluster tag that controls K3s' bundled
// traefik + servicelb (interim in-VPC app exposure). They are ON by default so
// workloads are reachable before the AWS Load Balancer Controller add-on ships;
// value "false" (case-insensitive) opts out for AWS parity. TODO: flip the
// default back off (parity) once the aws-lb-controller add-on lands — see
// docs/development/feature/eks-dataplane-ingress.md.
// managedIngressTagKey controls K3s built-in traefik+servicelb; "false" opts out for AWS parity.
const managedIngressTagKey = "spinifex.io/managed-ingress"
// builtinIngressEnabled reports whether a cluster keeps the K3s built-in ingress
@@ -340,9 +284,7 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
vpcID, err := s.deps.VPCSubnet.GetSubnetVPC(accountID, subnetIDs[0])
if err != nil {
// Subnet is a caller-supplied parameter — a resolve failure is a client
// fault (bad/foreign subnet), not an internal one. Surface it as such
// instead of collapsing to a retryable ServerInternal the SDK retries.
// Subnet resolve failure is a client fault (bad/foreign subnet).
slog.Error("CreateCluster: resolve subnet VPC failed",
"cluster", name, "accountID", accountID, "subnet", subnetIDs[0], "err", err)
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
@@ -370,26 +312,13 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID,
BuiltinIngress: builtinIngressEnabled(input.Tags),
CreatedAt: time.Now().UTC(),
}
// Atomically claim the cluster name before any launching/fallible work. A
// duplicate CreateCluster (the SDK's own retry, or a gateway re-publish that
// spawns a second concurrent handler) loses the claim and is rejected — or,
// for a prior FAILED attempt, reclaims it — so exactly one handler ever
// launches control-plane VMs for a given name.
// Claim the cluster name before any launching; duplicate/retry handlers lose the claim.
if err := s.claimClusterName(accountID, acctKV, meta); err != nil {
return nil, err
}
// Respond with the cluster in CREATING immediately and run the launch
// (SGs, NLB, OIDC, control-plane VM boot, egress, target registration,
// bootstrap, reconciler) off the request goroutine. A multi-minute control-
// plane boot must not occupy the spinifex-workers subscription or the gateway
// times out and re-publishes the request, spawning a duplicate concurrent
// handler (267.4 — the same retry the 267.3 claim guards against). AWS itself
// returns the cluster in CREATING here; the client polls DescribeCluster to
// ACTIVE/FAILED.
// Snapshot the CREATING response before spawning the launch: the goroutine
// owns meta from here on (it mutates and re-persists it), so the request
// goroutine must not touch meta again.
// Respond CREATING immediately; run the slow launch on a background goroutine.
// The goroutine owns meta from here on — do not touch it after this point.
out := &eks.CreateClusterOutput{Cluster: clusterMetaToAWS(meta)}
s.launchWG.Go(func() {
defer func() {
@@ -430,11 +359,7 @@ type clusterLaunchCtx struct {
acctKV nats.KeyValue
}
// failClusterLaunch records an asynchronous control-plane launch failure: it
// marks the cluster FAILED (with reason, so DescribeCluster surfaces the cause
// and the 267.3 reclaim path can recreate cleanly) and logs the stage. The
// launch runs after the CREATING response was already sent, so a failure can
// only surface as status, never as the CreateCluster return value.
// failClusterLaunch logs a launch failure and marks the cluster FAILED so DescribeCluster surfaces the cause.
func (s *EKSServiceImpl) failClusterLaunch(kv nats.KeyValue, name, accountID, stage string, err error) {
_ = logCreateErr(name, accountID, stage, err)
if mErr := MarkClusterFailed(kv, name, stage+": "+err.Error()); mErr != nil && !errors.Is(mErr, ErrClusterNotFound) {
@@ -442,11 +367,8 @@ func (s *EKSServiceImpl) failClusterLaunch(kv nats.KeyValue, name, accountID, st
}
}
// launchClusterInfra is CreateCluster's slow phase, run on a background
// goroutine after the CREATING response is sent: security groups, the cluster
// NLB, OIDC keypair, the control-plane VM(s), egress wiring, NLB target
// registration, the creator-admin AccessEntry, and the bootstrap/reconciler
// goroutines. Every failure path marks the cluster FAILED via failClusterLaunch.
// launchClusterInfra is CreateCluster's slow phase (SGs, NLB, OIDC, CP VMs, egress, bootstrap).
// Runs on a background goroutine; every failure marks the cluster FAILED.
func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) {
accountID := lc.accountID
callerPrincipalARN := lc.callerPrincipalARN
@@ -467,9 +389,6 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) {
}
meta.ResourcesVpcConfig.SecurityGroupIds = []string{cpSG, ngSG}
// The NLB's backing LB VM forwards the published endpoint to the apiserver
// from inside the VPC, so the control-plane SG must admit that hop or the
// NLB target stays unhealthy and the endpoint never serves.
vpcCIDR, err := s.deps.VPCSubnet.GetVPCCIDR(accountID, vpcID)
if err != nil {
s.failClusterLaunch(acctKV, name, accountID, "resolve vpc cidr", err)
@@ -479,24 +398,17 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) {
s.failClusterLaunch(acctKV, name, accountID, "ensure control-plane ingress", err)
return
}
// HA control planes run servers 2..N as join servers whose embedded etcd must
// peer with the quorum; without these self-referencing CP-SG rules a join
// registers but its etcd never replicates, so the node never reports Ready.
if err := EnsureControlPlaneHAIngress(s.deps.VPCSG, accountID, cpSG); err != nil {
s.failClusterLaunch(acctKV, name, accountID, "ensure control-plane HA ingress", err)
return
}
// Public access ⇒ internet-facing NLB (external-pool front-end IP, reachable
// on the LAN/edge network); private-only ⇒ internal NLB (VPC-only).
nlb, err := EnsureClusterNLB(s.deps.NLB, accountID, name, subnetIDs, publicAccess, publicCidrs)
if err != nil {
s.failClusterLaunch(acctKV, name, accountID, "ensure cluster NLB", err)
return
}
// Prefer the reachable front-end IP as the kubeconfig endpoint host so the
// apiserver serving cert (which SANs this IP) validates with TLS verification
// on. Fall back to the synthetic NLB DNS name when no IP was read back.
// Prefer the NLB front-end IP for TLS SAN validation; fall back to DNS name.
endpointHost := nlb.FrontendIP
if endpointHost == "" {
endpointHost = nlb.DNSName
@@ -506,11 +418,7 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) {
meta.NLBArn = nlb.LoadBalancerArn
meta.NLBTargetGroupArn = nlb.TargetGroupArn
// Persist the NLB ARNs now, before any further fallible step. Without this
// a failure between here and the final PutClusterMeta would leave the NLB +
// target group + listener + backing LB VM created but unrecorded, so
// DeleteCluster (which keys teardown off the persisted ARNs) could not
// reclaim them.
// Persist NLB ARNs early so DeleteCluster can reclaim them on any later failure.
if err := PutClusterMeta(acctKV, meta); err != nil {
s.failClusterLaunch(acctKV, name, accountID, "persist NLB arns", err)
return
@@ -534,9 +442,6 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) {
return
}
// Shared cluster token seeded into every control-plane server so the HA
// spread's servers 2..N join the first server's etcd quorum (and workers
// register). Single-CP clusters carry it too — harmless, one code path.
joinToken, err := GenerateK3sClusterToken()
if err != nil {
s.failClusterLaunch(acctKV, name, accountID, "generate k3s cluster token", err)
@@ -569,9 +474,7 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) {
s.failClusterLaunch(acctKV, name, accountID, "launch K3s VM", err)
return
}
// Mirror the primary ([0]) into the scalar fields the reconciler, egress
// wiring, and teardown read today. All CP nodes are NLB target-registered
// (failover); egress is still primary-only (separate hidden-pool /32 wire).
// Mirror primary CP ([0]) into scalar fields; all nodes are NLB-registered for failover.
primary := cpNodes[0]
meta.ControlPlaneNodes = cpNodes
meta.ControlPlaneSpreadGroup = spreadGroup
@@ -580,19 +483,12 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) {
meta.ControlPlaneENIIP = primary.ENIIP
meta.ControlPlaneMgmtIP = primary.MgmtIP
// Persist the CP VM + ENI + spread-group refs now, before any further fallible
// step. The VMs are live the moment placeControlPlane returns; without this a
// failure between here and the next PutClusterMeta (e.g. egress allocation)
// leaves them launched but unrecorded, so neither DeleteCluster nor the
// FAILED-cluster reclaim could reach them and the sys.medium VMs leak.
// Persist CP IDs immediately; VMs are live now and must not be orphaned on later failure.
if err := PutClusterMeta(acctKV, meta); err != nil {
s.failClusterLaunch(acctKV, name, accountID, "persist control-plane ids", err)
return
}
// Egress: the control-plane VM must pull container images. Allocate a
// hidden pool address and wire an egress-only SNAT for its /32 (no DNAT —
// the VM stays unreachable inbound; the NLB is its only front door).
egressIP, egressAllocID, err := s.allocateClusterEgressIP(accountID, name)
if err != nil {
s.failClusterLaunch(acctKV, name, accountID, "allocate cluster egress IP", err)
@@ -607,8 +503,7 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) {
ExternalIp: egressIP,
})
// Record the egress allocation before registering the target so a failed
// register (or anything after) leaves it recoverable by DeleteCluster.
// Persist egress allocation so DeleteCluster can reclaim it on later failure.
if err := PutClusterMeta(acctKV, meta); err != nil {
s.failClusterLaunch(acctKV, name, accountID, "persist egress allocation", err)
return
@@ -628,10 +523,7 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) {
return
}
// bootstrapClusterCreatorAdminPermissions (default true): grant the caller
// system:masters via an AccessEntry so the cluster creator can immediately
// authenticate through the token webhook (Q9). Keyed by the caller's exact
// principal ARN — the webhook looks the entry up by the same ARN STS returns.
// Seed creator system:masters AccessEntry so the token webhook can authenticate the creator immediately.
if bootstrapCreatorAdmin(input) && callerPrincipalARN != "" {
rec := newAccessEntryRecord(region, accountID, name, callerPrincipalARN, "",
[]string{"system:masters"}, AccessEntryTypeStandard, nil, time.Now().UTC())
@@ -789,17 +681,8 @@ func (s *EKSServiceImpl) DeleteCluster(input *eks.DeleteClusterInput, accountID
return &eks.DeleteClusterOutput{Cluster: clusterMetaToAWS(meta)}, nil
}
// claimClusterName atomically takes ownership of the cluster's meta key before
// CreateCluster does any launching work, closing the duplicate-handler race (an
// SDK retry or a gateway re-publish spawning a second concurrent CreateCluster).
// The fresh CREATING meta is laid down with kv.Create: the winner owns it. A
// loser finds the key already present and either rejects (a live cluster) or,
// for a prior FAILED attempt, reclaims it — flipping FAILED→CREATING via CAS so
// exactly one concurrent reclaimer wins, then purging the old attempt's infra
// (keeping the meta key as its lock) before laying down the fresh record. The
// old resource refs stay in the record until teardown confirms the infra is
// gone, so a crash mid-purge leaves them recoverable rather than orphaned.
// Returns an AWS-shaped error ready to hand back to the caller.
// claimClusterName atomically claims the cluster meta key before any launch work.
// A prior FAILED attempt is reclaimed via CAS (FAILED→CREATING); live clusters reject with ResourceInUse.
func (s *EKSServiceImpl) claimClusterName(accountID string, acctKV nats.KeyValue, meta *ClusterMeta) error {
name := meta.Name
data, err := json.Marshal(meta)
@@ -813,10 +696,7 @@ func (s *EKSServiceImpl) claimClusterName(accountID string, acctKV nats.KeyValue
return logCreateErr(name, accountID, "claim cluster name", cerr)
}
// Key already present. Reclaim only a prior FAILED attempt; flip its status
// to CREATING via CAS so exactly one concurrent reclaimer wins. The mutate
// changes only Status, so the old resource refs survive in the record for
// teardown below (and for a retry if we crash before clearing them).
// Key already present: reclaim a FAILED attempt via CAS; old refs survive for teardown.
var oldRefs *ClusterMeta
reclaimed := false
if err := casUpdateMeta(acctKV, name, func(m *ClusterMeta) bool {
@@ -1034,9 +914,7 @@ func (s *EKSServiceImpl) DeleteNodegroup(input *eks.DeleteNodegroupInput, accoun
// --- AccessEntry + AccessPolicy ---
// acctKVForCluster opens the per-account bucket and verifies the cluster exists,
// translating absence to the AWS ResourceNotFoundException shape. The
// AccessEntry handlers all need both.
// acctKVForCluster opens the per-account bucket and verifies the cluster exists.
func (s *EKSServiceImpl) acctKVForCluster(accountID, cluster string) (nats.KeyValue, error) {
if cluster == "" {
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
@@ -1072,8 +950,7 @@ func (s *EKSServiceImpl) CreateAccessEntry(input *eks.CreateAccessEntryInput, ac
entryType = AccessEntryTypeStandard
}
if entryType != AccessEntryTypeStandard {
// Node-type entries (EC2_LINUX/etc.) need the nodegroup join path; reject
// until Sprint 6d wires them.
// Non-standard types (EC2_LINUX etc.) are not yet implemented.
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
}
acctKV, err := s.acctKVForCluster(accountID, cluster)
spinifex/handlers/elbv2/agent_types.go
+10 -19
View File
@@ -5,16 +5,14 @@ import (
"github.com/mulgadc/spinifex/spinifex/lbagent"
)
// LBAgentHeartbeatInput is sent by the LB agent on each heartbeat tick.
// The agent includes its health report (HAProxy backend server statuses) so
// the daemon can update target health without polling.
// LBAgentHeartbeatInput is sent by the LB agent on each heartbeat tick,
// including its health report so the daemon can update target health.
type LBAgentHeartbeatInput struct {
LBID *string `locationName:"LBID" type:"string"`
Servers []*LBAgentServerStatus `locationName:"Servers" type:"list"`
}
// LBAgentServerStatus represents a single backend server's health status
// as reported by the LB agent. Maps to lbagent.ServerStatus for processing.
// LBAgentServerStatus represents a single backend server's health status as reported by the LB agent.
type LBAgentServerStatus struct {
Backend *string `locationName:"Backend" type:"string"`
Server *string `locationName:"Server" type:"string"`
@@ -32,11 +30,9 @@ type GetLBConfigInput struct {
LBID *string `locationName:"LBID" type:"string"`
}
// GetLBConfigOutput returns the pre-computed data-plane config and its hash,
// plus any TLS certificate files the config references. Engine selects the
// data plane the agent runs: "haproxy" (ALB) or "nginx" (NLB). HealthTargets
// is populated for NLBs only: nginx `stream` has no active upstream probing,
// so the agent probes these and reports the results via heartbeat.
// GetLBConfigOutput returns the pre-computed data-plane config, its hash, and
// any TLS certificate files. Engine is "haproxy" (ALB) or "nginx" (NLB).
// HealthTargets is NLB-only: the agent actively probes these targets.
type GetLBConfigOutput struct {
ConfigText *string `type:"string"`
ConfigHash *string `type:"string"`
@@ -45,12 +41,8 @@ type GetLBConfigOutput struct {
HealthTargets []*HealthTarget `locationName:"HealthTargets" type:"list"`
}
// HealthTarget is one backend the nginx agent must actively health-check.
// ServerName is daemon-computed (sanitizeName("srv", target.Id)) so it matches
// the name the health checker keys on — the agent echoes it back verbatim and
// needs no ELBv2 naming logic. Address is the ip:port to probe; Protocol is
// the target group's health-check protocol (TCP/HTTP/HTTPS); Path is the
// HTTP(S) health-check path.
// HealthTarget is one backend the nginx agent actively health-checks.
// ServerName matches sanitizeName("srv", target.Id); the agent echoes it back verbatim.
type HealthTarget struct {
ServerName *string `locationName:"ServerName" type:"string"`
Address *string `locationName:"Address" type:"string"`
@@ -58,9 +50,8 @@ type HealthTarget struct {
Path *string `locationName:"Path" type:"string"`
}
// CertFile is one TLS certificate PEM delivered to the LB agent. Path is the
// absolute destination (under lbagent.CertDir) the agent writes 0600 before
// reload; PEM is the combined cert+chain+key material.
// CertFile is one TLS certificate PEM delivered to the LB agent.
// Path is the absolute destination the agent writes 0600 before reload.
type CertFile struct {
Path *string `locationName:"Path" type:"string"`
PEM *string `locationName:"PEM" type:"string"`
spinifex/handlers/elbv2/arn_test.go
+3 -4
View File
@@ -242,10 +242,9 @@ func TestBuildMicrovmNICs_ExtraENIs(t *testing.T) {
assert.False(t, nics[2].IsDefault)
}
// TestBuildMicrovmNICs_MgmtRoute covers the same single-node/multi-node matrix
// as TestLBVMUserData_InternalSingleNodeFallback: internet-facing single-node
// must NOT emit a /32 via mgmt to AdvertiseIP because that steals the host's
// WAN return path for reply traffic, breaking same-chassis ingress.
// TestBuildMicrovmNICs_MgmtRoute verifies the single-node/multi-node mgmt-route matrix:
// internet-facing single-node must not emit a /32 via mgmt to AdvertiseIP, as that
// would steal the host's WAN return path and break same-chassis ingress.
func TestBuildMicrovmNICs_MgmtRoute(t *testing.T) {
mk := func() *ELBv2ServiceImpl {
return &ELBv2ServiceImpl{
spinifex/handlers/elbv2/haproxy.go
+38 -80
View File
@@ -12,12 +12,9 @@ import (
"github.com/mulgadc/spinifex/spinifex/lbagent"
)
// Every interpolated string field in the templates below MUST be regex-validated
// at its input boundary. A newline or HAProxy keyword in any unvalidated field
// would inject arbitrary directives into the rendered config. The current
// surface is HealthCheckPath and HealthCheckMatcher (validated in
// CreateTargetGroup); all other interpolated identifiers go through
// sanitizeName or are integers.
// All interpolated fields are regex-validated at the input boundary.
// HealthCheckPath and HealthCheckMatcher are validated in CreateTargetGroup;
// all other interpolated identifiers go through sanitizeName or are integers.
const haproxyConfigTemplate = `# Auto-generated by spinifex ELBv2 — do not edit
global
log stdout format short daemon
@@ -82,25 +79,22 @@ type HAProxyFrontend struct {
Rules []HAProxyRule
}
// HAProxyRule represents one routing rule emitted as ACL declarations + a
// use_backend directive. Multiple ACLs sharing a Name are OR'd by HAProxy;
// distinct ConditionNames in the use_backend "if" clause are AND'd.
// HAProxyRule represents one routing rule as ACL declarations + a use_backend
// directive. Same-name ACLs are OR'd; distinct ConditionNames are AND'd.
type HAProxyRule struct {
Backend string
ACLs []HAProxyACL
ConditionNames []string
}
// HAProxyACL is one acl line. Name may repeat across the rule's ACL slice to
// express OR semantics over multiple match expressions.
// HAProxyACL is one acl line. Repeated names express OR over multiple match expressions.
type HAProxyACL struct {
Name string
Expr string
}
// HAProxyBackend represents a target group backend, or a synthetic backend that
// emits a fixed response or a redirect (when FixedResponse / Redirect is set,
// the server fields are unused).
// HAProxyBackend represents a target group backend, or a synthetic redirect /
// fixed-response backend (server fields unused when those are set).
type HAProxyBackend struct {
Name string
HealthCheckPath string
@@ -110,8 +104,7 @@ type HAProxyBackend struct {
Redirect *HAProxyRedirect
}
// HAProxyFixedResponse renders an `http-request return` directive for a
// fixed-response listener default action. All fields are validated before use.
// HAProxyFixedResponse renders an `http-request return` directive. All fields are validated before use.
type HAProxyFixedResponse struct {
StatusCode string
ContentType string
@@ -119,8 +112,7 @@ type HAProxyFixedResponse struct {
}
// HAProxyRedirect renders an `http-request redirect` directive. Exactly one of
// Scheme (the scheme-only fast path, e.g. HTTP→HTTPS) or Location (a full
// rebuilt URL with HAProxy log-format fetches) is set.
// Scheme (scheme-only, e.g. HTTP→HTTPS) or Location (full rebuilt URL) is set.
type HAProxyRedirect struct {
Scheme string
Location string
@@ -137,22 +129,16 @@ type HAProxyServer struct {
Rise int64
}
// GenerateHAProxyConfig builds an HAProxy config string from the LB, its
// listeners, target groups, and per-listener rules. For NLBs it generates a
// TCP-mode config and ignores rules (NLB has no L7 routing). TLS termination
// is off (no certificates resolved) — use GenerateHAProxyConfigWithCerts for
// HTTPS listeners.
// GenerateHAProxyConfig builds an HAProxy config from the LB, listeners, and
// target groups. No TLS — use GenerateHAProxyConfigWithCerts for HTTPS listeners.
func GenerateHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgByArn map[string]*TargetGroupRecord, rulesByListener map[string][]*RuleRecord, bindAddr string) (string, error) {
config, _, err := GenerateHAProxyConfigWithCerts(lb, listeners, tgByArn, rulesByListener, bindAddr, nil)
return config, err
}
// GenerateHAProxyConfigWithCerts is GenerateHAProxyConfig plus TLS termination.
// certPEMByArn maps each listener certificate ARN to its combined PEM; secure
// frontends render `ssl crt <path>` and the returned certFiles map carries each
// path → PEM for delivery to the LB agent. A secure listener whose default cert
// ARN is absent from certPEMByArn renders without `ssl crt` (HAProxy would fail
// to bind, surfacing the missing material).
// certPEMByArn maps certificate ARNs to combined PEMs; secure frontends emit
// `ssl crt <path>` and the returned certFiles carries path→PEM for the LB agent.
func GenerateHAProxyConfigWithCerts(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgByArn map[string]*TargetGroupRecord, rulesByListener map[string][]*RuleRecord, bindAddr string, certPEMByArn map[string]string) (string, map[string]string, error) {
// NLBs (L4) render an nginx `stream` config — HAProxy load-balances no UDP.
if lb.Type == LoadBalancerTypeNetwork {
@@ -177,8 +163,7 @@ func generateALBConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgBy
}
// buildHAProxyConfig constructs the HAProxyConfig struct for an ALB (L7).
// rulesByListener provides per-listener rules sorted by priority. NLBs use the
// nginx data plane (generateNLBStreamConfig), not this builder.
// NLBs use the nginx data plane (generateNLBStreamConfig), not this builder.
func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgByArn map[string]*TargetGroupRecord, rulesByListener map[string][]*RuleRecord, bindAddr string, certPEMByArn map[string]string) (HAProxyConfig, error) {
socketDir := filepath.Join(os.TempDir(), "spinifex-haproxy")
// LoadBalancerID already carries the "lb-" prefix; must match the agent's socket path.
@@ -262,9 +247,7 @@ func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgB
if isForwardDefault {
defaultBackendName = sanitizeName("bk", da.TargetGroupArn)
} else {
// redirect / fixed-response (or any non-forward default, e.g. the
// shared-ingress 404): synthesize a backend so HAProxy has a valid
// default_backend instead of a dangling reference.
// Non-forward default: synthesize a backend for HAProxy's default_backend.
defaultBackendName = sanitizeName("bkdefault", l.ListenerArn)
}
@@ -276,10 +259,7 @@ func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgB
DefaultBackend: defaultBackendName,
}
// Secure listeners terminate TLS: resolve the default cert to a stable
// per-listener PEM path and stage it for delivery. Without resolved
// material the frontend renders plain (HAProxy then fails to bind,
// surfacing the gap rather than silently serving cleartext).
// Resolve the default cert and stage it for delivery to the agent.
if protocolRequiresCert(l.Protocol) {
if pem, path := resolveFrontendCert(lb, l, certPEMByArn); pem != "" {
frontend.CertPath = path
@@ -320,16 +300,13 @@ func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgB
return cfg, nil
}
// frontendCertPath returns the stable absolute path of a listener's combined
// PEM file under the agent's cert dir. Daemon-rendered and agent-written paths
// must match exactly.
// frontendCertPath returns the stable absolute PEM path under the agent's cert dir.
func frontendCertPath(lb *LoadBalancerRecord, l *ListenerRecord) string {
return filepath.Join(lbagent.CertDir, fmt.Sprintf("%s-%s.pem", lb.LoadBalancerID, l.ListenerID))
}
// resolveFrontendCert picks a listener's default certificate, resolves its PEM
// from certPEMByArn, and returns (pem, path). Returns ("", "") when the listener
// has no certificate or its default ARN is unresolved.
// resolveFrontendCert returns (pem, path) for the listener's default certificate,
// or ("", "") when absent or unresolved.
func resolveFrontendCert(lb *LoadBalancerRecord, l *ListenerRecord, certPEMByArn map[string]string) (pem, path string) {
if len(l.Certificates) == 0 || certPEMByArn == nil {
return "", ""
@@ -348,9 +325,7 @@ func resolveFrontendCert(lb *LoadBalancerRecord, l *ListenerRecord, certPEMByArn
return pem, frontendCertPath(lb, l)
}
// registerRuleBackend resolves the HAProxy backend a rule's single action
// routes to, registering a synthetic backend for redirect / fixed-response
// rule actions, and returns its name.
// registerRuleBackend registers the backend for a rule's single action and returns its name.
func registerRuleBackend(
r *RuleRecord,
l *ListenerRecord,
@@ -386,10 +361,8 @@ func registerRuleBackend(
}
}
// buildHAProxyRule converts a RuleRecord's conditions into the rendered
// ACL+use_backend shape, routing to backendName (already resolved/registered
// by registerRuleBackend). Returns an error if any condition value fails the
// defence-in-depth re-validation (HAProxy meta-character rejection).
// buildHAProxyRule converts a RuleRecord's conditions into ACL+use_backend shape.
// Returns an error if any condition value fails render-safe validation.
func buildHAProxyRule(r *RuleRecord, backendName string) (HAProxyRule, error) {
rule := HAProxyRule{Backend: backendName}
@@ -408,9 +381,8 @@ func buildHAProxyRule(r *RuleRecord, backendName string) (HAProxyRule, error) {
return rule, nil
}
// buildHAProxyACLExprs returns one or more HAProxy ACL expressions for a
// single rule condition. Multiple expressions represent OR semantics over the
// condition's values (HAProxy ORs same-name ACL declarations).
// buildHAProxyACLExprs returns HAProxy ACL expressions for a rule condition.
// Multiple expressions represent OR semantics over the condition's values.
func buildHAProxyACLExprs(c RuleCondition) ([]string, error) {
switch c.Field {
case RuleFieldHostHeader:
@@ -481,10 +453,7 @@ func hostOrPathExprs(prefix string, values []string) ([]string, error) {
}
// wildcardMatch maps a host/path pattern to a HAProxy match flag + literal.
// Supports leading *, trailing *, or no wildcard. Embedded `?` or interior `*`
// fall through to substring match (`sub`) — AWS treats `?` as single-char
// wildcard; HAProxy lacks an exact equivalent so we widen to substring (false
// positives over false negatives is the right failure mode for a router).
// `?` and interior `*` widen to substring — false positives over false negatives.
func wildcardMatch(v string) (flag, literal string) {
hasLead := strings.HasPrefix(v, "*")
hasTrail := strings.HasSuffix(v, "*")
@@ -502,8 +471,7 @@ func wildcardMatch(v string) (flag, literal string) {
}
}
// contentTypeRegex matches a conservative MIME type (type/subtype) for the
// fixed-response content-type header rendered into the HAProxy config.
// contentTypeRegex matches a conservative type/subtype MIME string.
var contentTypeRegex = regexp.MustCompile(`^[a-zA-Z0-9!#$&^_.+-]+/[a-zA-Z0-9!#$&^_.+-]+$`)
// validHTTPStatusCode accepts a 3-digit 2xx–5xx HTTP status.
@@ -524,9 +492,7 @@ func validContentType(s string) bool {
return contentTypeRegex.MatchString(s)
}
// validFixedResponseBody rejects bytes that would terminate the quoted HAProxy
// string or inject a directive. Spaces are allowed (the body renders inside
// double quotes).
// validFixedResponseBody rejects bytes that would terminate or inject into the HAProxy directive.
func validFixedResponseBody(s string) bool {
if s == "" {
return false
@@ -539,11 +505,9 @@ func validFixedResponseBody(s string) bool {
return true
}
// buildRedirect translates a stored RedirectAction into an HAProxy redirect
// directive in the context of listener l (which resolves the `#{protocol}` and
// `#{port}` placeholders). The common scheme-only redirect (e.g. HTTP→HTTPS
// with every other field left at its AWS default) renders as `redirect scheme`
// so HAProxy preserves host/path/query; anything else rebuilds a `location`.
// buildRedirect translates a RedirectAction into an HAProxy redirect directive.
// Scheme-only redirects (e.g. HTTP→HTTPS, all other fields default) render as
// `redirect scheme`; everything else rebuilds a `location`.
func buildRedirect(rd *RedirectAction, l *ListenerRecord) (*HAProxyRedirect, error) {
if rd == nil {
return nil, fmt.Errorf("redirect action missing config")
@@ -605,8 +569,7 @@ func orDefault(s, def string) string {
return s
}
// redirectScheme resolves the redirect scheme literal, mapping the `#{protocol}`
// placeholder (or an empty protocol) to the listener's own protocol.
// redirectScheme resolves the redirect scheme, mapping `#{protocol}` to the listener's protocol.
func redirectScheme(proto string, l *ListenerRecord) string {
switch proto {
case ProtocolHTTP, ProtocolHTTPS:
@@ -616,8 +579,7 @@ func redirectScheme(proto string, l *ListenerRecord) string {
}
}
// redirectPlaceholders substitutes the AWS redirect placeholders with HAProxy
// log-format fetches. `#{protocol}` and `#{port}` resolve from the listener.
// redirectPlaceholders substitutes AWS redirect placeholders with HAProxy log-format fetches.
func redirectPlaceholders(s string, l *ListenerRecord) string {
r := strings.NewReplacer(
"#{protocol}", strings.ToLower(l.Protocol),
@@ -629,10 +591,8 @@ func redirectPlaceholders(s string, l *ListenerRecord) string {
return r.Replace(s)
}
// validRedirectField rejects bytes that would break the HAProxy redirect
// directive. The known `#{...}` placeholders are stripped first, so any
// remaining `#`, `%`, bracket, quote, backslash, whitespace, or control byte
// (which would indicate an unknown placeholder or an injection attempt) fails.
// validRedirectField rejects bytes that would break the HAProxy redirect directive.
// Known `#{...}` placeholders are stripped first; any remaining forbidden byte fails.
func validRedirectField(s string) bool {
known := []string{"#{protocol}", "#{host}", "#{port}", "#{path}", "#{query}"}
tmp := s
@@ -651,10 +611,8 @@ func validRedirectField(s string) bool {
return true
}
// validateRenderSafe rejects bytes that could break ACL parsing or inject new
// directives. Mirrors the write-time validators in service_impl_rules.go;
// duplicated here so a future migration that bypasses the API cannot leak a
// malicious string into the generated config.
// validateRenderSafe rejects bytes that could break ACL parsing or inject directives.
// Duplicated from write-time validators so API-bypass cannot leak malicious strings.
func validateRenderSafe(v string) error {
if v == "" {
return fmt.Errorf("empty rule token")
@@ -678,7 +636,7 @@ func sanitizeACLToken(id string) string {
return sb.String()
}
// sanitizeName creates a safe identifier from an ARN or ID for use in HAProxy config.
// sanitizeName creates a safe HAProxy identifier from an ARN or ID.
func sanitizeName(prefix, id string) string {
// Use last segment of ARN or ID, replace non-alphanumeric with underscore
parts := strings.Split(id, "/")
spinifex/handlers/elbv2/haproxy_test.go
+3 -5
View File
@@ -161,11 +161,9 @@ func TestGenerateHAProxyConfig_SharedTargetGroup(t *testing.T) {
assert.Equal(t, 1, strings.Count(config, "balance roundrobin"))
}
// TestGenerateHAProxyConfig_FixedResponseDefault mirrors the shared-ingress
// shape: a listener whose default action is fixed-response (no target group),
// with host-header rules forwarding to per-app target groups. The generator
// must synthesize a default backend instead of emitting a dangling
// `default_backend bk_`, which makes HAProxy fail to start.
// TestGenerateHAProxyConfig_FixedResponseDefault covers a fixed-response default action
// with host-header rules forwarding to target groups. The generator must synthesize a
// default backend; a dangling `default_backend bk_` makes HAProxy fail to start.
func TestGenerateHAProxyConfig_FixedResponseDefault(t *testing.T) {
lb := &LoadBalancerRecord{LoadBalancerID: "lb-ingress"}
listenerArn := "arn:aws:elasticloadbalancing:ap-southeast-2:1:listener/app/wd-ingress/lb-ingress/lst-616760baf6a3031c3"
spinifex/handlers/elbv2/health_checker.go
+7 -17
View File
@@ -9,13 +9,8 @@ import (
"github.com/mulgadc/spinifex/spinifex/lbagent"
)
// healthChecker processes target health reports from LB agents. Reports arrive
// via LBAgentHeartbeat (the agent pushes health data on each heartbeat). This
// checker maps HAProxy backend server statuses back to registered targets and
// updates HealthState in the store.
//
// This mirrors the AWS model where the LB itself health-checks the targets,
// rather than the control plane probing them directly.
// healthChecker processes target health reports from LB agents pushed via
// LBAgentHeartbeat, mapping backend statuses back to targets in the store.
type healthChecker struct {
store *Store
@@ -37,7 +32,6 @@ func newHealthChecker(store *Store) *healthChecker {
}
// handleHealthReport unmarshals a JSON-encoded health report and processes it.
// This is the []byte entry point used by tests and any future wire-format callers.
func (hc *healthChecker) handleHealthReport(data []byte) {
var report lbagent.HealthReport
if err := json.Unmarshal(data, &report); err != nil {
@@ -48,8 +42,7 @@ func (hc *healthChecker) handleHealthReport(data []byte) {
hc.handleHealthReportDirect(report)
}
// handleHealthReportDirect processes a health report from an lb-agent without
// an intermediate JSON round-trip. Called directly from LBAgentHeartbeat.
// handleHealthReportDirect processes a health report without a JSON round-trip.
func (hc *healthChecker) handleHealthReportDirect(report lbagent.HealthReport) {
if len(report.Servers) == 0 {
return
@@ -61,8 +54,7 @@ func (hc *healthChecker) handleHealthReportDirect(report lbagent.HealthReport) {
serverUp[srv.Server] = srv.Status == "UP"
}
// Look up only the target groups attached to this LB's listeners.
// Falls back to scanning all target groups if LBID is missing.
// Look up target groups for this LB; fall back to all TGs if LBID is missing.
var tgs []*TargetGroupRecord
var err error
if report.LBID != "" {
@@ -75,8 +67,7 @@ func (hc *healthChecker) handleHealthReportDirect(report lbagent.HealthReport) {
return
}
// Collect changed TGs under the lock, then persist outside to avoid
// holding the mutex across KV store network I/O.
// Collect changed TGs under the lock; persist outside to avoid holding it across KV I/O.
hc.mu.Lock()
var changedTGs []*TargetGroupRecord
for _, tg := range tgs {
@@ -142,9 +133,8 @@ func (hc *healthChecker) handleHealthReportDirect(report lbagent.HealthReport) {
}
// evaluateHealth applies threshold logic to determine a target's new state.
// From "initial", a single healthy probe transitions to healthy (fast start,
// matching AWS behavior for newly registered targets). From "healthy" or
// "unhealthy", full threshold counts are required.
// From "initial", one healthy probe transitions to healthy (AWS fast-start);
// from "healthy"/"unhealthy", full threshold counts are required.
func evaluateHealth(current string, ctr *targetCounter, cfg HealthCheckConfig) (string, string) {
healthyThreshold := cfg.HealthyThreshold
if healthyThreshold == 0 {
spinifex/handlers/elbv2/launcher.go
+2 -5
View File
@@ -2,11 +2,8 @@ package handlers_elbv2
import "github.com/mulgadc/spinifex/spinifex/handlers/sysinstance"
// The system-instance launch contract now lives in the boot-agnostic
// handlers/sysinstance package so both ELBv2 and EKS can launch system VMs
// without an eks→elbv2 dependency. These aliases keep existing ELBv2 source
// (which builds direct-boot microVM inputs) compiling unchanged. ELBv2 always
// launches with BootMode=BootDirect (the zero value).
// These aliases re-export sysinstance types so ELBv2 and EKS share launch
// logic without an eks→elbv2 dependency. ELBv2 always uses BootMode=BootDirect.
type (
SystemInstanceLauncher = sysinstance.SystemInstanceLauncher
SystemInstanceInput = sysinstance.SystemInstanceInput
spinifex/handlers/elbv2/lifecycle_invariants_test.go
+12 -22
View File
@@ -11,13 +11,9 @@ import (
"github.com/stretchr/testify/require"
)
// TestRLC1_ELBv2DeleteIdempotentOnAbsent enforces Common Resource Lifecycle
// Contract rule #1 (idempotent delete) across the ELBv2 surface: deleting an
// absent resource is success, not a NotFound error. This mirrors AWS ELBv2
// delete semantics and is required for tofu destroy to converge on retry.
//
// Every ELBv2 delete endpoint MUST have a case here. When a new delete
// endpoint lands, add it; a missing case is an idempotency gap.
// TestRLC1_ELBv2DeleteIdempotentOnAbsent enforces RLC rule #1 (idempotent delete):
// deleting an absent ELBv2 resource is success, not NotFound. Every delete endpoint
// must have a case here — a missing case is an idempotency gap.
func TestRLC1_ELBv2DeleteIdempotentOnAbsent(t *testing.T) {
const arnBase = "arn:aws:elasticloadbalancing:us-east-1:123456789012:"
@@ -57,10 +53,9 @@ func TestRLC1_ELBv2DeleteIdempotentOnAbsent(t *testing.T) {
}
}
// TestRLC2_ELBv2NoOrphanAfterDeleteLB enforces ADR-0002 §5 no-orphan
// completeness: after DeleteLoadBalancer, no listener or rule owned by that LB
// may survive in the store. Uses two listeners (each with a rule) so a
// single-listener cascade can't pass by accident.
// TestRLC2_ELBv2NoOrphanAfterDeleteLB enforces ADR-0002 §5 no-orphan completeness:
// after DeleteLoadBalancer no listener or rule owned by the LB may remain. Two
// listeners are used so a single-listener cascade can't pass by accident.
func TestRLC2_ELBv2NoOrphanAfterDeleteLB(t *testing.T) {
svc := setupTestService(t)
@@ -102,13 +97,9 @@ func TestRLC2_ELBv2NoOrphanAfterDeleteLB(t *testing.T) {
require.Emptyf(t, rules, "ADR-0002 §5 no-orphan completeness: no rule owned by a deleted LB may remain")
}
// TestRLC3_ELBv2DeleteLBDoesNotBypassListenerCascade enforces ADR-0002 §5 no
// store-bypass cascade: DeleteLoadBalancer must tear listeners (and their
// rules) down through the shared deleteListenerCascade helper, never by
// calling store.DeleteListener directly — a direct store delete orphans the
// listener's rules. The store field is a concrete type with no injection seam,
// so the contract is enforced structurally against the source rather than with
// a runtime spy.
// TestRLC3_ELBv2DeleteLBDoesNotBypassListenerCascade enforces ADR-0002 §5 no-store-bypass
// cascade: DeleteLoadBalancer must use deleteListenerCascade, not store.DeleteListener
// directly (a direct call orphans rules). Enforced structurally against source.
func TestRLC3_ELBv2DeleteLBDoesNotBypassListenerCascade(t *testing.T) {
src, err := os.ReadFile("service_impl.go")
require.NoError(t, err)
@@ -148,10 +139,9 @@ func stripComments(src string) string {
return b.String()
}
// TestRLC4_ELBv2TGDeletableAfterLBTeardown enforces ADR-0002 §5 TG deletability
// after LB teardown: create LB+listener+rule→TG, delete the LB, then the target
// group must delete successfully rather than staying pinned as ResourceInUse by
// an orphaned rule or stale listener default action.
// TestRLC4_ELBv2TGDeletableAfterLBTeardown enforces ADR-0002 §5 TG deletability:
// after LB+listener+rule teardown, the target group must not remain pinned as
// ResourceInUse by an orphaned rule or stale listener default action.
func TestRLC4_ELBv2TGDeletableAfterLBTeardown(t *testing.T) {
svc := setupTestService(t)
spinifex/handlers/elbv2/nginx.go
+9 -17
View File
@@ -11,12 +11,9 @@ import (
"github.com/mulgadc/spinifex/spinifex/lbagent"
)
// NLB (L4) data plane. HAProxy load-balances no UDP in any release, so NLB
// listeners (TCP/UDP/TLS/TCP_UDP) render an nginx `stream` config instead;
// ALBs stay on HAProxy. See docs/development/feature/nlb-nginx-l4-dataplane.md.
//
// Health checks are passive only (max_fails / fail_timeout) — OSS nginx stream
// has no active upstream probing.
// NLB (L4) data plane. NLB listeners render an nginx `stream` config because HAProxy
// has no UDP support; ALBs stay on HAProxy. Health checks are passive only
// (max_fails/fail_timeout) — OSS nginx stream has no active upstream probing.
const nginxStreamConfigTemplate = `# Auto-generated by spinifex ELBv2 (NLB) — do not edit
load_module /usr/lib/nginx/modules/ngx_stream_module.so;
@@ -100,11 +97,9 @@ func engineForType(lbType string) string {
return EngineHAProxy
}
// generateNLBStreamConfig builds an nginx `stream` config for an NLB. Secure
// (TLS) listeners terminate TLS at nginx and forward plaintext to the target;
// the returned certFiles map carries each cert path → combined PEM for delivery
// to the LB agent. bindAddr is unused (nginx listens on all addresses, matching
// the HAProxy `bind *` behaviour).
// generateNLBStreamConfig builds an nginx `stream` config for an NLB; TLS listeners
// terminate at nginx. certFiles maps cert path → combined PEM for delivery to the
// agent. bindAddr is unused — nginx listens on all addresses like HAProxy's `bind *`.
func generateNLBStreamConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgByArn map[string]*TargetGroupRecord, _ string, certPEMByArn map[string]string) (string, map[string]string, error) {
cfg := nginxConfig{PIDPath: lbagent.NginxPIDPath}
var certFiles map[string]string
@@ -187,12 +182,9 @@ func nginxCertPath(lb *LoadBalancerRecord, l *ListenerRecord) string {
return filepath.Join(lbagent.NginxCertDir, fmt.Sprintf("%s-%s.pem", lb.LoadBalancerID, l.ListenerID))
}
// buildNLBHealthTargets computes the active health-check targets for an NLB's
// backends. nginx `stream` has no active upstream probing, so the agent probes
// these and reports the results via heartbeat. ServerName matches the health
// checker's key (sanitizeName("srv", target.Id)); the probe port resolves
// "traffic-port" to the target's port. Sorted by server name for deterministic
// delivery. The tgByArn map is the same set used to render the upstreams.
// buildNLBHealthTargets returns backends the agent actively probes (nginx stream
// has no built-in active probing). ServerName matches sanitizeName("srv", target.Id);
// sorted by name for deterministic delivery.
func buildNLBHealthTargets(tgByArn map[string]*TargetGroupRecord) []HealthTargetSpec {
var out []HealthTargetSpec
for _, tg := range tgByArn {
spinifex/handlers/elbv2/service_impl.go
+115 -299
View File
@@ -36,15 +36,11 @@ const (
// elbv2LBTag is the tag key storing the parent LB ARN on managed ENIs.
elbv2LBTag = tags.LBARNKey
// heartbeatPersistInterval controls how often a no-op heartbeat (no state
// change) writes the full LoadBalancerRecord back to KV. State transitions
// always persist immediately.
// heartbeatPersistInterval is how often a no-op heartbeat writes to KV.
heartbeatPersistInterval = 60 * time.Second
// Health check fields are interpolated verbatim into the HAProxy template
// (see haproxy.go); restrict them to characters that cannot terminate a
// directive or introduce a new one. Length caps bound abuse and match the
// surface AWS exposes for these fields.
// Health check fields are interpolated into the HAProxy template; restrict
// to characters that cannot terminate or inject directives.
maxHealthCheckPathLen = 1024
maxHealthCheckMatcherLen = 64
)
@@ -68,21 +64,9 @@ func validateHealthCheckMatcher(m string) error {
return nil
}
// resolveMgmtRoute returns the (gateway, target) pair for the host route the
// lb-agent uses to reach AWSGW from inside the LB microVM, or empty strings
// to skip the route. Consumed by buildMicrovmNICs to populate NIC[1]'s
// RouteDst/RouteVia for the netcfg fw_cfg blob.
//
// Multi-node (AWSGW on dedicated mgmt IP): MgmtRoute{Gateway,Target} are set
// by the daemon for both schemes — return them.
//
// Single-node (AWSGW on advertiseIP, MgmtRoute fields empty): internet-facing
// LBs reach AWSGW via VPC → external (their EIP gives OVN a SNAT pair for the
// reply), so they need no mgmt route — and adding a /32 to advertiseIP would
// steal the host's WAN return path. Internal LBs have no EIP and no SNAT pair,
// so the VPC egress reply targets the VM's private IP from the host's WAN with
// no route back → packet drops, lb-agent never heartbeats, LB sticks in
// provisioning. Force the mgmt route for internal scheme via the br-mgmt IP.
// resolveMgmtRoute returns the (gateway, target) for the lb-agent's host route to AWSGW.
// Multi-node: returns MgmtRoute{Gateway,Target}. Single-node internet-facing: no route
// (EIP provides SNAT). Single-node internal: route forced via br-mgmt IP.
func (s *ELBv2ServiceImpl) resolveMgmtRoute(scheme string) (gateway, target string) {
if s.MgmtRouteGateway != "" && s.MgmtRouteTarget != "" {
return s.MgmtRouteGateway, s.MgmtRouteTarget
@@ -132,10 +116,9 @@ type ELBv2ServiceImpl struct {
recoveryLBIndexMu sync.Mutex
recoveryLBIndex map[string]*LoadBalancerRecord
// launchWG tracks in-flight asynchronous LB-VM launches spawned by
// CreateLoadBalancer. Not awaited at shutdown (a multi-minute boot must not
// block Close); exposed via WaitLaunches purely so tests can join the
// background launch deterministically.
// launchWG tracks in-flight LB-VM launches from CreateLoadBalancer; not awaited
// at shutdown to avoid blocking Close on a multi-minute boot. Exposed via
// WaitLaunches so tests can join the background launch deterministically.
launchWG sync.WaitGroup
}
@@ -150,10 +133,7 @@ func NewELBv2ServiceImplWithNATS(cfg *config.Config, nc *nats.Conn) (*ELBv2Servi
return nil, fmt.Errorf("failed to create ELBv2 store: %w", err)
}
// ACM store shares the same JetStream KV — read cert PEM directly, no
// cross-service NATS hop. Non-fatal: a failure here only disables HTTPS
// termination (resolveCertPEM returns an error), it must not block the
// daemon from serving ELBv2 control-plane requests.
// ACM store shares JetStream KV. Non-fatal: failure only disables HTTPS termination.
acmStore, acmErr := handlers_acm.NewStore(nc)
if acmErr != nil {
slog.Warn("ELBv2: ACM store unavailable, HTTPS listeners cannot resolve certs", "err", acmErr)
@@ -192,29 +172,9 @@ func (s *ELBv2ServiceImpl) Close() {
}
}
// ResetTargetHealthOnStartup invalidates persisted target HealthState across
// all target groups by resetting every non-draining target to "initial". Run
// once during daemon startup before the service begins serving requests.
//
// Rationale: target HealthState is persisted in JetStream KV, but the
// healthChecker's per-target counters live only in process memory. After a
// daemon restart (e.g. host reboot), the counters are gone but the stored
// "healthy" claim remains — DescribeTargetHealth returns "healthy"
// immediately even though the daemon has zero active health observations
// and the LB system VM's lb-agent hasn't posted a fresh report yet. Tests
// that gate on WaitForTargetsHealthy advance prematurely, then ALB traffic
// lands on a backend that isn't actually up.
//
// Resetting to "initial" is the correct AWS-semantics representation of
// "we don't know yet, give us a moment" — the next lb-agent heartbeat
// drives the state machine forward through evaluateHealth.
//
// Multi-node caveat: in a multi-daemon cluster, each daemon that restarts
// resets the SHARED KV state. A sibling daemon's in-memory counters would
// then be out-of-sync with the persisted "initial" state until the next
// heartbeat re-converges. Acceptable trade-off — heartbeats arrive within
// one health-check interval (default 5-30s) and round-robin transiently
// excluding a backend is safer than serving traffic to a dead one.
// ResetTargetHealthOnStartup resets every non-draining target to "initial" so
// stale "healthy" state doesn't mislead DescribeTargetHealth after a restart;
// the next lb-agent heartbeat re-evaluates and converges within one interval.
func (s *ELBv2ServiceImpl) ResetTargetHealthOnStartup(ctx context.Context) error {
if s == nil || s.store == nil {
return nil
@@ -232,8 +192,7 @@ func (s *ELBv2ServiceImpl) ResetTargetHealthOnStartup(ctx context.Context) error
changed := false
for i := range tg.Targets {
t := &tg.Targets[i]
// Skip targets actively being drained — that state is driven by
// DeregisterTargets and must not be clobbered by a daemon restart.
// Draining state is driven by DeregisterTargets; don't clobber it.
if t.HealthState == TargetHealthDraining {
continue
}
@@ -261,8 +220,7 @@ func (s *ELBv2ServiceImpl) ResetTargetHealthOnStartup(ctx context.Context) error
return nil
}
// SetSystemInstanceTypeFunc sets a function that resolves the smallest available
// instance type. Called at request time so it adapts to node capacity.
// SetSystemInstanceTypeFunc sets a function that resolves the smallest available instance type.
func (s *ELBv2ServiceImpl) SetSystemInstanceTypeFunc(fn func() string) {
s.systemInstanceTypeMu.Lock()
defer s.systemInstanceTypeMu.Unlock()
@@ -271,9 +229,7 @@ func (s *ELBv2ServiceImpl) SetSystemInstanceTypeFunc(fn func() string) {
s.systemInstanceTypeResolved = false
}
// getSystemInstanceType returns the instance type for system VMs.
// Only caches non-empty results so the resolver retries when no capacity
// is available yet.
// getSystemInstanceType returns the instance type for system VMs, caching only non-empty results.
func (s *ELBv2ServiceImpl) getSystemInstanceType() string {
s.systemInstanceTypeMu.Lock()
defer s.systemInstanceTypeMu.Unlock()
@@ -296,9 +252,8 @@ func (s *ELBv2ServiceImpl) buildLBAgentEnv(lbID string) string {
lbID, s.GatewayURL, s.SystemAccessKey, s.SystemSecretKey, s.region)
}
// subnetCIDRForIP computes "ip/prefixlen" given a host IP and the subnet's
// CIDR block (e.g. "10.0.1.0/24" → "10.0.1.5/24"). Returns empty string on
// parse failure so the caller can log and continue without panicking.
// subnetCIDRForIP returns "ip/prefixlen" from a host IP and subnet CIDR block.
// Returns empty string on parse failure.
func subnetCIDRForIP(hostIP, subnetCIDR string) string {
_, ipNet, err := net.ParseCIDR(subnetCIDR)
if err != nil {
@@ -325,17 +280,9 @@ func subnetGatewayIP(subnetCIDR string) string {
return gwCopy.String()
}
// buildMicrovmNICs constructs the NIC slice for a direct-boot microvm launch.
//
// NIC[0] is the primary VPC ENI (IsDefault=true): MAC/IP from the ENI record,
// CIDR from ip+subnet prefix, gateway = subnet network address +1.
//
// NIC[1] is the management NIC (IsDefault=false): MAC and CIDR are left empty
// here — the daemon injects them after allocating the per-instance mgmt IP.
// RouteDst/RouteVia encode the host route the lb-agent needs to reach AWSGW
// via the management interface (same logic as resolveMgmtRoute).
//
// NIC[2+] are extra VPC ENIs for multi-subnet ALBs.
// buildMicrovmNICs constructs the NIC slice for a direct-boot microvm.
// NIC[0] is the primary VPC ENI; NIC[1] is the mgmt NIC (MAC/CIDR filled
// post-allocation); NIC[2+] are extra ENIs for multi-subnet ALBs.
func (s *ELBv2ServiceImpl) buildMicrovmNICs(primaryIP, primaryMAC, primarySubnetID, _, scheme string, extraENIs []ExtraENIInput, accountID string) []NICConfig {
// Resolve primary subnet CIDR for NIC[0] prefix and gateway.
primaryCIDR := ""
@@ -359,11 +306,8 @@ func (s *ELBv2ServiceImpl) buildMicrovmNICs(primaryIP, primaryMAC, primarySubnet
},
}
// NIC[1]: management NIC — daemon fills MAC/CIDR after IP allocation.
// RouteDst = AWSGW IP (destination), RouteVia = br-mgmt IP (next-hop).
// Honor resolveMgmtRoute's deliberate empty return for single-node
// internet-facing: adding a /32 to AdvertiseIP via mgmt steals the host's
// WAN return path for reply traffic, breaking same-chassis ingress.
// NIC[1]: mgmt NIC — daemon fills MAC/CIDR. Empty return from resolveMgmtRoute
// intentionally skips the route for single-node internet-facing LBs.
mgmtRouteGW, mgmtRouteTarget := s.resolveMgmtRoute(scheme)
nics = append(nics, NICConfig{
IsDefault: false,
@@ -394,10 +338,8 @@ func (s *ELBv2ServiceImpl) buildMicrovmNICs(primaryIP, primaryMAC, primarySubnet
return nics
}
// describeENIs resolves the supplied ENI IDs in one VPC describe call and
// returns them keyed by NetworkInterfaceId. Returns an empty map when the
// VPC service is unwired or the describe errors — callers must tolerate
// missing entries.
// describeENIs resolves ENI IDs in one VPC call, keyed by NetworkInterfaceId.
// Returns an empty map when the VPC service is unavailable or the call errors.
func (s *ELBv2ServiceImpl) describeENIs(eniIDs []string, accountID string) map[string]*ec2.NetworkInterface {
eniDetails := make(map[string]*ec2.NetworkInterface, len(eniIDs))
if s.VPCService == nil || len(eniIDs) == 0 {
@@ -422,11 +364,8 @@ func (s *ELBv2ServiceImpl) describeENIs(eniIDs []string, accountID string) map[s
return eniDetails
}
// buildExtraENIInputs threads every non-primary ENI in eniIDs into an
// ExtraENIInput, using eniDetails as the MAC/IP/SubnetID source. eniIDs[0]
// is the primary ENI and is intentionally excluded. ENIs missing from
// eniDetails are skipped (matching CreateLoadBalancer's original behaviour
// when DescribeNetworkInterfaces drops an entry).
// buildExtraENIInputs builds ExtraENIInput for all non-primary ENIs (eniIDs[0] excluded).
// ENIs missing from eniDetails are skipped.
func buildExtraENIInputs(eniIDs []string, eniDetails map[string]*ec2.NetworkInterface) []ExtraENIInput {
if len(eniIDs) <= 1 {
return nil
@@ -447,9 +386,8 @@ func buildExtraENIInputs(eniIDs []string, eniDetails map[string]*ec2.NetworkInte
return extras
}
// lbVMLaunch is the outcome of booting the system VM that backs a load
// balancer. failed is true when the VM could not be launched (missing system
// credentials or a launcher error) so the caller can mark the LB failed.
// lbVMLaunch is the outcome of booting the LB system VM. failed is true when
// the VM could not be launched.
type lbVMLaunch struct {
instanceID string
vpcIP string
@@ -458,12 +396,9 @@ type lbVMLaunch struct {
failed bool
}
// launchLBVM boots the system VM backing a load balancer with the given ENI
// set. The first ENI is the primary NIC; any additional ENIs are passed as
// ExtraENIs so the daemon sets up a tap + QEMU NIC per subnet, giving the LB VM
// data-plane presence in every subnet it spans. When no launcher is configured
// the call is a no-op (zero value, failed=false) so launcher-less deployments
// keep the LB Active. Shared by CreateLoadBalancer and the SetSubnets relaunch.
// launchLBVM boots the system VM for a load balancer. The first ENI is the
// primary NIC; extras give multi-subnet data-plane presence. No-op when no
// launcher is configured. Shared by CreateLoadBalancer and SetSubnets.
func (s *ELBv2ServiceImpl) launchLBVM(lbID, scheme string, eniIDs, subnets []string, accountID string) lbVMLaunch {
var res lbVMLaunch
if s.InstanceLauncher == nil || len(eniIDs) == 0 || len(subnets) == 0 {
@@ -522,10 +457,8 @@ func (s *ELBv2ServiceImpl) launchLBVM(lbID, scheme string, eniIDs, subnets []str
return res
}
// loadRecoveryLBIndex returns the instanceID->LB map, populated lazily on
// first successful call. Errors are not cached: a transient JetStream
// failure at startup must not condemn every subsequent recovery candidate
// in the same batch — the next caller retries.
// loadRecoveryLBIndex returns the instanceID→LB map, populated lazily.
// Errors are not cached so transient JetStream failures don't condemn recovery.
func (s *ELBv2ServiceImpl) loadRecoveryLBIndex() (map[string]*LoadBalancerRecord, error) {
s.recoveryLBIndexMu.Lock()
defer s.recoveryLBIndexMu.Unlock()
@@ -546,10 +479,8 @@ func (s *ELBv2ServiceImpl) loadRecoveryLBIndex() (map[string]*LoadBalancerRecord
return s.recoveryLBIndex, nil
}
// RebuildSystemInstanceInput reconstructs the launch input for an existing
// system instance during host-reboot recovery. The instance->LB map is
// memoised in s.recoveryLBIndex so N concurrent recovery candidates share a
// single ListLoadBalancers call instead of issuing N×L JetStream round-trips.
// RebuildSystemInstanceInput reconstructs the launch input for host-reboot recovery.
// The instance→LB map is memoised so concurrent recovery candidates share one ListLoadBalancers call.
func (s *ELBv2ServiceImpl) RebuildSystemInstanceInput(ctx RecoveryContext) (*SystemInstanceInput, error) {
index, err := s.loadRecoveryLBIndex()
if err != nil {
@@ -564,10 +495,7 @@ func (s *ELBv2ServiceImpl) RebuildSystemInstanceInput(ctx RecoveryContext) (*Sys
}
eniDetails := s.describeENIs(lb.ENIs, lb.AccountID)
// Multi-ENI rebuild needs an authoritative MAC/IP/subnet per extra, which
// only the VPC store supplies. Single-ENI ALBs can fall back to ctx.ENIMac
// + lb.VPCIP for the primary, so we only enforce the strict count match
// when extras are present.
// For multi-ENI LBs the VPC store must supply all MAC/IP/subnet details.
if len(lb.ENIs) > 1 && len(eniDetails) < len(lb.ENIs) {
return nil, fmt.Errorf("describe lb %s ENIs: got %d/%d", lb.LoadBalancerID, len(eniDetails), len(lb.ENIs))
}
@@ -583,10 +511,7 @@ func (s *ELBv2ServiceImpl) RebuildSystemInstanceInput(ctx RecoveryContext) (*Sys
extraENIs := buildExtraENIInputs(lb.ENIs, eniDetails)
nics := s.buildMicrovmNICs(lb.VPCIP, primaryMAC, lb.Subnets[0], lb.ENIs[0], lb.Scheme, extraENIs, lb.AccountID)
// Re-inject the daemon-allocated mgmt NIC values so writeFwCfgBlobs
// emits the same netcfg the original launch produced — buildMicrovmNICs
// leaves NIC[1] MAC/CIDR blank because they are only known after the
// daemon's per-instance mgmt IP allocation.
// Re-inject mgmt NIC MAC/CIDR — buildMicrovmNICs leaves them blank.
if len(nics) > 1 && ctx.MgmtMAC != "" {
nics[1].MAC = ctx.MgmtMAC
if ctx.MgmtIP != "" {
@@ -630,10 +555,8 @@ func (s *ELBv2ServiceImpl) resolveENIBindAddr(lb *LoadBalancerRecord) string {
return ""
}
// updateStoredConfig generates the HAProxy config for an ALB, hashes it, and
// stores both ConfigText and ConfigHash on the LB record. The agent pulls this
// on its next heartbeat when it detects a hash change. Safe to call when
// instanceLauncher is nil or the LB has no VM.
// updateStoredConfig generates the data-plane config, hashes it, and stores it
// on the LB record. The agent fetches it on the next heartbeat when the hash changes.
func (s *ELBv2ServiceImpl) updateStoredConfig(lb *LoadBalancerRecord) error {
if lb.InstanceID == "" {
return nil
@@ -701,8 +624,7 @@ func (s *ELBv2ServiceImpl) updateStoredConfig(lb *LoadBalancerRecord) error {
lb.ConfigText = configContent
lb.ConfigHash = hash
lb.CertFiles = certFiles
// NLBs run nginx, which has no active upstream probing — the agent
// probes these targets and reports health. ALBs report via HAProxy stats.
// NLBs: the agent actively probes targets (nginx has no upstream probing).
if lb.Type == LoadBalancerTypeNetwork {
lb.HealthTargets = buildNLBHealthTargets(tgByArn)
} else {
@@ -719,9 +641,8 @@ func (s *ELBv2ServiceImpl) updateStoredConfig(lb *LoadBalancerRecord) error {
return nil
}
// resolveListenerCerts resolves every distinct certificate ARN referenced by
// the listeners to its combined PEM via the ACM store. Returns nil when no
// listener carries a certificate (the common HTTP-only case).
// resolveListenerCerts resolves each distinct certificate ARN to its combined PEM.
// Returns nil for HTTP-only listeners.
func (s *ELBv2ServiceImpl) resolveListenerCerts(listeners []*ListenerRecord, accountID string) (map[string]string, error) {
var out map[string]string
for _, l := range listeners {
@@ -742,10 +663,8 @@ func (s *ELBv2ServiceImpl) resolveListenerCerts(listeners []*ListenerRecord, acc
return out, nil
}
// resolveCertPEM loads a certificate by ARN from the ACM store and returns its
// combined PEM (leaf + chain + key) in the order HAProxy `ssl crt` expects.
// Ownership is enforced: a cert belonging to another account is treated as
// absent.
// resolveCertPEM loads a certificate from ACM and returns its combined PEM
// (leaf + chain + key). Cross-account certs are treated as absent.
func (s *ELBv2ServiceImpl) resolveCertPEM(arn, accountID string) (string, error) {
if s.acmStore == nil {
return "", errors.New(awserrors.ErrorELBv2CertificateNotFound)
@@ -770,13 +689,9 @@ func (s *ELBv2ServiceImpl) resolveCertPEM(arn, accountID string) (string, error)
return b.String(), nil
}
// validateListenerCerts confirms every certificate ARN resolves in the ACM
// store and is owned by the account, rejecting an unknown ARN at the API
// boundary (CreateListener/ModifyListener) with ErrorELBv2CertificateNotFound.
// Without this, an unresolvable ARN is accepted into the listener and only
// fails later at config-render time, which aborts updateStoredConfig wholesale
// and silently freezes the LB's data-plane convergence. Skipped when the ACM
// store is unavailable (cannot validate; matches the nil-safe resolve path).
// validateListenerCerts confirms every certificate ARN resolves in the ACM store
// and is owned by the account. Rejects at the API boundary to avoid silently
// freezing data-plane convergence at config-render time.
func (s *ELBv2ServiceImpl) validateListenerCerts(certs []ListenerCertificate, accountID string) error {
if s.acmStore == nil {
return nil
@@ -789,9 +704,8 @@ func (s *ELBv2ServiceImpl) validateListenerCerts(certs []ListenerCertificate, ac
return nil
}
// configCertHash hashes the rendered config plus every delivered cert file
// (path + content, in sorted path order) so a cert rotation changes the hash
// and triggers an agent reload even when the config text is unchanged.
// configCertHash hashes the config and all cert files so a cert rotation triggers
// an agent reload even when the config text is unchanged.
func configCertHash(configContent string, certFiles map[string]string) string {
h := sha256.New()
h.Write([]byte(configContent))
@@ -827,7 +741,7 @@ func (s *ELBv2ServiceImpl) updateStoredConfigForTargetGroup(tgArn string) error
}
}
// Rules can independently forward to a TG even if the listener default does not.
// Rules may forward to a TG even if the listener default action does not.
allRules, rulesErr := s.store.ListRules()
if rulesErr != nil {
slog.Error("updateStoredConfigForTargetGroup: failed to list rules", "err", rulesErr)
@@ -860,10 +774,8 @@ func (s *ELBv2ServiceImpl) updateStoredConfigForTargetGroup(tgArn string) error
return nil
}
// LBAgentHeartbeat processes a heartbeat from an LB agent. On first heartbeat,
// transitions LB from provisioning → active. Always updates LastHeartbeat and
// processes the health report. Returns the current config hash so the agent
// knows whether to fetch new config.
// LBAgentHeartbeat processes a heartbeat from an LB agent. On first heartbeat
// transitions LB provisioning→active, processes health, and returns the config hash.
func (s *ELBv2ServiceImpl) LBAgentHeartbeat(input *LBAgentHeartbeatInput, accountID string) (*LBAgentHeartbeatOutput, error) {
if input.LBID == nil || *input.LBID == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -877,10 +789,7 @@ func (s *ELBv2ServiceImpl) LBAgentHeartbeat(input *LBAgentHeartbeatInput, accoun
return nil, errors.New(awserrors.ErrorServerInternal)
}
if lb == nil || (lb.AccountID != accountID && accountID != utils.GlobalAccountID) {
// The heartbeat reached AWSGW (so the mgmt return path works) but
// cannot transition any LB. Without this the reject is silent and a
// stuck-in-provisioning LB looks identical to one whose heartbeat
// never arrived.
// Log to distinguish a stuck-in-provisioning LB from one whose heartbeat never arrived.
slog.Warn("LBAgentHeartbeat: LB not found or account mismatch",
"lbId", lbID, "accountId", accountID, "found", lb != nil)
return nil, errors.New(awserrors.ErrorELBv2LoadBalancerNotFound)
@@ -900,21 +809,14 @@ func (s *ELBv2ServiceImpl) LBAgentHeartbeat(input *LBAgentHeartbeatInput, accoun
switch {
case stateChanged:
// Activation is the LB's readiness signal: InstanceID is set and the
// agent is alive to consume config. The reactive updateStoredConfig
// calls during the create burst no-op while the VM is still launching
// (InstanceID empty), so build the data-plane config now from the
// listeners/targets created during provisioning. updateStoredConfig
// persists the record (state, config, hash, health targets) in one
// write, so the ConfigHash returned below is fresh.
// Build data-plane config on activation: reactive calls during create
// no-op while InstanceID is empty, so we build it now from existing listeners/targets.
if err := s.updateStoredConfig(lb); err != nil {
slog.Error("LBAgentHeartbeat: failed to build config on activation", "lbId", lbID, "err", err)
return nil, errors.New(awserrors.ErrorServerInternal)
}
case heartbeatStale:
// Steady state: refresh the heartbeat timestamp only. Avoids writing
// the full record (including ConfigText) every 5 seconds when nothing
// changed.
// Refresh the heartbeat timestamp only; avoid writing the full record on every tick.
if err := s.store.PutLoadBalancer(lb); err != nil {
slog.Error("LBAgentHeartbeat: failed to persist LB", "lbId", lbID, "err", err)
return nil, errors.New(awserrors.ErrorServerInternal)
@@ -975,8 +877,7 @@ func healthTargetsToSDK(specs []HealthTargetSpec) []*HealthTarget {
return out
}
// certFilesToSDK converts the stored path→PEM map into the sorted CertFile
// slice delivered to the agent. Sorted for deterministic output.
// certFilesToSDK converts the stored path→PEM map into a sorted CertFile slice.
func certFilesToSDK(certFiles map[string]string) []*CertFile {
if len(certFiles) == 0 {
return nil
@@ -1031,11 +932,9 @@ func (s *ELBv2ServiceImpl) resolveTargetIP(instanceID, accountID string) string
return ""
}
// resolveRegisteredTargetIP returns the private IP to route to for a target
// being registered, based on the target group's target type. For ip targets the
// supplied ID is the IP itself; for instance targets it is resolved via the
// instance's primary ENI. Returns an error when the ID's shape does not match
// the target type.
// resolveRegisteredTargetIP returns the private IP for a target being registered.
// For ip-type targets the ID is the IP; for instance targets it is resolved via
// the primary ENI. Returns an error if the ID doesn't match the target type.
func (s *ELBv2ServiceImpl) resolveRegisteredTargetIP(targetType, id, accountID string) (string, error) {
switch targetType {
case TargetTypeIP:
@@ -1071,12 +970,8 @@ const (
elbv2ResourceListenerRule = "listener-rule"
)
// elbv2ResourceTypeFromArn extracts the resource type from an ELBv2 ARN.
// ELBv2 ARNs have the form
// "arn:aws:elasticloadbalancing:{region}:{account}:{type}/...". Returns one
// of "loadbalancer", "targetgroup", "listener", "listener-rule" — anything
// else yields ErrorInvalidParameterValue so callers can surface a proper API
// error.
// elbv2ResourceTypeFromArn extracts the resource type from an ELBv2 ARN,
// returning one of "loadbalancer", "targetgroup", "listener", "listener-rule".
func elbv2ResourceTypeFromArn(arn string) (string, error) {
parts := strings.SplitN(arn, ":", 6)
if len(parts) < 6 || parts[0] != "arn" || parts[2] != "elasticloadbalancing" {
@@ -1096,9 +991,7 @@ func elbv2ResourceTypeFromArn(arn string) (string, error) {
}
}
// isCompatibleProtocol checks whether a listener protocol is compatible with a
// target group protocol. ALB: HTTP/HTTPS listeners can forward to HTTP or HTTPS
// TGs. NLB: TCP→TCP, UDP→UDP, TLS→TCP or TLS, TCP_UDP→TCP_UDP.
// isCompatibleProtocol reports whether a listener protocol is compatible with a target group protocol.
func isCompatibleProtocol(listenerProto, tgProto string) bool {
switch listenerProto {
case ProtocolHTTP, ProtocolHTTPS:
@@ -1166,11 +1059,8 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
}
dnsName := fmt.Sprintf("%s%s-%s.%s.elb.spinifex.local", dnsPrefix, name, lbID, s.region)
// Atomically claim the LB name before any ENI/VM work. A duplicate
// CreateLoadBalancer (SDK retry / gateway re-publish that spawned a second
// handler) loses the claim and never launches an LB VM; a claim left by a
// crashed prior create (no live record) is reclaimed. Every failure path
// below releases the claim so a failed create does not block the name.
// Atomically claim the name before ENI/VM work. SDK retries lose the claim;
// orphaned claims from crashed creates are reclaimed. Every failure releases it.
claimOK, claimDup, claimErr := s.store.ClaimLBName(name, accountID, lbID)
if claimErr != nil {
slog.Error("CreateLoadBalancer: name claim failed", "name", name, "err", claimErr)
@@ -1205,10 +1095,9 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
vpcID := ""
var nlbManagedSGID string
if s.VPCService != nil && len(subnets) > 0 {
// NLBs reject customer SGs (rejected above), so the ENIs would fall back
// to the VPC default SG whose intra-SG-only ingress drops inbound
// listener traffic. Mint a dedicated managed SG up front and attach it to
// every LB ENI; CreateListener opens the listener ports on it.
// NLBs reject customer SGs, so mint a dedicated managed SG for all LB ENIs;
// CreateListener opens listener ports on it. Without this, ENIs fall back
// to the VPC default SG and inbound listener traffic is dropped.
eniGroups := securityGroups
if lbType == LoadBalancerTypeNetwork {
sgID, sgErr := s.createNLBManagedSG(lbID, lbArn, subnets[0], accountID)
@@ -1234,18 +1123,15 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
},
},
}
// SG enforcement gates inbound traffic by the ENI's port-group
// membership, which is set at ENI creation. For NLBs this is the
// managed SG minted above; for ALBs it is the customer SGs. Empty
// Groups intentionally falls back to the VPC default SG inside
// eni.CreateNetworkInterface.
// SG enforcement gates inbound traffic via ENI port-group membership set at
// creation. NLBs use the managed SG; ALBs use customer SGs. Empty Groups
// falls back to the VPC default SG inside CreateNetworkInterface.
if len(eniGroups) > 0 {
eniIn.Groups = aws.StringSlice(eniGroups)
}
eniOut, eniErr := s.VPCService.CreateNetworkInterface(eniIn, accountID)
if eniErr != nil {
// All-or-nothing: tear down the ENIs/SG already created so a
// failed create leaks nothing and frees the name claim.
// Rollback so a partial ENI creation doesn't leak resources.
s.rollbackLBInfra(eniIDs, nlbManagedSGID, name, accountID)
slog.Error("CreateLoadBalancer: failed to create ENI", "subnet", subnetID, "err", eniErr)
return nil, errors.New(awserrors.ErrorELBv2SubnetNotFound)
@@ -1268,24 +1154,14 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
}
}
// Determine the data-plane state synchronously. When a launcher is configured
// the LB is backed by a system VM whose boot is a multi-minute job: running it
// inline head-of-line blocks the shared gateway responder and the gateway
// times out and re-publishes the create (267.4). So the VM boot runs on a
// background goroutine and the create returns provisioning immediately, which
// matches AWS (CreateLoadBalancer returns the LB provisioning; the AZ/VpcId in
// the response come from the ENIs created synchronously above). Launcher-less
// deployments (no InstanceLauncher / no ENIs) have no data plane to boot and
// are active on the spot.
// VM boot is async to avoid blocking the gateway responder. The create
// returns provisioning immediately; launcher-less deployments are active on the spot.
state := StateActive
willLaunch := s.InstanceLauncher != nil && len(eniIDs) > 0 && len(subnets) > 0
if willLaunch {
state = StateProvisioning
// Internal LBs only leave provisioning once the lb-agent heartbeats,
// which needs the mgmt return route. If it can't be resolved the LB
// hangs in provisioning forever — record the failed state up front so the
// broken return path is visible immediately. The VM still boots (matching
// the synchronous path) so the data plane exists for diagnosis.
// Internal LBs without a mgmt return route would hang in provisioning;
// mark failed immediately so the broken path is visible.
if scheme == SchemeInternal {
if gw, tgt := s.resolveMgmtRoute(scheme); gw == "" || tgt == "" {
slog.Error("CreateLoadBalancer: internal LB has no mgmt return route; marking failed (lb-agent cannot heartbeat AWSGW)",
@@ -1297,11 +1173,8 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
}
}
// Attributes are intentionally left nil — DescribeLoadBalancerAttributes
// derives per-type defaults from lb.Type via DefaultLoadBalancerAttributes.
// Callers that want a non-default value must call ModifyLoadBalancerAttributes.
// InstanceID/VPCIP/HostPorts and the per-AZ PublicIP are filled in by the
// asynchronous launch below once the VM boots.
// Attributes left nil — defaults derived from lb.Type on read.
// InstanceID/VPCIP/HostPorts are filled by the async launch.
record := &LoadBalancerRecord{
LoadBalancerArn: lbArn,
LoadBalancerID: lbID,
@@ -1325,15 +1198,13 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
if err := s.store.PutLoadBalancer(record); err != nil {
slog.Error("CreateLoadBalancer: failed to persist record", "lbId", lbID, "err", err)
// The ENIs and managed SG exist but no record was persisted to own them —
// roll them back so they are not orphaned.
// Rollback so unowned ENIs/SG don't leak.
s.rollbackLBInfra(eniIDs, nlbManagedSGID, name, accountID)
return nil, errors.New(awserrors.ErrorServerInternal)
}
if willLaunch {
// Snapshot the launch inputs; the goroutine must not read the record the
// caller is about to return.
// Snapshot inputs so the goroutine doesn't alias the caller's record.
lc := lbLaunchCtx{lbID: lbID, lbArn: lbArn, scheme: scheme, eniIDs: eniIDs, subnets: subnets, accountID: accountID}
s.launchWG.Go(func() {
defer func() {
@@ -1346,8 +1217,6 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
})
}
// Agent heartbeat will transition provisioning → active on first contact.
slog.Info("CreateLoadBalancer accepted", "name", name, "lbArn", lbArn, "enis", len(eniIDs), "state", state, "accountID", accountID)
return &elbv2.CreateLoadBalancerOutput{
@@ -1365,11 +1234,8 @@ type lbLaunchCtx struct {
accountID string
}
// launchLBVMAsync boots the LB-VM on a background goroutine and folds the result
// back into the persisted record: on success the instance/VPC-IP/host-ports (and
// the per-AZ public IP) are recorded and the LB stays provisioning until the
// lb-agent heartbeats; on failure the record is marked failed so the API
// reflects the broken data plane. Runs after CreateLoadBalancer has returned.
// launchLBVMAsync boots the LB-VM and folds the result into the persisted record.
// On failure marks the LB failed; on success stays provisioning until lb-agent heartbeats.
func (s *ELBv2ServiceImpl) launchLBVMAsync(lc lbLaunchCtx) {
launch := s.launchLBVM(lc.lbID, lc.scheme, lc.eniIDs, lc.subnets, lc.accountID)
@@ -1398,12 +1264,8 @@ func (s *ELBv2ServiceImpl) launchLBVMAsync(lc lbLaunchCtx) {
}
}
// rollbackLBInfra tears down the infrastructure a partial CreateLoadBalancer
// created before it could persist an owning record: every ENI minted so far, the
// NLB managed front-end SG (if any), and the name claim. Without this an
// ENI/SG-create-then-persist-fail path orphans those resources with no record
// for DeleteLoadBalancer or any reclaim to reach. Each step is best-effort and
// idempotent — a NotFound is success.
// rollbackLBInfra tears down ENIs, the NLB managed SG, and the name claim created
// by a partial CreateLoadBalancer. Each step is best-effort and idempotent.
func (s *ELBv2ServiceImpl) rollbackLBInfra(eniIDs []string, nlbManagedSGID, name, accountID string) {
if s.VPCService != nil {
for _, eniID := range eniIDs {
@@ -1418,8 +1280,7 @@ func (s *ELBv2ServiceImpl) rollbackLBInfra(eniIDs []string, nlbManagedSGID, name
s.releaseLBNameClaim(name, accountID)
}
// markLBFailed reloads the LB record and flips it to the failed state, used by
// the async launch panic recovery.
// markLBFailed reloads the LB record and flips it to the failed state.
func (s *ELBv2ServiceImpl) markLBFailed(lbArn string) {
record, err := s.store.GetLoadBalancerByArn(lbArn)
if err != nil || record == nil {
@@ -1442,17 +1303,11 @@ func (s *ELBv2ServiceImpl) DeleteLoadBalancer(input *elbv2.DeleteLoadBalancerInp
return nil, errors.New(awserrors.ErrorServerInternal)
}
if lb == nil || lb.AccountID != accountID {
// Idempotent: AWS ELBv2 delete returns success on an absent (or
// not-owned) load balancer, so tofu destroy retries converge.
// Idempotent: return success for absent/unowned LBs.
return &elbv2.DeleteLoadBalancerOutput{}, nil
}
// Delete all listeners (and their rules) for this LB before tearing down
// backing artifacts. Cascade through the shared helper, not
// store.DeleteListener directly, so rules don't survive and pin their
// target groups as ResourceInUse after the LB is gone. A list or cascade
// failure aborts the delete with the record intact, so tofu retries
// converge instead of leaving orphaned rules behind.
// Cascade-delete all listeners and their rules so no orphan pins a TG as ResourceInUse.
listeners, err := s.store.ListListenersByLB(lb.LoadBalancerArn)
if err != nil {
slog.Error("DeleteLoadBalancer: failed to list listeners for cascade", "lbArn", lb.LoadBalancerArn, "err", err)
@@ -1465,16 +1320,10 @@ func (s *ELBv2ServiceImpl) DeleteLoadBalancer(input *elbv2.DeleteLoadBalancerInp
}
}
// Reap the LB VM's floating-IP dnat_and_snat deterministically. The VM/ENI
// teardown paths reap it only conditionally — DeleteNetworkInterface defers
// EIP-owned addresses to the EIP lifecycle, and that lifecycle's reap runs
// in the async terminate goroutine below — so without this, stale rules
// accumulate on the VPC router across repeated create/delete cycles.
// Reap floating-IP NAT now; conditional VM/ENI teardown paths can miss it.
s.reapFloatingIPNAT(lb)
// Terminate ALB VM in background (VM termination also cleans up tap device).
// Must be async because VM shutdown can take seconds (QMP powerdown + QEMU exit)
// and we don't want to block the NATS request handler.
// Terminate ALB VM in background; shutdown can take seconds (QMP powerdown).
if lb.InstanceID != "" && s.InstanceLauncher != nil {
instanceID := lb.InstanceID
go func() {
@@ -1514,20 +1363,16 @@ func (s *ELBv2ServiceImpl) DeleteLoadBalancer(input *elbv2.DeleteLoadBalancerInp
return &elbv2.DeleteLoadBalancerOutput{}, nil
}
// releaseLBNameClaim drops the per-account LB name claim. Failure is logged but
// not fatal: a leaked claim is reclaimed by the crash-orphan path on the next
// same-name create (its owner lbID resolves to no live record).
// releaseLBNameClaim drops the per-account LB name claim. Failures are logged but
// not fatal; leaked claims are reclaimed by the crash-orphan path on next create.
func (s *ELBv2ServiceImpl) releaseLBNameClaim(name, accountID string) {
if err := s.store.ReleaseLBName(name, accountID); err != nil {
slog.Warn("failed to release LB name claim", "name", name, "accountID", accountID, "err", err)
}
}
// reapFloatingIPNAT removes the OVN dnat_and_snat rule mapping an
// internet-facing LB's floating (public) IP to its VM's VPC IP. DeleteEIP
// matches on external_ip+logical_ip and is idempotent, so a redundant publish
// from the EIP/ENI teardown paths is harmless. Internal LBs carry no floating
// IP and are a no-op.
// reapFloatingIPNAT removes the OVN dnat_and_snat rule for an internet-facing
// LB's floating IP. Idempotent; internal LBs are a no-op.
func (s *ELBv2ServiceImpl) reapFloatingIPNAT(lb *LoadBalancerRecord) {
publicIP := ""
for _, az := range lb.AvailZones {
@@ -1785,15 +1630,11 @@ func (s *ELBv2ServiceImpl) DeleteTargetGroup(input *elbv2.DeleteTargetGroupInput
return nil, errors.New(awserrors.ErrorServerInternal)
}
if tg == nil || tg.AccountID != accountID {
// Idempotent: AWS ELBv2 delete returns success on an absent (or
// not-owned) target group, so tofu destroy retries converge.
// Idempotent: return success for absent/unowned TGs.
return &elbv2.DeleteTargetGroupOutput{}, nil
}
// Only a *live* listener or rule pins the target group. A listener (or
// rule) whose owning load balancer no longer exists is treated as already
// torn down, not a live reference, so an orphan left by a partial teardown
// can never pin this target group as ResourceInUse permanently.
// Only live listeners/rules (whose LB still exists) pin the TG as ResourceInUse.
lbs, err := s.store.ListLoadBalancers()
if err != nil {
slog.Error("DeleteTargetGroup: failed to list load balancers for in-use check", "err", err)
@@ -2074,10 +1915,8 @@ func (s *ELBv2ServiceImpl) DescribeTargetHealth(input *elbv2.DescribeTargetHealt
// --- Listener operations ---
// listenerActionFromSDK converts an SDK listener action into the stored form,
// preserving fixed-response config so HAProxy generation and tofu read-back see
// the full action (a forward-only conversion drops the fixed-response default
// and produces a dangling backend + perpetual plan diff).
// listenerActionFromSDK converts an SDK listener action to stored form,
// preserving fixed-response/redirect config to avoid dangling backends.
func listenerActionFromSDK(a *elbv2.Action) ListenerAction {
action := ListenerAction{}
if a.Type != nil {
@@ -2124,11 +1963,8 @@ func listenerActionFromSDK(a *elbv2.Action) ListenerAction {
return action
}
// validateListenerAction enforces the per-type action contract for listener
// default actions and rule actions. Forward actions are validated against the
// target group elsewhere; this covers redirect (status code + render-safe
// fields) so invalid input fails at the API instead of being silently
// defaulted by the renderer.
// validateListenerAction enforces the per-type action contract. Redirect fields
// are validated here so invalid input fails at the API, not at render time.
func validateListenerAction(a ListenerAction) error {
if a.Type == ActionTypeRedirect {
if a.Redirect == nil {
@@ -2139,9 +1975,7 @@ func validateListenerAction(a ListenerAction) error {
return nil
}
// validateRedirectAction rejects an unsupported status code or any field that
// would break the HAProxy redirect directive once the known AWS placeholders
// are stripped.
// validateRedirectAction rejects an unsupported status code or render-unsafe fields.
func validateRedirectAction(rd *RedirectAction) error {
if rd == nil {
return errors.New(awserrors.ErrorMissingParameter)
@@ -2159,11 +1993,8 @@ func validateRedirectAction(rd *RedirectAction) error {
return nil
}
// buildListenerCertificates validates and normalises the certificate set and
// SSL policy for a listener of the given protocol. Secure protocols (HTTPS,
// TLS) require at least one certificate and receive a default SslPolicy when
// unset; non-secure protocols must carry neither certificates nor an SslPolicy.
// Exactly one certificate is marked default.
// buildListenerCertificates validates certificates and SSL policy for a listener.
// Secure protocols require exactly one default cert and receive a default SslPolicy when unset.
func buildListenerCertificates(protocol string, in []*elbv2.Certificate, sslPolicy *string) ([]ListenerCertificate, string, error) {
policy := ""
if sslPolicy != nil {
@@ -2361,11 +2192,9 @@ func (s *ELBv2ServiceImpl) CreateListener(input *elbv2.CreateListenerInput, acco
}, nil
}
// rollbackListener removes a just-persisted listener whose post-persist
// data-plane wiring failed, so a failed CreateListener leaves no orphan record
// and no dangling SG authorization. authorizedCIDRs is non-empty only when the
// NLB front-end port was already opened (config-update failure path), in which
// case the port is revoked first. Best-effort: cleanup errors are logged.
// rollbackListener removes a listener persisted before post-persist wiring failed.
// authorizedCIDRs is non-empty when the NLB port was already opened — it is
// revoked first. Cleanup errors are logged but not returned.
func (s *ELBv2ServiceImpl) rollbackListener(record *ListenerRecord, lb *LoadBalancerRecord, protocol string, port int64, authorizedCIDRs []string, accountID string) {
if len(authorizedCIDRs) > 0 {
if err := s.revokeNLBListenerPort(lb, protocol, port, authorizedCIDRs, accountID); err != nil {
@@ -2407,8 +2236,7 @@ func (s *ELBv2ServiceImpl) DeleteListener(input *elbv2.DeleteListenerInput, acco
return nil, errors.New(awserrors.ErrorServerInternal)
}
if listener == nil || listener.AccountID != accountID {
// Idempotent: AWS ELBv2 delete returns success on an absent (or
// not-owned) listener, so tofu destroy retries converge.
// Idempotent: return success for absent/unowned listeners.
return &elbv2.DeleteListenerOutput{}, nil
}
@@ -2548,9 +2376,7 @@ func (s *ELBv2ServiceImpl) ModifyListener(input *elbv2.ModifyListenerInput, acco
}
}
// Certificates / SSL policy. The effective protocol after any change drives
// the requirement: switching to a non-secure protocol clears cert material;
// staying on or switching to a secure protocol validates and defaults it.
// Cert/policy: switching to non-secure clears material; staying/switching to secure validates it.
if !protocolRequiresCert(updated.Protocol) {
if len(input.Certificates) > 0 || (input.SslPolicy != nil && *input.SslPolicy != "") {
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
@@ -2639,12 +2465,8 @@ func (s *ELBv2ServiceImpl) DescribeListeners(input *elbv2.DescribeListenersInput
// --- Tag operations ---
// DescribeTags returns tags for one or more ELBv2 resources (load balancers,
// target groups, listeners, listener rules). Tag data is read from the existing
// record stores — Spinifex doesn't have a separate tag KV. Untagged resources
// return an empty Tags slice (matches AWS behaviour). Cross-account or unknown
// ARNs return the per-resource not-found error so existence isn't leaked across
// accounts.
// DescribeTags returns tags for ELBv2 resources read from the record stores.
// Cross-account or unknown ARNs return a not-found error.
func (s *ELBv2ServiceImpl) DescribeTags(input *elbv2.DescribeTagsInput, accountID string) (*elbv2.DescribeTagsOutput, error) {
if input == nil || len(input.ResourceArns) == 0 {
return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -2734,8 +2556,7 @@ func (s *ELBv2ServiceImpl) DescribeTags(input *elbv2.DescribeTagsInput, accountI
}, nil
}
// tagsFromSDK converts an SDK Tag slice into a tag map, skipping entries with a
// nil key or value. Returns a non-nil (possibly empty) map for storage.
// tagsFromSDK converts an SDK Tag slice into a tag map, skipping nil key/value entries.
func tagsFromSDK(tags []*elbv2.Tag) map[string]string {
m := make(map[string]string, len(tags))
for _, tag := range tags {
@@ -2746,9 +2567,7 @@ func tagsFromSDK(tags []*elbv2.Tag) map[string]string {
return m
}
// tagsMapToSDK converts a tag map into the SDK Tag slice with deterministic
// (sorted-by-key) ordering. Returns nil for empty/nil input so the response
// matches AWS for untagged resources.
// tagsMapToSDK converts a tag map into a sorted SDK Tag slice; returns nil for empty input.
func tagsMapToSDK(tags map[string]string) []*elbv2.Tag {
if len(tags) == 0 {
return nil
@@ -2784,9 +2603,7 @@ func (s *ELBv2ServiceImpl) lbRecordToSDK(r *LoadBalancerRecord) *elbv2.LoadBalan
lb.SecurityGroups = append(lb.SecurityGroups, aws.String(sg))
}
// Build a subnet → private IP map by looking up each ENI. This lets each
// AvailabilityZone entry surface the private IP of the ENI that lives in
// its subnet, not just the public IP recorded at create time.
// Build subnet → private IP map so each AZ entry surfaces the ENI's private IP.
privateIPBySubnet := map[string]string{}
if s.VPCService != nil && len(r.ENIs) > 0 {
eniPtrs := make([]*string, 0, len(r.ENIs))
@@ -2873,9 +2690,8 @@ func (s *ELBv2ServiceImpl) listenerRecordToSDK(r *ListenerRecord) *elbv2.Listene
return listener
}
// listenerActionToSDK converts a stored action into the AWS SDK shape,
// preserving forward / fixed-response / redirect detail so DescribeListeners
// and DescribeRules round-trip cleanly. Shared by listeners and rules.
// listenerActionToSDK converts a stored action to the AWS SDK shape.
// Shared by listeners and rules; preserves all action types for round-trip fidelity.
func listenerActionToSDK(a ListenerAction) *elbv2.Action {
action := &elbv2.Action{Type: aws.String(a.Type)}
if a.TargetGroupArn != "" {
spinifex/handlers/elbv2/service_impl_attributes.go
+3 -4
View File
@@ -50,10 +50,9 @@ func (r attrResource[R]) load() (*R, error) {
return rec, nil
}
// modifyResourceAttributes validates and merges submitted attributes into the
// record, persisting only when something actually changed. rawCount is the
// number of attributes the caller submitted before nil filtering, used to
// distinguish "nothing sent" from "everything sent was invalid".
// modifyResourceAttributes validates and merges submitted attributes, persisting only
// on change. rawCount is pre-nil-filter count, used to distinguish "nothing sent"
// from "everything sent was invalid".
func modifyResourceAttributes[R any](res attrResource[R], pairs []attrPair, rawCount int) ([]attrPair, error) {
if res.arn == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
spinifex/handlers/elbv2/service_impl_health_test.go
+3 -4
View File
@@ -161,10 +161,9 @@ func TestRegisterTargets_MultipleInstances(t *testing.T) {
assert.Equal(t, "10.0.1.51", tg.Targets[1].PrivateIP)
}
// TestResetTargetHealthOnStartup_TransitionsAllNonDrainingToInitial verifies
// that the startup reset rewrites every non-draining target's HealthState
// back to "initial" so a daemon restart can't surface stale "healthy" claims
// before the lb-agent has posted a fresh report.
// TestResetTargetHealthOnStartup_TransitionsAllNonDrainingToInitial verifies that
// startup resets non-draining targets to "initial", preventing stale "healthy"
// claims before the lb-agent has posted a fresh report after a daemon restart.
func TestResetTargetHealthOnStartup_TransitionsAllNonDrainingToInitial(t *testing.T) {
store := setupTestNATS(t)
svc := &ELBv2ServiceImpl{store: store}
spinifex/handlers/elbv2/service_impl_lb_network.go
+18 -47
View File
@@ -11,9 +11,7 @@ import (
"github.com/mulgadc/spinifex/spinifex/awserrors"
)
// SetIpAddressType sets the load balancer's IP address type. Spinifex ALBs are
// IPv4-only, so the only accepted value is "ipv4"; dualstack variants are
// rejected with InvalidConfigurationRequest. The call is idempotent.
// SetIpAddressType sets the LB IP address type; only "ipv4" is accepted.
func (s *ELBv2ServiceImpl) SetIpAddressType(input *elbv2.SetIpAddressTypeInput, accountID string) (*elbv2.SetIpAddressTypeOutput, error) {
if input == nil || input.LoadBalancerArn == nil || *input.LoadBalancerArn == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -47,11 +45,8 @@ func (s *ELBv2ServiceImpl) SetIpAddressType(input *elbv2.SetIpAddressTypeInput,
}, nil
}
// SetSecurityGroups replaces the security groups associated with an
// (application) load balancer. The new groups are re-attached to every ENI the
// ALB spans via ModifyNetworkInterfaceAttribute, which validates them against
// the ENI's VPC and pushes the change to the live data-plane port before the
// record is persisted.
// SetSecurityGroups replaces the security groups on an ALB, re-attaching them
// to every ENI via ModifyNetworkInterfaceAttribute before persisting.
func (s *ELBv2ServiceImpl) SetSecurityGroups(input *elbv2.SetSecurityGroupsInput, accountID string) (*elbv2.SetSecurityGroupsOutput, error) {
if input == nil || input.LoadBalancerArn == nil || *input.LoadBalancerArn == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -82,10 +77,7 @@ func (s *ELBv2ServiceImpl) SetSecurityGroups(input *elbv2.SetSecurityGroupsInput
sgs = append(sgs, *sg)
}
// Re-attach the new groups to each ALB ENI. This validates the groups
// against the ENI's VPC and fires the live port-SG update; a failure here
// (e.g. unknown SG) aborts before the record is persisted. All ENIs share
// the LB's VPC and groups, so a successful first apply implies the rest.
// Re-attach to each ENI; failure aborts before the record is persisted.
if s.VPCService != nil {
for _, eniID := range lb.ENIs {
if _, err := s.VPCService.ModifyNetworkInterfaceAttribute(&ec2.ModifyNetworkInterfaceAttributeInput{
@@ -109,13 +101,8 @@ func (s *ELBv2ServiceImpl) SetSecurityGroups(input *elbv2.SetSecurityGroupsInput
}, nil
}
// SetSubnets enables the subnets (and their backing ENIs) for a load balancer.
// It is a full add+remove: a subnet in the request but not on the LB gets a new
// managed ENI; a subnet on the LB but absent from the request has its ENI
// removed. The system VM binds its data-plane taps at boot and the launcher
// exposes no live ENI hotplug, so the new ENI set is applied by relaunching the
// LB VM — a brief data-plane interruption. The record is mutated only after the
// new ENIs exist and the VM has been relaunched.
// SetSubnets does a full add+remove of the LB's subnets and their ENIs.
// Because ENI hotplug is not supported, the LB VM is relaunched with the new ENI set.
func (s *ELBv2ServiceImpl) SetSubnets(input *elbv2.SetSubnetsInput, accountID string) (*elbv2.SetSubnetsOutput, error) {
if input == nil || input.LoadBalancerArn == nil || *input.LoadBalancerArn == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -157,8 +144,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(input *elbv2.SetSubnetsInput, accountID st
return s.setSubnetsOutput(lb), nil // idempotent — no change
}
// Without a VPC service we cannot manage ENIs; record the requested subnet
// set so the API stays consistent for launcher-less / test deployments.
// No VPC service: just record the subnet set (launcher-less / test deployments).
if s.VPCService == nil {
lb.Subnets = desired
lb.AvailZones = rebuildAvailZones(desired, lb.AvailZones, nil)
@@ -169,8 +155,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(input *elbv2.SetSubnetsInput, accountID st
return s.setSubnetsOutput(lb), nil
}
// Create ENIs for the added subnets first; roll them back on any failure so
// a partial apply never leaks ENIs or mutates the live LB.
// Create ENIs for added subnets; roll back on failure to avoid leaks.
newENIBySubnet := make(map[string]string, len(toAdd))
newAZBySubnet := make(map[string]string, len(toAdd))
rollbackNewENIs := func() {
@@ -203,8 +188,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(input *elbv2.SetSubnetsInput, accountID st
}
}
// Terminate the LB VM before reshaping its ENI set, then relaunch it on the
// new set.
// Terminate the LB VM before reshaping its ENI set, then relaunch on the new set.
if lb.InstanceID != "" && s.InstanceLauncher != nil {
if err := s.InstanceLauncher.TerminateSystemInstance(lb.InstanceID); err != nil {
rollbackNewENIs()
@@ -213,10 +197,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(input *elbv2.SetSubnetsInput, accountID st
}
}
// TerminateSystemInstance tears down the VM but leaves its ENIs marked
// in-use, so detach the existing ENIs explicitly: retained ones must be free
// to re-attach to the relaunched VM, and removed ones must be detached before
// they can be deleted.
// Detach all ENIs explicitly: TerminateSystemInstance doesn't clear in-use status.
for _, eniID := range current {
if detachErr := s.VPCService.DetachENI(accountID, eniID); detachErr != nil {
slog.Warn("SetSubnets: failed to detach ENI before relaunch", "eni", eniID, "err", detachErr)
@@ -256,8 +237,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(input *elbv2.SetSubnetsInput, accountID st
return s.setSubnetsOutput(lb), nil
}
// desiredSubnetSet flattens the request's Subnets and SubnetMappings into a
// de-duplicated, order-preserving subnet-ID list.
// desiredSubnetSet flattens Subnets and SubnetMappings into a deduplicated, ordered list.
func desiredSubnetSet(input *elbv2.SetSubnetsInput) []string {
seen := make(map[string]bool)
var out []string
@@ -280,9 +260,7 @@ func desiredSubnetSet(input *elbv2.SetSubnetsInput) []string {
return out
}
// subnetENIMap pairs each of the LB's current subnets with its backing ENI.
// CreateLoadBalancer and SetSubnets write Subnets and ENIs in lockstep, so the
// parallel arrays are authoritative for which ENI lives in which subnet.
// subnetENIMap pairs each current subnet with its ENI using the parallel Subnets/ENIs arrays.
func subnetENIMap(lb *LoadBalancerRecord) map[string]string {
m := make(map[string]string, len(lb.Subnets))
for i, sn := range lb.Subnets {
@@ -293,9 +271,7 @@ func subnetENIMap(lb *LoadBalancerRecord) map[string]string {
return m
}
// createLBENI creates a single managed ENI for the LB in the given subnet,
// tagged and security-grouped like the create-time ENIs. Returns the ENI ID and
// its availability zone.
// createLBENI creates a managed ENI in the given subnet, returning the ENI ID and AZ.
func (s *ELBv2ServiceImpl) createLBENI(subnetID string, lb *LoadBalancerRecord, accountID string) (eniID, az string, err error) {
eniIn := &ec2.CreateNetworkInterfaceInput{
SubnetId: aws.String(subnetID),
@@ -321,10 +297,8 @@ func (s *ELBv2ServiceImpl) createLBENI(subnetID string, lb *LoadBalancerRecord,
return aws.StringValue(eni.NetworkInterfaceId), aws.StringValue(eni.AvailabilityZone), nil
}
// rebuildAvailZones produces the AvailZoneInfo list for the new subnet set in
// order, preserving the existing zone name for retained subnets and using
// newAZBySubnet for added ones. PublicIP is cleared — it is re-derived from the
// relaunched VM by the caller.
// rebuildAvailZones builds the AZ list for the new subnet set, preserving existing
// zone names and using newAZBySubnet for additions. PublicIP is cleared.
func rebuildAvailZones(subnets []string, existing []AvailZoneInfo, newAZBySubnet map[string]string) []AvailZoneInfo {
bySubnet := make(map[string]AvailZoneInfo, len(existing))
for _, az := range existing {
@@ -341,9 +315,8 @@ func rebuildAvailZones(subnets []string, existing []AvailZoneInfo, newAZBySubnet
return out
}
// lbStateAfterLaunch mirrors CreateLoadBalancer's post-launch state gate: a VM
// that came up enters provisioning until the lb-agent heartbeats; an internal
// LB with no mgmt return route is failed loud; a failed launch is failed.
// lbStateAfterLaunch returns the post-launch state: provisioning if the VM came up,
// failed if the launch failed or if an internal LB has no mgmt return route.
func (s *ELBv2ServiceImpl) lbStateAfterLaunch(launch lbVMLaunch, scheme string) string {
if launch.instanceID == "" {
if launch.failed {
@@ -361,9 +334,7 @@ func (s *ELBv2ServiceImpl) lbStateAfterLaunch(launch lbVMLaunch, scheme string)
return StateProvisioning
}
// setSubnetsOutput builds the SetSubnets response from the persisted record,
// reusing the SDK availability-zone projection (which surfaces per-AZ private
// IPs) so the response matches DescribeLoadBalancers.
// setSubnetsOutput builds the SetSubnets response from the persisted record.
func (s *ELBv2ServiceImpl) setSubnetsOutput(lb *LoadBalancerRecord) *elbv2.SetSubnetsOutput {
sdk := s.lbRecordToSDK(lb)
return &elbv2.SetSubnetsOutput{
spinifex/handlers/elbv2/service_impl_listener_certs.go
+2 -4
View File
@@ -21,10 +21,8 @@ func listenerCertsToSDK(certs []ListenerCertificate) []*elbv2.Certificate {
return out
}
// AddListenerCertificates attaches additional (SNI) certificates to a secure
// listener. The default certificate is set at create time and is unaffected;
// added certificates are non-default. Re-adding an existing certificate is a
// no-op.
// AddListenerCertificates attaches additional (SNI) certificates to a secure listener.
// Added certificates are non-default; re-adding an existing certificate is a no-op.
func (s *ELBv2ServiceImpl) AddListenerCertificates(input *elbv2.AddListenerCertificatesInput, accountID string) (*elbv2.AddListenerCertificatesOutput, error) {
if input == nil || input.ListenerArn == nil || *input.ListenerArn == "" {
return nil, errors.New(awserrors.ErrorMissingParameter)

Some files were not shown because too many files have changed in this diff Show More