diff --git a/cmd/lb-agent/main.go b/cmd/lb-agent/main.go index ac6f5f593..0de7de776 100644 --- a/cmd/lb-agent/main.go +++ b/cmd/lb-agent/main.go @@ -49,12 +49,13 @@ func main() { slog.Error("--gateway is required (or set LB_GATEWAY_URL)") os.Exit(1) } - if accessKey == "" || secretKey == "" { - slog.Error("--access-key and --secret-key are required (or set LB_ACCESS_KEY / LB_SECRET_KEY)") - os.Exit(1) + // Empty keys select IMDS instance-role credentials (the SDK default chain + // falls back to the in-VM IMDS endpoint); static keys sign as-is. + credSource := "imds" + if accessKey != "" && secretKey != "" { + credSource = "static" } - - slog.Info("Starting LB agent", "lbId", lbID, "gateway", gatewayURL) + slog.Info("Starting LB agent", "lbId", lbID, "gateway", gatewayURL, "credSource", credSource) agent, err := lbagent.New(lbID, gatewayURL, accessKey, secretKey, region) if err != nil { diff --git a/go.mod b/go.mod index 049d4c40a..7268ee833 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,9 @@ go 1.26.4 require ( github.com/aws/aws-sdk-go v1.55.8 + github.com/aws/aws-sdk-go-v2 v1.42.0 + github.com/aws/aws-sdk-go-v2/config v1.32.26 + github.com/aws/aws-sdk-go-v2/credentials v1.19.25 github.com/charmbracelet/bubbles v1.0.0 github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/lipgloss v1.1.0 @@ -42,7 +45,16 @@ require ( github.com/antithesishq/antithesis-sdk-go v0.7.0 // indirect github.com/armon/go-metrics v0.4.1 // indirect github.com/atotto/clipboard v0.1.4 // indirect - github.com/aws/aws-sdk-go-v2 v1.42.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.2.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.31.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.43.4 // indirect github.com/aws/smithy-go v1.27.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index 1b41209bf..74df9105b 100644 --- a/go.sum +++ b/go.sum @@ -529,14 +529,18 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13/go.mod h1:8cIfkE9M github.com/aws/aws-sdk-go-v2/config v1.18.14/go.mod h1:0pI6JQBHKwd0JnwAZS3VCapLKMO++UL2BOkWwyyzTnA= github.com/aws/aws-sdk-go-v2/config v1.27.4/go.mod h1:zq2FFXK3A416kiukwpsd+rD4ny6JC7QSkp4QdN1Mp2g= github.com/aws/aws-sdk-go-v2/config v1.29.12/go.mod h1:xse1YTjmORlb/6fhkWi8qJh3cvZi4JoVNhc+NbJt4kI= +github.com/aws/aws-sdk-go-v2/config v1.32.26 h1:JI+W5B3jUA8UBz2ggbICGd9UCR6/+SB21G8EFl0SFTQ= +github.com/aws/aws-sdk-go-v2/config v1.32.26/go.mod h1:RLE2Ls/wRstvdSz1GPrIWNnXcKZ/znDdWyMuiQxdBoY= github.com/aws/aws-sdk-go-v2/credentials v1.13.14/go.mod h1:85ckagDuzdIOnZRwws1eLKnymJs3ZM1QwVC1XcuNGOY= github.com/aws/aws-sdk-go-v2/credentials v1.17.4/go.mod h1:+30tpwrkOgvkJL1rUZuRLoxcJwtI/OkeBLYnHxJtVe0= github.com/aws/aws-sdk-go-v2/credentials v1.17.65/go.mod h1:4zyjAuGOdikpNYiSGpsGz8hLGmUzlY8pc8r9QQ/RXYQ= -github.com/aws/aws-sdk-go-v2/credentials v1.19.24 h1:2hQqYCV9yqyePQ9o6dCrZc/zO8U3TwPr9mIKlZnPu/I= -github.com/aws/aws-sdk-go-v2/credentials v1.19.24/go.mod h1:IDwpACtwqHLISdzfwUUNq4P9DsB/h5BLg4FwJPNfqFY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.25 h1:TzPVjfUZ1hsKafvYE+DIzKXIik2KufQxsPHanlkttbo= +github.com/aws/aws-sdk-go-v2/credentials v1.19.25/go.mod h1:K4hw0buguVvtC74HnVfTRr0LzQQHAWPqJbBU9QGk2Pg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 h1:r6qZHbT+wxgWO/e9vYNUEtg7lv5+UN3pRqKhLXvnArg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29/go.mod h1:QRnaRcTVGKPGRy8w78HMQtKUGRYcnMZAANATkeVA6Mo= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2/go.mod h1:wRQv0nN6v9wDXuWThpovGQjqF1HFdcgWjporw14lS8k= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= @@ -567,15 +571,23 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29 h1:hiME6pBzC7OTl github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29/go.mod h1:G7RP+uhagpKtKhd1BM9N6JQqjCcGEU47K5lBVZQyRQw= github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0 h1:ta8csKy5vN91F3i5gGR85lFV0srBqySEji7Jroes6rE= github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0/go.mod h1:77ZAgynvx1txMvDG8gGWoWkO1augYDxkp9JElWFgjQU= +github.com/aws/aws-sdk-go-v2/service/signin v1.2.1 h1:BeJmkm5YOZs6lGRGcNoIuLSoTTtGLLCEqlSiRKYodfM= +github.com/aws/aws-sdk-go-v2/service/signin v1.2.1/go.mod h1:LxYujSTLPRlp2vTtcUO/+1ilrew8ytt6SvQyOgejzFQ= github.com/aws/aws-sdk-go-v2/service/sso v1.12.3/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw= github.com/aws/aws-sdk-go-v2/service/sso v1.20.1/go.mod h1:RsYqzYr2F2oPDdpy+PdhephuZxTfjHQe7SOBcZGoAU8= github.com/aws/aws-sdk-go-v2/service/sso v1.25.2/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.4 h1:i465b/3c7xJd++pobNIDOggouekCuiWOnB0goQJy+94= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.4/go.mod h1:Lk7PlmoTYryQmyBG0EXqj5BcUbj3whXdU2s3yGI3EAc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.3/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1/go.mod h1:YjAPFn4kGFqKC54VsHs5fn5B6d+PCY2tziEa3U/GB5Y= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.7 h1:xbmJAnBbyYPkTzoCNCF/bpJ6ymQHRdXX1vquYfDIGYk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.7/go.mod h1:Q5N6icH+KJZDLh+ESNwzdv6cZ6vLFF/egy3IOxWhmz4= github.com/aws/aws-sdk-go-v2/service/sts v1.18.4/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s= github.com/aws/aws-sdk-go-v2/service/sts v1.28.1/go.mod h1:uQ7YYKZt3adCRrdCBREm1CD3efFLOUNH77MrUCvx5oA= github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.4 h1:Np0vmL7op0Zs5xGacYMMX3v5O5pvZ46xhb5LwDgPj8M= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.4/go.mod h1:r8wkDOuLaaMFqFiYAb8dGY2A3gJCOujMc6CFOVC4Zhc= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= diff --git a/internal/eksgw/client.go b/internal/eksgw/client.go index 328a1987a..26a4d3f45 100644 --- a/internal/eksgw/client.go +++ b/internal/eksgw/client.go @@ -1,10 +1,11 @@ // Package eksgw is the on-VM SigV4 client used by EKS helpers (eks-gateway-publish, // eks-token-webhook) to POST over HTTPS to the AWS gateway instead of NATS. -// Signs with Predastore credentials for service "eks". +// Signs for service "eks" with either static keys or IMDS instance-role creds. package eksgw import ( "bytes" + "context" "crypto/sha256" "crypto/tls" "crypto/x509" @@ -16,33 +17,41 @@ import ( "strings" "time" - "github.com/mulgadc/predastore/auth" + "github.com/mulgadc/spinifex/internal/gwsign" "github.com/mulgadc/spinifex/internal/tlsconfig" ) // Client posts SigV4-signed requests to the AWS gateway for the "eks" service. type Client struct { baseURL string - accessKey string - secretKey string + signer *gwsign.Signer region string httpClient *http.Client } // New builds a client. caPath optionally pins the gateway TLS CA; when empty the // client relies on the system trust store. region defaults to us-east-1 when -// empty (SigV4 requires a non-empty region). +// empty (SigV4 requires a non-empty region). When accessKey/secretKey are empty +// the client signs with IMDS instance-role credentials served in-VM. func New(baseURL, caPath, accessKey, secretKey, region string) (*Client, error) { if baseURL == "" { return nil, fmt.Errorf("eksgw: baseURL is required") } - if accessKey == "" || secretKey == "" { - return nil, fmt.Errorf("eksgw: access key and secret key are required") - } if region == "" { region = "us-east-1" } + var signer *gwsign.Signer + if accessKey != "" && secretKey != "" { + signer = gwsign.NewStatic(accessKey, secretKey) + } else { + s, err := gwsign.NewIMDS(context.Background(), region) + if err != nil { + return nil, fmt.Errorf("eksgw: init IMDS signer: %w", err) + } + signer = s + } + tlsCfg := &tls.Config{ MinVersion: tls.VersionTLS13, CurvePreferences: tlsconfig.Curves, @@ -60,10 +69,9 @@ func New(baseURL, caPath, accessKey, secretKey, region string) (*Client, error) } return &Client{ - baseURL: strings.TrimRight(baseURL, "/"), - accessKey: accessKey, - secretKey: secretKey, - region: region, + baseURL: strings.TrimRight(baseURL, "/"), + signer: signer, + region: region, httpClient: &http.Client{ Timeout: 10 * time.Second, Transport: &http.Transport{TLSClientConfig: tlsCfg, MaxIdleConns: 2, IdleConnTimeout: 30 * time.Second}, @@ -94,7 +102,7 @@ func (c *Client) do(method, path string, body []byte) ([]byte, error) { } sum := sha256.Sum256(body) - if err := auth.SignReq(req, c.accessKey, c.secretKey, hex.EncodeToString(sum[:]), "eks", c.region); err != nil { + if err := c.signer.Sign(req, hex.EncodeToString(sum[:]), "eks", c.region); err != nil { return nil, fmt.Errorf("sign request: %w", err) } diff --git a/internal/gwsign/signer.go b/internal/gwsign/signer.go new file mode 100644 index 000000000..42e34f7d6 --- /dev/null +++ b/internal/gwsign/signer.go @@ -0,0 +1,58 @@ +// Package gwsign SigV4-signs on-VM requests to the AWS gateway. Credentials +// come from either fixed long-lived keys or the AWS SDK default credential +// chain, which resolves the in-VM IMDS instance-role credentials and rotates +// them automatically. A session token, when present, is signed as +// X-Amz-Security-Token so the gateway can verify temporary (ASIA) credentials. +package gwsign + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" +) + +// Signer holds a credential provider and signs requests in place. +type Signer struct { + provider aws.CredentialsProvider +} + +// NewStatic returns a Signer using fixed long-lived keys (no session token). +func NewStatic(accessKey, secretKey string) *Signer { + return &Signer{provider: credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")} +} + +// NewIMDS returns a Signer backed by the AWS SDK default credential chain. In a +// guest VM with no env/profile credentials the chain falls back to the IMDS +// endpoint (169.254.169.254), yielding scoped, auto-rotating instance-role +// credentials. region sets the chain's default region. +func NewIMDS(ctx context.Context, region string) (*Signer, error) { + cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) + if err != nil { + return nil, fmt.Errorf("load AWS default config: %w", err) + } + if cfg.Credentials == nil { + return nil, fmt.Errorf("default credential chain resolved no provider") + } + return &Signer{provider: cfg.Credentials}, nil +} + +// Sign signs r in place for service/region. payloadHash is the hex SHA-256 of +// the body (or a SigV4 sentinel). Credentials are retrieved per call so rotated +// IMDS credentials take effect without a restart; the SDK signer sets +// X-Amz-Security-Token when the credentials carry a session token. +func (s *Signer) Sign(r *http.Request, payloadHash, service, region string) error { + creds, err := s.provider.Retrieve(r.Context()) + if err != nil { + return fmt.Errorf("retrieve credentials: %w", err) + } + // The gateway recovers the payload hash from this header for non-S3 + // services; the SDK signer alone does not set it for direct callers. + r.Header.Set("X-Amz-Content-Sha256", payloadHash) + return v4.NewSigner().SignHTTP(r.Context(), creds, r, payloadHash, service, region, time.Now().UTC()) +} diff --git a/internal/gwsign/signer_test.go b/internal/gwsign/signer_test.go new file mode 100644 index 000000000..5ca6dbfc5 --- /dev/null +++ b/internal/gwsign/signer_test.go @@ -0,0 +1,94 @@ +package gwsign + +import ( + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/mulgadc/predastore/auth" +) + +const ( + testAccessKey = "AKIAIOSFODNN7EXAMPLE" + testSecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + testRegion = "us-east-1" + testService = "elasticloadbalancing" +) + +// TestSign_StaticVerifies confirms a statically-keyed Signer produces a request +// the gateway's SigV4 verifier accepts, with X-Amz-Content-Sha256 set to the +// body hash. +func TestSign_StaticVerifies(t *testing.T) { + body := []byte("Action=LBAgentHeartbeat") + sum := sha256.Sum256(body) + payloadHash := hex.EncodeToString(sum[:]) + + var verifyErr error + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + req, err := auth.ParseReq(r) + if err != nil { + verifyErr = err + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + verifyErr = req.Verify(testSecretKey, testService, testRegion, auth.WithBodyHash(payloadHash)) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + req, _ := http.NewRequest(http.MethodPost, srv.URL, strings.NewReader(string(body))) + if err := NewStatic(testAccessKey, testSecretKey).Sign(req, payloadHash, testService, testRegion); err != nil { + t.Fatalf("Sign: %v", err) + } + if got := req.Header.Get("X-Amz-Content-Sha256"); got != payloadHash { + t.Fatalf("X-Amz-Content-Sha256 = %q, want %q", got, payloadHash) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do: %v", err) + } + resp.Body.Close() + if verifyErr != nil { + t.Fatalf("server-side verify failed: %v", verifyErr) + } +} + +// TestSign_SessionTokenHeader confirms temporary credentials surface as a signed +// X-Amz-Security-Token header — the path the gateway uses to verify ASIA +// instance-role credentials. +func TestSign_SessionTokenHeader(t *testing.T) { + const token = "FwoGZXIvYXdzEXAMPLEsessiontoken" + s := &Signer{provider: credentials.NewStaticCredentialsProvider(testAccessKey, testSecretKey, token)} + + body := []byte("Action=LBAgentHeartbeat") + sum := sha256.Sum256(body) + req, _ := http.NewRequest(http.MethodPost, "https://gw:9999/", strings.NewReader(string(body))) + if err := s.Sign(req, hex.EncodeToString(sum[:]), testService, testRegion); err != nil { + t.Fatalf("Sign: %v", err) + } + + if got := req.Header.Get("X-Amz-Security-Token"); got != token { + t.Fatalf("X-Amz-Security-Token = %q, want %q", got, token) + } + authHdr := req.Header.Get("Authorization") + if !strings.Contains(authHdr, "x-amz-security-token") { + t.Fatalf("session token not in SignedHeaders: %q", authHdr) + } +} + +// TestNewIMDS_BuildsProvider confirms the IMDS constructor resolves a provider +// without performing network I/O (the chain retrieves lazily at first sign). +func TestNewIMDS_BuildsProvider(t *testing.T) { + s, err := NewIMDS(t.Context(), testRegion) + if err != nil { + t.Fatalf("NewIMDS: %v", err) + } + if s.provider == nil { + t.Fatal("expected non-nil credentials provider") + } +} diff --git a/scripts/images/eks-node/k3s-first-boot.sh b/scripts/images/eks-node/k3s-first-boot.sh index 72166df9f..156b4efc6 100755 --- a/scripts/images/eks-node/k3s-first-boot.sh +++ b/scripts/images/eks-node/k3s-first-boot.sh @@ -48,7 +48,9 @@ set -a . "${ENVFILE}" set +a -for v in EKS_GATEWAY_URL EKS_ACCESS_KEY EKS_SECRET_KEY EKS_ACCOUNT_ID EKS_CLUSTER_NAME EKS_NLB_ENDPOINT; do +# EKS_ACCESS_KEY/EKS_SECRET_KEY are intentionally not required: when absent, +# eks-gateway-publish signs with IMDS instance-role creds via the SDK chain. +for v in EKS_GATEWAY_URL EKS_ACCOUNT_ID EKS_CLUSTER_NAME EKS_NLB_ENDPOINT; do eval "val=\${$v:-}" [ -n "${val}" ] || die "env ${v} not set" done diff --git a/scripts/images/eks-node/mulga-eks-state-report.sh b/scripts/images/eks-node/mulga-eks-state-report.sh index 19a2ccd88..36da1cb33 100755 --- a/scripts/images/eks-node/mulga-eks-state-report.sh +++ b/scripts/images/eks-node/mulga-eks-state-report.sh @@ -26,9 +26,9 @@ set -a . "${ENVFILE}" set +a +# EKS_ACCESS_KEY/EKS_SECRET_KEY are optional: when absent, eks-gateway-publish +# signs with IMDS instance-role credentials via the AWS SDK default chain. : "${EKS_GATEWAY_URL:?}" -: "${EKS_ACCESS_KEY:?}" -: "${EKS_SECRET_KEY:?}" : "${EKS_ACCOUNT_ID:?}" : "${EKS_CLUSTER_NAME:?}" diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index 08c914280..fbde34376 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -53,6 +53,7 @@ import ( handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs" handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks" handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/instancetypes" "github.com/mulgadc/spinifex/spinifex/network/external/dhcp" "github.com/mulgadc/spinifex/spinifex/network/host" @@ -252,6 +253,10 @@ type Daemon struct { // stateWriteMu: serialises WriteState to prevent races on the .tmp staging file. stateWriteMu sync.Mutex + // iamEnsurerMu guards the lazily-built system-role IAM service (systemRoleEnsurer). + iamEnsurerMu sync.Mutex + iamEnsurerCached handlers_iam.SystemInstanceRoleEnsurer + mu sync.Mutex } @@ -1408,6 +1413,13 @@ func (d *Daemon) startCluster() error { // Route system VM launches through NATS so they fan out across the cluster. d.elbv2Service.InstanceLauncher = handlers_elbv2.NewNATSSystemInstanceLauncher(d.natsConn, 0) + // Provide a lazily-built KV-backed IAM service so an LB VM gets a system + // instance profile and authenticates with IMDS instance-role creds. The + // provider is resolved at LB-launch time, not now, so it cannot race the + // NATS KV backend coming up; absent (no master key) the LB VM falls back to + // baked static creds. + d.elbv2Service.IAMProvider = d.systemRoleEnsurer + d.wireLBAgentConfig() d.elbv2Service.SetSystemInstanceTypeFunc(func() string { diff --git a/spinifex/daemon/daemon_system_instance.go b/spinifex/daemon/daemon_system_instance.go index c4151ab65..eca90b8ab 100644 --- a/spinifex/daemon/daemon_system_instance.go +++ b/spinifex/daemon/daemon_system_instance.go @@ -34,6 +34,23 @@ func resolveENIAccount(owner, fallback string) string { return fallback } +// recordENIInstanceOwner stamps the attached instance's owning account onto the +// ENI record. System VMs run in the system account but plug into a customer ENI, +// so without this IMDS would resolve the instance and its IAM role under the ENI +// account and serve no credentials. Best-effort: a miss leaves IMDS falling back +// to the ENI account, which only matters for the cross-account system path. +func (d *Daemon) recordENIInstanceOwner(eniAccountID, eniID, instanceOwnerID string) { + if d.vpcService == nil || eniID == "" || instanceOwnerID == eniAccountID { + return + } + if err := d.vpcService.UpdateENI(eniAccountID, eniID, func(r *handlers_ec2_vpc.ENIRecord) { + r.InstanceOwnerId = instanceOwnerID + }); err != nil { + slog.Warn("LaunchSystemInstance: failed to record ENI instance owner", + "eniId", eniID, "instanceOwner", instanceOwnerID, "err", err) + } +} + // LaunchSystemInstance creates and starts a system-managed VM (ELBv2 LB). // The VM is owned by the system account (GlobalAccountID), is not visible to // customer DescribeInstances calls, and always boots via direct kernel boot @@ -78,6 +95,9 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput) if input.SubnetID != "" { runInput.SubnetId = aws.String(input.SubnetID) } + if input.IamInstanceProfileArn != "" { + runInput.IamInstanceProfile = &ec2.IamInstanceProfileSpecification{Arn: aws.String(input.IamInstanceProfileArn)} + } // Create VM via instance service instance, ec2Instance, err := d.instanceService.RunInstance(runInput) @@ -127,6 +147,7 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput) if _, attachErr := d.vpcService.AttachENI(eniAccountID, instance.ENIId, instance.ID, 0); attachErr != nil { slog.Warn("LaunchSystemInstance: failed to attach ENI", "eniId", instance.ENIId, "instanceId", instance.ID, "err", attachErr) } + d.recordENIInstanceOwner(eniAccountID, instance.ENIId, accountID) } // Attach any additional pre-created ENIs (multi-subnet ALB VMs). // Use the launcher input slice as the source of truth so the VM @@ -143,6 +164,7 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput) d.cleanupFailedSystemInstance(instance, instanceType) return nil, fmt.Errorf("attach extra ENI %s: %w", extra.ENIID, attachErr) } + d.recordENIInstanceOwner(extraAccount, extra.ENIID, accountID) } instance.ExtraENIs = append(instance.ExtraENIs, vm.ExtraENI{ ENIID: extra.ENIID, @@ -172,6 +194,7 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput) if _, attachErr := d.vpcService.AttachENI(accountID, instance.ENIId, instance.ID, 0); attachErr != nil { slog.Warn("LaunchSystemInstance: failed to attach auto-created ENI", "eniId", instance.ENIId, "instanceId", instance.ID, "err", attachErr) } + d.recordENIInstanceOwner(accountID, instance.ENIId, accountID) } // Allocate public IP for internet-facing ALBs. Route through the EIP @@ -421,6 +444,9 @@ func (d *Daemon) launchAMISystemInstance(input *sysinstance.SystemInstanceInput) if input.UserData != "" { runInput.UserData = aws.String(base64.StdEncoding.EncodeToString([]byte(input.UserData))) } + if input.IamInstanceProfileArn != "" { + runInput.IamInstanceProfile = &ec2.IamInstanceProfileSpecification{Arn: aws.String(input.IamInstanceProfileArn)} + } if input.ManagedBy != "" { runInput.TagSpecifications = []*ec2.TagSpecification{{ ResourceType: aws.String("instance"), diff --git a/spinifex/daemon/eks_deps.go b/spinifex/daemon/eks_deps.go index 31e1837a0..64f59a86e 100644 --- a/spinifex/daemon/eks_deps.go +++ b/spinifex/daemon/eks_deps.go @@ -11,6 +11,41 @@ import ( "log/slog" ) +// systemRoleEnsurer lazily builds — and memoizes — the KV-backed IAM service +// that find-or-creates the system instance roles/profiles backing IMDS +// instance-role credentials. It is built on first use, NOT at daemon startup: +// the NATS KV backend has no responders until JetStream is ready, so an eager +// build races boot and, on a node that loses, fails permanently with no retry, +// silently dropping every system VM it launches to baked static creds. By +// first-use (an LB/cluster create) the backend is up. Returns nil (caller falls +// back to static creds) until a build succeeds; retried on each call until then, +// cached once it works. +func (d *Daemon) systemRoleEnsurer() handlers_iam.SystemInstanceRoleEnsurer { + d.iamEnsurerMu.Lock() + defer d.iamEnsurerMu.Unlock() + if d.iamEnsurerCached != nil { + return d.iamEnsurerCached + } + masterKey, err := handlers_iam.LoadMasterKey(filepath.Join(filepath.Dir(d.configPath), "master.key")) + if err != nil || masterKey == nil { + slog.Warn("System role ensurer: master key unavailable; system VMs fall back to baked static creds", + "err", err) + return nil + } + clusterSize := 1 + if d.clusterConfig != nil { + clusterSize = len(d.clusterConfig.Nodes) + } + iamSvc, iamErr := handlers_iam.NewIAMServiceImpl(d.natsConn, masterKey, clusterSize) + if iamErr != nil { + slog.Warn("System role ensurer: IAM service init failed (retried on next launch); system VMs fall back to baked static creds", + "err", iamErr) + return nil + } + d.iamEnsurerCached = iamSvc + return iamSvc +} + // buildEKSServiceDeps assembles the EKSServiceDeps. All collaborators must // already be initialised. MasterKey is loaded best-effort from the config dir. func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps { @@ -65,21 +100,10 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps { } // A KV-backed IAM service (sharing the gateway's buckets over NATS) lets EKS - // find-or-create the node-role instance profile workers need for ECR pulls. - // Only wired when the master key is present; a typed-nil would defeat the - // service's own nil guard. - if masterKey != nil { - clusterSize := 1 - if d.clusterConfig != nil { - clusterSize = len(d.clusterConfig.Nodes) - } - if iamSvc, iamErr := handlers_iam.NewIAMServiceImpl(d.natsConn, masterKey, clusterSize); iamErr != nil { - slog.Warn("EKS: IAM service init failed; nodegroup workers launch without an instance profile, internal ECR pulls will fail", - "err", iamErr) - } else { - deps.IAM = iamSvc - } - } + // find-or-create the node-role + CP instance profiles. Provided lazily so it + // resolves at cluster-launch time rather than racing the NATS KV backend at + // daemon startup; absent (no master key) workers/CP fall back accordingly. + deps.IAMProvider = d.systemRoleEnsurer return deps } diff --git a/spinifex/handlers/ec2/instance/service_impl_test.go b/spinifex/handlers/ec2/instance/service_impl_test.go index d11573d31..bfe45619d 100644 --- a/spinifex/handlers/ec2/instance/service_impl_test.go +++ b/spinifex/handlers/ec2/instance/service_impl_test.go @@ -2121,6 +2121,41 @@ func TestPrepareRunInstances_HappyPathNoENI(t *testing.T) { } } +// An instance profile passed at launch is persisted on every prepared VM and +// echoed on the ec2.Instance so the in-VM IMDS endpoint resolves a role and +// serves its credentials (mirrors the RunInstance singular path). +func TestPrepareRunInstances_PersistsIamInstanceProfile(t *testing.T) { + types, _ := defaultPrepareInstanceTypes() + prov := &fakeResourceCapacityProvider{ + instanceTypes: types, + canAllocFn: func(_ *ec2.InstanceTypeInfo, count int) int { return count }, + } + svc := &InstanceServiceImpl{ + config: &config.Config{}, + instanceTypes: types, + amiLoader: &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ + "ami-1": {ImageOwnerAlias: "acc"}, + }}, + resourceMgr: prov, + } + const profileARN = "arn:aws:iam::000000000000:instance-profile/spinifex-eks-server" + _, instances, _, err := svc.PrepareRunInstances(&ec2.RunInstancesInput{ + InstanceType: aws.String("t3.micro"), + ImageId: aws.String("ami-1"), + MinCount: aws.Int64(2), + MaxCount: aws.Int64(2), + IamInstanceProfile: &ec2.IamInstanceProfileSpecification{Arn: aws.String(profileARN)}, + }, "acc", "") + require.NoError(t, err) + require.Len(t, instances, 2) + for _, inst := range instances { + assert.Equal(t, profileARN, inst.IamInstanceProfileArn) + assert.NotEmpty(t, inst.IamInstanceProfileAssociationId) + require.NotNil(t, inst.Instance.IamInstanceProfile) + assert.Equal(t, profileARN, aws.StringValue(inst.Instance.IamInstanceProfile.Arn)) + } +} + // A targeted launch consumes slots from the reservation (not the general pool) // and stamps the reservation id onto every prepared VM so terminate can restore. func TestPrepareRunInstances_ConsumesReservation(t *testing.T) { diff --git a/spinifex/handlers/ec2/vpc/eni.go b/spinifex/handlers/ec2/vpc/eni.go index 76ccbd892..2adbda45f 100644 --- a/spinifex/handlers/ec2/vpc/eni.go +++ b/spinifex/handlers/ec2/vpc/eni.go @@ -24,22 +24,27 @@ const ( // ENIRecord represents a stored Elastic Network Interface type ENIRecord struct { - NetworkInterfaceId string `json:"network_interface_id"` - SubnetId string `json:"subnet_id"` - VpcId string `json:"vpc_id"` - AvailabilityZone string `json:"availability_zone"` - PrivateIpAddress string `json:"private_ip_address"` - MacAddress string `json:"mac_address"` - Description string `json:"description"` - Status string `json:"status"` // available, in-use - AttachmentId string `json:"attachment_id,omitempty"` - InstanceId string `json:"instance_id,omitempty"` - DeviceIndex int64 `json:"device_index"` - PublicIpAddress string `json:"public_ip_address,omitempty"` // Auto-assigned or EIP - PublicIpPool string `json:"public_ip_pool,omitempty"` // Pool name the public IP came from - SecurityGroupIds []string `json:"security_group_ids,omitempty"` - Tags map[string]string `json:"tags"` - CreatedAt time.Time `json:"created_at"` + NetworkInterfaceId string `json:"network_interface_id"` + SubnetId string `json:"subnet_id"` + VpcId string `json:"vpc_id"` + AvailabilityZone string `json:"availability_zone"` + PrivateIpAddress string `json:"private_ip_address"` + MacAddress string `json:"mac_address"` + Description string `json:"description"` + Status string `json:"status"` // available, in-use + AttachmentId string `json:"attachment_id,omitempty"` + InstanceId string `json:"instance_id,omitempty"` + // InstanceOwnerId is the account that owns the attached instance, mirroring + // AWS's Attachment.InstanceOwnerId. It differs from the ENI's own account only + // for system VMs (LB/EKS) that plug into a customer-account ENI; IMDS resolves + // the instance and its IAM role under this account. Empty means same-account. + InstanceOwnerId string `json:"instance_owner_id,omitempty"` + DeviceIndex int64 `json:"device_index"` + PublicIpAddress string `json:"public_ip_address,omitempty"` // Auto-assigned or EIP + PublicIpPool string `json:"public_ip_pool,omitempty"` // Pool name the public IP came from + SecurityGroupIds []string `json:"security_group_ids,omitempty"` + Tags map[string]string `json:"tags"` + CreatedAt time.Time `json:"created_at"` // AttachmentStatus carries the hot-plug transition state independent of // Status. AWS-parity field: "" (not transitioning), "attaching", diff --git a/spinifex/handlers/eks/k3s_server_role.go b/spinifex/handlers/eks/k3s_server_role.go new file mode 100644 index 000000000..6555abe49 --- /dev/null +++ b/spinifex/handlers/eks/k3s_server_role.go @@ -0,0 +1,43 @@ +package handlers_eks + +import ( + "log/slog" + + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" +) + +const ( + // eksServerSystemRoleName is the Spinifex-managed instance role attached to + // the k3s control-plane VM. IMDS serves it so the VM's gateway publishes are + // signed with scoped, rotating credentials instead of a baked static key. + eksServerSystemRoleName = "spinifex-eks-server" + + // eksServerInlinePolicyName is the inline policy granting the CP VM the + // internal gateway actions its bootstrap/state-report/addon-fetch need. + eksServerInlinePolicyName = "spinifex-eks-server-internal" + + // eksServerInlinePolicy grants only the internal gateway actions the CP VM + // calls: PublishInternal (bootstrap/state) and ListInternalAddons (addon + // fetch). The gateway evaluates these per request against the role's policies. + eksServerInlinePolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["eks:PublishInternal","eks:ListInternalAddons"],"Resource":"*"}]}` +) + +// ensureCPInstanceProfile returns the CP instance-profile ARN, or "" to signal +// the caller should fall back to static creds. IAM unwired or a transient +// failure both fall back rather than brick a cluster launch — the static-key +// path still authenticates (it is what shipped before IMDS creds). +func (s *EKSServiceImpl) ensureCPInstanceProfile(accountID string) string { + iam := s.iamEnsurer() + if iam == nil { + slog.Warn("EKS: IAM service unwired; CP VM falls back to baked static gateway creds") + return "" + } + profileARN, err := handlers_iam.EnsureSystemInstanceProfile(iam, accountID, + eksServerSystemRoleName, eksServerInlinePolicyName, eksServerInlinePolicy) + if err != nil { + slog.Warn("EKS: ensure CP instance profile failed; falling back to baked static gateway creds", + "err", err) + return "" + } + return profileARN +} diff --git a/spinifex/handlers/eks/k3s_server_role_test.go b/spinifex/handlers/eks/k3s_server_role_test.go new file mode 100644 index 000000000..12528fed6 --- /dev/null +++ b/spinifex/handlers/eks/k3s_server_role_test.go @@ -0,0 +1,91 @@ +package handlers_eks + +import ( + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testSysAcct = "000000000000" + +// TestEnsureK3sServerInstanceProfile_CreatesRolePolicyProfile asserts the CP +// role is created with the EC2 trust policy, carries the internal gateway +// permissions, and its instance profile is returned for launch attachment. +func TestEnsureK3sServerInstanceProfile_CreatesRolePolicyProfile(t *testing.T) { + f := newFakeEnsurer() + s := &EKSServiceImpl{deps: EKSServiceDeps{IAM: f}} + + arn := s.ensureCPInstanceProfile(testSysAcct) + assert.Equal(t, "arn:aws:iam::"+testSysAcct+":instance-profile/"+eksServerSystemRoleName, arn) + + assert.Equal(t, 1, f.createRoleCalls) + assert.Contains(t, f.lastTrustDoc, "ec2.amazonaws.com") + assert.Contains(t, f.lastTrustDoc, "sts:AssumeRole") + + policy := f.rolePolicies[eksServerSystemRoleName] + assert.Contains(t, policy, "eks:PublishInternal") + assert.Contains(t, policy, "eks:ListInternalAddons") + + require.NotNil(t, f.profiles[eksServerSystemRoleName]) + assert.Len(t, f.profiles[eksServerSystemRoleName].Roles, 1) +} + +// TestEnsureK3sServerInstanceProfile_ExistingRoleConverges asserts a re-launch +// against a pre-existing role does not re-create it but still re-asserts the +// inline policy so a stale role converges onto the current permissions. +func TestEnsureK3sServerInstanceProfile_ExistingRoleConverges(t *testing.T) { + f := newFakeEnsurer() + f.roles[eksServerSystemRoleName] = &iam.Role{ + RoleName: aws.String(eksServerSystemRoleName), + Arn: aws.String("arn:aws:iam::" + testSysAcct + ":role/" + eksServerSystemRoleName), + } + s := &EKSServiceImpl{deps: EKSServiceDeps{IAM: f}} + + arn := s.ensureCPInstanceProfile(testSysAcct) + assert.NotEmpty(t, arn) + assert.Zero(t, f.createRoleCalls) + assert.Contains(t, f.rolePolicies[eksServerSystemRoleName], "eks:PublishInternal") +} + +// TestEnsureCPInstanceProfile_NilIAMFallsBack asserts an unwired IAM service +// yields "" so the caller falls back to baked static creds rather than failing. +func TestEnsureCPInstanceProfile_NilIAMFallsBack(t *testing.T) { + s := &EKSServiceImpl{deps: EKSServiceDeps{}} + assert.Equal(t, "", s.ensureCPInstanceProfile(testSysAcct)) +} + +// TestEnsureCPInstanceProfile_UsesLazyProvider asserts the CP profile is created +// via the lazily-resolved IAMProvider when the eager deps.IAM is unset — the +// daemon wires only the provider so the build cannot race the NATS KV backend. +func TestEnsureCPInstanceProfile_UsesLazyProvider(t *testing.T) { + f := newFakeEnsurer() + s := &EKSServiceImpl{deps: EKSServiceDeps{ + IAMProvider: func() handlers_iam.SystemInstanceRoleEnsurer { return f }, + }} + arn := s.ensureCPInstanceProfile(testSysAcct) + assert.Equal(t, "arn:aws:iam::"+testSysAcct+":instance-profile/"+eksServerSystemRoleName, arn) + assert.Equal(t, 1, f.createRoleCalls) +} + +// TestEnsureCPInstanceProfile_ProviderNotReadyFallsBack asserts a provider that +// returns nil (NATS KV backend not yet up) yields "" so the launch falls back to +// static creds and retries on the next call rather than bricking the cluster. +func TestEnsureCPInstanceProfile_ProviderNotReadyFallsBack(t *testing.T) { + s := &EKSServiceImpl{deps: EKSServiceDeps{ + IAMProvider: func() handlers_iam.SystemInstanceRoleEnsurer { return nil }, + }} + assert.Equal(t, "", s.ensureCPInstanceProfile(testSysAcct)) +} + +// TestEKSServerTrustPolicyIsAssumable guards that the trust doc the CP role +// ships with is exactly the EC2 service-principal shape AssumeRoleForInstance +// matches; a drift here silently breaks IMDS credential minting. +func TestEKSServerTrustPolicyIsAssumable(t *testing.T) { + assert.True(t, strings.Contains(handlers_iam.EC2InstanceTrustPolicy, `"Service":"ec2.amazonaws.com"`)) + assert.True(t, strings.Contains(handlers_iam.EC2InstanceTrustPolicy, `"Action":"sts:AssumeRole"`)) +} diff --git a/spinifex/handlers/eks/k3s_server_vm.go b/spinifex/handlers/eks/k3s_server_vm.go index 63871024c..a4fd6c4d8 100644 --- a/spinifex/handlers/eks/k3s_server_vm.go +++ b/spinifex/handlers/eks/k3s_server_vm.go @@ -124,10 +124,17 @@ type K3sServerInput struct { // addon pod specs (EKS_ADDON_GATEWAY_URL). Those pods run on workers, which // cannot reach the mgmt GatewayURL, so they target this public address. AddonGatewayURL string - AccessKey string - SecretKey string - GatewayCACert string - InstanceType string + // AccessKey/SecretKey are the static system SigV4 creds baked into the VM's + // first-boot env. Empty selects IMDS instance-role creds instead, which + // requires IamInstanceProfileArn to be set so IMDS serves a role. + AccessKey string + SecretKey string + // IamInstanceProfileArn attaches a system instance profile to the CP VM so + // the in-VM IMDS endpoint serves rotating role credentials. When set, the + // static AccessKey/SecretKey are omitted from user-data. + IamInstanceProfileArn string + GatewayCACert string + InstanceType string // TargetNodeID pins the VM to a specific host for HA spread; empty = local node. TargetNodeID string // JoinToken is the shared k3s cluster token so HA servers join the etcd quorum. @@ -196,16 +203,17 @@ func LaunchK3sServerVM( userData := buildK3sUserData(in) sysOut, err := instSvc.LaunchSystemInstanceOnNode(in.TargetNodeID, &sysinstance.SystemInstanceInput{ - BootMode: sysinstance.BootAMI, - ManagedBy: tags.ManagedByEKS, - InstanceType: instanceType, - ImageID: amiID, - AccountID: in.AccountID, - ENIID: eniID, - ENIMac: aws.StringValue(eniOut.NetworkInterface.MacAddress), - ENIIP: eniIP, - SubnetID: in.SubnetID, - UserData: userData, + BootMode: sysinstance.BootAMI, + ManagedBy: tags.ManagedByEKS, + InstanceType: instanceType, + ImageID: amiID, + AccountID: in.AccountID, + ENIID: eniID, + ENIMac: aws.StringValue(eniOut.NetworkInterface.MacAddress), + ENIIP: eniIP, + SubnetID: in.SubnetID, + UserData: userData, + IamInstanceProfileArn: in.IamInstanceProfileArn, }) if err != nil { rollbackK3sENI(vpcSvc, in.AccountID, eniID) @@ -366,10 +374,10 @@ func validateK3sServerInput(in K3sServerInput) error { return errors.New("eks: K3sServerInput empty GatewayURL") case in.AddonGatewayURL == "": return errors.New("eks: K3sServerInput empty AddonGatewayURL") - case in.AccessKey == "": - return errors.New("eks: K3sServerInput empty AccessKey") - case in.SecretKey == "": - return errors.New("eks: K3sServerInput empty SecretKey") + case in.AccessKey == "" && in.IamInstanceProfileArn == "": + return errors.New("eks: K3sServerInput needs static creds (AccessKey/SecretKey) or an instance profile (IamInstanceProfileArn)") + case (in.AccessKey == "") != (in.SecretKey == ""): + return errors.New("eks: K3sServerInput AccessKey and SecretKey must both be set or both empty") case strings.TrimSpace(in.GatewayCACert) == "": return errors.New("eks: K3sServerInput empty GatewayCACert") case in.ServerURL != "" && in.JoinToken == "": @@ -412,21 +420,30 @@ func buildK3sUserData(in K3sServerInput) string { role = "server-join" } - envBody := strings.Join([]string{ + envLines := []string{ "SPINIFEX_K3S_ROLE=" + role, "EKS_GATEWAY_URL=" + in.GatewayURL, "EKS_ADDON_GATEWAY_URL=" + in.AddonGatewayURL, "EKS_GATEWAY_CA=" + k3sGatewayCAPath, - "EKS_ACCESS_KEY=" + in.AccessKey, - "EKS_SECRET_KEY=" + in.SecretKey, - "EKS_REGION=" + in.Region, - "EKS_VPC_ID=" + in.VpcID, - "EKS_ELB_SUBNET_IDS=" + strings.Join(in.ELBSubnetIDs, ","), - "EKS_ACCOUNT_ID=" + in.ClusterAccountID, - "EKS_CLUSTER_NAME=" + in.ClusterName, - "EKS_NLB_ENDPOINT=" + nlbEndpoint, - "EKS_OIDC_ISSUER=" + in.OIDCIssuer, - }, "\n") + } + // Static SigV4 creds only when supplied; otherwise the gateway helpers fall + // back to the AWS SDK chain, which reads IMDS instance-role credentials. + if in.AccessKey != "" { + envLines = append(envLines, + "EKS_ACCESS_KEY="+in.AccessKey, + "EKS_SECRET_KEY="+in.SecretKey, + ) + } + envLines = append(envLines, + "EKS_REGION="+in.Region, + "EKS_VPC_ID="+in.VpcID, + "EKS_ELB_SUBNET_IDS="+strings.Join(in.ELBSubnetIDs, ","), + "EKS_ACCOUNT_ID="+in.ClusterAccountID, + "EKS_CLUSTER_NAME="+in.ClusterName, + "EKS_NLB_ENDPOINT="+nlbEndpoint, + "EKS_OIDC_ISSUER="+in.OIDCIssuer, + ) + envBody := strings.Join(envLines, "\n") // First server uses cluster-init (embedded etcd); join servers set `server: ` + token. // etcd-expose-metrics: surfaces etcd fsync/commit latency on 127.0.0.1:2381/metrics. @@ -497,7 +514,7 @@ func buildK3sUserData(in K3sServerInput) string { k3sConfig := strings.Join(configLines, "\n") files := []userDataFile{ - // 0600: contains system SigV4 secret key. + // 0600: may contain a system SigV4 secret key (static-cred mode). {Path: k3sFirstBootEnvPath, Perms: "0600", Body: envBody}, {Path: k3sOIDCSigningKeyPath, Perms: "0600", Body: strings.TrimRight(in.OIDCPrivateKeyPEM, "\n")}, {Path: k3sOIDCPublicKeyPath, Perms: "0644", Body: strings.TrimRight(in.OIDCPublicKeyPEM, "\n")}, diff --git a/spinifex/handlers/eks/k3s_server_vm_test.go b/spinifex/handlers/eks/k3s_server_vm_test.go index 589afb6e1..e114599a8 100644 --- a/spinifex/handlers/eks/k3s_server_vm_test.go +++ b/spinifex/handlers/eks/k3s_server_vm_test.go @@ -217,6 +217,39 @@ func TestLaunchK3sServerVM_EmptyInputsRejected(t *testing.T) { } } +func TestValidateK3sServerInput_ProfileOnlyAccepted(t *testing.T) { + in := validK3sInput() + in.AccessKey = "" + in.SecretKey = "" + in.IamInstanceProfileArn = "arn:aws:iam::000000000000:instance-profile/spinifex-eks-server" + require.NoError(t, validateK3sServerInput(in)) +} + +func TestValidateK3sServerInput_NoCredsNoProfileRejected(t *testing.T) { + in := validK3sInput() + in.AccessKey = "" + in.SecretKey = "" + require.Error(t, validateK3sServerInput(in)) +} + +func TestBuildK3sUserData_StaticCredsBakeKeys(t *testing.T) { + ud := buildK3sUserData(validK3sInput()) + assert.Contains(t, ud, "EKS_ACCESS_KEY=AKIAEXAMPLE") + assert.Contains(t, ud, "EKS_SECRET_KEY=s3cr3t-key") +} + +func TestBuildK3sUserData_ProfileModeOmitsKeys(t *testing.T) { + in := validK3sInput() + in.AccessKey = "" + in.SecretKey = "" + in.IamInstanceProfileArn = "arn:aws:iam::000000000000:instance-profile/spinifex-eks-server" + ud := buildK3sUserData(in) + assert.NotContains(t, ud, "EKS_ACCESS_KEY=") + assert.NotContains(t, ud, "EKS_SECRET_KEY=") + // Non-credential env still present. + assert.Contains(t, ud, "EKS_CLUSTER_NAME=alpha") +} + func TestLaunchK3sServerVM_AMINotFound(t *testing.T) { vpc, inst := &fakeK3sVPC{}, &fakeK3sInst{} ami := &fakeK3sAMI{describeOut: &ec2.DescribeImagesOutput{}} diff --git a/spinifex/handlers/eks/nodegroup.go b/spinifex/handlers/eks/nodegroup.go index a95fd5c0f..a2f4afc5f 100644 --- a/spinifex/handlers/eks/nodegroup.go +++ b/spinifex/handlers/eks/nodegroup.go @@ -473,7 +473,7 @@ func (s *EKSServiceImpl) launchOneWorker(rec *NodegroupRecord, meta *ClusterMeta // Attach the node role's instance profile so IMDS serves the role to the ECR // credential provider; without it worker pulls from the internal ECR get 401. - if s.deps.IAM != nil && rec.NodeRole != "" { + if s.iamEnsurer() != nil && rec.NodeRole != "" { profileARN, perr := s.ensureNodeInstanceProfile(accountID, rec.NodeRole) if perr != nil { return "", fmt.Errorf("ensure node instance profile: %w", perr) @@ -526,7 +526,7 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string } profileName := roleName - out, err := s.deps.IAM.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{ + out, err := s.iamEnsurer().GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{ InstanceProfileName: aws.String(profileName), }) if err == nil { @@ -537,7 +537,7 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string return "", fmt.Errorf("get instance profile %q: %w", profileName, err) } - created, err := s.deps.IAM.CreateInstanceProfile(accountID, &iam.CreateInstanceProfileInput{ + created, err := s.iamEnsurer().CreateInstanceProfile(accountID, &iam.CreateInstanceProfileInput{ InstanceProfileName: aws.String(profileName), }) if err != nil { @@ -545,7 +545,7 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string if err.Error() != awserrors.ErrorIAMEntityAlreadyExists { return "", fmt.Errorf("create instance profile %q: %w", profileName, err) } - got, gerr := s.deps.IAM.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{ + got, gerr := s.iamEnsurer().GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{ InstanceProfileName: aws.String(profileName), }) if gerr != nil { @@ -565,7 +565,7 @@ func (s *EKSServiceImpl) attachRoleToProfile(accountID, profileName, roleName, p if alreadyAttached { return profileARN, nil } - _, err := s.deps.IAM.AddRoleToInstanceProfile(accountID, &iam.AddRoleToInstanceProfileInput{ + _, err := s.iamEnsurer().AddRoleToInstanceProfile(accountID, &iam.AddRoleToInstanceProfileInput{ InstanceProfileName: aws.String(profileName), RoleName: aws.String(roleName), }) diff --git a/spinifex/handlers/eks/nodegroup_instance_profile_test.go b/spinifex/handlers/eks/nodegroup_instance_profile_test.go index f744fd41f..60b321f72 100644 --- a/spinifex/handlers/eks/nodegroup_instance_profile_test.go +++ b/spinifex/handlers/eks/nodegroup_instance_profile_test.go @@ -19,10 +19,48 @@ type fakeInstanceProfileEnsurer struct { addRoleErr error createCalls int addCalls int + + // Role store + capture for the system-role path (ensureSystemRole). + roles map[string]*iam.Role + createRoleCalls int + lastTrustDoc string + rolePolicies map[string]string // roleName -> last PutRolePolicy document } func newFakeEnsurer() *fakeInstanceProfileEnsurer { - return &fakeInstanceProfileEnsurer{profiles: map[string]*iam.InstanceProfile{}} + return &fakeInstanceProfileEnsurer{ + profiles: map[string]*iam.InstanceProfile{}, + roles: map[string]*iam.Role{}, + rolePolicies: map[string]string{}, + } +} + +func (f *fakeInstanceProfileEnsurer) GetRole(accountID string, in *iam.GetRoleInput) (*iam.GetRoleOutput, error) { + r, ok := f.roles[aws.StringValue(in.RoleName)] + if !ok { + return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) + } + return &iam.GetRoleOutput{Role: r}, nil +} + +func (f *fakeInstanceProfileEnsurer) CreateRole(accountID string, in *iam.CreateRoleInput) (*iam.CreateRoleOutput, error) { + f.createRoleCalls++ + f.lastTrustDoc = aws.StringValue(in.AssumeRolePolicyDocument) + name := aws.StringValue(in.RoleName) + if _, ok := f.roles[name]; ok { + return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists) + } + r := &iam.Role{ + RoleName: aws.String(name), + Arn: aws.String("arn:aws:iam::" + accountID + ":role/" + name), + } + f.roles[name] = r + return &iam.CreateRoleOutput{Role: r}, nil +} + +func (f *fakeInstanceProfileEnsurer) PutRolePolicy(_ string, in *iam.PutRolePolicyInput) (*iam.PutRolePolicyOutput, error) { + f.rolePolicies[aws.StringValue(in.RoleName)] = aws.StringValue(in.PolicyDocument) + return &iam.PutRolePolicyOutput{}, nil } func (f *fakeInstanceProfileEnsurer) GetInstanceProfile(_ string, in *iam.GetInstanceProfileInput) (*iam.GetInstanceProfileOutput, error) { diff --git a/spinifex/handlers/eks/service_impl.go b/spinifex/handlers/eks/service_impl.go index 94123a551..febea1079 100644 --- a/spinifex/handlers/eks/service_impl.go +++ b/spinifex/handlers/eks/service_impl.go @@ -22,6 +22,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/admin" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/handlers/sysinstance" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" @@ -67,8 +68,14 @@ type EKSServiceDeps struct { // IAM backs a nodegroup's node role with an instance profile so workers expose // the role over IMDS for the ECR credential provider. Nil disables the wiring // (workers launch without a profile and cannot pull from the internal ECR). + // Tests set this directly; production prefers IAMProvider. IAM instanceProfileEnsurer + // IAMProvider lazily resolves the IAM ensurer at cluster-launch time so it + // cannot race the NATS KV backend at daemon startup. Preferred over IAM when + // set; its concrete service satisfies instanceProfileEnsurer. + IAMProvider func() handlers_iam.SystemInstanceRoleEnsurer + // VPCMgr / NATGW / RouteTable compose the managed control-plane VPC ("Set B") // from the real EC2 VPC-family APIs under the system account. The daemon // adapts its concrete VPC / NAT-gateway / route-table services onto these. @@ -93,6 +100,23 @@ type WorkerLauncher interface { TerminateWorkerInstances(instanceIDs []string, accountID string) error } +// iamEnsurer resolves the IAM ensurer, preferring the test-injected deps.IAM, +// then the lazy IAMProvider (built at launch time to dodge the daemon-startup +// NATS-KV race). Returns nil when neither yields a service, so callers fall back +// to static creds / skip the profile. The provider's concrete service satisfies +// instanceProfileEnsurer (identical method set to SystemInstanceRoleEnsurer). +func (s *EKSServiceImpl) iamEnsurer() instanceProfileEnsurer { + if s.deps.IAM != nil { + return s.deps.IAM + } + if s.deps.IAMProvider != nil { + if e := s.deps.IAMProvider(); e != nil { + return e + } + } + return nil +} + // instanceProfileEnsurer is the narrow IAM surface EKS needs to find-or-create the // instance profile that fronts a nodegroup's node role. Real EKS creates this // profile implicitly for a node role; Spinifex does the same at worker launch. @@ -100,6 +124,13 @@ type instanceProfileEnsurer interface { GetInstanceProfile(accountID string, input *iam.GetInstanceProfileInput) (*iam.GetInstanceProfileOutput, error) CreateInstanceProfile(accountID string, input *iam.CreateInstanceProfileInput) (*iam.CreateInstanceProfileOutput, error) AddRoleToInstanceProfile(accountID string, input *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error) + + // Role find-or-create surface for the system-managed control-plane role. + // Unlike worker node roles (customer-supplied), the k3s server role is + // created by Spinifex with the IMDS-scoped gateway permissions it needs. + GetRole(accountID string, input *iam.GetRoleInput) (*iam.GetRoleOutput, error) + CreateRole(accountID string, input *iam.CreateRoleInput) (*iam.CreateRoleOutput, error) + PutRolePolicy(accountID string, input *iam.PutRolePolicyInput) (*iam.PutRolePolicyOutput, error) } // eipProvisioner is the narrow EIP surface for allocating a CP VM egress IP. @@ -597,7 +628,7 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) { // an unresolved AZ falls back to auto-discovery rather than risk a dup-AZ error. elbSubnets := dedupSubnetsByAZ(s.deps.VPCSubnet, accountID, lc.subnetIDs) - cpNodes, spreadGroup, err := s.placeControlPlane(sysAcct, name, K3sServerInput{ + serverIn := K3sServerInput{ AccountID: sysAcct, ClusterAccountID: accountID, ClusterName: name, @@ -614,11 +645,21 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) { OIDCPublicKeyPEM: pubPEM, GatewayURL: s.deps.SystemGatewayURL, AddonGatewayURL: s.deps.GatewayBaseURL, - AccessKey: s.deps.SystemAccessKey, - SecretKey: s.deps.SystemSecretKey, GatewayCACert: s.deps.GatewayCACert, JoinToken: joinToken, - }) + } + + // Prefer IMDS instance-role creds: attach a system instance profile so the + // CP VM authenticates with scoped, rotating credentials and no static secret + // rides in user-data. Falls back to baked system keys when IAM is unwired. + if profileARN := s.ensureCPInstanceProfile(sysAcct); profileARN != "" { + serverIn.IamInstanceProfileArn = profileARN + } else { + serverIn.AccessKey = s.deps.SystemAccessKey + serverIn.SecretKey = s.deps.SystemSecretKey + } + + cpNodes, spreadGroup, err := s.placeControlPlane(sysAcct, name, serverIn) if err != nil { if errors.Is(err, ErrEKSServerAMINotFound) { s.failClusterLaunch(acctKV, name, accountID, meta, "eks-server AMI not found", err) diff --git a/spinifex/handlers/elbv2/arn_test.go b/spinifex/handlers/elbv2/arn_test.go index 5fecf467b..1eb725947 100644 --- a/spinifex/handlers/elbv2/arn_test.go +++ b/spinifex/handlers/elbv2/arn_test.go @@ -49,7 +49,7 @@ func TestBuildLBAgentEnv(t *testing.T) { region: "ap-southeast-2", } // With no mgmt bridge, the lb-agent falls back to the WAN GatewayURL. - env := svc.buildLBAgentEnv("lb-abc123") + env := svc.buildLBAgentEnv("lb-abc123", true) lines := strings.Split(strings.TrimRight(env, "\n"), "\n") kvs := make(map[string]string, len(lines)) @@ -66,6 +66,23 @@ func TestBuildLBAgentEnv(t *testing.T) { assert.Equal(t, "ap-southeast-2", kvs["LB_REGION"]) } +// TestBuildLBAgentEnv_IMDSModeOmitsKeys verifies that when a profile is attached +// (staticCreds=false) the static keys are dropped so the lb-agent signs with +// IMDS instance-role credentials via the SDK chain. +func TestBuildLBAgentEnv_IMDSModeOmitsKeys(t *testing.T) { + svc := &ELBv2ServiceImpl{ + GatewayURL: "https://10.0.0.1:9999", + SystemAccessKey: "AKIAIOSFODNN7EXAMPLE", + SystemSecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + region: "ap-southeast-2", + } + env := svc.buildLBAgentEnv("lb-abc123", false) + assert.NotContains(t, env, "LB_ACCESS_KEY") + assert.NotContains(t, env, "LB_SECRET_KEY") + assert.Contains(t, env, "LB_LB_ID=lb-abc123\n") + assert.Contains(t, env, "LB_REGION=ap-southeast-2\n") +} + // TestBuildLBAgentEnv_UsesMgmtGateway verifies every lb-agent (customer or // system) heartbeats over the on-link mgmt-bridge URL, not the WAN GatewayURL, // so the control-plane heartbeat survives a reboot that strands the data plane. @@ -77,12 +94,12 @@ func TestBuildLBAgentEnv_UsesMgmtGateway(t *testing.T) { SystemSecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", region: "ap-southeast-2", } - env := svc.buildLBAgentEnv("lb-abc123") + env := svc.buildLBAgentEnv("lb-abc123", true) assert.Contains(t, env, "LB_GATEWAY_URL=https://192.168.50.1:9999\n") // With no mgmt bridge, the lb-agent falls back to the WAN URL. svc.MgmtBridgeIP = "" - env = svc.buildLBAgentEnv("lb-abc123") + env = svc.buildLBAgentEnv("lb-abc123", true) assert.Contains(t, env, "LB_GATEWAY_URL=https://10.0.0.1:9999\n") } diff --git a/spinifex/handlers/elbv2/lb_role.go b/spinifex/handlers/elbv2/lb_role.go new file mode 100644 index 000000000..bc46cb868 --- /dev/null +++ b/spinifex/handlers/elbv2/lb_role.go @@ -0,0 +1,49 @@ +package handlers_elbv2 + +import ( + "log/slog" + + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" +) + +const ( + // lbAgentSystemRoleName is the Spinifex-managed instance role attached to an + // LB VM. IMDS serves it so the lb-agent signs its gateway polls with scoped, + // rotating credentials instead of the baked system static key. + lbAgentSystemRoleName = "spinifex-lb-agent" + + // lbAgentInlinePolicyName is the inline policy granting the lb-agent the two + // gateway actions it calls. + lbAgentInlinePolicyName = "spinifex-lb-agent-internal" + + // lbAgentInlinePolicy grants only the actions the lb-agent invokes: + // LBAgentHeartbeat (target health) and GetLBConfig (poll listener/target + // config). The gateway evaluates these per request against the role policies. + lbAgentInlinePolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["elasticloadbalancing:LBAgentHeartbeat","elasticloadbalancing:GetLBConfig"],"Resource":"*"}]}` +) + +// ensureLBInstanceProfile returns the LB instance-profile ARN, or "" to signal +// the caller should fall back to static creds. accountID must be the account the +// LB VM runs under (the system account) — IMDS resolves the profile under the +// instance's account, so a role created anywhere else is invisible to it. The +// resulting system-account principal satisfies the LBAgentHeartbeat / GetLBConfig +// guard via its GlobalAccountID branch. IAM unwired or a transient failure both +// fall back rather than block an LB launch. +func (s *ELBv2ServiceImpl) ensureLBInstanceProfile(accountID string) string { + iam := s.IAM + if iam == nil && s.IAMProvider != nil { + iam = s.IAMProvider() + } + if iam == nil { + slog.Warn("ELBv2: IAM service unwired; LB VM falls back to baked static gateway creds") + return "" + } + profileARN, err := handlers_iam.EnsureSystemInstanceProfile(iam, accountID, + lbAgentSystemRoleName, lbAgentInlinePolicyName, lbAgentInlinePolicy) + if err != nil { + slog.Warn("ELBv2: ensure LB instance profile failed; falling back to baked static gateway creds", + "accountId", accountID, "err", err) + return "" + } + return profileARN +} diff --git a/spinifex/handlers/elbv2/lb_role_test.go b/spinifex/handlers/elbv2/lb_role_test.go new file mode 100644 index 000000000..0450f2e22 --- /dev/null +++ b/spinifex/handlers/elbv2/lb_role_test.go @@ -0,0 +1,89 @@ +package handlers_elbv2 + +import ( + "errors" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/mulgadc/spinifex/spinifex/awserrors" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/stretchr/testify/assert" +) + +// fakeLBEnsurer is a minimal SystemInstanceRoleEnsurer recording the account the +// role and instance profile are created under. GetRole/GetInstanceProfile always +// miss so EnsureSystemInstanceProfile takes the create path. +type fakeLBEnsurer struct { + roleAcct string + profileAcct string +} + +func (f *fakeLBEnsurer) GetRole(_ string, _ *iam.GetRoleInput) (*iam.GetRoleOutput, error) { + return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) +} + +func (f *fakeLBEnsurer) CreateRole(acct string, in *iam.CreateRoleInput) (*iam.CreateRoleOutput, error) { + f.roleAcct = acct + name := aws.StringValue(in.RoleName) + return &iam.CreateRoleOutput{Role: &iam.Role{ + RoleName: aws.String(name), + Arn: aws.String("arn:aws:iam::" + acct + ":role/" + name), + }}, nil +} + +func (f *fakeLBEnsurer) PutRolePolicy(_ string, _ *iam.PutRolePolicyInput) (*iam.PutRolePolicyOutput, error) { + return &iam.PutRolePolicyOutput{}, nil +} + +func (f *fakeLBEnsurer) GetInstanceProfile(_ string, _ *iam.GetInstanceProfileInput) (*iam.GetInstanceProfileOutput, error) { + return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) +} + +func (f *fakeLBEnsurer) CreateInstanceProfile(acct string, in *iam.CreateInstanceProfileInput) (*iam.CreateInstanceProfileOutput, error) { + f.profileAcct = acct + name := aws.StringValue(in.InstanceProfileName) + return &iam.CreateInstanceProfileOutput{InstanceProfile: &iam.InstanceProfile{ + InstanceProfileName: aws.String(name), + Arn: aws.String("arn:aws:iam::" + acct + ":instance-profile/" + name), + }}, nil +} + +func (f *fakeLBEnsurer) AddRoleToInstanceProfile(_ string, _ *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error) { + return &iam.AddRoleToInstanceProfileOutput{}, nil +} + +var _ handlers_iam.SystemInstanceRoleEnsurer = (*fakeLBEnsurer)(nil) + +// TestEnsureLBInstanceProfile_UsesLazyProvider asserts the LB profile resolves via +// the lazy IAMProvider when the eager IAM field is unset — the daemon wires only +// the provider so the IAM build cannot race the NATS KV backend at startup. +func TestEnsureLBInstanceProfile_UsesLazyProvider(t *testing.T) { + f := &fakeLBEnsurer{} + s := &ELBv2ServiceImpl{IAMProvider: func() handlers_iam.SystemInstanceRoleEnsurer { return f }} + + arn := s.ensureLBInstanceProfile(utils.GlobalAccountID) + assert.Equal(t, "arn:aws:iam::"+utils.GlobalAccountID+":instance-profile/"+lbAgentSystemRoleName, arn) +} + +// TestEnsureLBInstanceProfile_CreatesInSystemAccount asserts the role and profile +// are created under the account the LB VM runs in (the system account). IMDS +// resolves the profile under the instance's account, so a role created in the LB +// owner account would be invisible to it and the lb-agent would get no creds. +func TestEnsureLBInstanceProfile_CreatesInSystemAccount(t *testing.T) { + f := &fakeLBEnsurer{} + s := &ELBv2ServiceImpl{IAM: f} + + s.ensureLBInstanceProfile(utils.GlobalAccountID) + assert.Equal(t, utils.GlobalAccountID, f.roleAcct) + assert.Equal(t, utils.GlobalAccountID, f.profileAcct) +} + +// TestEnsureLBInstanceProfile_NotReadyFallsBack asserts an unwired IAM (no field, +// provider returns nil because the KV backend is not up yet) yields "" so the LB +// VM falls back to baked static creds and retries on the next launch. +func TestEnsureLBInstanceProfile_NotReadyFallsBack(t *testing.T) { + s := &ELBv2ServiceImpl{IAMProvider: func() handlers_iam.SystemInstanceRoleEnsurer { return nil }} + assert.Equal(t, "", s.ensureLBInstanceProfile(utils.GlobalAccountID)) +} diff --git a/spinifex/handlers/elbv2/service_impl.go b/spinifex/handlers/elbv2/service_impl.go index 4455ad809..62fc8e0c9 100644 --- a/spinifex/handlers/elbv2/service_impl.go +++ b/spinifex/handlers/elbv2/service_impl.go @@ -23,6 +23,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/config" handlers_acm "github.com/mulgadc/spinifex/spinifex/handlers/acm" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" "github.com/mulgadc/spinifex/spinifex/network/topology" "github.com/mulgadc/spinifex/spinifex/tags" "github.com/mulgadc/spinifex/spinifex/utils" @@ -90,20 +91,24 @@ var _ ELBv2Service = (*ELBv2ServiceImpl)(nil) // ELBv2ServiceImpl implements ELBv2 operations with NATS JetStream persistence. type ELBv2ServiceImpl struct { - config *config.Config - store *Store - acmStore *handlers_acm.Store // resolves listener cert ARNs → PEM; nil-safe (HTTPS unavailable when nil) - nc *nats.Conn // NATS connection for JetStream KV store - VPCService *handlers_ec2_vpc.VPCServiceImpl // nil-safe: ENI ops skipped when nil (e.g. in tests) - InstanceLauncher SystemInstanceLauncher // nil-safe: system VM ops skipped when nil - SystemAccessKey string // System account access key for ALB agent SigV4 auth - SystemSecretKey string // System account secret key for ALB agent SigV4 auth - GatewayURL string // AWS gateway URL for ALB agent outbound connections - MgmtRouteGateway string // br-mgmt IP (next-hop for mgmt route); empty when AWSGW is on 0.0.0.0 - MgmtRouteTarget string // AWSGW bind IP to route via mgmt NIC - MgmtBridgeIP string // br-mgmt IP, populated whenever br-mgmt exists (single + multi node) for the internal-scheme fallback route - AdvertiseIP string // AdvertiseIP / WAN gateway, populated whenever set; used as the internal-scheme fallback route target on single-node - CACert string // PEM-encoded CA certificate delivered to microvm guests via fw_cfg + config *config.Config + store *Store + acmStore *handlers_acm.Store // resolves listener cert ARNs → PEM; nil-safe (HTTPS unavailable when nil) + nc *nats.Conn // NATS connection for JetStream KV store + VPCService *handlers_ec2_vpc.VPCServiceImpl // nil-safe: ENI ops skipped when nil (e.g. in tests) + InstanceLauncher SystemInstanceLauncher // nil-safe: system VM ops skipped when nil + IAM handlers_iam.SystemInstanceRoleEnsurer // nil-safe: LB VM falls back to baked static creds when nil (tests set directly) + // IAMProvider lazily resolves the IAM ensurer at launch time so it cannot + // race the NATS KV backend at daemon startup. Preferred over IAM when set. + IAMProvider func() handlers_iam.SystemInstanceRoleEnsurer + SystemAccessKey string // System account access key for ALB agent SigV4 auth + SystemSecretKey string // System account secret key for ALB agent SigV4 auth + GatewayURL string // AWS gateway URL for ALB agent outbound connections + MgmtRouteGateway string // br-mgmt IP (next-hop for mgmt route); empty when AWSGW is on 0.0.0.0 + MgmtRouteTarget string // AWSGW bind IP to route via mgmt NIC + MgmtBridgeIP string // br-mgmt IP, populated whenever br-mgmt exists (single + multi node) for the internal-scheme fallback route + AdvertiseIP string // AdvertiseIP / WAN gateway, populated whenever set; used as the internal-scheme fallback route target on single-node + CACert string // PEM-encoded CA certificate delivered to microvm guests via fw_cfg nodeID string region string systemInstanceType string // instance type for system VMs; resolved lazily via systemInstanceTypeFunc @@ -251,7 +256,13 @@ func (s *ELBv2ServiceImpl) getSystemInstanceType() string { // br-mgmt NIC (the mgmt-bridge URL): the heartbeat is control-plane and must // survive a host reboot that strands the OVN/EIP data plane, so it never rides // the WAN. mgmtGatewayURL falls back to the WAN URL when no mgmt bridge exists. -func (s *ELBv2ServiceImpl) buildLBAgentEnv(lbID string) string { +func (s *ELBv2ServiceImpl) buildLBAgentEnv(lbID string, staticCreds bool) string { + // staticCreds false ⇒ no LB_ACCESS_KEY/LB_SECRET_KEY: the agent's signer + // falls back to the AWS SDK chain, which reads IMDS instance-role creds. + if !staticCreds { + return fmt.Sprintf("LB_LB_ID=%s\nLB_GATEWAY_URL=%s\nLB_REGION=%s\n", + lbID, s.mgmtGatewayURL(), s.region) + } return fmt.Sprintf("LB_LB_ID=%s\nLB_GATEWAY_URL=%s\nLB_ACCESS_KEY=%s\nLB_SECRET_KEY=%s\nLB_REGION=%s\n", lbID, s.mgmtGatewayURL(), s.SystemAccessKey, s.SystemSecretKey, s.region) } @@ -451,19 +462,27 @@ func (s *ELBv2ServiceImpl) launchLBVM(lbID, scheme string, eniIDs, subnets []str return res } + // Prefer IMDS instance-role creds: attach a system instance profile so the + // lb-agent authenticates with scoped, rotating credentials and no static + // secret rides in fw_cfg. Falls back to baked system keys when IAM is unwired. + // The role lives in the system account because the LB VM (and its ENI) run + // there — IMDS resolves the profile under the instance's account. + profileARN := s.ensureLBInstanceProfile(utils.GlobalAccountID) + nics := s.buildMicrovmNICs(primaryIP, primaryMAC, subnets[0], eniIDs[0], scheme, extraENIInputs, accountID) launchInput := &SystemInstanceInput{ - InstanceType: s.getSystemInstanceType(), - SubnetID: subnets[0], - ENIID: eniIDs[0], - ENIMac: primaryMAC, - ENIIP: primaryIP, - ExtraENIs: extraENIInputs, - Scheme: scheme, - AccountID: accountID, - NICs: nics, - LBAgentEnv: s.buildLBAgentEnv(lbID), - CACert: s.CACert, + InstanceType: s.getSystemInstanceType(), + SubnetID: subnets[0], + ENIID: eniIDs[0], + ENIMac: primaryMAC, + ENIIP: primaryIP, + ExtraENIs: extraENIInputs, + Scheme: scheme, + AccountID: accountID, + NICs: nics, + LBAgentEnv: s.buildLBAgentEnv(lbID, profileARN == ""), + CACert: s.CACert, + IamInstanceProfileArn: profileARN, } // Dev-mode only: forward HTTP/HTTPS ports from host for local testing. // In production (VPC networking), traffic reaches the LB VM's VPC IP directly. @@ -551,18 +570,24 @@ func (s *ELBv2ServiceImpl) RebuildSystemInstanceInput(ctx RecoveryContext) (*Sys } } + // Re-ensure the instance profile so a recovered LB VM keeps IMDS creds; the + // ensure is idempotent and converges on the existing role/profile. The role + // lives in the system account where the LB VM runs, not the LB owner account. + profileARN := s.ensureLBInstanceProfile(utils.GlobalAccountID) + return &SystemInstanceInput{ - InstanceType: ctx.InstanceType, - SubnetID: lb.Subnets[0], - ENIID: lb.ENIs[0], - ENIMac: primaryMAC, - ENIIP: lb.VPCIP, - ExtraENIs: extraENIs, - Scheme: lb.Scheme, - AccountID: lb.AccountID, - NICs: nics, - LBAgentEnv: s.buildLBAgentEnv(lb.LoadBalancerID), - CACert: s.CACert, + InstanceType: ctx.InstanceType, + SubnetID: lb.Subnets[0], + ENIID: lb.ENIs[0], + ENIMac: primaryMAC, + ENIIP: lb.VPCIP, + ExtraENIs: extraENIs, + Scheme: lb.Scheme, + AccountID: lb.AccountID, + NICs: nics, + LBAgentEnv: s.buildLBAgentEnv(lb.LoadBalancerID, profileARN == ""), + CACert: s.CACert, + IamInstanceProfileArn: profileARN, }, nil } diff --git a/spinifex/handlers/iam/system_role.go b/spinifex/handlers/iam/system_role.go new file mode 100644 index 000000000..cc7b0425b --- /dev/null +++ b/spinifex/handlers/iam/system_role.go @@ -0,0 +1,115 @@ +package handlers_iam + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/mulgadc/spinifex/spinifex/awserrors" +) + +// EC2InstanceTrustPolicy is the trust document AssumeRoleForInstance requires: +// the EC2 service principal may assume the role so IMDS can mint instance-role +// credentials for the VM. +const EC2InstanceTrustPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}` + +// SystemInstanceRoleEnsurer is the narrow IAM surface needed to find-or-create a +// Spinifex-managed instance role and the instance profile that fronts it. +type SystemInstanceRoleEnsurer interface { + GetRole(accountID string, input *iam.GetRoleInput) (*iam.GetRoleOutput, error) + CreateRole(accountID string, input *iam.CreateRoleInput) (*iam.CreateRoleOutput, error) + PutRolePolicy(accountID string, input *iam.PutRolePolicyInput) (*iam.PutRolePolicyOutput, error) + GetInstanceProfile(accountID string, input *iam.GetInstanceProfileInput) (*iam.GetInstanceProfileOutput, error) + CreateInstanceProfile(accountID string, input *iam.CreateInstanceProfileInput) (*iam.CreateInstanceProfileOutput, error) + AddRoleToInstanceProfile(accountID string, input *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error) +} + +// EnsureSystemInstanceProfile find-or-creates a system-managed role (EC2 trust + +// the given inline permission policy) plus an instance profile carrying it, +// returning the profile ARN to attach at launch so IMDS serves the role. +// Idempotent: a re-launch re-asserts the policy, and a racing create converges +// via the EntityAlreadyExists re-read. The role/profile share a name. +func EnsureSystemInstanceProfile(e SystemInstanceRoleEnsurer, accountID, roleName, policyName, policyDoc string) (string, error) { + if err := ensureSystemRole(e, accountID, roleName); err != nil { + return "", err + } + if _, err := e.PutRolePolicy(accountID, &iam.PutRolePolicyInput{ + RoleName: aws.String(roleName), + PolicyName: aws.String(policyName), + PolicyDocument: aws.String(policyDoc), + }); err != nil { + return "", fmt.Errorf("put role policy %q on %q: %w", policyName, roleName, err) + } + return ensureSystemInstanceProfile(e, accountID, roleName) +} + +// ensureSystemRole creates the role with the EC2 trust policy when absent. A +// racing create that lost (EntityAlreadyExists) is tolerated as success. +func ensureSystemRole(e SystemInstanceRoleEnsurer, accountID, roleName string) error { + if _, err := e.GetRole(accountID, &iam.GetRoleInput{RoleName: aws.String(roleName)}); err == nil { + return nil + } else if err.Error() != awserrors.ErrorIAMNoSuchEntity { + return fmt.Errorf("get role %q: %w", roleName, err) + } + + _, err := e.CreateRole(accountID, &iam.CreateRoleInput{ + RoleName: aws.String(roleName), + AssumeRolePolicyDocument: aws.String(EC2InstanceTrustPolicy), + Description: aws.String("Spinifex-managed system instance role"), + }) + if err == nil || err.Error() == awserrors.ErrorIAMEntityAlreadyExists { + return nil + } + return fmt.Errorf("create role %q: %w", roleName, err) +} + +// ensureSystemInstanceProfile guarantees an instance profile named for the role +// exists and carries it, returning the profile ARN. Concurrent launches converge +// on the same profile (EntityAlreadyExists / LimitExceeded treated as success). +func ensureSystemInstanceProfile(e SystemInstanceRoleEnsurer, accountID, roleName string) (string, error) { + out, err := e.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{ + InstanceProfileName: aws.String(roleName), + }) + if err == nil { + return attachSystemRole(e, accountID, roleName, + aws.StringValue(out.InstanceProfile.Arn), len(out.InstanceProfile.Roles) > 0) + } + if err.Error() != awserrors.ErrorIAMNoSuchEntity { + return "", fmt.Errorf("get instance profile %q: %w", roleName, err) + } + + created, err := e.CreateInstanceProfile(accountID, &iam.CreateInstanceProfileInput{ + InstanceProfileName: aws.String(roleName), + }) + if err != nil { + if err.Error() != awserrors.ErrorIAMEntityAlreadyExists { + return "", fmt.Errorf("create instance profile %q: %w", roleName, err) + } + got, gerr := e.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{ + InstanceProfileName: aws.String(roleName), + }) + if gerr != nil { + return "", fmt.Errorf("re-get instance profile %q: %w", roleName, gerr) + } + return attachSystemRole(e, accountID, roleName, + aws.StringValue(got.InstanceProfile.Arn), len(got.InstanceProfile.Roles) > 0) + } + return attachSystemRole(e, accountID, roleName, + aws.StringValue(created.InstanceProfile.Arn), false) +} + +// attachSystemRole adds roleName to the profile unless already attached. A +// LimitExceeded error means a racing launch attached it first — treated success. +func attachSystemRole(e SystemInstanceRoleEnsurer, accountID, roleName, profileARN string, alreadyAttached bool) (string, error) { + if alreadyAttached { + return profileARN, nil + } + _, err := e.AddRoleToInstanceProfile(accountID, &iam.AddRoleToInstanceProfileInput{ + InstanceProfileName: aws.String(roleName), + RoleName: aws.String(roleName), + }) + if err != nil && err.Error() != awserrors.ErrorIAMLimitExceeded { + return "", fmt.Errorf("add role %q to instance profile %q: %w", roleName, roleName, err) + } + return profileARN, nil +} diff --git a/spinifex/handlers/iam/system_role_test.go b/spinifex/handlers/iam/system_role_test.go new file mode 100644 index 000000000..9bae83e0d --- /dev/null +++ b/spinifex/handlers/iam/system_role_test.go @@ -0,0 +1,130 @@ +package handlers_iam + +import ( + "errors" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeSystemRoleEnsurer is an in-memory SystemInstanceRoleEnsurer that records +// calls and serves role/profile stores keyed by name. +type fakeSystemRoleEnsurer struct { + roles map[string]*iam.Role + profiles map[string]*iam.InstanceProfile + policies map[string]string + createRoleCalls int + createProfCalls int + addRoleCalls int + lastTrustDoc string +} + +func newFakeSystemRoleEnsurer() *fakeSystemRoleEnsurer { + return &fakeSystemRoleEnsurer{ + roles: map[string]*iam.Role{}, + profiles: map[string]*iam.InstanceProfile{}, + policies: map[string]string{}, + } +} + +func (f *fakeSystemRoleEnsurer) GetRole(_ string, in *iam.GetRoleInput) (*iam.GetRoleOutput, error) { + r, ok := f.roles[aws.StringValue(in.RoleName)] + if !ok { + return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) + } + return &iam.GetRoleOutput{Role: r}, nil +} + +func (f *fakeSystemRoleEnsurer) CreateRole(acct string, in *iam.CreateRoleInput) (*iam.CreateRoleOutput, error) { + f.createRoleCalls++ + f.lastTrustDoc = aws.StringValue(in.AssumeRolePolicyDocument) + name := aws.StringValue(in.RoleName) + if _, ok := f.roles[name]; ok { + return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists) + } + r := &iam.Role{RoleName: aws.String(name), Arn: aws.String("arn:aws:iam::" + acct + ":role/" + name)} + f.roles[name] = r + return &iam.CreateRoleOutput{Role: r}, nil +} + +func (f *fakeSystemRoleEnsurer) PutRolePolicy(_ string, in *iam.PutRolePolicyInput) (*iam.PutRolePolicyOutput, error) { + f.policies[aws.StringValue(in.RoleName)] = aws.StringValue(in.PolicyDocument) + return &iam.PutRolePolicyOutput{}, nil +} + +func (f *fakeSystemRoleEnsurer) GetInstanceProfile(_ string, in *iam.GetInstanceProfileInput) (*iam.GetInstanceProfileOutput, error) { + p, ok := f.profiles[aws.StringValue(in.InstanceProfileName)] + if !ok { + return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) + } + return &iam.GetInstanceProfileOutput{InstanceProfile: p}, nil +} + +func (f *fakeSystemRoleEnsurer) CreateInstanceProfile(acct string, in *iam.CreateInstanceProfileInput) (*iam.CreateInstanceProfileOutput, error) { + f.createProfCalls++ + name := aws.StringValue(in.InstanceProfileName) + if _, ok := f.profiles[name]; ok { + return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists) + } + p := &iam.InstanceProfile{ + InstanceProfileName: aws.String(name), + Arn: aws.String("arn:aws:iam::" + acct + ":instance-profile/" + name), + } + f.profiles[name] = p + return &iam.CreateInstanceProfileOutput{InstanceProfile: p}, nil +} + +func (f *fakeSystemRoleEnsurer) AddRoleToInstanceProfile(_ string, in *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error) { + f.addRoleCalls++ + p, ok := f.profiles[aws.StringValue(in.InstanceProfileName)] + if !ok { + return nil, errors.New(awserrors.ErrorIAMNoSuchEntity) + } + p.Roles = []*iam.Role{{RoleName: in.RoleName}} + return &iam.AddRoleToInstanceProfileOutput{}, nil +} + +const ( + testRoleAcct = "000000000007" + testRoleName = "spinifex-system-role" + testPolicyName = "spinifex-system-internal" + testPolicyDoc = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["svc:DoThing"],"Resource":"*"}]}` +) + +// TestEnsureSystemInstanceProfile_CreatesAll asserts the role (EC2 trust), inline +// policy, and instance profile are all created and the profile ARN is returned. +func TestEnsureSystemInstanceProfile_CreatesAll(t *testing.T) { + f := newFakeSystemRoleEnsurer() + arn, err := EnsureSystemInstanceProfile(f, testRoleAcct, testRoleName, testPolicyName, testPolicyDoc) + require.NoError(t, err) + assert.Equal(t, "arn:aws:iam::"+testRoleAcct+":instance-profile/"+testRoleName, arn) + + assert.Equal(t, 1, f.createRoleCalls) + assert.Equal(t, 1, f.createProfCalls) + assert.Equal(t, 1, f.addRoleCalls) + assert.True(t, strings.Contains(f.lastTrustDoc, "ec2.amazonaws.com")) + assert.Equal(t, testPolicyDoc, f.policies[testRoleName]) + require.NotNil(t, f.profiles[testRoleName]) + assert.Len(t, f.profiles[testRoleName].Roles, 1) +} + +// TestEnsureSystemInstanceProfile_Idempotent asserts a re-run against existing +// role+profile creates nothing new but still re-asserts the inline policy. +func TestEnsureSystemInstanceProfile_Idempotent(t *testing.T) { + f := newFakeSystemRoleEnsurer() + _, err := EnsureSystemInstanceProfile(f, testRoleAcct, testRoleName, testPolicyName, testPolicyDoc) + require.NoError(t, err) + + _, err = EnsureSystemInstanceProfile(f, testRoleAcct, testRoleName, testPolicyName, testPolicyDoc) + require.NoError(t, err) + assert.Equal(t, 1, f.createRoleCalls, "role created once") + assert.Equal(t, 1, f.createProfCalls, "profile created once") + assert.Equal(t, 1, f.addRoleCalls, "role attached once") +} + +var _ SystemInstanceRoleEnsurer = (*fakeSystemRoleEnsurer)(nil) diff --git a/spinifex/handlers/imds/credentials.go b/spinifex/handlers/imds/credentials.go index eb3e5d011..a7b5b3cda 100644 --- a/spinifex/handlers/imds/credentials.go +++ b/spinifex/handlers/imds/credentials.go @@ -64,7 +64,7 @@ func (c *credCache) get(eni *eniFacts, roleName, roleARN string, now time.Time) } c.mu.Unlock() - out, err := c.sts.AssumeRoleForInstance(eni.accountID, roleARN, eni.instanceID, credDurationSeconds) + out, err := c.sts.AssumeRoleForInstance(eni.iamAccountID(), roleARN, eni.instanceID, credDurationSeconds) if err != nil { return nil, err } @@ -81,7 +81,7 @@ func (c *credCache) get(eni *eniFacts, roleName, roleARN string, now time.Time) SecretAccessKey: aws.StringValue(out.Credentials.SecretAccessKey), Token: aws.StringValue(out.Credentials.SessionToken), Expiration: expiration.UTC().Format(time.RFC3339), - AccountId: eni.accountID, + AccountId: eni.iamAccountID(), }) if err != nil { return nil, fmt.Errorf("marshal instance credential: %w", err) diff --git a/spinifex/handlers/imds/cross_account_owner_test.go b/spinifex/handlers/imds/cross_account_owner_test.go new file mode 100644 index 000000000..55e1cbea7 --- /dev/null +++ b/spinifex/handlers/imds/cross_account_owner_test.go @@ -0,0 +1,63 @@ +package handlers_imds + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A same-account ENI leaves InstanceOwnerId empty, so iamAccountID falls back to +// the ENI's own account — the common EC2 case where instance and ENI co-own. +func TestIAMAccountID_FallsBackToENIAccount(t *testing.T) { + eni := &eniFacts{accountID: "111122223333"} + assert.Equal(t, "111122223333", eni.iamAccountID()) +} + +// A system VM (LB/EKS) plugs into a customer ENI: the ENI's own account differs +// from the account that owns the instance and its IAM role. iamAccountID returns +// the instance-owner account so the role lookup hits the right tenant. +func TestIAMAccountID_PrefersInstanceOwner(t *testing.T) { + eni := &eniFacts{accountID: "000000000001", instanceAccountID: "000000000000"} + assert.Equal(t, "000000000000", eni.iamAccountID()) +} + +// eniFactsFromRecord projects the persisted InstanceOwnerId onto the fact the +// IAM/STS resolution keys off, so a cross-account attach survives the round-trip. +func TestEniFactsFromRecord_CarriesInstanceOwner(t *testing.T) { + rec := &eniRecord{ + NetworkInterfaceId: "eni-lb", + InstanceId: "i-sysvm", + InstanceOwnerId: "000000000000", + } + facts := eniFactsFromRecord("000000000001", rec) + assert.Equal(t, "000000000001", facts.accountID) + assert.Equal(t, "000000000000", facts.instanceAccountID) + assert.Equal(t, "000000000000", facts.iamAccountID()) +} + +// resolveInstance must describe the instance under its owning account, not the +// ENI account — a system VM is invisible under the customer-account fan-out, so +// keying the describe off the ENI account would resolve no instance at all. +func TestResolveInstance_DescribesUnderInstanceOwner(t *testing.T) { + r, lookup := newTestResolver(t) + lookup.facts = &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::000000000000:instance-profile/spinifex-lb-agent"} + + eni := &eniFacts{accountID: "000000000001", instanceAccountID: "000000000000", instanceID: "i-sysvm"} + inst, err := r.resolveInstance(eni) + require.NoError(t, err) + require.NotNil(t, inst) + assert.Equal(t, "000000000000", lookup.lastAccount) +} + +// Without an instance-owner stamp the describe routes to the ENI account, the +// historical same-account behaviour the fallback preserves. +func TestResolveInstance_FallsBackToENIAccount(t *testing.T) { + r, lookup := newTestResolver(t) + lookup.facts = &instanceFacts{} + + eni := &eniFacts{accountID: "111122223333", instanceID: "i-app"} + _, err := r.resolveInstance(eni) + require.NoError(t, err) + assert.Equal(t, "111122223333", lookup.lastAccount) +} diff --git a/spinifex/handlers/imds/metadata.go b/spinifex/handlers/imds/metadata.go index 769a665f4..17929c52a 100644 --- a/spinifex/handlers/imds/metadata.go +++ b/spinifex/handlers/imds/metadata.go @@ -274,7 +274,7 @@ func (s *IMDSServiceImpl) serveMetaDataRoot(w http.ResponseWriter, eni *eniFacts keys = append(keys, "public-keys/") } if inst.iamInstanceProfileArn != "" { - if p, err := s.iam.ResolveInstanceProfile(eni.accountID, inst.iamInstanceProfileArn); err == nil && p != nil { + if p, err := s.iam.ResolveInstanceProfile(eni.iamAccountID(), inst.iamInstanceProfileArn); err == nil && p != nil { keys = append(keys, "iam/") } } @@ -339,7 +339,7 @@ func (s *IMDSServiceImpl) serveInstanceIdentityDocument(w http.ResponseWriter, e return } doc := instanceIdentityDocument{ - AccountID: eni.accountID, + AccountID: eni.iamAccountID(), Architecture: inst.architecture, AvailabilityZone: eni.availabilityZone, ImageID: inst.imageID, @@ -412,9 +412,9 @@ func (s *IMDSServiceImpl) serveRoleCredentials(w http.ResponseWriter, eni *eniFa return } - role, err := s.iam.GetRole(eni.accountID, &iam.GetRoleInput{RoleName: aws.String(profile.RoleName)}) + role, err := s.iam.GetRole(eni.iamAccountID(), &iam.GetRoleInput{RoleName: aws.String(profile.RoleName)}) if err != nil || role == nil || role.Role == nil || role.Role.Arn == nil { - slog.Error("IMDS: resolve role ARN failed", "account_id", eni.accountID, "role", profile.RoleName, "err", err) + slog.Error("IMDS: resolve role ARN failed", "account_id", eni.iamAccountID(), "role", profile.RoleName, "err", err) w.WriteHeader(http.StatusInternalServerError) return } @@ -422,7 +422,7 @@ func (s *IMDSServiceImpl) serveRoleCredentials(w http.ResponseWriter, eni *eniFa body, err := s.creds.get(eni, profile.RoleName, *role.Role.Arn, s.now()) if err != nil { // Mirror the AWS Code:"Failed" JSON body so SDKs report a recognisable error. - slog.Warn("IMDS: AssumeRoleForInstance failed", "account_id", eni.accountID, "role", profile.RoleName, "instance_id", eni.instanceID, "err", err) + slog.Warn("IMDS: AssumeRoleForInstance failed", "account_id", eni.iamAccountID(), "role", profile.RoleName, "instance_id", eni.instanceID, "err", err) writeJSON(w, map[string]string{ "Code": "Failed", "LastUpdated": s.now().UTC().Format(time.RFC3339), @@ -453,13 +453,13 @@ func (s *IMDSServiceImpl) servePublicKeys(w http.ResponseWriter, eni *eniFacts, case "0", "0/": writeText(w, "openssh-key") case "0/openssh-key": - material, err := s.pubKeys.GetPublicKey(eni.accountID, inst.keyName) + material, err := s.pubKeys.GetPublicKey(eni.iamAccountID(), inst.keyName) if err != nil { if err.Error() == awserrors.ErrorInvalidKeyPairNotFound { w.WriteHeader(http.StatusNotFound) return } - slog.Error("IMDS: public key material fetch failed", "account_id", eni.accountID, "key_name", inst.keyName, "instance_id", eni.instanceID, "err", err) + slog.Error("IMDS: public key material fetch failed", "account_id", eni.iamAccountID(), "key_name", inst.keyName, "instance_id", eni.instanceID, "err", err) w.WriteHeader(http.StatusInternalServerError) return } @@ -602,12 +602,12 @@ func (s *IMDSServiceImpl) profileFor(eni *eniFacts) (*resolvedProfile, error) { if inst == nil || inst.iamInstanceProfileArn == "" { return nil, nil } - profile, err := s.iam.ResolveInstanceProfile(eni.accountID, inst.iamInstanceProfileArn) + profile, err := s.iam.ResolveInstanceProfile(eni.iamAccountID(), inst.iamInstanceProfileArn) if err != nil { if err.Error() == awserrors.ErrorIAMNoSuchEntity { return nil, nil // profile deleted; treat as no role } - slog.Error("IMDS: resolve instance profile failed", "account_id", eni.accountID, "arn", inst.iamInstanceProfileArn, "err", err) + slog.Error("IMDS: resolve instance profile failed", "account_id", eni.iamAccountID(), "arn", inst.iamInstanceProfileArn, "err", err) return nil, err } if profile == nil { diff --git a/spinifex/handlers/imds/resolver.go b/spinifex/handlers/imds/resolver.go index 3177b7a93..ff167c849 100644 --- a/spinifex/handlers/imds/resolver.go +++ b/spinifex/handlers/imds/resolver.go @@ -19,16 +19,28 @@ const kvBucketVPCs = "spinifex-vpc-vpcs" // eniFacts is the ENI fields served by the IMDS metadata surface, read live from the ENI record. type eniFacts struct { - eniID string - accountID string - instanceID string - vpcID string - subnetID string - privateIP string - publicIP string - mac string - availabilityZone string - securityGroupIDs []string + eniID string + accountID string + instanceAccountID string + instanceID string + vpcID string + subnetID string + privateIP string + publicIP string + mac string + availabilityZone string + securityGroupIDs []string +} + +// iamAccountID is the account that owns this ENI's attached instance and its IAM +// resources. System VMs (LB/EKS) run in the system account but plug into a +// customer-account ENI, so the instance and its role live in a different account +// than the ENI; falls back to the ENI account for the same-account common case. +func (e *eniFacts) iamAccountID() string { + if e.instanceAccountID != "" { + return e.instanceAccountID + } + return e.accountID } // instanceFacts carries metadata fields not present on the ENI record, fetched via instanceLookup. @@ -64,6 +76,7 @@ type eniRecord struct { PrivateIpAddress string `json:"private_ip_address"` MacAddress string `json:"mac_address"` InstanceId string `json:"instance_id,omitempty"` + InstanceOwnerId string `json:"instance_owner_id,omitempty"` PublicIpAddress string `json:"public_ip_address,omitempty"` SecurityGroupIds []string `json:"security_group_ids,omitempty"` } @@ -136,16 +149,17 @@ func (r *metadataResolver) findENIByID(eniID string) (string, []byte, error) { // fact subset the metadata surface serves. func eniFactsFromRecord(accountID string, rec *eniRecord) *eniFacts { return &eniFacts{ - eniID: rec.NetworkInterfaceId, - accountID: accountID, - instanceID: rec.InstanceId, - vpcID: rec.VpcId, - subnetID: rec.SubnetId, - privateIP: rec.PrivateIpAddress, - publicIP: rec.PublicIpAddress, - mac: rec.MacAddress, - availabilityZone: rec.AvailabilityZone, - securityGroupIDs: rec.SecurityGroupIds, + eniID: rec.NetworkInterfaceId, + accountID: accountID, + instanceAccountID: rec.InstanceOwnerId, + instanceID: rec.InstanceId, + vpcID: rec.VpcId, + subnetID: rec.SubnetId, + privateIP: rec.PrivateIpAddress, + publicIP: rec.PublicIpAddress, + mac: rec.MacAddress, + availabilityZone: rec.AvailabilityZone, + securityGroupIDs: rec.SecurityGroupIds, } } @@ -154,7 +168,7 @@ func (r *metadataResolver) resolveInstance(eni *eniFacts) (*instanceFacts, error if eni.instanceID == "" { return nil, nil } - return r.lookup.describe(eni.accountID, eni.instanceID) + return r.lookup.describe(eni.iamAccountID(), eni.instanceID) } // resolveSGNames maps SG IDs to group names for /security-groups. Best-effort; falls back to IDs. diff --git a/spinifex/handlers/imds/resolver_test.go b/spinifex/handlers/imds/resolver_test.go index dfb11214c..c8521df6d 100644 --- a/spinifex/handlers/imds/resolver_test.go +++ b/spinifex/handlers/imds/resolver_test.go @@ -14,13 +14,15 @@ import ( // fakeLookup is a programmable instanceLookup so resolver tests never need the // account-scoped instance fan-out. type fakeLookup struct { - facts *instanceFacts - err error - calls int + facts *instanceFacts + err error + calls int + lastAccount string } -func (f *fakeLookup) describe(_, _ string) (*instanceFacts, error) { +func (f *fakeLookup) describe(accountID, _ string) (*instanceFacts, error) { f.calls++ + f.lastAccount = accountID return f.facts, f.err } diff --git a/spinifex/handlers/sysinstance/launcher.go b/spinifex/handlers/sysinstance/launcher.go index 2a6f1e03f..113f49bd3 100644 --- a/spinifex/handlers/sysinstance/launcher.go +++ b/spinifex/handlers/sysinstance/launcher.go @@ -81,6 +81,11 @@ type SystemInstanceInput struct { // UserData is the raw (un-encoded) cloud-init user-data for a BootAMI guest. // The launcher base64-encodes it for the RunInstances path. UserData string `json:"user_data,omitempty"` + + // IamInstanceProfileArn attaches an instance profile so the in-VM IMDS + // endpoint serves the profile's role credentials. Empty leaves the VM + // without a profile (the agent then needs static keys). + IamInstanceProfileArn string `json:"iam_instance_profile_arn,omitempty"` } // ExtraENIInput describes an additional pre-created ENI to attach to a system diff --git a/spinifex/lbagent/agent.go b/spinifex/lbagent/agent.go index 8a1cf511e..b8e151819 100644 --- a/spinifex/lbagent/agent.go +++ b/spinifex/lbagent/agent.go @@ -5,6 +5,7 @@ package lbagent import ( "bytes" + "context" "crypto/sha256" "crypto/tls" "encoding/hex" @@ -22,7 +23,7 @@ import ( "syscall" "time" - "github.com/mulgadc/predastore/auth" + "github.com/mulgadc/spinifex/internal/gwsign" "github.com/mulgadc/spinifex/internal/tlsconfig" ) @@ -60,9 +61,8 @@ type Agent struct { certDir string // dir for TLS cert PEM files; defaults to CertDir (overridable in tests) socketPath string // HAProxy stats socket - accessKey string - secretKey string - client *http.Client + signer *gwsign.Signer + client *http.Client localConfigHash string engine string // data-plane engine of the last applied config @@ -86,13 +86,23 @@ func New(lbID, gatewayURL, accessKey, secretKey, region string) (*Agent, error) if gatewayURL == "" { return nil, fmt.Errorf("gatewayURL is required") } - if accessKey == "" || secretKey == "" { - return nil, fmt.Errorf("access key and secret key are required") - } if region == "" { return nil, fmt.Errorf("region is required") } + // Static keys (when present in user-data) sign as-is; otherwise fall back to + // the IMDS instance-role credentials served in-VM, which rotate. + var signer *gwsign.Signer + if accessKey != "" && secretKey != "" { + signer = gwsign.NewStatic(accessKey, secretKey) + } else { + s, err := gwsign.NewIMDS(context.Background(), region) + if err != nil { + return nil, fmt.Errorf("init IMDS signer: %w", err) + } + signer = s + } + // Use system CA trust store (deployment CA provisioned via fw_cfg on the microvm). client := &http.Client{ Timeout: 10 * time.Second, @@ -114,8 +124,7 @@ func New(lbID, gatewayURL, accessKey, secretKey, region string) (*Agent, error) pidPath: DefaultPIDPath, certDir: CertDir, socketPath: fmt.Sprintf("/tmp/spinifex-haproxy/%s.sock", lbID), - accessKey: accessKey, - secretKey: secretKey, + signer: signer, client: client, stopCh: make(chan struct{}), reloadFn: reloadHAProxy, @@ -309,7 +318,7 @@ func (a *Agent) signedPost(params url.Values) ([]byte, error) { sum := sha256.Sum256([]byte(body)) payloadHash := hex.EncodeToString(sum[:]) - if err := auth.SignReq(req, a.accessKey, a.secretKey, payloadHash, "elasticloadbalancing", a.region); err != nil { + if err := a.signer.Sign(req, payloadHash, "elasticloadbalancing", a.region); err != nil { return nil, fmt.Errorf("sign request: %w", err) } diff --git a/spinifex/lbagent/agent_test.go b/spinifex/lbagent/agent_test.go index 43e4f9417..f1c215630 100644 --- a/spinifex/lbagent/agent_test.go +++ b/spinifex/lbagent/agent_test.go @@ -44,14 +44,15 @@ func TestNew_EmptyGatewayURL(t *testing.T) { } } -func TestNew_EmptyCredentials(t *testing.T) { - _, err := New("lb-test", "https://gw:9999", "", "SECRET", "us-east-1") - if err == nil { - t.Fatal("expected error for empty access key") +func TestNew_EmptyCredentials_SelectsIMDS(t *testing.T) { + // Empty static keys fall back to the IMDS instance-role credential chain; + // construction succeeds (the chain resolves lazily at first sign). + agent, err := New("lb-test", "https://gw:9999", "", "", "us-east-1") + if err != nil { + t.Fatalf("New with empty keys (IMDS mode): %v", err) } - _, err = New("lb-test", "https://gw:9999", "AKID", "", "us-east-1") - if err == nil { - t.Fatal("expected error for empty secret key") + if agent.signer == nil { + t.Fatal("expected signer to be set in IMDS mode") } } diff --git a/tests/e2e/lb/lb_test.go b/tests/e2e/lb/lb_test.go index c7e48dd18..04207459a 100644 --- a/tests/e2e/lb/lb_test.go +++ b/tests/e2e/lb/lb_test.go @@ -701,6 +701,10 @@ func runUDPNLBSuite(t *testing.T, c *harness.AWSClient, f *sharedFixture) { // The key assertion: NLB targets reach healthy via the agent's active // prober. Pre-feature this timed out (nginx reported empty server lists). harness.WaitForTargetsHealthy(t, c, tgArn, 2, label, 2*time.Minute) + // Targets only reach healthy if the lb-agent authenticated to the gateway; + // with static keys dropped from user-data that auth is IMDS-only, so assert + // the cred provenance explicitly (no baked keys, assumed-role principal). + assertLBAgentAuthViaIMDS(t) eni := lbENI(t, c, "net", lb) priv := privateIP(eni) @@ -1649,6 +1653,49 @@ func dumpDaemonLogs(t *testing.T, dir, label string) { "journalctl", "-u", "spinifex-daemon", "--no-pager", "-n", "200") } +// assertLBAgentAuthViaIMDS proves the LB VM authenticated to the gateway with +// IMDS-served instance-role credentials rather than baked static keys. The LB VM +// is a system-account instance plugged into a customer-account ENI, so it is +// invisible to the customer AWS API; the gateway journal on the cluster nodes is +// the only surface that observes its principal. A recent assumed-role (ASIA) +// auth proves the cred type is an instance role — a regression to the three +// boot-path defects strands the agent on an empty role list, so no recent +// assumed-role auth appears and this fails. Skips when no node is SSH-reachable. +// +// The window is time-bounded on purpose: console logs accumulate across every VM +// the node has ever run (including pre-fix failures), so a flat-file scan for the +// old error would false-fail; the gateway's recent successes do not have that +// problem. +func assertLBAgentAuthViaIMDS(t *testing.T) { + t.Helper() + env := harness.LoadEnv(t) + if len(env.NodeIPs) == 0 { + t.Skip("no node IPs in env; cannot observe the LB VM gateway principal") + } + ssh := harness.NewPeerSSH() + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + sawAssumedRole, anyReachable := false, false + for _, ip := range env.NodeIPs { + gw, err := ssh.Run(ctx, ip, + "sudo journalctl -u spinifex-awsgw --no-pager --since '5 minutes ago' 2>/dev/null | "+ + `grep -F '"principalType":"assumed-role"' | grep -F '"accessKey":"ASIA' | tail -1`) + if err != nil { + continue // node unreachable; another node carries the heartbeat + } + anyReachable = true + if strings.TrimSpace(string(gw)) != "" { + sawAssumedRole = true + } + } + if !anyReachable { + t.Skip("no node SSH-reachable; cannot observe the LB VM gateway principal") + } + require.True(t, sawAssumedRole, + "no assumed-role (ASIA) gateway auth on any node — LB agent did not authenticate via IMDS creds") +} + func base64Encode(s string) string { return base64.StdEncoding.EncodeToString([]byte(s)) }