feat(ecs): ecs-agent skeleton + containerd image pull (Sprint 4c) (#414)

* feat(ecs): ecs-agent skeleton + containerd image pull (Sprint 4c)

Add the ecs-agent binary that runs inside an ECS container instance: it
resolves host identity from IMDS, registers on the Layer-2 NATS bus, and
heartbeats while alive. Image pulls go through a containerd-backed
ImagePuller behind an interface, with an ECR resolver that mints
GetAuthorizationToken creds from the instance role.

Extract the IMDS-credential and ECR-auth flow out of
ecr-credential-provider into internal/imdscreds and internal/ecrauth so
the agent and the kubelet provider share one implementation.

Add spinifex/handlers/ecs/bus as the single source of truth for Layer-2
subject builders and wire payloads, shared by the agent now and the
scheduler in Sprint 4e.

* build: pin genproto via replace to fix containerd/v2 ambiguous import

go.opencensus.io -> grpc@v1.33.2 drags the pre-split monolithic genproto
into the module graph; it still ships googleapis/rpc/status, which collides
with the split genproto/googleapis/rpc module that containerd/ttrpc needs.

The main module never imports the monolith, so a plain require bump is
dropped by 'go mod tidy'. Pin it with a replace to a post-split version so
the fix survives tidy and contributors can run 'go mod tidy' freely.

* fix(ecs): agent boot failsafes from runtime verify

- Build the ECR gateway HTTP client lazily on first image pull instead of
  at boot. A missing/malformed gateway CA no longer kills the agent before
  it can register + heartbeat, matching the best-effort containerd path.
- Actively probe containerd with IsServing in runtime.New. The client dials
  lazily, so a dead socket previously slipped past the boot-time
  'containerd unavailable' warn and only surfaced on the first pull.
- Make the heartbeat interval configurable via ECS_HEARTBEAT_INTERVAL
  (default 30s) for test/verify and slow-link tuning.
This commit is contained in:
Josh Siviour
2026-06-24 21:18:03 +10:00
committed by GitHub
parent ea03efe415
commit 93db7e4d1b
32 changed files with 4753 additions and 212 deletions
+5 -1
View File
@@ -68,6 +68,10 @@ build-lb-agent:
@echo -e "\n....Building lb-agent (static)"
CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host `build`)
@echo -e "\n....Building ecs-agent (static)"
CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
ifndef IMAGE
$(error IMAGE is required. Usage: make build-system-image IMAGE=lb)
@@ -287,7 +291,7 @@ distro-arm64:
distro-clean:
rm -rf dist/
.PHONY: build build-ui build-installer build-lb-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness manifest-check manifest-lint manifest-lint-update \
.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-harness manifest-check manifest-lint manifest-lint-update \
deploy reinstall clean \
install-system install-go install-aws quickinstall \
lint fix govulncheck \
+11 -174
View File
@@ -13,9 +13,6 @@ package main
import (
"bufio"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/json"
"fmt"
"io"
@@ -25,11 +22,8 @@ import (
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ecr"
"github.com/mulgadc/spinifex/internal/tlsconfig"
"github.com/mulgadc/spinifex/internal/ecrauth"
"github.com/mulgadc/spinifex/internal/imdscreds"
_ "github.com/mulgadc/spinifex/internal/fipsboot"
)
@@ -44,7 +38,6 @@ const (
defaultEnvFile = "/etc/spinifex-eks/agent.env"
defaultGatewayCA = "/etc/spinifex-eks/gateway-ca.pem"
defaultIMDSBase = "http://169.254.169.254/latest"
imdsTokenTTL = "21600"
)
// CredentialProviderRequest is the kubelet exec-provider request on STDIN.
@@ -80,7 +73,7 @@ type config struct {
func main() {
cfg := loadConfig(defaultEnvFile)
httpClient, err := gatewayHTTPClient(cfg.GatewayCA)
httpClient, err := ecrauth.GatewayHTTPClient(cfg.GatewayCA)
if err != nil {
slog.Error("ecr-credential-provider: build gateway client", "err", err)
emitEmpty(os.Stdout)
@@ -106,155 +99,35 @@ func run(in io.Reader, out io.Writer, cfg config, httpClient *http.Client) error
if req.Image == "" {
return fmt.Errorf("request image is empty")
}
imageHost := hostFromImage(req.Image)
imageHost := ecrauth.HostFromImage(req.Image)
creds, err := fetchIMDSCredentials(httpClient, cfg.IMDSBase)
creds, err := imdscreds.Fetch(httpClient, cfg.IMDSBase)
if err != nil {
return fmt.Errorf("fetch IMDS credentials: %w", err)
}
username, password, proxyHost, expiresAt, err := getAuthToken(cfg, httpClient, creds)
tok, err := ecrauth.GetAuthorizationToken(cfg.Region, cfg.GatewayURL, httpClient,
creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)
if err != nil {
return fmt.Errorf("get authorization token: %w", err)
}
auth := AuthConfig{Username: username, Password: password}
auth := AuthConfig{Username: tok.Username, Password: tok.Password}
authMap := map[string]AuthConfig{imageHost: auth}
if proxyHost != "" && proxyHost != imageHost {
authMap[proxyHost] = auth
if tok.ProxyHost != "" && tok.ProxyHost != imageHost {
authMap[tok.ProxyHost] = auth
}
resp := CredentialProviderResponse{
APIVersion: credProviderAPIVersion,
Kind: "CredentialProviderResponse",
CacheKeyType: "Registry",
CacheDuration: cacheDurationFrom(expiresAt),
CacheDuration: cacheDurationFrom(tok.ExpiresAt),
Auth: authMap,
}
return json.NewEncoder(out).Encode(&resp)
}
// imdsCreds mirrors the IMDS security-credentials JSON shape.
type imdsCreds struct {
Code string `json:"Code"`
AccessKeyID string `json:"AccessKeyId"`
SecretAccessKey string `json:"SecretAccessKey"`
Token string `json:"Token"`
Expiration string `json:"Expiration"`
}
// fetchIMDSCredentials resolves the node role name then its credentials via
// IMDSv2 (PUT token, then GET with the token header).
func fetchIMDSCredentials(client *http.Client, base string) (imdsCreds, error) {
base = strings.TrimRight(base, "/")
token := imdsV2Token(client, base)
roleName, err := imdsGet(client, base+"/meta-data/iam/security-credentials/", token)
if err != nil {
return imdsCreds{}, fmt.Errorf("fetch role name: %w", err)
}
roleName = strings.TrimSpace(roleName)
if roleName == "" {
return imdsCreds{}, fmt.Errorf("IMDS returned empty role name")
}
body, err := imdsGet(client, base+"/meta-data/iam/security-credentials/"+roleName, token)
if err != nil {
return imdsCreds{}, fmt.Errorf("fetch role credentials: %w", err)
}
var creds imdsCreds
if err := json.Unmarshal([]byte(body), &creds); err != nil {
return imdsCreds{}, fmt.Errorf("decode role credentials: %w", err)
}
if creds.AccessKeyID == "" || creds.SecretAccessKey == "" {
return imdsCreds{}, fmt.Errorf("IMDS credentials missing access/secret key")
}
return creds, nil
}
// imdsV2Token requests an IMDSv2 session token; returns "" if the service does
// not enforce v2 (tokenless v1 then applies).
func imdsV2Token(client *http.Client, base string) string {
req, err := http.NewRequest(http.MethodPut, base+"/api/token", nil)
if err != nil {
return ""
}
req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", imdsTokenTTL)
resp, err := client.Do(req)
if err != nil {
return ""
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return ""
}
body, _ := io.ReadAll(resp.Body)
return strings.TrimSpace(string(body))
}
func imdsGet(client *http.Client, url, token string) (string, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return "", err
}
if token != "" {
req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
}
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", fmt.Errorf("IMDS %s returned %d: %s", url, resp.StatusCode, strings.TrimSpace(string(body)))
}
return string(body), nil
}
// getAuthToken builds an aws-sdk ECR client pointed at the gateway and calls
// GetAuthorizationToken, decoding the AWS:<jwt> token into Basic-auth parts. It
// returns the username, password, the ProxyEndpoint host, and the token expiry.
func getAuthToken(cfg config, httpClient *http.Client, creds imdsCreds) (username, password, proxyHost string, expiresAt time.Time, err error) {
region := cfg.Region
if region == "" {
region = "us-east-1"
}
sess, serr := session.NewSession(&aws.Config{
Region: aws.String(region),
Endpoint: aws.String(cfg.GatewayURL),
Credentials: credentials.NewStaticCredentials(creds.AccessKeyID, creds.SecretAccessKey, creds.Token),
HTTPClient: httpClient,
DisableSSL: aws.Bool(false),
})
if serr != nil {
return "", "", "", time.Time{}, fmt.Errorf("new session: %w", serr)
}
out, gerr := ecr.New(sess).GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
if gerr != nil {
return "", "", "", time.Time{}, fmt.Errorf("GetAuthorizationToken: %w", gerr)
}
if len(out.AuthorizationData) == 0 || out.AuthorizationData[0].AuthorizationToken == nil {
return "", "", "", time.Time{}, fmt.Errorf("GetAuthorizationToken returned no authorization data")
}
data := out.AuthorizationData[0]
raw, derr := base64.StdEncoding.DecodeString(aws.StringValue(data.AuthorizationToken))
if derr != nil {
return "", "", "", time.Time{}, fmt.Errorf("decode authorization token: %w", derr)
}
user, pass, ok := strings.Cut(string(raw), ":")
if !ok {
return "", "", "", time.Time{}, fmt.Errorf("authorization token not in user:password form")
}
if data.ExpiresAt != nil {
expiresAt = aws.TimeValue(data.ExpiresAt)
}
return user, pass, hostFromImage(aws.StringValue(data.ProxyEndpoint)), expiresAt, nil
}
// cacheDurationFrom derives a Go-duration string from the token expiry minus a
// safety margin, falling back to the default when expiry is unknown or too near.
func cacheDurationFrom(expiresAt time.Time) string {
@@ -268,30 +141,6 @@ func cacheDurationFrom(expiresAt time.Time) string {
return d.Round(time.Second).String()
}
// gatewayHTTPClient builds an HTTP client trusting the gateway CA at caPath.
// When caPath is empty it relies on the system trust store.
func gatewayHTTPClient(caPath string) (*http.Client, error) {
tlsCfg := &tls.Config{
MinVersion: tls.VersionTLS13,
CurvePreferences: tlsconfig.Curves,
}
if caPath != "" {
pem, err := os.ReadFile(caPath)
if err != nil {
return nil, fmt.Errorf("read gateway CA %q: %w", caPath, err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(pem) {
return nil, fmt.Errorf("gateway CA %q has no usable certificates", caPath)
}
tlsCfg.RootCAs = pool
}
return &http.Client{
Timeout: 15 * time.Second,
Transport: &http.Transport{TLSClientConfig: tlsCfg, MaxIdleConns: 2, IdleConnTimeout: 30 * time.Second},
}, nil
}
// loadConfig reads the cloud-init env file then lets real env vars override. The
// IMDS base is fixed (not in the env file) but overridable for tests.
func loadConfig(envFile string) config {
@@ -341,18 +190,6 @@ func parseEnvFile(path string) map[string]string {
return out
}
// hostFromImage returns the host[:port] portion of an image ref or URL. It strips
// any scheme and trims at the first path separator.
func hostFromImage(image string) string {
if i := strings.Index(image, "://"); i >= 0 {
image = image[i+3:]
}
if i := strings.IndexByte(image, '/'); i >= 0 {
image = image[:i]
}
return image
}
// emitEmpty writes a valid CredentialProviderResponse with no auth entries.
func emitEmpty(out io.Writer) {
resp := CredentialProviderResponse{
-14
View File
@@ -122,17 +122,3 @@ func TestRun_ReturnsBasicAuth(t *testing.T) {
t.Errorf("password = %q, want testjwt", auth.Password)
}
}
func TestHostFromImage(t *testing.T) {
cases := map[string]string{
"acct.dkr.ecr.us-east-1.spinifex.internal/demo:latest": "acct.dkr.ecr.us-east-1.spinifex.internal",
"https://acct.dkr.ecr.us-east-1.spinifex.internal": "acct.dkr.ecr.us-east-1.spinifex.internal",
"host:9999/repo:tag": "host:9999",
"plainhost": "plainhost",
}
for in, want := range cases {
if got := hostFromImage(in); got != want {
t.Errorf("hostFromImage(%q) = %q, want %q", in, got, want)
}
}
}
+167
View File
@@ -0,0 +1,167 @@
package main
import (
"bufio"
"context"
"fmt"
"log/slog"
"net/http"
"os"
goruntime "runtime"
"strconv"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
)
// version is the agent build version, reported in the register message.
// Overridable via -ldflags "-X main.version=...".
var version = "dev"
// Agent wires the ecs-agent's runtime seams: a NATS publisher for the Layer-2
// bus, a container runtime, and an ECR resolver. It registers the host as a
// container instance at boot and heartbeats while alive. Task assignment and
// container lifecycle land in Sprint 4e.
type Agent struct {
cfg config
id identity
puller ctrruntime.ImagePuller
resolver ctrruntime.Resolver
reg *registrar
hb *heartbeater
nc *nats.Conn
closers []func() error
}
// newAgent assembles an Agent from already-built seams. Tests use this directly
// with fakes; New builds the production seams and delegates here.
func newAgent(cfg config, id identity, pub publisher, puller ctrruntime.ImagePuller, resolver ctrruntime.Resolver) *Agent {
return &Agent{
cfg: cfg,
id: id,
puller: puller,
resolver: resolver,
reg: newRegistrar(pub, id),
hb: newHeartbeater(pub, id, cfg.Heartbeat, nil),
}
}
// New builds an Agent from config: it resolves the host identity from IMDS,
// connects to NATS, builds the ECR resolver and (best-effort) the containerd
// runtime. A containerd connect failure is logged, not fatal — registration and
// heartbeat still run so the instance is visible while the runtime recovers.
// The ECR gateway client is built lazily on first image pull (not here), so a
// missing or malformed gateway CA does not stop the agent from registering.
func New(cfg config) (*Agent, error) {
imdsClient := &http.Client{Timeout: 5 * time.Second}
meta, err := fetchInstanceMetadata(imdsClient, cfg.IMDSBase)
if err != nil {
return nil, fmt.Errorf("instance metadata: %w", err)
}
host, _ := os.Hostname()
id := identity{
AccountID: meta.AccountID,
ClusterName: cfg.ClusterName,
InstanceID: meta.InstanceID,
AZ: meta.AZ,
Hostname: host,
Capacity: detectCapacity(),
AgentVersion: version,
}
creds := credentials.NewIMDSProvider(imdsClient, cfg.IMDSBase)
resolver := newLazyECRResolver(creds, cfg.Region, cfg.GatewayURL, cfg.GatewayCA)
var puller ctrruntime.ImagePuller
if p, perr := ctrruntime.New(cfg.ContainerdSocket); perr != nil {
slog.Warn("ecs-agent: containerd unavailable at boot, image pulls disabled", "err", perr)
} else {
puller = p
}
nc, err := nats.Connect(cfg.NATSURL, nats.Name("ecs-agent/"+id.InstanceID))
if err != nil {
return nil, fmt.Errorf("connect nats %s: %w", cfg.NATSURL, err)
}
a := newAgent(cfg, id, nc, puller, resolver)
a.nc = nc
if puller != nil {
a.closers = append(a.closers, puller.Close)
}
return a, nil
}
// Run registers the instance, starts the heartbeat loop, and blocks until ctx is
// cancelled, then tears down.
func (a *Agent) Run(ctx context.Context) error {
if err := a.reg.Register(); err != nil {
slog.Error("ecs-agent: initial registration failed", "err", err)
} else {
slog.Info("ecs-agent: registered",
"cluster", a.id.ClusterName, "instance", a.id.InstanceID,
"subject", bus.RegisterSubject(a.id.AccountID, a.id.ClusterName, a.id.InstanceID))
}
go a.hb.Run(ctx)
<-ctx.Done()
return a.Stop()
}
// Stop drains NATS and closes the runtime. Safe to call once.
func (a *Agent) Stop() error {
var firstErr error
for _, c := range a.closers {
if err := c(); err != nil && firstErr == nil {
firstErr = err
}
}
if a.nc != nil {
_ = a.nc.Drain()
a.nc.Close()
}
return firstErr
}
// detectCapacity reports the host's total CPU units (1 vCPU = 1024) and memory.
func detectCapacity() bus.InstanceCapacity {
return bus.InstanceCapacity{
CPU: goruntime.NumCPU() * 1024,
MemoryMiB: memTotalMiB(),
}
}
// memTotalMiB reads MemTotal from /proc/meminfo, returning 0 if unavailable.
func memTotalMiB() int {
f, err := os.Open("/proc/meminfo")
if err != nil {
return 0
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
if !strings.HasPrefix(line, "MemTotal:") {
continue
}
fields := strings.Fields(line)
if len(fields) < 2 {
return 0
}
kb, err := strconv.Atoi(fields[1])
if err != nil {
return 0
}
return kb / 1024
}
return 0
}
+59
View File
@@ -0,0 +1,59 @@
package main
import (
"context"
"testing"
"time"
ctrruntime "github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
)
func TestAgent_RunRegistersThenStopsOnContext(t *testing.T) {
pub := newCapturePub()
cfg := config{Heartbeat: 5 * time.Millisecond}
id := testIdentity()
puller := &ctrruntime.FakePuller{}
a := newAgent(cfg, id, pub, puller, nil)
a.closers = append(a.closers, puller.Close)
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- a.Run(ctx) }()
// Give it time to register + emit a heartbeat or two.
time.Sleep(30 * time.Millisecond)
cancel()
select {
case err := <-done:
if err != nil {
t.Fatalf("Run returned error: %v", err)
}
case <-time.After(time.Second):
t.Fatal("Run did not return after cancel")
}
regSubj := bus.RegisterSubject(id.AccountID, id.ClusterName, id.InstanceID)
if _, ok := pub.msgs[regSubj]; !ok {
t.Errorf("no register message on %s", regSubj)
}
hbSubj := bus.HeartbeatSubject(id.AccountID, id.ClusterName, id.InstanceID)
if _, ok := pub.msgs[hbSubj]; !ok {
t.Errorf("no heartbeat message on %s", hbSubj)
}
if !puller.Closed {
t.Error("Stop did not close the puller")
}
}
func TestDetectCapacity_Positive(t *testing.T) {
c := detectCapacity()
if c.CPU <= 0 {
t.Errorf("CPU = %d, want > 0", c.CPU)
}
// MemoryMiB may be 0 on platforms without /proc/meminfo; just assert non-negative.
if c.MemoryMiB < 0 {
t.Errorf("MemoryMiB = %d, want >= 0", c.MemoryMiB)
}
}
+97
View File
@@ -0,0 +1,97 @@
package main
import (
"bufio"
"os"
"strings"
"time"
)
const (
defaultEnvFile = "/etc/spinifex-ecs/agent.env"
defaultGatewayCA = "/etc/spinifex-ecs/gateway-ca.pem"
defaultIMDSBase = "http://169.254.169.254/latest"
defaultContainerdSocket = "/run/containerd/containerd.sock"
defaultNATSURL = "nats://127.0.0.1:4222"
defaultHeartbeat = 30 * time.Second
)
// config holds the static settings the agent reads at boot. Cluster identity
// (account ID, instance ID) is discovered at runtime from IMDS, not configured;
// only the cluster *name* the instance was launched into is static.
type config struct {
GatewayURL string
GatewayCA string
Region string
ClusterName string
NATSURL string
IMDSBase string
ContainerdSocket string
Heartbeat time.Duration
}
// loadConfig reads the cloud-init env file then lets real env vars override.
func loadConfig(envFile string) config {
env := parseEnvFile(envFile)
get := func(key string) string {
if v := os.Getenv(key); v != "" {
return v
}
return env[key]
}
cfg := config{
GatewayURL: get("ECS_GATEWAY_URL"),
GatewayCA: get("ECS_GATEWAY_CA"),
Region: get("ECS_REGION"),
ClusterName: get("ECS_CLUSTER"),
NATSURL: get("ECS_NATS_URL"),
IMDSBase: get("ECS_IMDS_BASE"),
ContainerdSocket: get("ECS_CONTAINERD_SOCKET"),
}
if cfg.GatewayCA == "" {
cfg.GatewayCA = defaultGatewayCA
}
if cfg.IMDSBase == "" {
cfg.IMDSBase = defaultIMDSBase
}
if cfg.ContainerdSocket == "" {
cfg.ContainerdSocket = defaultContainerdSocket
}
if cfg.NATSURL == "" {
cfg.NATSURL = defaultNATSURL
}
if cfg.ClusterName == "" {
cfg.ClusterName = "default"
}
cfg.Heartbeat = defaultHeartbeat
if v := get("ECS_HEARTBEAT_INTERVAL"); v != "" {
if d, err := time.ParseDuration(v); err == nil && d > 0 {
cfg.Heartbeat = d
}
}
return cfg
}
// parseEnvFile reads a simple KEY=value file; missing files yield an empty map.
func parseEnvFile(path string) map[string]string {
out := map[string]string{}
f, err := os.Open(path)
if err != nil {
return out
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
key, val, ok := strings.Cut(line, "=")
if !ok {
continue
}
out[strings.TrimSpace(key)] = strings.TrimSpace(val)
}
return out
}
+88
View File
@@ -0,0 +1,88 @@
package main
import (
"os"
"path/filepath"
"testing"
"time"
)
func TestLoadConfig_Defaults(t *testing.T) {
// Ensure env vars don't leak in from the host.
for _, k := range []string{"ECS_GATEWAY_URL", "ECS_GATEWAY_CA", "ECS_REGION",
"ECS_CLUSTER", "ECS_NATS_URL", "ECS_IMDS_BASE", "ECS_CONTAINERD_SOCKET",
"ECS_HEARTBEAT_INTERVAL"} {
t.Setenv(k, "")
}
cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
if cfg.GatewayCA != defaultGatewayCA {
t.Errorf("GatewayCA = %q, want default", cfg.GatewayCA)
}
if cfg.IMDSBase != defaultIMDSBase {
t.Errorf("IMDSBase = %q, want default", cfg.IMDSBase)
}
if cfg.ContainerdSocket != defaultContainerdSocket {
t.Errorf("ContainerdSocket = %q, want default", cfg.ContainerdSocket)
}
if cfg.NATSURL != defaultNATSURL {
t.Errorf("NATSURL = %q, want default", cfg.NATSURL)
}
if cfg.ClusterName != "default" {
t.Errorf("ClusterName = %q, want default", cfg.ClusterName)
}
if cfg.Heartbeat != defaultHeartbeat {
t.Errorf("Heartbeat = %v, want %v", cfg.Heartbeat, defaultHeartbeat)
}
}
func TestLoadConfig_FileThenEnvOverride(t *testing.T) {
dir := t.TempDir()
envFile := filepath.Join(dir, "agent.env")
body := "# comment\nECS_GATEWAY_URL=https://gw.file\nECS_REGION=us-west-2\nECS_CLUSTER=prod\n"
if err := os.WriteFile(envFile, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
t.Setenv("ECS_GATEWAY_URL", "https://gw.env")
t.Setenv("ECS_NATS_URL", "nats://10.0.0.1:4222")
cfg := loadConfig(envFile)
if cfg.GatewayURL != "https://gw.env" {
t.Errorf("GatewayURL = %q, want env override", cfg.GatewayURL)
}
if cfg.Region != "us-west-2" {
t.Errorf("Region = %q, want from file", cfg.Region)
}
if cfg.ClusterName != "prod" {
t.Errorf("ClusterName = %q, want from file", cfg.ClusterName)
}
if cfg.NATSURL != "nats://10.0.0.1:4222" {
t.Errorf("NATSURL = %q, want env", cfg.NATSURL)
}
}
func TestLoadConfig_HeartbeatOverride(t *testing.T) {
t.Setenv("ECS_HEARTBEAT_INTERVAL", "5s")
if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != 5*time.Second {
t.Errorf("Heartbeat = %v, want 5s", cfg.Heartbeat)
}
// Garbage value falls back to the default, not zero.
t.Setenv("ECS_HEARTBEAT_INTERVAL", "not-a-duration")
if cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env")); cfg.Heartbeat != defaultHeartbeat {
t.Errorf("Heartbeat = %v, want default on bad value", cfg.Heartbeat)
}
}
func TestParseEnvFile_SkipsBlankAndComments(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "f.env")
if err := os.WriteFile(p, []byte("\n# c\nA=1\nnokeyval\nB = 2 \n"), 0o600); err != nil {
t.Fatal(err)
}
m := parseEnvFile(p)
if m["A"] != "1" || m["B"] != "2" {
t.Errorf("parse mismatch: %#v", m)
}
if _, ok := m["nokeyval"]; ok {
t.Errorf("line without = should be skipped")
}
}
+59
View File
@@ -0,0 +1,59 @@
package credentials
import (
"context"
"net/http"
"sync"
"time"
"github.com/mulgadc/spinifex/internal/imdscreds"
)
// refreshMargin is how long before expiry the cache is considered stale.
const refreshMargin = 5 * time.Minute
// IMDSProvider fetches instance-role credentials from IMDS and caches them until
// they near expiry. Safe for concurrent use.
type IMDSProvider struct {
client *http.Client
base string
mu sync.Mutex
cached Credentials
}
var _ CredentialsProvider = (*IMDSProvider)(nil)
// NewIMDSProvider builds a provider against the given IMDS base URL (e.g.
// "http://169.254.169.254/latest"). A nil client uses http.DefaultClient.
func NewIMDSProvider(client *http.Client, base string) *IMDSProvider {
if client == nil {
client = http.DefaultClient
}
return &IMDSProvider{client: client, base: base}
}
// Retrieve returns cached credentials when still valid, else fetches fresh ones.
func (p *IMDSProvider) Retrieve(ctx context.Context) (Credentials, error) {
p.mu.Lock()
defer p.mu.Unlock()
if p.cached.Valid(refreshMargin) {
return p.cached, nil
}
if err := ctx.Err(); err != nil {
return Credentials{}, err
}
raw, err := imdscreds.Fetch(p.client, p.base)
if err != nil {
return Credentials{}, err
}
p.cached = Credentials{
AccessKeyID: raw.AccessKeyID,
SecretAccessKey: raw.SecretAccessKey,
SessionToken: raw.SessionToken,
Expiration: raw.Expiration,
}
return p.cached, nil
}
+104
View File
@@ -0,0 +1,104 @@
package credentials
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
)
func imdsStub(t *testing.T, hits *int32, creds map[string]string) *httptest.Server {
t.Helper()
mux := http.NewServeMux()
mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("v2-token"))
})
mux.HandleFunc("/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/meta-data/iam/security-credentials/" {
_, _ = w.Write([]byte("node-role"))
return
}
if hits != nil {
atomic.AddInt32(hits, 1)
}
_ = json.NewEncoder(w).Encode(creds)
})
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)
return srv
}
func TestRetrieve_FetchesAndCaches(t *testing.T) {
var hits int32
exp := time.Now().Add(6 * time.Hour).UTC().Round(time.Second)
srv := imdsStub(t, &hits, map[string]string{
"AccessKeyId": "AKIA",
"SecretAccessKey": "secret",
"Token": "session",
"Expiration": exp.Format(time.RFC3339),
})
p := NewIMDSProvider(srv.Client(), srv.URL)
got, err := p.Retrieve(context.Background())
if err != nil {
t.Fatalf("Retrieve: %v", err)
}
if got.AccessKeyID != "AKIA" || !got.Expiration.Equal(exp) {
t.Errorf("creds mismatch: %+v", got)
}
// Second call within validity window must not re-hit IMDS.
if _, err := p.Retrieve(context.Background()); err != nil {
t.Fatalf("Retrieve 2: %v", err)
}
if hits != 1 {
t.Errorf("IMDS credential hits = %d, want 1 (cached)", hits)
}
}
func TestRetrieve_RefetchesWhenStale(t *testing.T) {
var hits int32
// Expiry inside the refresh margin → never cached, always refetched.
srv := imdsStub(t, &hits, map[string]string{
"AccessKeyId": "AKIA",
"SecretAccessKey": "secret",
"Expiration": time.Now().Add(time.Minute).UTC().Format(time.RFC3339),
})
p := NewIMDSProvider(srv.Client(), srv.URL)
if _, err := p.Retrieve(context.Background()); err != nil {
t.Fatal(err)
}
if _, err := p.Retrieve(context.Background()); err != nil {
t.Fatal(err)
}
if hits != 2 {
t.Errorf("hits = %d, want 2 (stale refetch)", hits)
}
}
func TestRetrieve_CancelledContext(t *testing.T) {
srv := imdsStub(t, nil, map[string]string{"AccessKeyId": "A", "SecretAccessKey": "B"})
p := NewIMDSProvider(srv.Client(), srv.URL)
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := p.Retrieve(ctx); err == nil {
t.Fatal("expected error from cancelled context")
}
}
func TestCredentials_Valid(t *testing.T) {
if (Credentials{}).Valid(0) {
t.Error("empty creds should be invalid")
}
noExp := Credentials{AccessKeyID: "a", SecretAccessKey: "b"}
if !noExp.Valid(time.Hour) {
t.Error("zero expiry should be treated valid")
}
soon := Credentials{AccessKeyID: "a", SecretAccessKey: "b", Expiration: time.Now().Add(time.Minute)}
if soon.Valid(5 * time.Minute) {
t.Error("expiry inside margin should be invalid")
}
}
+36
View File
@@ -0,0 +1,36 @@
// Package credentials supplies AWS SigV4 credentials to the ecs-agent. The agent
// signs gateway calls (ECR GetAuthorizationToken, instance register/heartbeat)
// with the host's instance-role credentials, fetched from IMDS and cached until
// near expiry.
package credentials
import (
"context"
"time"
)
// Credentials is a SigV4 credential set with an expiry.
type Credentials struct {
AccessKeyID string
SecretAccessKey string
SessionToken string
Expiration time.Time
}
// Valid reports whether the credentials are usable now, leaving a margin so a
// caller does not start a request with a token about to expire mid-flight.
func (c Credentials) Valid(margin time.Duration) bool {
if c.AccessKeyID == "" || c.SecretAccessKey == "" {
return false
}
if c.Expiration.IsZero() {
return true
}
return time.Now().Add(margin).Before(c.Expiration)
}
// CredentialsProvider yields current credentials, refreshing as needed. Retrieve
// must be safe for concurrent use.
type CredentialsProvider interface {
Retrieve(ctx context.Context) (Credentials, error)
}
+78
View File
@@ -0,0 +1,78 @@
package main
import (
"context"
"net/http"
"sync"
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
"github.com/mulgadc/spinifex/cmd/ecs-agent/runtime"
"github.com/mulgadc/spinifex/internal/ecrauth"
)
// ecrResolver implements runtime.Resolver by minting an ECR authorization token
// per pull: it fetches the host's IMDS credentials, SigV4-signs
// GetAuthorizationToken against the gateway, and returns the AWS:<jwt> as a
// user/password pair for containerd.
//
// The gateway HTTP client is built lazily on first Authorize, not at agent boot:
// a missing or malformed gateway CA only matters when an image is actually
// pulled, so the agent can still register + heartbeat on a host with absent or
// wrong ECR config.
type ecrResolver struct {
creds credentials.CredentialsProvider
region string
gatewayURL string
caPath string
mu sync.Mutex
httpClient *http.Client
}
var _ runtime.Resolver = (*ecrResolver)(nil)
// newECRResolver builds a resolver with an injected HTTP client (tests). The
// client is used as-is and never rebuilt from caPath.
func newECRResolver(creds credentials.CredentialsProvider, region, gatewayURL string, httpClient *http.Client) *ecrResolver {
return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, httpClient: httpClient}
}
// newLazyECRResolver builds a resolver that constructs its gateway HTTP client
// from caPath on first use (production path).
func newLazyECRResolver(creds credentials.CredentialsProvider, region, gatewayURL, caPath string) *ecrResolver {
return &ecrResolver{creds: creds, region: region, gatewayURL: gatewayURL, caPath: caPath}
}
// client returns the gateway HTTP client, building it from caPath on first call
// and caching the result. An injected client (newECRResolver) is returned as-is.
func (e *ecrResolver) client() (*http.Client, error) {
e.mu.Lock()
defer e.mu.Unlock()
if e.httpClient != nil {
return e.httpClient, nil
}
c, err := ecrauth.GatewayHTTPClient(e.caPath)
if err != nil {
return nil, err
}
e.httpClient = c
return c, nil
}
// Authorize returns the ECR token's user/password and proxy endpoint for ref.
func (e *ecrResolver) Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error) {
c, err := e.creds.Retrieve(ctx)
if err != nil {
return "", "", "", err
}
httpClient, err := e.client()
if err != nil {
return "", "", "", err
}
tok, err := ecrauth.GetAuthorizationToken(e.region, e.gatewayURL, httpClient,
c.AccessKeyID, c.SecretAccessKey, c.SessionToken)
if err != nil {
return "", "", "", err
}
return tok.Username, tok.Password, tok.ProxyHost, nil
}
+102
View File
@@ -0,0 +1,102 @@
package main
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"
"github.com/mulgadc/spinifex/cmd/ecs-agent/credentials"
)
// stubCreds is a credentials.CredentialsProvider returning fixed creds or error.
type stubCreds struct {
c credentials.Credentials
err error
}
func (s stubCreds) Retrieve(context.Context) (credentials.Credentials, error) {
return s.c, s.err
}
func gatewayStub(t *testing.T, token, proxy string, status int) *httptest.Server {
t.Helper()
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.Contains(r.Header.Get("X-Amz-Target"), "GetAuthorizationToken") {
w.WriteHeader(http.StatusBadRequest)
return
}
if status != http.StatusOK {
w.WriteHeader(status)
return
}
w.Header().Set("Content-Type", "application/x-amz-json-1.1")
_ = json.NewEncoder(w).Encode(map[string]any{
"authorizationData": []map[string]any{{
"authorizationToken": base64.StdEncoding.EncodeToString([]byte(token)),
"proxyEndpoint": proxy,
}},
})
}))
t.Cleanup(srv.Close)
return srv
}
func trusting(srv *httptest.Server) *http.Client {
pool := x509.NewCertPool()
pool.AddCert(srv.Certificate())
return &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
}
func TestECRResolver_Authorize(t *testing.T) {
srv := gatewayStub(t, "AWS:jwt", "https://123456789012.dkr.ecr.us-east-1.spinifex.internal", http.StatusOK)
creds := stubCreds{c: credentials.Credentials{
AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
Expiration: time.Now().Add(time.Hour),
}}
r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
user, pass, endpoint, err := r.Authorize(context.Background(), "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1")
if err != nil {
t.Fatalf("Authorize: %v", err)
}
if user != "AWS" || pass != "jwt" {
t.Errorf("got %q:%q, want AWS:jwt", user, pass)
}
if endpoint != "123456789012.dkr.ecr.us-east-1.spinifex.internal" {
t.Errorf("endpoint = %q", endpoint)
}
}
func TestECRResolver_CredsError(t *testing.T) {
r := newECRResolver(stubCreds{err: errors.New("no imds")}, "us-east-1", "https://x", http.DefaultClient)
if _, _, _, err := r.Authorize(context.Background(), "ref"); err == nil {
t.Fatal("expected creds error to propagate")
}
}
func TestECRResolver_GatewayError(t *testing.T) {
srv := gatewayStub(t, "", "", http.StatusInternalServerError)
creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
if _, _, _, err := r.Authorize(context.Background(), "ref"); err == nil {
t.Fatal("expected gateway error")
}
}
func TestECRResolver_LazyClient_BadCADeferredToAuthorize(t *testing.T) {
creds := stubCreds{c: credentials.Credentials{AccessKeyID: "A", SecretAccessKey: "B"}}
// Construction with a bogus CA path must not fail — only Authorize does.
r := newLazyECRResolver(creds, "us-east-1", "https://gw", filepath.Join(t.TempDir(), "absent.pem"))
if _, _, _, err := r.Authorize(context.Background(), "ref"); err == nil {
t.Fatal("expected lazy gateway-client build to fail on missing CA")
}
}
+70
View File
@@ -0,0 +1,70 @@
package main
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"time"
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
)
// heartbeater periodically emits instance-heartbeat messages so the scheduler
// can tell live instances from dead ones.
type heartbeater struct {
pub publisher
id identity
interval time.Duration
// runningTasks reports the current running-task count; nil means zero.
runningTasks func() int
}
func newHeartbeater(pub publisher, id identity, interval time.Duration, running func() int) *heartbeater {
if interval <= 0 {
interval = defaultHeartbeat
}
return &heartbeater{pub: pub, id: id, interval: interval, runningTasks: running}
}
// beat publishes a single Heartbeat message.
func (h *heartbeater) beat() error {
n := 0
if h.runningTasks != nil {
n = h.runningTasks()
}
msg := bus.Heartbeat{
AccountID: h.id.AccountID,
ClusterName: h.id.ClusterName,
InstanceID: h.id.InstanceID,
Status: bus.StatusActive,
RunningTasks: n,
SentAt: time.Now().UTC(),
}
data, err := json.Marshal(&msg)
if err != nil {
return fmt.Errorf("marshal heartbeat: %w", err)
}
subj := bus.HeartbeatSubject(h.id.AccountID, h.id.ClusterName, h.id.InstanceID)
if err := h.pub.Publish(subj, data); err != nil {
return fmt.Errorf("publish heartbeat %s: %w", subj, err)
}
return nil
}
// Run beats every interval until ctx is cancelled. A failed beat is logged and
// retried on the next tick rather than killing the loop.
func (h *heartbeater) Run(ctx context.Context) {
ticker := time.NewTicker(h.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := h.beat(); err != nil {
slog.Warn("ecs-agent: heartbeat failed", "err", err)
}
}
}
}
+39
View File
@@ -0,0 +1,39 @@
// ecs-agent runs inside a Spinifex ECS container instance (a guest VM booted from
// the ECS-AMI, with containerd baked in). It registers the host with the ECS
// scheduler over the Layer-2 NATS bus, heartbeats while alive, and pulls task
// images from the internal ECR registry through containerd. Task assignment and
// container lifecycle land in Sprint 4e.
//
// Static config (gateway URL, CA, region, cluster, NATS, containerd socket) is
// read from the cloud-init env file /etc/spinifex-ecs/agent.env (KEY=value);
// real env vars override it. Host identity (account, instance, AZ) comes from
// IMDS at boot.
package main
import (
"context"
"log/slog"
"os"
"os/signal"
"syscall"
_ "github.com/mulgadc/spinifex/internal/fipsboot"
)
func main() {
cfg := loadConfig(defaultEnvFile)
agent, err := New(cfg)
if err != nil {
slog.Error("ecs-agent: startup failed", "err", err)
os.Exit(1)
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
if err := agent.Run(ctx); err != nil {
slog.Error("ecs-agent: run failed", "err", err)
os.Exit(1)
}
}
+97
View File
@@ -0,0 +1,97 @@
package main
import (
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
)
// instanceMetadata is the IMDS-derived identity of the host the agent runs on.
type instanceMetadata struct {
AccountID string
InstanceID string
AZ string
}
// fetchInstanceMetadata reads the instance-id, AZ and account-id from IMDSv2.
// base is the IMDS root, e.g. "http://169.254.169.254/latest".
func fetchInstanceMetadata(client *http.Client, base string) (instanceMetadata, error) {
token, err := imdsToken(client, base)
if err != nil {
return instanceMetadata{}, err
}
instanceID, err := imdsGet(client, base+"/meta-data/instance-id", token)
if err != nil {
return instanceMetadata{}, fmt.Errorf("instance-id: %w", err)
}
az, err := imdsGet(client, base+"/meta-data/placement/availability-zone", token)
if err != nil {
return instanceMetadata{}, fmt.Errorf("availability-zone: %w", err)
}
doc, err := imdsGet(client, base+"/dynamic/instance-identity/document", token)
if err != nil {
return instanceMetadata{}, fmt.Errorf("identity document: %w", err)
}
var ident struct {
AccountID string `json:"accountId"`
}
if err := json.Unmarshal([]byte(doc), &ident); err != nil {
return instanceMetadata{}, fmt.Errorf("parse identity document: %w", err)
}
return instanceMetadata{
AccountID: ident.AccountID,
InstanceID: strings.TrimSpace(instanceID),
AZ: strings.TrimSpace(az),
}, nil
}
// imdsToken fetches an IMDSv2 session token.
func imdsToken(client *http.Client, base string) (string, error) {
req, err := http.NewRequest(http.MethodPut, base+"/api/token", nil)
if err != nil {
return "", err
}
req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("imds token: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("imds token: status %d", resp.StatusCode)
}
return string(body), nil
}
// imdsGet performs a token-authenticated IMDS GET.
func imdsGet(client *http.Client, url, token string) (string, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return "", err
}
if token != "" {
req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
}
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("status %d", resp.StatusCode)
}
return string(body), nil
}
+76
View File
@@ -0,0 +1,76 @@
package main
import (
"net/http"
"net/http/httptest"
"testing"
)
func metadataStub(t *testing.T, enforceV2 bool) *httptest.Server {
t.Helper()
mux := http.NewServeMux()
mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPut {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
_, _ = w.Write([]byte("v2-token"))
})
guard := func(w http.ResponseWriter, r *http.Request) bool {
if enforceV2 && r.Header.Get("X-Aws-Ec2-Metadata-Token") != "v2-token" {
w.WriteHeader(http.StatusUnauthorized)
return false
}
return true
}
mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) {
if guard(w, r) {
_, _ = w.Write([]byte("i-0abc123"))
}
})
mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) {
if guard(w, r) {
_, _ = w.Write([]byte("us-east-1a"))
}
})
mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) {
if guard(w, r) {
_, _ = w.Write([]byte(`{"accountId":"123456789012","region":"us-east-1"}`))
}
})
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)
return srv
}
func TestFetchInstanceMetadata_OK(t *testing.T) {
srv := metadataStub(t, true)
got, err := fetchInstanceMetadata(srv.Client(), srv.URL)
if err != nil {
t.Fatalf("fetch: %v", err)
}
if got.InstanceID != "i-0abc123" || got.AZ != "us-east-1a" || got.AccountID != "123456789012" {
t.Errorf("metadata mismatch: %+v", got)
}
}
func TestFetchInstanceMetadata_TokenRequired(t *testing.T) {
// Server returns 405 for non-PUT token; a GET-only client would 401 downstream.
srv := metadataStub(t, true)
if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err != nil {
t.Fatalf("v2 path should succeed: %v", err)
}
}
func TestFetchInstanceMetadata_BadDocument(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("t")) })
mux.HandleFunc("/meta-data/instance-id", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("i-1")) })
mux.HandleFunc("/meta-data/placement/availability-zone", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("z")) })
mux.HandleFunc("/dynamic/instance-identity/document", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)
if _, err := fetchInstanceMetadata(srv.Client(), srv.URL); err == nil {
t.Fatal("expected parse error on bad identity document")
}
}
+66
View File
@@ -0,0 +1,66 @@
package main
import (
"encoding/json"
"fmt"
"time"
"github.com/nats-io/nats.go"
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
)
// identity is the container instance's stable identity, assembled at boot from
// IMDS (account, instance, AZ) and config (cluster). It is the join key the
// scheduler uses to track this instance.
type identity struct {
AccountID string
ClusterName string
InstanceID string
AZ string
Hostname string
Capacity bus.InstanceCapacity
AgentVersion string
}
// publisher is the subset of *nats.Conn the agent uses to emit bus messages.
// Narrowed to an interface so tests can drive it without a live server.
type publisher interface {
Publish(subject string, data []byte) error
}
var _ publisher = (*nats.Conn)(nil)
// registrar emits the boot-time instance-register message.
type registrar struct {
pub publisher
id identity
}
func newRegistrar(pub publisher, id identity) *registrar {
return &registrar{pub: pub, id: id}
}
// Register publishes the RegisterInstance message on the cluster's register
// subject. The scheduler records the container instance on receipt (Sprint 4e).
func (r *registrar) Register() error {
msg := bus.RegisterInstance{
AccountID: r.id.AccountID,
ClusterName: r.id.ClusterName,
InstanceID: r.id.InstanceID,
AZ: r.id.AZ,
Hostname: r.id.Hostname,
Capacity: r.id.Capacity,
AgentVersion: r.id.AgentVersion,
RegisteredAt: time.Now().UTC(),
}
data, err := json.Marshal(&msg)
if err != nil {
return fmt.Errorf("marshal register: %w", err)
}
subj := bus.RegisterSubject(r.id.AccountID, r.id.ClusterName, r.id.InstanceID)
if err := r.pub.Publish(subj, data); err != nil {
return fmt.Errorf("publish register %s: %w", subj, err)
}
return nil
}
+132
View File
@@ -0,0 +1,132 @@
package main
import (
"context"
"encoding/json"
"errors"
"sync"
"testing"
"time"
"github.com/mulgadc/spinifex/spinifex/handlers/ecs/bus"
)
// capturePub records published messages; ErrOn forces Publish to fail.
type capturePub struct {
mu sync.Mutex
msgs map[string][]byte
count int
errOn bool
}
func newCapturePub() *capturePub { return &capturePub{msgs: map[string][]byte{}} }
func (c *capturePub) Publish(subject string, data []byte) error {
c.mu.Lock()
defer c.mu.Unlock()
if c.errOn {
return errors.New("publish boom")
}
c.count++
cp := make([]byte, len(data))
copy(cp, data)
c.msgs[subject] = cp
return nil
}
func (c *capturePub) calls() int {
c.mu.Lock()
defer c.mu.Unlock()
return c.count
}
func testIdentity() identity {
return identity{
AccountID: "123456789012",
ClusterName: "default",
InstanceID: "i-abc123",
AZ: "us-east-1a",
Hostname: "ecs-host",
Capacity: bus.InstanceCapacity{CPU: 2048, MemoryMiB: 4096},
AgentVersion: "test",
}
}
func TestRegistrar_Register(t *testing.T) {
pub := newCapturePub()
id := testIdentity()
if err := newRegistrar(pub, id).Register(); err != nil {
t.Fatalf("Register: %v", err)
}
subj := bus.RegisterSubject(id.AccountID, id.ClusterName, id.InstanceID)
raw, ok := pub.msgs[subj]
if !ok {
t.Fatalf("no message on %s; got %v", subj, pub.msgs)
}
var got bus.RegisterInstance
if err := json.Unmarshal(raw, &got); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if got.InstanceID != id.InstanceID || got.Capacity.CPU != 2048 {
t.Errorf("payload mismatch: %+v", got)
}
if got.RegisteredAt.IsZero() {
t.Error("RegisteredAt not set")
}
}
func TestRegistrar_PublishError(t *testing.T) {
pub := newCapturePub()
pub.errOn = true
if err := newRegistrar(pub, testIdentity()).Register(); err == nil {
t.Fatal("expected publish error")
}
}
func TestHeartbeater_Beat(t *testing.T) {
pub := newCapturePub()
id := testIdentity()
h := newHeartbeater(pub, id, time.Second, func() int { return 3 })
if err := h.beat(); err != nil {
t.Fatalf("beat: %v", err)
}
subj := bus.HeartbeatSubject(id.AccountID, id.ClusterName, id.InstanceID)
var got bus.Heartbeat
if err := json.Unmarshal(pub.msgs[subj], &got); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if got.Status != bus.StatusActive || got.RunningTasks != 3 {
t.Errorf("payload mismatch: %+v", got)
}
}
func TestHeartbeater_NilRunningTasksIsZero(t *testing.T) {
pub := newCapturePub()
h := newHeartbeater(pub, testIdentity(), 0, nil)
if h.interval != defaultHeartbeat {
t.Errorf("interval = %v, want default", h.interval)
}
if err := h.beat(); err != nil {
t.Fatal(err)
}
}
func TestHeartbeater_RunStopsOnContext(t *testing.T) {
pub := newCapturePub()
h := newHeartbeater(pub, testIdentity(), 5*time.Millisecond, nil)
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { h.Run(ctx); close(done) }()
time.Sleep(25 * time.Millisecond)
cancel()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("Run did not stop on context cancel")
}
if pub.calls() == 0 {
t.Error("expected at least one heartbeat before cancel")
}
}
+100
View File
@@ -0,0 +1,100 @@
package runtime
import (
"context"
"fmt"
"time"
containerd "github.com/containerd/containerd/v2/client"
"github.com/containerd/containerd/v2/core/remotes"
"github.com/containerd/containerd/v2/core/remotes/docker"
"github.com/containerd/containerd/v2/pkg/namespaces"
)
// ecsNamespace isolates ecs-agent images/containers in containerd, mirroring the
// "moby"/"k8s.io" convention other runtimes use.
const ecsNamespace = "ecs"
// servingProbeTimeout bounds the boot-time daemon liveness check.
const servingProbeTimeout = 3 * time.Second
// containerdPuller drives a real containerd daemon over its unix socket.
type containerdPuller struct {
client *containerd.Client
}
var _ ImagePuller = (*containerdPuller)(nil)
// New connects to the containerd daemon at socket (e.g.
// "/run/containerd/containerd.sock") and returns an ImagePuller. The containerd
// client dials lazily, so New actively probes the daemon with IsServing and
// returns an error if it is unreachable — otherwise a dead socket would only
// surface on the first pull, and the agent's boot-time "containerd unavailable"
// log would never fire.
func New(socket string) (ImagePuller, error) {
client, err := containerd.New(socket, containerd.WithDefaultNamespace(ecsNamespace))
if err != nil {
return nil, fmt.Errorf("connect containerd %s: %w", socket, err)
}
ctx, cancel := context.WithTimeout(context.Background(), servingProbeTimeout)
defer cancel()
serving, err := client.IsServing(ctx)
if err != nil || !serving {
_ = client.Close()
return nil, fmt.Errorf("containerd %s not serving: %w", socket, err)
}
return &containerdPuller{client: client}, nil
}
// Pull resolves credentials via r, then pulls and unpacks the image.
func (p *containerdPuller) Pull(ctx context.Context, spec PullSpec, r Resolver) (Image, error) {
ctx = namespaces.WithNamespace(ctx, ecsNamespace)
resolver, err := newDockerResolver(ctx, spec.Ref, r)
if err != nil {
return Image{}, err
}
img, err := p.client.Pull(ctx, spec.Ref,
containerd.WithPullUnpack,
containerd.WithResolver(resolver),
)
if err != nil {
return Image{}, fmt.Errorf("pull %s: %w", spec.Ref, err)
}
target := img.Target()
return Image{
Ref: spec.Ref,
Digest: target.Digest.String(),
Size: target.Size,
}, nil
}
// Close releases the containerd client connection.
func (p *containerdPuller) Close() error {
if p.client == nil {
return nil
}
return p.client.Close()
}
// newDockerResolver builds a containerd remote resolver whose per-host
// credentials come from r (the ECR token resolver). Credentials are fetched once
// for ref and reused for every host containerd asks about during the pull.
func newDockerResolver(ctx context.Context, ref string, r Resolver) (remotes.Resolver, error) {
creds := func(string) (string, string, error) { return "", "", nil }
if r != nil {
user, pass, _, err := r.Authorize(ctx, ref)
if err != nil {
return nil, fmt.Errorf("authorize %s: %w", ref, err)
}
creds = func(string) (string, string, error) { return user, pass, nil }
}
authorizer := docker.NewDockerAuthorizer(docker.WithAuthCreds(creds))
return docker.NewResolver(docker.ResolverOptions{
Hosts: docker.ConfigureDefaultRegistries(docker.WithAuthorizer(authorizer)),
}), nil
}
+36
View File
@@ -0,0 +1,36 @@
// Package runtime is the ecs-agent's container-runtime seam. The agent pulls
// task images and (from Sprint 4e) runs containers through an ImagePuller; the
// real implementation drives containerd, but the interface lets tests and the
// scheduler-only build run against a fake.
package runtime
import "context"
// PullSpec describes one image to pull.
type PullSpec struct {
// Ref is the full image reference, e.g.
// "123456789012.dkr.ecr.us-east-1.spinifex.internal/app:latest".
Ref string
}
// Image identifies a pulled image in the runtime's content store.
type Image struct {
Ref string
Digest string
Size int64
}
// Resolver supplies registry credentials for a given image reference. The
// ecs-agent backs this with ECR GetAuthorizationToken (see ecrresolver.go).
type Resolver interface {
// Authorize returns the username, password and registry endpoint to use for
// ref. An empty user/pass means anonymous pull.
Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error)
}
// ImagePuller pulls images into the local runtime. Close releases the runtime
// client connection.
type ImagePuller interface {
Pull(ctx context.Context, spec PullSpec, r Resolver) (Image, error)
Close() error
}
+59
View File
@@ -0,0 +1,59 @@
package runtime
import (
"context"
"fmt"
"sync"
)
// FakePuller is an in-memory ImagePuller for tests and the scheduler-only build.
// It records pull requests and replays a programmed result or error.
type FakePuller struct {
mu sync.Mutex
Pulls []PullSpec
Authzd []string
Result Image
Err error
Closed bool
OnPull func(PullSpec) (Image, error)
}
var _ ImagePuller = (*FakePuller)(nil)
// Pull records the spec, exercises the resolver, and returns the programmed
// result. If OnPull is set it takes precedence over Result/Err.
func (f *FakePuller) Pull(ctx context.Context, spec PullSpec, r Resolver) (Image, error) {
f.mu.Lock()
f.Pulls = append(f.Pulls, spec)
f.mu.Unlock()
if r != nil {
user, _, _, err := r.Authorize(ctx, spec.Ref)
if err != nil {
return Image{}, fmt.Errorf("authorize %s: %w", spec.Ref, err)
}
f.mu.Lock()
f.Authzd = append(f.Authzd, user)
f.mu.Unlock()
}
if f.OnPull != nil {
return f.OnPull(spec)
}
if f.Err != nil {
return Image{}, f.Err
}
res := f.Result
if res.Ref == "" {
res.Ref = spec.Ref
}
return res, nil
}
// Close marks the puller closed.
func (f *FakePuller) Close() error {
f.mu.Lock()
defer f.mu.Unlock()
f.Closed = true
return nil
}
+67
View File
@@ -0,0 +1,67 @@
package runtime
import (
"context"
"errors"
"testing"
)
// staticResolver is a test Resolver returning fixed creds or an error.
type staticResolver struct {
user, pass, endpoint string
err error
}
func (s staticResolver) Authorize(_ context.Context, _ string) (string, string, string, error) {
return s.user, s.pass, s.endpoint, s.err
}
func TestFakePuller_RecordsAndDefaultsRef(t *testing.T) {
f := &FakePuller{}
img, err := f.Pull(context.Background(), PullSpec{Ref: "repo/app:1"}, staticResolver{user: "AWS"})
if err != nil {
t.Fatalf("Pull: %v", err)
}
if img.Ref != "repo/app:1" {
t.Errorf("Ref defaulted wrong: %q", img.Ref)
}
if len(f.Pulls) != 1 || f.Pulls[0].Ref != "repo/app:1" {
t.Errorf("Pulls = %+v", f.Pulls)
}
if len(f.Authzd) != 1 || f.Authzd[0] != "AWS" {
t.Errorf("Authzd = %+v", f.Authzd)
}
}
func TestFakePuller_ResolverError(t *testing.T) {
f := &FakePuller{}
_, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, staticResolver{err: errors.New("boom")})
if err == nil {
t.Fatal("expected resolver error to propagate")
}
}
func TestFakePuller_ProgrammedError(t *testing.T) {
f := &FakePuller{Err: errors.New("pull failed")}
if _, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil); err == nil {
t.Fatal("expected programmed error")
}
}
func TestFakePuller_OnPullOverrides(t *testing.T) {
want := Image{Ref: "x", Digest: "sha256:abc", Size: 42}
f := &FakePuller{Err: errors.New("ignored"), OnPull: func(PullSpec) (Image, error) {
return want, nil
}}
got, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil)
if err != nil || got != want {
t.Fatalf("got %+v, %v; want %+v", got, err, want)
}
}
func TestFakePuller_Close(t *testing.T) {
f := &FakePuller{}
if err := f.Close(); err != nil || !f.Closed {
t.Fatalf("Close: err=%v closed=%v", err, f.Closed)
}
}
+50 -8
View File
@@ -7,6 +7,7 @@ require (
github.com/charmbracelet/bubbles v1.0.0
github.com/charmbracelet/bubbletea v1.3.10
github.com/charmbracelet/lipgloss v1.1.0
github.com/containerd/containerd/v2 v2.3.2
github.com/go-chi/chi/v5 v5.3.0
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/uuid v1.6.0
@@ -35,6 +36,8 @@ require (
atomicgo.dev/cursor v0.2.0 // indirect
atomicgo.dev/keyboard v0.2.9 // indirect
atomicgo.dev/schedule v0.1.0 // indirect
github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect
github.com/Microsoft/hcsshim v0.15.0-rc.1 // indirect
github.com/antithesishq/antithesis-sdk-go v0.7.0 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/atotto/clipboard v0.1.4 // indirect
@@ -55,13 +58,26 @@ require (
github.com/clipperhouse/displaywidth v0.9.0 // indirect
github.com/clipperhouse/stringish v0.1.1 // indirect
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
github.com/containerd/cgroups/v3 v3.1.3 // indirect
github.com/containerd/console v1.0.5 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/containerd/containerd/api v1.11.1 // indirect
github.com/containerd/continuity v0.5.0 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
github.com/containerd/fifo v1.1.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/platforms v1.0.0-rc.4 // indirect
github.com/containerd/plugin v1.1.0 // indirect
github.com/containerd/ttrpc v1.2.8 // indirect
github.com/containerd/typeurl/v2 v2.2.3 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgraph-io/badger/v4 v4.9.2 // indirect
github.com/dgraph-io/ristretto/v2 v2.4.0 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
github.com/fatih/color v1.19.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
github.com/go-logr/logr v1.4.3 // indirect
@@ -70,6 +86,8 @@ require (
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.30.1 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/google/flatbuffers v25.12.19+incompatible // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/gookit/color v1.6.0 // indirect
@@ -94,9 +112,15 @@ require (
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.21 // indirect
github.com/mdlayher/packet v1.1.2 // indirect
github.com/mdlayher/socket v0.4.1 // indirect
github.com/mdlayher/socket v0.5.1 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/sys/mountinfo v0.7.2 // indirect
github.com/moby/sys/sequential v0.6.0 // indirect
github.com/moby/sys/signal v0.7.1 // indirect
github.com/moby/sys/user v0.4.0 // indirect
github.com/moby/sys/userns v0.1.0 // indirect
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/termenv v0.16.0 // indirect
@@ -104,8 +128,12 @@ require (
github.com/nats-io/jwt/v2 v2.8.2 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pierrec/lz4/v4 v4.1.14 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/opencontainers/runtime-spec v1.3.0 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.5 // indirect
@@ -113,6 +141,7 @@ require (
github.com/quic-go/quic-go v0.60.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/sagikazarmark/locafero v0.12.0 // indirect
github.com/sirupsen/logrus v1.9.4 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
@@ -120,10 +149,12 @@ require (
github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
go.etcd.io/bbolt v1.4.3 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel v1.42.0 // indirect
go.opentelemetry.io/otel/metric v1.42.0 // indirect
go.opentelemetry.io/otel/trace v1.42.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/mod v0.36.0 // indirect
@@ -134,7 +165,18 @@ require (
golang.org/x/time v0.15.0 // indirect
golang.org/x/tools v0.45.0 // indirect
golang.org/x/vuln v1.1.4 // indirect
google.golang.org/protobuf v1.36.11 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260618152121-87f3d3e198d3 // indirect
google.golang.org/grpc v1.81.1 // indirect
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
)
// go.opencensus.io -> grpc@v1.33.2 drags in the pre-split monolithic genproto,
// which still ships google.golang.org/genproto/googleapis/rpc/status and so
// collides with the split genproto/googleapis/rpc module that containerd/ttrpc
// needs ("ambiguous import"). The main module never imports the monolith, so a
// plain require is dropped by `go mod tidy`; pin it via replace to a post-split
// version (no rpc/status) so the fix survives tidy. Remove if grpc/opencensus
// stop requiring the monolith.
replace google.golang.org/genproto => google.golang.org/genproto v0.0.0-20260622175928-b703f567277d
tool golang.org/x/vuln/cmd/govulncheck
+2616 -13
View File
File diff suppressed because it is too large Load Diff
+119
View File
@@ -0,0 +1,119 @@
// Package ecrauth mints ECR registry credentials by SigV4-signing
// GetAuthorizationToken against the Spinifex AWS gateway's ECR endpoint and
// decoding the returned AWS:<jwt> token into Basic-auth parts. It is shared by
// the in-guest binaries that pull images from the internal registry: the EKS
// ecr-credential-provider and the ECS ecs-agent.
package ecrauth
import (
"crypto/tls"
"crypto/x509"
"encoding/base64"
"fmt"
"net/http"
"os"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ecr"
"github.com/mulgadc/spinifex/internal/tlsconfig"
)
// defaultRegion is used when the caller supplies no region; SigV4 still needs a
// scope and the gateway does not enforce a specific value.
const defaultRegion = "us-east-1"
// Token is the decoded result of GetAuthorizationToken: Basic-auth parts plus
// the registry proxy host and token expiry.
type Token struct {
Username string
Password string
ProxyHost string
ExpiresAt time.Time
}
// GetAuthorizationToken builds an aws-sdk ECR client pointed at the gateway and
// calls GetAuthorizationToken with the supplied static credentials, decoding the
// AWS:<jwt> token into Basic-auth parts. httpClient must trust the gateway CA.
func GetAuthorizationToken(region, gatewayURL string, httpClient *http.Client, akid, secret, sessionToken string) (Token, error) {
if region == "" {
region = defaultRegion
}
sess, err := session.NewSession(&aws.Config{
Region: aws.String(region),
Endpoint: aws.String(gatewayURL),
Credentials: credentials.NewStaticCredentials(akid, secret, sessionToken),
HTTPClient: httpClient,
DisableSSL: aws.Bool(false),
})
if err != nil {
return Token{}, fmt.Errorf("new session: %w", err)
}
out, err := ecr.New(sess).GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
if err != nil {
return Token{}, fmt.Errorf("GetAuthorizationToken: %w", err)
}
if len(out.AuthorizationData) == 0 || out.AuthorizationData[0].AuthorizationToken == nil {
return Token{}, fmt.Errorf("GetAuthorizationToken returned no authorization data")
}
data := out.AuthorizationData[0]
raw, err := base64.StdEncoding.DecodeString(aws.StringValue(data.AuthorizationToken))
if err != nil {
return Token{}, fmt.Errorf("decode authorization token: %w", err)
}
user, pass, ok := strings.Cut(string(raw), ":")
if !ok {
return Token{}, fmt.Errorf("authorization token not in user:password form")
}
tok := Token{
Username: user,
Password: pass,
ProxyHost: HostFromImage(aws.StringValue(data.ProxyEndpoint)),
}
if data.ExpiresAt != nil {
tok.ExpiresAt = aws.TimeValue(data.ExpiresAt)
}
return tok, nil
}
// GatewayHTTPClient builds an HTTP client trusting the gateway CA at caPath.
// When caPath is empty it relies on the system trust store.
func GatewayHTTPClient(caPath string) (*http.Client, error) {
tlsCfg := &tls.Config{
MinVersion: tls.VersionTLS13,
CurvePreferences: tlsconfig.Curves,
}
if caPath != "" {
pem, err := os.ReadFile(caPath)
if err != nil {
return nil, fmt.Errorf("read gateway CA %q: %w", caPath, err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(pem) {
return nil, fmt.Errorf("gateway CA %q has no usable certificates", caPath)
}
tlsCfg.RootCAs = pool
}
return &http.Client{
Timeout: 15 * time.Second,
Transport: &http.Transport{TLSClientConfig: tlsCfg, MaxIdleConns: 2, IdleConnTimeout: 30 * time.Second},
}, nil
}
// HostFromImage returns the host[:port] portion of an image ref or URL. It strips
// any scheme and trims at the first path separator.
func HostFromImage(image string) string {
if i := strings.Index(image, "://"); i >= 0 {
image = image[i+3:]
}
if i := strings.IndexByte(image, '/'); i >= 0 {
image = image[:i]
}
return image
}
+107
View File
@@ -0,0 +1,107 @@
package ecrauth
import (
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
func TestHostFromImage(t *testing.T) {
cases := map[string]string{
"acct.dkr.ecr.us-east-1.spinifex.internal/demo:latest": "acct.dkr.ecr.us-east-1.spinifex.internal",
"https://acct.dkr.ecr.us-east-1.spinifex.internal": "acct.dkr.ecr.us-east-1.spinifex.internal",
"host:9999/repo:tag": "host:9999",
"plainhost": "plainhost",
}
for in, want := range cases {
if got := HostFromImage(in); got != want {
t.Errorf("HostFromImage(%q) = %q, want %q", in, got, want)
}
}
}
// gatewayStub serves the ECR GetAuthorizationToken X-Amz-Target dispatch,
// returning a base64 AWS:<jwt> token like the real gateway.
func gatewayStub(t *testing.T, token, proxy string, status int) *httptest.Server {
t.Helper()
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.Contains(r.Header.Get("X-Amz-Target"), "GetAuthorizationToken") {
w.WriteHeader(http.StatusBadRequest)
return
}
if status != http.StatusOK {
w.WriteHeader(status)
return
}
w.Header().Set("Content-Type", "application/x-amz-json-1.1")
_ = json.NewEncoder(w).Encode(map[string]any{
"authorizationData": []map[string]any{{
"authorizationToken": base64.StdEncoding.EncodeToString([]byte(token)),
"proxyEndpoint": proxy,
}},
})
}))
t.Cleanup(srv.Close)
return srv
}
func clientTrusting(srv *httptest.Server) *http.Client {
pool := x509.NewCertPool()
pool.AddCert(srv.Certificate())
return &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
}
func TestGetAuthorizationToken_OK(t *testing.T) {
srv := gatewayStub(t, "AWS:testjwt", "https://123456789012.dkr.ecr.us-east-1.spinifex.internal", http.StatusOK)
tok, err := GetAuthorizationToken("us-east-1", srv.URL, clientTrusting(srv), "AKIA", "secret", "session")
if err != nil {
t.Fatalf("GetAuthorizationToken: %v", err)
}
if tok.Username != "AWS" || tok.Password != "testjwt" {
t.Errorf("got %q:%q, want AWS:testjwt", tok.Username, tok.Password)
}
if tok.ProxyHost != "123456789012.dkr.ecr.us-east-1.spinifex.internal" {
t.Errorf("ProxyHost = %q", tok.ProxyHost)
}
}
func TestGetAuthorizationToken_BadTokenForm(t *testing.T) {
srv := gatewayStub(t, "no-colon-token", "https://x", http.StatusOK)
if _, err := GetAuthorizationToken("us-east-1", srv.URL, clientTrusting(srv), "AKIA", "secret", ""); err == nil {
t.Fatal("expected error for token not in user:password form")
}
}
func TestGetAuthorizationToken_ServerError(t *testing.T) {
srv := gatewayStub(t, "", "", http.StatusInternalServerError)
if _, err := GetAuthorizationToken("us-east-1", srv.URL, clientTrusting(srv), "AKIA", "secret", ""); err == nil {
t.Fatal("expected error on gateway 500")
}
}
func TestGatewayHTTPClient(t *testing.T) {
// Empty CA path: system trust store, no error.
if _, err := GatewayHTTPClient(""); err != nil {
t.Fatalf("empty CA path: %v", err)
}
// Missing file: error.
if _, err := GatewayHTTPClient(filepath.Join(t.TempDir(), "nope.pem")); err == nil {
t.Fatal("expected error for missing CA file")
}
// Garbage PEM: error.
bad := filepath.Join(t.TempDir(), "bad.pem")
if err := os.WriteFile(bad, []byte("not a cert"), 0o600); err != nil {
t.Fatal(err)
}
if _, err := GatewayHTTPClient(bad); err == nil {
t.Fatal("expected error for CA file with no certificates")
}
}
+118
View File
@@ -0,0 +1,118 @@
// Package imdscreds fetches AWS instance-role credentials from the in-guest
// IMDS endpoint (169.254.169.254) using IMDSv2. It is shared by the binaries
// that run inside a Spinifex guest and need the node/instance role credentials:
// the EKS ecr-credential-provider and the ECS ecs-agent.
package imdscreds
import (
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
)
// imdsTokenTTL is the IMDSv2 session-token lifetime requested on the PUT.
const imdsTokenTTL = "21600"
// Credentials is a resolved instance-role credential set with parsed expiry.
type Credentials struct {
AccessKeyID string
SecretAccessKey string
SessionToken string
Expiration time.Time
}
// imdsCreds mirrors the IMDS security-credentials JSON shape.
type imdsCreds struct {
Code string `json:"Code"`
AccessKeyID string `json:"AccessKeyId"`
SecretAccessKey string `json:"SecretAccessKey"`
Token string `json:"Token"`
Expiration string `json:"Expiration"`
}
// Fetch resolves the instance role name then its credentials via IMDSv2
// (PUT token, then GET with the token header). base is the IMDS root, e.g.
// http://169.254.169.254/latest.
func Fetch(client *http.Client, base string) (Credentials, error) {
base = strings.TrimRight(base, "/")
token := v2Token(client, base)
roleName, err := get(client, base+"/meta-data/iam/security-credentials/", token)
if err != nil {
return Credentials{}, fmt.Errorf("fetch role name: %w", err)
}
roleName = strings.TrimSpace(roleName)
if roleName == "" {
return Credentials{}, fmt.Errorf("IMDS returned empty role name")
}
body, err := get(client, base+"/meta-data/iam/security-credentials/"+roleName, token)
if err != nil {
return Credentials{}, fmt.Errorf("fetch role credentials: %w", err)
}
var raw imdsCreds
if err := json.Unmarshal([]byte(body), &raw); err != nil {
return Credentials{}, fmt.Errorf("decode role credentials: %w", err)
}
if raw.AccessKeyID == "" || raw.SecretAccessKey == "" {
return Credentials{}, fmt.Errorf("IMDS credentials missing access/secret key")
}
creds := Credentials{
AccessKeyID: raw.AccessKeyID,
SecretAccessKey: raw.SecretAccessKey,
SessionToken: raw.Token,
}
if raw.Expiration != "" {
// IMDS emits RFC3339; a parse failure leaves a zero expiry (treated as
// unknown by callers) rather than failing the whole fetch.
if exp, perr := time.Parse(time.RFC3339, raw.Expiration); perr == nil {
creds.Expiration = exp
}
}
return creds, nil
}
// v2Token requests an IMDSv2 session token; returns "" if the service does not
// enforce v2 (tokenless v1 then applies).
func v2Token(client *http.Client, base string) string {
req, err := http.NewRequest(http.MethodPut, base+"/api/token", nil)
if err != nil {
return ""
}
req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", imdsTokenTTL)
resp, err := client.Do(req)
if err != nil {
return ""
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return ""
}
body, _ := io.ReadAll(resp.Body)
return strings.TrimSpace(string(body))
}
// get issues an IMDS GET, attaching the IMDSv2 token header when present.
func get(client *http.Client, url, token string) (string, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return "", err
}
if token != "" {
req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
}
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", fmt.Errorf("IMDS %s returned %d: %s", url, resp.StatusCode, strings.TrimSpace(string(body)))
}
return string(body), nil
}
+80
View File
@@ -0,0 +1,80 @@
package imdscreds
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
)
// imdsStub serves the IMDSv2 token + role + credentials endpoints.
func imdsStub(t *testing.T, creds map[string]string, enforceV2 bool) *httptest.Server {
t.Helper()
mux := http.NewServeMux()
mux.HandleFunc("/api/token", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPut {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
_, _ = w.Write([]byte("v2-token"))
})
mux.HandleFunc("/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
if enforceV2 && r.Header.Get("X-Aws-Ec2-Metadata-Token") != "v2-token" {
w.WriteHeader(http.StatusUnauthorized)
return
}
if r.URL.Path == "/meta-data/iam/security-credentials/" {
_, _ = w.Write([]byte("node-role"))
return
}
_ = json.NewEncoder(w).Encode(creds)
})
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)
return srv
}
func TestFetch_OK(t *testing.T) {
exp := time.Now().Add(6 * time.Hour).UTC().Round(time.Second)
srv := imdsStub(t, map[string]string{
"Code": "Success",
"AccessKeyId": "AKIATEST",
"SecretAccessKey": "secrettest",
"Token": "sessiontoken",
"Expiration": exp.Format(time.RFC3339),
}, true)
got, err := Fetch(srv.Client(), srv.URL)
if err != nil {
t.Fatalf("Fetch: %v", err)
}
if got.AccessKeyID != "AKIATEST" || got.SecretAccessKey != "secrettest" || got.SessionToken != "sessiontoken" {
t.Errorf("creds mismatch: %+v", got)
}
if !got.Expiration.Equal(exp) {
t.Errorf("Expiration = %v, want %v", got.Expiration, exp)
}
}
func TestFetch_MissingKeys(t *testing.T) {
srv := imdsStub(t, map[string]string{"Code": "Success"}, false)
if _, err := Fetch(srv.Client(), srv.URL); err == nil {
t.Fatal("expected error when access/secret key absent")
}
}
func TestFetch_BadExpirationIsZero(t *testing.T) {
srv := imdsStub(t, map[string]string{
"AccessKeyId": "AKIA",
"SecretAccessKey": "secret",
"Expiration": "not-a-time",
}, false)
got, err := Fetch(srv.Client(), srv.URL)
if err != nil {
t.Fatalf("Fetch: %v", err)
}
if !got.Expiration.IsZero() {
t.Errorf("Expiration = %v, want zero on parse failure", got.Expiration)
}
}
+6 -2
View File
@@ -266,8 +266,12 @@ ecs.bus.<accountID>.<clusterName>.instance-register.<instanceID> # agent → sc
ecs.bus.<accountID>.<clusterName>.service-reconcile # scheduler-internal tick
```
This layer lands with the scheduler + agent bodies. The Phase 5 per-AZ NATS
cutover inserts `{azID}` after `ecs.` → `ecs.<azID>.bus.<accountID>.<clusterName>.*`.
Sprint 4c lands the agent-side publishers for `instance-register` (at boot) and
`instance-heartbeat` (every 30s); the scheduler-side subscribers, `assign`, and
`task-state` land in Sprint 4e. Subject builders and wire payloads live in
`spinifex/handlers/ecs/bus` so both the agent and the scheduler share one source
of truth. The Phase 5 per-AZ NATS cutover inserts `{azID}` after `ecs.` →
`ecs.<azID>.bus.<accountID>.<clusterName>.*`.
### Cross-service calls
+41
View File
@@ -0,0 +1,41 @@
package bus
import "time"
// InstanceCapacity is a container instance's total registrable resources.
type InstanceCapacity struct {
CPU int `json:"cpu"` // total vCPU units (1 vCPU = 1024)
MemoryMiB int `json:"memoryMiB"` // total memory in MiB
}
// RegisterInstance is published on RegisterSubject when an ecs-agent boots and
// joins its cluster. The scheduler records a container-instance entry keyed by
// InstanceID (handlers_ecs.InstanceKey).
type RegisterInstance struct {
AccountID string `json:"accountId"`
ClusterName string `json:"clusterName"`
InstanceID string `json:"instanceId"`
AZ string `json:"availabilityZone"`
Hostname string `json:"hostname"`
Capacity InstanceCapacity `json:"capacity"`
AgentVersion string `json:"agentVersion"`
RegisteredAt time.Time `json:"registeredAt"`
}
// Heartbeat is published on HeartbeatSubject every ~30s while the agent is
// healthy. A missed heartbeat past the scheduler's TTL marks the instance
// DISCONNECTED (wired in Sprint 4e).
type Heartbeat struct {
AccountID string `json:"accountId"`
ClusterName string `json:"clusterName"`
InstanceID string `json:"instanceId"`
Status string `json:"status"` // ACTIVE | DRAINING
RunningTasks int `json:"runningTasks"`
SentAt time.Time `json:"sentAt"`
}
// Instance status values reported in Heartbeat.Status.
const (
StatusActive = "ACTIVE"
StatusDraining = "DRAINING"
)
+43
View File
@@ -0,0 +1,43 @@
// Package bus defines the Layer-2 internal scheduler↔agent NATS subjects and
// wire payloads for the ECS control plane (ecs-v1.md Q14). Both the ecs-agent
// (publisher) and the spinifex scheduler (subscriber, Sprint 4e) import it so the
// subject shape and message schema have a single source of truth.
//
// Cluster identity is "<accountID>.<clusterName>" (matching the KV layout), not a
// UUID. The Phase 5 per-AZ NATS cutover inserts "{azID}" after "ecs." — keep
// these builders the one place that knows the shape so that change stays local.
package bus
import "fmt"
// Prefix returns "ecs.bus.<accountID>.<clusterName>".
func Prefix(accountID, clusterName string) string {
return fmt.Sprintf("ecs.bus.%s.%s", accountID, clusterName)
}
// RegisterSubject is published by the agent at boot (agent → scheduler).
func RegisterSubject(accountID, clusterName, instanceID string) string {
return fmt.Sprintf("%s.instance-register.%s", Prefix(accountID, clusterName), instanceID)
}
// HeartbeatSubject is published by the agent every 30s (agent → scheduler).
func HeartbeatSubject(accountID, clusterName, instanceID string) string {
return fmt.Sprintf("%s.instance-heartbeat.%s", Prefix(accountID, clusterName), instanceID)
}
// AssignSubject is the per-instance task-assignment subject (scheduler → agent).
// The agent does not subscribe to it until Sprint 4e.
func AssignSubject(accountID, clusterName, instanceID string) string {
return fmt.Sprintf("%s.assign.%s", Prefix(accountID, clusterName), instanceID)
}
// TaskStateSubject is the per-task state-report subject (agent → scheduler),
// published from Sprint 4e.
func TaskStateSubject(accountID, clusterName, taskID string) string {
return fmt.Sprintf("%s.task-state.%s", Prefix(accountID, clusterName), taskID)
}
// ServiceReconcileSubject is the scheduler-internal reconcile tick.
func ServiceReconcileSubject(accountID, clusterName string) string {
return fmt.Sprintf("%s.service-reconcile", Prefix(accountID, clusterName))
}
@@ -0,0 +1,25 @@
package bus
import "testing"
func TestSubjectBuilders(t *testing.T) {
const (
acct = "123456789012"
cluster = "default"
inst = "i-abc123"
task = "t-deadbeef"
)
cases := map[string]struct{ got, want string }{
"prefix": {Prefix(acct, cluster), "ecs.bus.123456789012.default"},
"register": {RegisterSubject(acct, cluster, inst), "ecs.bus.123456789012.default.instance-register.i-abc123"},
"heartbeat": {HeartbeatSubject(acct, cluster, inst), "ecs.bus.123456789012.default.instance-heartbeat.i-abc123"},
"assign": {AssignSubject(acct, cluster, inst), "ecs.bus.123456789012.default.assign.i-abc123"},
"taskstate": {TaskStateSubject(acct, cluster, task), "ecs.bus.123456789012.default.task-state.t-deadbeef"},
"reconcile": {ServiceReconcileSubject(acct, cluster), "ecs.bus.123456789012.default.service-reconcile"},
}
for name, c := range cases {
if c.got != c.want {
t.Errorf("%s = %q, want %q", name, c.got, c.want)
}
}
}