From a4a4288d7d37eeafeba9052d0a9d2ed1fdcb7813 Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Thu, 18 Jun 2026 17:29:41 +1000 Subject: [PATCH] feat(ecr): auth bridge + GetAuthorizationToken JWT (Sprint 2c) (#380) * feat(ecr): auth bridge + GetAuthorizationToken JWT (Sprint 2c) Mint a self-contained ES256 JWT from GetAuthorizationToken and verify it on the /v2 OCI registry via a new ECRAuthBridgeMiddleware (Bearer | Basic AWS:jwt). Signing keys live in the cluster-replicated awsgw-keys KV bucket under jwt-signing/{kid}, encrypted with the IAM master key. The registry is now scoped per-request to the token's account (replacing the fixed bootstrap account); bucket-ensure is cached per account. Cross-account token/host mismatch is rejected 403; unauthenticated /v2 access gets a 401 Bearer challenge. * fix(ecr): use Basic auth challenge on /v2 (not Bearer token-endpoint flow) ECR authenticates the OCI registry with Basic AWS:; a Bearer realm made docker chase an unimplemented /v2/token endpoint and fail login. --- spinifex/gateway/ecr.go | 1 + spinifex/gateway/ecr/context.go | 22 +++ spinifex/gateway/ecr/registry.go | 67 +++++-- spinifex/gateway/ecr_getauthorizationtoken.go | 55 ++++++ .../gateway/ecr_getauthorizationtoken_test.go | 75 ++++++++ spinifex/gateway/ecrapi.go | 6 + spinifex/gateway/ecrauth/ecrauth_test.go | 166 +++++++++++++++++ spinifex/gateway/ecrauth/signingkey.go | 170 ++++++++++++++++++ spinifex/gateway/ecrauth/token.go | 127 +++++++++++++ spinifex/gateway/ecrauth_middleware.go | 99 ++++++++++ spinifex/gateway/ecrauth_middleware_test.go | 165 +++++++++++++++++ spinifex/gateway/gateway.go | 11 ++ spinifex/services/awsgw/awsgw.go | 47 +++-- 13 files changed, 979 insertions(+), 32 deletions(-) create mode 100644 spinifex/gateway/ecr/context.go create mode 100644 spinifex/gateway/ecr_getauthorizationtoken.go create mode 100644 spinifex/gateway/ecr_getauthorizationtoken_test.go create mode 100644 spinifex/gateway/ecrauth/ecrauth_test.go create mode 100644 spinifex/gateway/ecrauth/signingkey.go create mode 100644 spinifex/gateway/ecrauth/token.go create mode 100644 spinifex/gateway/ecrauth_middleware.go create mode 100644 spinifex/gateway/ecrauth_middleware_test.go diff --git a/spinifex/gateway/ecr.go b/spinifex/gateway/ecr.go index bde6c145e..7b04283d6 100644 --- a/spinifex/gateway/ecr.go +++ b/spinifex/gateway/ecr.go @@ -20,6 +20,7 @@ import ( func (gw *GatewayConfig) mountOCIRegistry(r chi.Router) { r.Route("/v2", func(v2 chi.Router) { v2.Use(gw.hostMatch) + v2.Use(gw.ecrAuthBridge) v2.Get("/", gateway_ecr.APIVersion) if gw.ECRRegistry != nil { v2.HandleFunc("/*", gw.ECRRegistry.ServeHTTP) diff --git a/spinifex/gateway/ecr/context.go b/spinifex/gateway/ecr/context.go new file mode 100644 index 000000000..09078e058 --- /dev/null +++ b/spinifex/gateway/ecr/context.go @@ -0,0 +1,22 @@ +package gateway_ecr + +import "context" + +// ctxKey is a private context key type for this package. +type ctxKey string + +// ctxAuthAccount carries the accountID resolved from a verified ECR token by the +// /v2 auth bridge. The Registry scopes per-request storage to it. +const ctxAuthAccount ctxKey = "ecr.authAccount" + +// WithAuthAccount returns a context carrying the auth-bridge–resolved account. +// The gateway auth bridge calls this after verifying a registry token. +func WithAuthAccount(ctx context.Context, accountID string) context.Context { + return context.WithValue(ctx, ctxAuthAccount, accountID) +} + +// authAccount returns the auth-bridge–resolved account from ctx, if any. +func authAccount(ctx context.Context) string { + a, _ := ctx.Value(ctxAuthAccount).(string) + return a +} diff --git a/spinifex/gateway/ecr/registry.go b/spinifex/gateway/ecr/registry.go index e5575619b..a4fa663fa 100644 --- a/spinifex/gateway/ecr/registry.go +++ b/spinifex/gateway/ecr/registry.go @@ -46,23 +46,58 @@ type Registry struct { Meta ecr.MetaStore AccountID string - // bucketMu guards bucketReady, which caches that the account's predastore - // bucket has been provisioned. Only success is cached: a failed ensure is - // retried on the next request. - bucketMu sync.Mutex - bucketReady bool + // buckets caches which account predastore buckets have been provisioned. + // Shared across per-request Registry copies so concurrent callers converge. + buckets *bucketCache +} + +// bucketCache records, per account, that the predastore bucket exists. Only +// success is cached: a failed ensure is retried on the next request. +type bucketCache struct { + mu sync.Mutex + ready map[string]bool } // NewRegistry wires a Registry to its predastore object store and the metadata -// store (a NATS client in production) for the given account. +// store (a NATS client in production). accountID is the fallback account used +// when a request carries no auth-bridge account (e.g. unit tests). func NewRegistry(store objectstore.ObjectStore, meta ecr.MetaStore, accountID string) *Registry { - return &Registry{Store: store, Meta: meta, AccountID: accountID} + return &Registry{ + Store: store, + Meta: meta, + AccountID: accountID, + buckets: &bucketCache{ready: make(map[string]bool)}, + } } -// ServeHTTP dispatches a /v2/* request by manually parsing the path: OCI repo -// names contain slashes, so the {name} segment is everything between "/v2/" and -// the trailing "/blobs", "/manifests" or "/tags" marker. +// forAccount returns a shallow Registry copy scoped to account. Store, Meta and +// the bucket cache are shared by pointer; only the request-scoped AccountID +// differs, so every handler method transparently operates on the caller's +// account without threading it through each signature. +func (reg *Registry) forAccount(account string) *Registry { + cp := *reg + cp.AccountID = account + return &cp +} + +// ServeHTTP resolves the caller account from the auth-bridge context (falling +// back to the configured default), then dispatches on a per-request scoped copy. func (reg *Registry) ServeHTTP(w http.ResponseWriter, r *http.Request) { + account := authAccount(r.Context()) + if account == "" { + account = reg.AccountID + } + if account == "" { + reg.internal(w, "resolve account", errors.New("no authenticated account")) + return + } + reg.forAccount(account).serve(w, r) +} + +// serve dispatches a /v2/* request by manually parsing the path: OCI repo names +// contain slashes, so the {name} segment is everything between "/v2/" and the +// trailing "/blobs", "/manifests" or "/tags" marker. +func (reg *Registry) serve(w http.ResponseWriter, r *http.Request) { if err := reg.ensureBucket(); err != nil { reg.internal(w, "ensure bucket", err) return @@ -714,18 +749,18 @@ func (reg *Registry) bucket() string { return ecr.AccountBucket(reg.AccountID) } // ensureBucket provisions the account's predastore bucket on first use. ECR // storage is account-managed: the bucket appears transparently rather than -// being created by an operator. The result is cached once provisioned; a -// backend failure is left uncached so the next request retries. +// being created by an operator. Success is cached per account; a backend +// failure is left uncached so the next request retries. func (reg *Registry) ensureBucket() error { - reg.bucketMu.Lock() - defer reg.bucketMu.Unlock() - if reg.bucketReady { + reg.buckets.mu.Lock() + defer reg.buckets.mu.Unlock() + if reg.buckets.ready[reg.AccountID] { return nil } if err := reg.Store.EnsureBucket(reg.bucket()); err != nil { return err } - reg.bucketReady = true + reg.buckets.ready[reg.AccountID] = true return nil } diff --git a/spinifex/gateway/ecr_getauthorizationtoken.go b/spinifex/gateway/ecr_getauthorizationtoken.go new file mode 100644 index 000000000..bbad0c4e4 --- /dev/null +++ b/spinifex/gateway/ecr_getauthorizationtoken.go @@ -0,0 +1,55 @@ +package gateway + +import ( + "encoding/base64" + "errors" + "log/slog" + "net/http" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ecr" + "github.com/mulgadc/spinifex/spinifex/awserrors" + gateway_ecrapi "github.com/mulgadc/spinifex/spinifex/gateway/ecrapi" + gateway_ecrauth "github.com/mulgadc/spinifex/spinifex/gateway/ecrauth" +) + +// handleGetAuthorizationToken mints a self-contained ES256 ECR token for the +// SigV4-authenticated caller and returns it in the AWS GetAuthorizationToken +// shape. The token is base64("AWS:"); docker stores it and replays it as +// Basic auth against the /v2 registry, where the auth bridge verifies it. +func (gw *GatewayConfig) handleGetAuthorizationToken(w http.ResponseWriter, r *http.Request) error { + if gw.ECRTokenIssuer == nil { + return errors.New(awserrors.ErrorNotImplemented) + } + ctx := r.Context() + accountID, _ := ctx.Value(ctxAccountID).(string) + if accountID == "" { + slog.Error("GetAuthorizationToken: no account ID in auth context") + return errors.New(awserrors.ErrorServerInternal) + } + accessKey, _ := ctx.Value(ctxAccessKey).(string) + principalType, _ := ctx.Value(ctxPrincipalType).(string) + + token, expiresAt, err := gw.ECRTokenIssuer.Mint(gateway_ecrauth.Principal{ + AccountID: accountID, + ARN: eksCallerPrincipalARN(r), + Type: principalType, + AccessKeyID: accessKey, + }) + if err != nil { + slog.Error("GetAuthorizationToken: mint failed", "err", err) + return errors.New(awserrors.ErrorServerInternal) + } + + authToken := base64.StdEncoding.EncodeToString([]byte("AWS:" + token)) + proxyEndpoint := "https://" + accountID + ".dkr.ecr." + gw.Region + "." + gw.InternalSuffix + + gateway_ecrapi.WriteJSONResponse(w, &ecr.GetAuthorizationTokenOutput{ + AuthorizationData: []*ecr.AuthorizationData{{ + AuthorizationToken: aws.String(authToken), + ProxyEndpoint: aws.String(proxyEndpoint), + ExpiresAt: aws.Time(expiresAt), + }}, + }) + return nil +} diff --git a/spinifex/gateway/ecr_getauthorizationtoken_test.go b/spinifex/gateway/ecr_getauthorizationtoken_test.go new file mode 100644 index 000000000..8cf9ebcdb --- /dev/null +++ b/spinifex/gateway/ecr_getauthorizationtoken_test.go @@ -0,0 +1,75 @@ +package gateway + +import ( + "context" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHandleGetAuthorizationToken_MintsUsableToken(t *testing.T) { + iss, verify := newECRAuth(t) + gw := &GatewayConfig{ + Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, + ECRTokenIssuer: iss, ECRTokenVerifier: verify, DisableLogging: true, + } + + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("{}")) + ctx := context.WithValue(req.Context(), ctxAccountID, ecrTestAccount) + ctx = context.WithValue(ctx, ctxPrincipalType, principalTypeUser) + w := httptest.NewRecorder() + + require.NoError(t, gw.handleGetAuthorizationToken(w, req.WithContext(ctx))) + require.Equal(t, http.StatusOK, w.Code) + + var out struct { + AuthorizationData []struct { + AuthorizationToken string `json:"authorizationToken"` + ProxyEndpoint string `json:"proxyEndpoint"` + ExpiresAt float64 `json:"expiresAt"` + } `json:"authorizationData"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &out)) + require.Len(t, out.AuthorizationData, 1) + + ad := out.AuthorizationData[0] + assert.Equal(t, "https://"+ecrTestAccount+".dkr.ecr."+ecrTestRegion+"."+ecrTestSuffix, ad.ProxyEndpoint) + assert.Positive(t, ad.ExpiresAt) + + // authorizationToken decodes to "AWS:" and the jwt verifies to the account. + decoded, err := base64.StdEncoding.DecodeString(ad.AuthorizationToken) + require.NoError(t, err) + user, jwtStr, found := strings.Cut(string(decoded), ":") + require.True(t, found) + assert.Equal(t, "AWS", user) + + claims, err := verify.Verify(jwtStr) + require.NoError(t, err) + assert.Equal(t, ecrTestAccount, claims.AccountID) +} + +func TestHandleGetAuthorizationToken_NoIssuerNotImplemented(t *testing.T) { + gw := &GatewayConfig{DisableLogging: true} + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("{}")) + ctx := context.WithValue(req.Context(), ctxAccountID, ecrTestAccount) + err := gw.handleGetAuthorizationToken(httptest.NewRecorder(), req.WithContext(ctx)) + require.Error(t, err) +} + +func TestECRRequest_GetAuthorizationTokenDispatched(t *testing.T) { + iss, verify := newECRAuth(t) + gw := &GatewayConfig{ + Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, + ECRTokenIssuer: iss, ECRTokenVerifier: verify, DisableLogging: true, + } + w := httptest.NewRecorder() + req := setupECRRequest("AmazonEC2ContainerRegistry_V20150921.GetAuthorizationToken", "{}") + require.NoError(t, gw.ECR_Request(w, req)) + assert.Equal(t, http.StatusOK, w.Code) +} diff --git a/spinifex/gateway/ecrapi.go b/spinifex/gateway/ecrapi.go index 4a5d04cde..db18bd039 100644 --- a/spinifex/gateway/ecrapi.go +++ b/spinifex/gateway/ecrapi.go @@ -39,6 +39,12 @@ func (gw *GatewayConfig) ECR_Request(w http.ResponseWriter, r *http.Request) err return err } + // GetAuthorizationToken mints a registry token from the SigV4 auth context; + // it is served inline rather than relayed onto a NATS subject. + if action == "GetAuthorizationToken" { + return gw.handleGetAuthorizationToken(w, r) + } + accountID, _ := r.Context().Value(ctxAccountID).(string) if accountID == "" { slog.Error("ECR_Request: no account ID in auth context") diff --git a/spinifex/gateway/ecrauth/ecrauth_test.go b/spinifex/gateway/ecrauth/ecrauth_test.go new file mode 100644 index 000000000..c21e78217 --- /dev/null +++ b/spinifex/gateway/ecrauth/ecrauth_test.go @@ -0,0 +1,166 @@ +package gateway_ecrauth + +import ( + "crypto/ecdsa" + "crypto/rand" + "strings" + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testAudience = "ecr.ap-southeast-2.mulga.internal" + +var testMasterKey = func() []byte { + k := make([]byte, 32) + if _, err := rand.Read(k); err != nil { + panic(err) + } + return k +}() + +func samplePrincipal() Principal { + return Principal{ + AccountID: "000000000001", + ARN: "arn:aws:iam::000000000001:user/dev", + Type: "IAMUser", + AccessKeyID: "AKIAVAOB203DGNBR04XP", + } +} + +func TestLoadOrCreateSigningKey_CreatesThenReloads(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + + key1, verify1, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + require.NoError(t, err) + require.NotEmpty(t, key1.Kid) + require.Contains(t, verify1, key1.Kid) + + // Second call must reload the same persisted key, not mint a new one. + key2, verify2, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + require.NoError(t, err) + assert.Equal(t, key1.Kid, key2.Kid, "persisted signing key must be reused") + assert.Len(t, verify2, 1) +} + +func TestLoadOrCreateSigningKey_StoresEncrypted(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + key, _, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + require.NoError(t, err) + + kv, err := js.KeyValue(SigningBucket) + require.NoError(t, err) + entry, err := kv.Get(signingKeyName(key.Kid)) + require.NoError(t, err) + assert.NotContains(t, string(entry.Value()), "PRIVATE KEY", + "signing key must be encrypted at rest") +} + +func TestLoadOrCreateSigningKey_EmptyMasterKeyRejected(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + _, _, err := LoadOrCreateSigningKey(js, nil, 1) + require.Error(t, err) +} + +func TestIssuerVerifier_RoundTrip(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + key, verify, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + require.NoError(t, err) + + iss := NewIssuer(key, testAudience) + tok, exp, err := iss.Mint(samplePrincipal()) + require.NoError(t, err) + assert.WithinDuration(t, time.Now().Add(DefaultTokenTTL), exp, time.Minute) + + claims, err := NewVerifier(verify, testAudience).Verify(tok) + require.NoError(t, err) + assert.Equal(t, "000000000001", claims.AccountID) + assert.Equal(t, "arn:aws:iam::000000000001:user/dev", claims.Subject) + assert.Equal(t, "IAMUser", claims.PrincipalType) +} + +func TestIssuer_MintRequiresAccountID(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + key, _, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + require.NoError(t, err) + + p := samplePrincipal() + p.AccountID = "" + _, _, err = NewIssuer(key, testAudience).Mint(p) + require.Error(t, err) +} + +func TestVerifier_RejectsWrongAudience(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + key, verify, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + require.NoError(t, err) + + tok, _, err := NewIssuer(key, testAudience).Mint(samplePrincipal()) + require.NoError(t, err) + + _, err = NewVerifier(verify, "ecr.us-east-1.mulga.internal").Verify(tok) + require.Error(t, err) +} + +func TestVerifier_RejectsUnknownKid(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + key, _, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + require.NoError(t, err) + + tok, _, err := NewIssuer(key, testAudience).Mint(samplePrincipal()) + require.NoError(t, err) + + // Verifier with an empty key set cannot resolve the kid. + _, err = NewVerifier(map[string]*ecdsa.PublicKey{}, testAudience).Verify(tok) + require.Error(t, err) +} + +func TestVerifier_RejectsExpiredToken(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + key, verify, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + require.NoError(t, err) + + // Hand-mint an already-expired token with the same signing key. + claims := Claims{ + RegisteredClaims: jwt.RegisteredClaims{ + Issuer: tokenIssuer, + Audience: jwt.ClaimStrings{testAudience}, + IssuedAt: jwt.NewNumericDate(time.Now().Add(-2 * time.Hour)), + ExpiresAt: jwt.NewNumericDate(time.Now().Add(-time.Hour)), + }, + AccountID: "000000000001", + } + raw := jwt.NewWithClaims(jwt.SigningMethodES256, claims) + raw.Header["kid"] = key.Kid + signed, err := raw.SignedString(key.priv) + require.NoError(t, err) + + _, err = NewVerifier(verify, testAudience).Verify(signed) + require.Error(t, err) +} + +func TestVerifier_RejectsNonES256(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + _, verify, err := LoadOrCreateSigningKey(js, testMasterKey, 1) + require.NoError(t, err) + + // HS256 token must be refused regardless of signature. + claims := Claims{ + RegisteredClaims: jwt.RegisteredClaims{ + Issuer: tokenIssuer, + Audience: jwt.ClaimStrings{testAudience}, + ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Hour)), + }, + AccountID: "000000000001", + } + raw := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + signed, err := raw.SignedString([]byte(strings.Repeat("k", 32))) + require.NoError(t, err) + + _, err = NewVerifier(verify, testAudience).Verify(signed) + require.Error(t, err) +} diff --git a/spinifex/gateway/ecrauth/signingkey.go b/spinifex/gateway/ecrauth/signingkey.go new file mode 100644 index 000000000..bf1a23a67 --- /dev/null +++ b/spinifex/gateway/ecrauth/signingkey.go @@ -0,0 +1,170 @@ +// Package gateway_ecrauth implements the ECR auth bridge: it mints and verifies +// the self-contained ES256 JWT that GetAuthorizationToken issues and the +// /v2/* registry surface accepts (Authorization: Bearer | Basic AWS:). +// Signing keys live in the cluster-replicated JetStream KV bucket awsgw-keys +// under jwt-signing/{kid}, encrypted at rest with the IAM master key. +package gateway_ecrauth + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + "strings" + + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/nats-io/nats.go" +) + +const ( + // SigningBucket is the cluster-replicated KV bucket holding awsgw signing keys. + SigningBucket = "awsgw-keys" + // signingKeyPrefix namespaces JWT signing keys within SigningBucket. + signingKeyPrefix = "jwt-signing/" + // signingKeyHistory keeps one revision; rotation writes new kids, not revisions. + signingKeyHistory = 1 +) + +// SigningKey is an ES256 (P-256) private key plus its key id. kid is the +// base64url SHA-256 of the SPKI DER, matching the EKS OIDC convention so the +// same JWK tooling applies. +type SigningKey struct { + Kid string + priv *ecdsa.PrivateKey +} + +func signingKeyName(kid string) string { return signingKeyPrefix + kid } + +// kidFor derives the key id from a public key as base64url(sha256(SPKI DER)). +func kidFor(pub *ecdsa.PublicKey) (string, error) { + der, err := x509.MarshalPKIXPublicKey(pub) + if err != nil { + return "", fmt.Errorf("marshal pkix public key: %w", err) + } + sum := sha256.Sum256(der) + return base64.RawURLEncoding.EncodeToString(sum[:]), nil +} + +// LoadOrCreateSigningKey opens (or creates) the awsgw-keys KV bucket, loads +// every stored signing key into the kid->public-key verification set, and +// returns the active signing key. On first run it generates and persists one. +// The active key is the lexically-greatest kid so all nodes converge on the +// same signer without coordination. +func LoadOrCreateSigningKey(js nats.JetStreamContext, masterKey []byte, replicas int) (*SigningKey, map[string]*ecdsa.PublicKey, error) { + if js == nil { + return nil, nil, errors.New("ecrauth: nil JetStream context") + } + if len(masterKey) == 0 { + return nil, nil, errors.New("ecrauth: empty master key") + } + + kv, err := js.CreateKeyValue(&nats.KeyValueConfig{ + Bucket: SigningBucket, + History: signingKeyHistory, + Replicas: replicas, + }) + if err != nil { + if kv, err = js.KeyValue(SigningBucket); err != nil { + return nil, nil, fmt.Errorf("open %s bucket: %w", SigningBucket, err) + } + } + + keys, err := kv.Keys() + if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + return nil, nil, fmt.Errorf("list signing keys: %w", err) + } + + verify := make(map[string]*ecdsa.PublicKey) + var active *SigningKey + for _, name := range keys { + if !strings.HasPrefix(name, signingKeyPrefix) { + continue + } + sk, err := loadSigningKey(kv, name, masterKey) + if err != nil { + return nil, nil, err + } + verify[sk.Kid] = &sk.priv.PublicKey + if active == nil || sk.Kid > active.Kid { + active = sk + } + } + + if active == nil { + active, err = generateSigningKey(kv, masterKey) + if err != nil { + return nil, nil, err + } + verify[active.Kid] = &active.priv.PublicKey + } + return active, verify, nil +} + +// loadSigningKey decrypts and parses one stored signing key. The kid is the +// KV key suffix; the stored bytes are the AES-256-GCM-encrypted PKCS#8 PEM. +func loadSigningKey(kv nats.KeyValue, name string, masterKey []byte) (*SigningKey, error) { + entry, err := kv.Get(name) + if err != nil { + return nil, fmt.Errorf("kv get %s: %w", name, err) + } + pemStr, err := handlers_iam.DecryptSecret(string(entry.Value()), masterKey) + if err != nil { + return nil, fmt.Errorf("decrypt signing key %s: %w", name, err) + } + priv, err := parseECPrivateKeyPEM(pemStr) + if err != nil { + return nil, err + } + return &SigningKey{Kid: strings.TrimPrefix(name, signingKeyPrefix), priv: priv}, nil +} + +// generateSigningKey creates a fresh ES256 key, persists the encrypted PEM under +// jwt-signing/{kid}, and returns it. +func generateSigningKey(kv nats.KeyValue, masterKey []byte) (*SigningKey, error) { + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("generate ES256 key: %w", err) + } + kid, err := kidFor(&priv.PublicKey) + if err != nil { + return nil, err + } + pkcs8, err := x509.MarshalPKCS8PrivateKey(priv) + if err != nil { + return nil, fmt.Errorf("marshal pkcs8 private key: %w", err) + } + pemBytes := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}) + ciphertext, err := handlers_iam.EncryptSecret(string(pemBytes), masterKey) + if err != nil { + return nil, fmt.Errorf("encrypt signing key: %w", err) + } + if _, err := kv.Put(signingKeyName(kid), []byte(ciphertext)); err != nil { + return nil, fmt.Errorf("kv put %s: %w", signingKeyName(kid), err) + } + return &SigningKey{Kid: kid, priv: priv}, nil +} + +// parseECPrivateKeyPEM decodes a PKCS#8 PEM into a P-256 ECDSA private key. +func parseECPrivateKeyPEM(pemStr string) (*ecdsa.PrivateKey, error) { + block, _ := pem.Decode([]byte(pemStr)) + if block == nil { + return nil, errors.New("ecrauth: no PEM block in signing key") + } + keyAny, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("parse pkcs8 private key: %w", err) + } + priv, ok := keyAny.(*ecdsa.PrivateKey) + if !ok { + return nil, fmt.Errorf("ecrauth: unexpected key type %T (want *ecdsa.PrivateKey)", keyAny) + } + if priv.Curve != elliptic.P256() { + return nil, fmt.Errorf("ecrauth: unexpected curve %s (want P-256)", priv.Curve.Params().Name) + } + return priv, nil +} diff --git a/spinifex/gateway/ecrauth/token.go b/spinifex/gateway/ecrauth/token.go new file mode 100644 index 000000000..f7e6971c0 --- /dev/null +++ b/spinifex/gateway/ecrauth/token.go @@ -0,0 +1,127 @@ +package gateway_ecrauth + +import ( + "crypto/ecdsa" + "errors" + "fmt" + "slices" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/google/uuid" +) + +const ( + // tokenIssuer is the fixed iss claim on every minted ECR token. + tokenIssuer = "awsgw" + // DefaultTokenTTL is the ECR token lifetime (AWS parity: 12h). + DefaultTokenTTL = 12 * time.Hour +) + +// Claims is the ECR token claim set: registered OIDC claims plus the mulga +// principal attributes the bridge reads to scope a request. accountID is the +// authority for /v2 account scoping; accessKeyID is audit-only. +type Claims struct { + jwt.RegisteredClaims + + AccountID string `json:"mulga:accountID"` + PrincipalType string `json:"mulga:principalType,omitempty"` + AccessKeyID string `json:"mulga:accessKeyID,omitempty"` +} + +// Principal is the SigV4-authenticated caller GetAuthorizationToken mints a +// token for. +type Principal struct { + AccountID string + ARN string + Type string + AccessKeyID string +} + +// Issuer mints ES256 ECR tokens bound to an audience (ecr.{region}.{suffix}). +type Issuer struct { + key *SigningKey + audience string + ttl time.Duration +} + +// NewIssuer returns an Issuer signing with key for the given audience. +func NewIssuer(key *SigningKey, audience string) *Issuer { + return &Issuer{key: key, audience: audience, ttl: DefaultTokenTTL} +} + +// Mint returns a signed ES256 JWT for p and its expiry. The kid header lets any +// verifier resolve the public key from the shared signing-key set. +func (i *Issuer) Mint(p Principal) (token string, expiresAt time.Time, err error) { + if i.key == nil { + return "", time.Time{}, errors.New("ecrauth: issuer has no signing key") + } + if p.AccountID == "" { + return "", time.Time{}, errors.New("ecrauth: mint requires account ID") + } + now := time.Now() + exp := now.Add(i.ttl) + claims := Claims{ + RegisteredClaims: jwt.RegisteredClaims{ + Issuer: tokenIssuer, + Subject: p.ARN, + Audience: jwt.ClaimStrings{i.audience}, + IssuedAt: jwt.NewNumericDate(now), + ExpiresAt: jwt.NewNumericDate(exp), + ID: uuid.NewString(), + }, + AccountID: p.AccountID, + PrincipalType: p.Type, + AccessKeyID: p.AccessKeyID, + } + tok := jwt.NewWithClaims(jwt.SigningMethodES256, claims) + tok.Header["kid"] = i.key.Kid + signed, err := tok.SignedString(i.key.priv) + if err != nil { + return "", time.Time{}, fmt.Errorf("sign ECR token: %w", err) + } + return signed, exp, nil +} + +// Verifier validates ECR tokens against the shared kid->public-key set and a +// fixed expected audience. +type Verifier struct { + keys map[string]*ecdsa.PublicKey + audience string +} + +// NewVerifier returns a Verifier over keys for the expected audience. +func NewVerifier(keys map[string]*ecdsa.PublicKey, audience string) *Verifier { + return &Verifier{keys: keys, audience: audience} +} + +// Verify parses and validates raw: ES256, unexpired, known kid, matching iss and +// aud, non-empty accountID. It returns the claims on success. +func (v *Verifier) Verify(raw string) (*Claims, error) { + claims := &Claims{} + parser := jwt.NewParser( + jwt.WithValidMethods([]string{"ES256"}), + jwt.WithExpirationRequired(), + ) + _, err := parser.ParseWithClaims(raw, claims, func(t *jwt.Token) (any, error) { + kid, _ := t.Header["kid"].(string) + pub, ok := v.keys[kid] + if !ok { + return nil, fmt.Errorf("unknown kid %q", kid) + } + return pub, nil + }) + if err != nil { + return nil, err + } + if claims.Issuer != tokenIssuer { + return nil, fmt.Errorf("unexpected issuer %q", claims.Issuer) + } + if !slices.Contains(claims.Audience, v.audience) { + return nil, errors.New("audience mismatch") + } + if claims.AccountID == "" { + return nil, errors.New("missing accountID claim") + } + return claims, nil +} diff --git a/spinifex/gateway/ecrauth_middleware.go b/spinifex/gateway/ecrauth_middleware.go new file mode 100644 index 000000000..6b32bf8d3 --- /dev/null +++ b/spinifex/gateway/ecrauth_middleware.go @@ -0,0 +1,99 @@ +package gateway + +import ( + "context" + "encoding/base64" + "log/slog" + "net/http" + "strings" + + gateway_ecr "github.com/mulgadc/spinifex/spinifex/gateway/ecr" +) + +// ecrAuthBridge authenticates /v2/* requests with an ECR token. It accepts a +// single Authorization header carrying "Bearer " or "Basic AWS:", +// verifies the JWT, enforces that the token account matches the host-derived +// account, and stashes the resolved account/principal on the request context. +// Unauthenticated or invalid requests get a 401 Bearer challenge. +// +// A nil verifier disables the bridge (registry mounts open), matching the nil +// ECRRegistry fallback used by unit tests of unrelated routes. +func (gw *GatewayConfig) ecrAuthBridge(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if gw.ECRTokenVerifier == nil { + next.ServeHTTP(w, r) + return + } + + authz := r.Header.Values("Authorization") + if len(authz) > 1 { + gateway_ecr.WriteError(w, http.StatusBadRequest, "UNAUTHORIZED", "multiple Authorization headers") + return + } + raw := "" + if len(authz) == 1 { + raw = authz[0] + } + + token, ok := extractECRToken(raw) + if !ok { + gw.writeECRChallenge(w, r) + return + } + claims, err := gw.ECRTokenVerifier.Verify(token) + if err != nil { + slog.Debug("ECR auth bridge: token verify failed", "err", err) + gw.writeECRChallenge(w, r) + return + } + + // Cross-account guard: a token is account-scoped, so it must match the + // account in the registry host it is presented against. + if target, _ := r.Context().Value(ctxTargetAccount).(string); target != "" && target != claims.AccountID { + gateway_ecr.WriteError(w, http.StatusForbidden, "DENIED", "token account does not match registry host") + return + } + + ctx := gateway_ecr.WithAuthAccount(r.Context(), claims.AccountID) + ctx = context.WithValue(ctx, ctxAuthPrincipal, claims.Subject) + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +// extractECRToken pulls the JWT from a single Authorization header value, +// accepting "Bearer " or "Basic base64(AWS:)". ok is false for any +// other scheme, malformed value, or empty token. SigV4 is not accepted on /v2. +func extractECRToken(authz string) (string, bool) { + scheme, rest, found := strings.Cut(authz, " ") + if !found { + return "", false + } + switch { + case strings.EqualFold(scheme, "Bearer"): + rest = strings.TrimSpace(rest) + return rest, rest != "" + case strings.EqualFold(scheme, "Basic"): + decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(rest)) + if err != nil { + return "", false + } + user, pass, found := strings.Cut(string(decoded), ":") + if !found || user != "AWS" || pass == "" { + return "", false + } + return pass, true + default: + return "", false + } +} + +// writeECRChallenge emits the 401 Basic challenge. ECR authenticates the OCI +// registry with Basic AWS: rather than the OAuth2 token-endpoint flow, +// so the scheme must be Basic: a Bearer challenge makes docker chase a +// /v2/token endpoint that does not exist. The realm uses the request Host so +// docker negotiates against the address it actually dialed. +func (gw *GatewayConfig) writeECRChallenge(w http.ResponseWriter, r *http.Request) { + realm := "https://" + r.Host + "/" + w.Header().Set("WWW-Authenticate", `Basic realm="`+realm+`"`) + gateway_ecr.WriteError(w, http.StatusUnauthorized, "UNAUTHORIZED", "authentication required") +} diff --git a/spinifex/gateway/ecrauth_middleware_test.go b/spinifex/gateway/ecrauth_middleware_test.go new file mode 100644 index 000000000..0eaa2fcb5 --- /dev/null +++ b/spinifex/gateway/ecrauth_middleware_test.go @@ -0,0 +1,165 @@ +package gateway + +import ( + "context" + "crypto/rand" + "encoding/base64" + "net/http" + "net/http/httptest" + "testing" + + gateway_ecrauth "github.com/mulgadc/spinifex/spinifex/gateway/ecrauth" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + ecrTestRegion = "ap-southeast-2" + ecrTestSuffix = "mulga.internal" + ecrTestAudience = "ecr." + ecrTestRegion + "." + ecrTestSuffix + ecrTestAccount = "000000000001" +) + +func ecrTestMasterKey(t *testing.T) []byte { + t.Helper() + k := make([]byte, 32) + _, err := rand.Read(k) + require.NoError(t, err) + return k +} + +// newECRAuth builds a wired issuer+verifier over a freshly generated signing key. +func newECRAuth(t *testing.T) (*gateway_ecrauth.Issuer, *gateway_ecrauth.Verifier) { + t.Helper() + _, _, js := testutil.StartTestJetStream(t) + key, verify, err := gateway_ecrauth.LoadOrCreateSigningKey(js, ecrTestMasterKey(t), 1) + require.NoError(t, err) + return gateway_ecrauth.NewIssuer(key, ecrTestAudience), gateway_ecrauth.NewVerifier(verify, ecrTestAudience) +} + +func mintBasic(t *testing.T, iss *gateway_ecrauth.Issuer, account string) string { + t.Helper() + tok, _, err := iss.Mint(gateway_ecrauth.Principal{AccountID: account, ARN: "arn:aws:iam::" + account + ":user/dev"}) + require.NoError(t, err) + return "Basic " + base64.StdEncoding.EncodeToString([]byte("AWS:"+tok)) +} + +func okHandler() (http.Handler, *bool) { + called := new(bool) + h := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + *called = true + w.WriteHeader(http.StatusOK) + }) + return h, called +} + +func TestExtractECRToken(t *testing.T) { + good := base64.StdEncoding.EncodeToString([]byte("AWS:abc.def.ghi")) + cases := []struct { + name string + authz string + want string + ok bool + }{ + {"bearer", "Bearer abc.def.ghi", "abc.def.ghi", true}, + {"basic AWS", "Basic " + good, "abc.def.ghi", true}, + {"basic wrong user", "Basic " + base64.StdEncoding.EncodeToString([]byte("root:x")), "", false}, + {"basic empty pass", "Basic " + base64.StdEncoding.EncodeToString([]byte("AWS:")), "", false}, + {"basic bad base64", "Basic !!!notbase64", "", false}, + {"unknown scheme", "AWS4-HMAC-SHA256 cred", "", false}, + {"empty", "", "", false}, + {"bearer empty", "Bearer ", "", false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, ok := extractECRToken(c.authz) + assert.Equal(t, c.ok, ok) + assert.Equal(t, c.want, got) + }) + } +} + +func TestECRAuthBridge_NilVerifierPassesThrough(t *testing.T) { + gw := &GatewayConfig{} + next, called := okHandler() + w := httptest.NewRecorder() + gw.ecrAuthBridge(next).ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v2/", nil)) + assert.True(t, *called) + assert.Equal(t, http.StatusOK, w.Code) +} + +func TestECRAuthBridge_NoAuthChallenges401(t *testing.T) { + _, verify := newECRAuth(t) + gw := &GatewayConfig{Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, ECRTokenVerifier: verify} + next, called := okHandler() + + req := httptest.NewRequest(http.MethodGet, "/v2/", nil) + req.Host = ecrTestAccount + ".dkr.ecr." + ecrTestRegion + "." + ecrTestSuffix + w := httptest.NewRecorder() + gw.ecrAuthBridge(next).ServeHTTP(w, req) + + assert.False(t, *called, "handler must not run unauthenticated") + assert.Equal(t, http.StatusUnauthorized, w.Code) + // ECR uses Basic auth, not the Bearer token-endpoint flow. + assert.Equal(t, `Basic realm="https://`+req.Host+`/"`, w.Header().Get("WWW-Authenticate")) +} + +func TestECRAuthBridge_ValidTokenPassesThrough(t *testing.T) { + iss, verify := newECRAuth(t) + gw := &GatewayConfig{Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, ECRTokenVerifier: verify} + next, called := okHandler() + + req := httptest.NewRequest(http.MethodGet, "/v2/", nil) + req.Header.Set("Authorization", mintBasic(t, iss, ecrTestAccount)) + w := httptest.NewRecorder() + gw.ecrAuthBridge(next).ServeHTTP(w, req) + + assert.True(t, *called) + assert.Equal(t, http.StatusOK, w.Code) +} + +func TestECRAuthBridge_CrossAccountForbidden(t *testing.T) { + iss, verify := newECRAuth(t) + gw := &GatewayConfig{Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, ECRTokenVerifier: verify} + next, called := okHandler() + + // Token is for ecrTestAccount, but the host targets a different account. + req := httptest.NewRequest(http.MethodGet, "/v2/", nil) + req.Header.Set("Authorization", mintBasic(t, iss, ecrTestAccount)) + ctx := context.WithValue(req.Context(), ctxTargetAccount, "999999999999") + w := httptest.NewRecorder() + gw.ecrAuthBridge(next).ServeHTTP(w, req.WithContext(ctx)) + + assert.False(t, *called) + assert.Equal(t, http.StatusForbidden, w.Code) +} + +func TestECRAuthBridge_MultipleAuthHeadersRejected(t *testing.T) { + _, verify := newECRAuth(t) + gw := &GatewayConfig{Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, ECRTokenVerifier: verify} + next, called := okHandler() + + req := httptest.NewRequest(http.MethodGet, "/v2/", nil) + req.Header.Add("Authorization", "Bearer a.b.c") + req.Header.Add("Authorization", "Bearer d.e.f") + w := httptest.NewRecorder() + gw.ecrAuthBridge(next).ServeHTTP(w, req) + + assert.False(t, *called) + assert.Equal(t, http.StatusBadRequest, w.Code) +} + +func TestECRAuthBridge_InvalidTokenChallenges401(t *testing.T) { + _, verify := newECRAuth(t) + gw := &GatewayConfig{Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, ECRTokenVerifier: verify} + next, called := okHandler() + + req := httptest.NewRequest(http.MethodGet, "/v2/", nil) + req.Header.Set("Authorization", "Bearer not.a.realtoken") + w := httptest.NewRecorder() + gw.ecrAuthBridge(next).ServeHTTP(w, req) + + assert.False(t, *called) + assert.Equal(t, http.StatusUnauthorized, w.Code) +} diff --git a/spinifex/gateway/gateway.go b/spinifex/gateway/gateway.go index 131d740be..d6abadb9b 100644 --- a/spinifex/gateway/gateway.go +++ b/spinifex/gateway/gateway.go @@ -20,6 +20,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/awsec2query" "github.com/mulgadc/spinifex/spinifex/awserrors" gateway_ecr "github.com/mulgadc/spinifex/spinifex/gateway/ecr" + gateway_ecrauth "github.com/mulgadc/spinifex/spinifex/gateway/ecrauth" "github.com/mulgadc/spinifex/spinifex/gateway/policy" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" handlers_sts "github.com/mulgadc/spinifex/spinifex/handlers/sts" @@ -52,6 +53,11 @@ const ( ctxTargetAccount contextKey = "host.targetAccount" // ctxTargetRegion carries the region parsed from the same registry host. ctxTargetRegion contextKey = "host.targetRegion" + + // ctxAuthPrincipal carries the verified ECR token subject (principal ARN). + // The resolved account is stashed via gateway_ecr.WithAuthAccount so the + // registry package can read it without sharing this package's key type. + ctxAuthPrincipal contextKey = "ecr.authPrincipal" ) // Values stored under ctxPrincipalType. Downstream handlers that interpret @@ -81,6 +87,11 @@ type GatewayConfig struct { // ECRRegistry serves the OCI Distribution v2 (/v2/*) surface. Nil falls back // to the 501 stub (e.g. in unit tests of unrelated routes). ECRRegistry *gateway_ecr.Registry + // ECRTokenIssuer mints GetAuthorizationToken JWTs; ECRTokenVerifier validates + // them on /v2/*. Both nil disables the auth bridge (registry mounts open, as + // in unit tests of unrelated routes). + ECRTokenIssuer *gateway_ecrauth.Issuer + ECRTokenVerifier *gateway_ecrauth.Verifier } var supportedServices = map[string]bool{ diff --git a/spinifex/services/awsgw/awsgw.go b/spinifex/services/awsgw/awsgw.go index a39e3b036..ba0d0286a 100644 --- a/spinifex/services/awsgw/awsgw.go +++ b/spinifex/services/awsgw/awsgw.go @@ -17,6 +17,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/gateway" gateway_ecr "github.com/mulgadc/spinifex/spinifex/gateway/ecr" + gateway_ecrauth "github.com/mulgadc/spinifex/spinifex/gateway/ecrauth" "github.com/mulgadc/spinifex/spinifex/handlers/ecr" handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" handlers_sts "github.com/mulgadc/spinifex/spinifex/handlers/sts" @@ -201,9 +202,8 @@ func launchService(config *config.ClusterConfig) error { // OCI Distribution v2 registry: blob/manifest bytes stream straight to // predastore from the gateway; repo/tag/manifest metadata and in-progress - // uploads are owned by the daemon and reached over NATS request/reply. - // Account scoping uses the bootstrap account until the /v2 token-auth - // bridge lands. + // uploads are owned by the daemon and reached over NATS request/reply. The + // /v2 auth bridge resolves the per-request account from a verified token. ecrStore := objectstore.NewS3ObjectStoreFromConfig( admin.DialTarget(nodeConfig.Predastore.Host), nodeConfig.Predastore.Region, @@ -212,20 +212,35 @@ func launchService(config *config.ClusterConfig) error { ) ecrRegistry := gateway_ecr.NewRegistry(ecrStore, ecr.NewNATSMetaStore(natsConn), config.Bootstrap.AccountID) + // ECR auth bridge: load (or first-run create) the ES256 signing key from the + // cluster-replicated awsgw-keys KV bucket, then build the token issuer + // (GetAuthorizationToken) and verifier (/v2 Authorization). + js, err := natsConn.JetStream() + if err != nil { + return fmt.Errorf("ECR auth bridge: JetStream context: %w", err) + } + signingKey, verifyKeys, err := gateway_ecrauth.LoadOrCreateSigningKey(js, masterKey, len(config.Nodes)) + if err != nil { + return fmt.Errorf("ECR auth bridge: load signing key: %w", err) + } + ecrAudience := "ecr." + nodeConfig.Region + "." + config.AWS.InternalSuffix + gw := gateway.GatewayConfig{ - Debug: nodeConfig.AWSGW.Debug, - DisableLogging: false, - NATSConn: natsConn, - Config: nodeConfig.AWSGW.Config, - ExpectedNodes: len(config.Nodes), - Region: nodeConfig.Region, - InternalSuffix: config.AWS.InternalSuffix, - AZ: nodeConfig.AZ, - IAMService: iamService, - STSService: stsService, - Version: version, - Commit: commit, - ECRRegistry: ecrRegistry, + Debug: nodeConfig.AWSGW.Debug, + DisableLogging: false, + NATSConn: natsConn, + Config: nodeConfig.AWSGW.Config, + ExpectedNodes: len(config.Nodes), + Region: nodeConfig.Region, + InternalSuffix: config.AWS.InternalSuffix, + AZ: nodeConfig.AZ, + IAMService: iamService, + STSService: stsService, + Version: version, + Commit: commit, + ECRRegistry: ecrRegistry, + ECRTokenIssuer: gateway_ecrauth.NewIssuer(signingKey, ecrAudience), + ECRTokenVerifier: gateway_ecrauth.NewVerifier(verifyKeys, ecrAudience), } if throttleCfg.Enabled {