From 68eeff069e38853863664a9c074df9aff066b7d0 Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Wed, 16 Sep 2026 16:45:24 +1000 Subject: [PATCH] feat: project EKS access policies into Kubernetes RBAC (#1049) AssociateAccessPolicy recorded a policy association but never granted it in the cluster. Bake static ClusterRoleBindings for the mulga: groups into the eks-node image, project a principal's cluster-scoped associated policies onto those groups in the TokenReview webhook, and reject namespace scope at association time since there is no namespace-scoped RBAC writer. --- scripts/images/eks-node/k3s-prestart.sh | 10 +++ scripts/images/eks-node/k3s-prestart_test.sh | 14 +++- scripts/images/eks-node/manifest.conf | 1 + .../rbac/eks-access-policy-bindings.yaml | 50 +++++++++++ spinifex/awserrors/awserrors.go | 4 + spinifex/handlers/eks/access_entry.go | 10 +++ spinifex/handlers/eks/service_impl.go | 4 + spinifex/handlers/eks/service_impl_test.go | 60 ++++++++++++++ spinifex/handlers/eks/token_review.go | 36 +++++++- spinifex/handlers/eks/token_review_test.go | 83 +++++++++++++++++++ 10 files changed, 270 insertions(+), 2 deletions(-) create mode 100644 scripts/images/eks-node/rbac/eks-access-policy-bindings.yaml diff --git a/scripts/images/eks-node/k3s-prestart.sh b/scripts/images/eks-node/k3s-prestart.sh index 859a95e28..8bf0df7ab 100644 --- a/scripts/images/eks-node/k3s-prestart.sh +++ b/scripts/images/eks-node/k3s-prestart.sh @@ -23,6 +23,8 @@ KONN_AGENT_MANIFEST_DST=${KONN_AGENT_MANIFEST_DST:-/var/lib/rancher/k3s/server/m COREDNS_SRC=${COREDNS_SRC:-/usr/share/spinifex-eks/coredns/coredns-mulga.yaml} COREDNS_SKIP=${COREDNS_SKIP:-/var/lib/rancher/k3s/server/manifests/coredns.yaml.skip} COREDNS_DST=${COREDNS_DST:-/var/lib/rancher/k3s/server/manifests/coredns-mulga.yaml} +ACCESS_POLICY_BINDINGS_SRC=${ACCESS_POLICY_BINDINGS_SRC:-/usr/share/spinifex-eks/rbac/eks-access-policy-bindings.yaml} +ACCESS_POLICY_BINDINGS_DST=${ACCESS_POLICY_BINDINGS_DST:-/var/lib/rancher/k3s/server/manifests/eks-access-policy-bindings.yaml} TOKEN_WEBHOOK_KUBECONFIG=${TOKEN_WEBHOOK_KUBECONFIG:-/etc/spinifex-eks/token-webhook.kubeconfig} WAIT_SECS=${WAIT_SECS:-30} @@ -91,6 +93,14 @@ if [ -f "${COREDNS_SRC}" ]; then cp "${COREDNS_SRC}" "${COREDNS_DST}" fi +# Static ClusterRoleBindings for the EKS access-policy groups (mulga:eks-admin/ +# edit/view). No templating, so a plain idempotent copy; guarded on the baked +# manifest so an older image without it just leaves the associations unbound. +if [ -f "${ACCESS_POLICY_BINDINGS_SRC}" ]; then + mkdir -p "$(dirname "${ACCESS_POLICY_BINDINGS_DST}")" + cp "${ACCESS_POLICY_BINDINGS_SRC}" "${ACCESS_POLICY_BINDINGS_DST}" +fi + # The apiserver is configured with --authentication-token-webhook-config-file # pointing at the kubeconfig eks-token-webhook writes at startup. `before k3s` # orders the webhook's start but it backgrounds, so wait (bounded) for the diff --git a/scripts/images/eks-node/k3s-prestart_test.sh b/scripts/images/eks-node/k3s-prestart_test.sh index 58e0763f0..18ceefc23 100644 --- a/scripts/images/eks-node/k3s-prestart_test.sh +++ b/scripts/images/eks-node/k3s-prestart_test.sh @@ -32,12 +32,16 @@ reset_env() { COREDNS_SRC="${CASE}/coredns-mulga.yaml.src" COREDNS_SKIP="${CASE}/manifests/coredns.yaml.skip" COREDNS_DST="${CASE}/manifests/coredns-mulga.yaml" + ACCESS_POLICY_BINDINGS_SRC="${CASE}/eks-access-policy-bindings.yaml.src" + ACCESS_POLICY_BINDINGS_DST="${CASE}/manifests/eks-access-policy-bindings.yaml" TOKEN_WEBHOOK_KUBECONFIG="${CASE}/token-webhook.kubeconfig" WAIT_SECS=0 STDERR="${CASE}/stderr" export BLOCK_MARKER K3S_CONFIG K3S_CONFIG_SKEL FIRST_BOOT_ENVFILE \ KONN_AGENT_MANIFEST_SRC KONN_AGENT_MANIFEST_DST \ - COREDNS_SRC COREDNS_SKIP COREDNS_DST TOKEN_WEBHOOK_KUBECONFIG WAIT_SECS + COREDNS_SRC COREDNS_SKIP COREDNS_DST \ + ACCESS_POLICY_BINDINGS_SRC ACCESS_POLICY_BINDINGS_DST \ + TOKEN_WEBHOOK_KUBECONFIG WAIT_SECS : > "${K3S_CONFIG}" printf 'server-ca\n' > "${TOKEN_WEBHOOK_KUBECONFIG}" } @@ -122,6 +126,14 @@ invoke [ "$(cat "${COREDNS_DST}" 2>/dev/null)" = "kind: DaemonSet" ] \ && pass "coredns: DaemonSet staged" || fail "coredns: DaemonSet not staged" +# --- Case 6b: EKS access-policy ClusterRoleBindings staged when bundle present --- +reset_env access_policy_bindings +printf 'kind: ClusterRoleBinding\n' > "${ACCESS_POLICY_BINDINGS_SRC}" +invoke +[ "${rc}" -eq 0 ] && pass "access-policy-bindings: exit 0" || fail "access-policy-bindings: expected exit 0 (rc=${rc})" +[ "$(cat "${ACCESS_POLICY_BINDINGS_DST}" 2>/dev/null)" = "kind: ClusterRoleBinding" ] \ + && pass "access-policy-bindings: manifest staged" || fail "access-policy-bindings: manifest not staged" + # --- Case 7: token-webhook kubeconfig absent -> bounded wait, warns, exit 0 --- reset_env webhook_wait rm -f "${TOKEN_WEBHOOK_KUBECONFIG}" diff --git a/scripts/images/eks-node/manifest.conf b/scripts/images/eks-node/manifest.conf index e8ebff2ec..7ba890da3 100644 --- a/scripts/images/eks-node/manifest.conf +++ b/scripts/images/eks-node/manifest.conf @@ -66,6 +66,7 @@ scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/20-node.yaml:/usr/share scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/30-storageclass.yaml:/usr/share/spinifex-eks/addons/aws-ebs-csi-driver/1.40.1/30-storageclass.yaml \ scripts/images/eks-node/addons/nvidia-device-plugin/0.17.4/10-daemonset.yaml:/usr/share/spinifex-eks/addons/nvidia-device-plugin/0.17.4/10-daemonset.yaml \ scripts/images/eks-node/coredns/coredns-mulga.yaml:/usr/share/spinifex-eks/coredns/coredns-mulga.yaml \ +scripts/images/eks-node/rbac/eks-access-policy-bindings.yaml:/usr/share/spinifex-eks/rbac/eks-access-policy-bindings.yaml \ scripts/images/eks-node/cloud-init-ec2.cfg:/etc/cloud/cloud.cfg.d/90-spinifex-ec2-imds.cfg \ scripts/images/eks-node/mulga-eks-etcd-snapshot.sh:/etc/periodic/daily/mulga-eks-etcd-snapshot \ scripts/images/eks-node/mulga-eks-etcd-snapshot.sh:/etc/periodic/15min/mulga-eks-etcd-snapshot" diff --git a/scripts/images/eks-node/rbac/eks-access-policy-bindings.yaml b/scripts/images/eks-node/rbac/eks-access-policy-bindings.yaml new file mode 100644 index 000000000..7c1ba66f4 --- /dev/null +++ b/scripts/images/eks-node/rbac/eks-access-policy-bindings.yaml @@ -0,0 +1,50 @@ +# Static ClusterRoleBindings for the three AWS-managed EKS access policies +# that have no default Kubernetes binding. AmazonEKSClusterAdminPolicy needs +# no entry here: it maps to system:masters, which Kubernetes already binds to +# cluster-admin. These exist from first boot, independent of any cluster or +# access-entry association, so AssociateAccessPolicy's group projection has +# something to bind to as soon as it names one of these groups. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: mulga-eks-admin + labels: + app.kubernetes.io/managed-by: spinifex-eks +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: "mulga:eks-admin" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: admin +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: mulga-eks-edit + labels: + app.kubernetes.io/managed-by: spinifex-eks +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: "mulga:eks-edit" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: edit +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: mulga-eks-view + labels: + app.kubernetes.io/managed-by: spinifex-eks +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: "mulga:eks-view" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: view diff --git a/spinifex/awserrors/awserrors.go b/spinifex/awserrors/awserrors.go index a9aee1fd9..314932f9c 100644 --- a/spinifex/awserrors/awserrors.go +++ b/spinifex/awserrors/awserrors.go @@ -384,6 +384,10 @@ var ( ErrorResourceNotFound = "ResourceNotFound" ErrorEKSResourceInUse = "ResourceInUseException" ErrorEKSResourceNotFound = "ResourceNotFoundException" + // ErrorEKSInvalidParameter aliases ErrorECSInvalidParameter: real EKS and + // real ECS both use the restJson1 wire code "InvalidParameterException", + // so this gives EKS call sites their own name for the same registered code. + ErrorEKSInvalidParameter = ErrorECSInvalidParameter // ErrorACMResourceInUse aliases ErrorEKSResourceInUse: real ACM's // DeleteCertificate and real EKS's CreateCluster both legitimately use the // wire code "ResourceInUseException", so this is a distinct, self-documenting diff --git a/spinifex/handlers/eks/access_entry.go b/spinifex/handlers/eks/access_entry.go index 43c1b5402..44b15d89f 100644 --- a/spinifex/handlers/eks/access_entry.go +++ b/spinifex/handlers/eks/access_entry.go @@ -34,6 +34,16 @@ var supportedAccessPolicies = map[string]string{ "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy": "view", } +// accessPolicyGroups maps each supported policy ARN to the Kubernetes group +// its cluster-scope association projects: system:masters for the cluster-admin +// policy, a synthetic mulga: group bound by a static manifest for the rest. +var accessPolicyGroups = map[string]string{ + "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy": "system:masters", + "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminPolicy": "mulga:eks-admin", + "arn:aws:eks::aws:cluster-access-policy/AmazonEKSEditPolicy": "mulga:eks-edit", + "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy": "mulga:eks-view", +} + // ErrAccessEntryNotFound is returned when no entry exists for the principal. // Callers translate it to ResourceNotFoundException at the service boundary. var ErrAccessEntryNotFound = errors.New("eks: access entry not found") diff --git a/spinifex/handlers/eks/service_impl.go b/spinifex/handlers/eks/service_impl.go index 029df672e..938cfd1d6 100644 --- a/spinifex/handlers/eks/service_impl.go +++ b/spinifex/handlers/eks/service_impl.go @@ -1691,6 +1691,10 @@ func (s *EKSServiceImpl) AssociateAccessPolicy(ctx context.Context, input *eks.A if err != nil { return nil, errors.New(awserrors.ErrorInvalidParameterValue) } + if scope.Type == accessScopeNamespace { + return nil, awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, + "accessScope.type: namespace-scoped access policies are not supported, only cluster is supported") + } acctKV, err := s.acctKVForCluster(ctx, accountID, cluster) if err != nil { return nil, err diff --git a/spinifex/handlers/eks/service_impl_test.go b/spinifex/handlers/eks/service_impl_test.go index d9545ddec..affc01c22 100644 --- a/spinifex/handlers/eks/service_impl_test.go +++ b/spinifex/handlers/eks/service_impl_test.go @@ -370,6 +370,66 @@ func TestAccessPolicy_AssociateRejectsUnsupportedPolicyAndScope(t *testing.T) { require.EqualError(t, err, awserrors.ErrorInvalidParameterValue) } +// Namespace scope has no RBAC writer (Change 3): it must fail at association +// time with a message naming accessScope.type, not read back as a durable +// grant that never actually authorizes anything. +func TestAccessPolicy_AssociateRejectsNamespaceScope(t *testing.T) { + svc := setupTestService(t) + seedTestCluster(t, svc, "c1") + _, err := svc.CreateAccessEntry(context.Background(), &eks.CreateAccessEntryInput{ + ClusterName: aws.String("c1"), PrincipalArn: aws.String(testPrincipalARN), + }, testAccountID) + require.NoError(t, err) + + _, err = svc.AssociateAccessPolicy(context.Background(), &eks.AssociateAccessPolicyInput{ + ClusterName: aws.String("c1"), + PrincipalArn: aws.String(testPrincipalARN), + PolicyArn: aws.String("arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy"), + AccessScope: &eks.AccessScope{Type: aws.String("namespace"), Namespaces: aws.StringSlice([]string{"team-a"})}, + }, testAccountID) + require.Error(t, err) + code, message, ok := awserrors.ResolveErrorDetail(err) + require.True(t, ok) + assert.Equal(t, awserrors.ErrorEKSInvalidParameter, code) + assert.Contains(t, message, "accessScope.type") +} + +// DisassociateAccessPolicy already removes the association from the record; +// this confirms the projection built on top of it reflects the removal too. +func TestAccessPolicy_DisassociateRemovesProjectedGroup(t *testing.T) { + svc := setupTestService(t) + seedTestCluster(t, svc, "c1") + _, err := svc.CreateAccessEntry(context.Background(), &eks.CreateAccessEntryInput{ + ClusterName: aws.String("c1"), PrincipalArn: aws.String(testPrincipalARN), + }, testAccountID) + require.NoError(t, err) + + const viewPolicy = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy" + _, err = svc.AssociateAccessPolicy(context.Background(), &eks.AssociateAccessPolicyInput{ + ClusterName: aws.String("c1"), + PrincipalArn: aws.String(testPrincipalARN), + PolicyArn: aws.String(viewPolicy), + AccessScope: &eks.AccessScope{Type: aws.String("cluster")}, + }, testAccountID) + require.NoError(t, err) + + js := testutil.NewJetStream(t, svc.deps.NATSConn) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + require.NoError(t, err) + rec, err := GetAccessEntryRecord(t.Context(), kv, "c1", testPrincipalARN) + require.NoError(t, err) + assert.Equal(t, []string{"mulga:eks-view"}, effectiveGroups(rec)) + + _, err = svc.DisassociateAccessPolicy(context.Background(), &eks.DisassociateAccessPolicyInput{ + ClusterName: aws.String("c1"), PrincipalArn: aws.String(testPrincipalARN), PolicyArn: aws.String(viewPolicy), + }, testAccountID) + require.NoError(t, err) + + rec, err = GetAccessEntryRecord(t.Context(), kv, "c1", testPrincipalARN) + require.NoError(t, err) + assert.Empty(t, effectiveGroups(rec)) +} + func TestListAccessPolicies_ReturnsSupportedCatalogue(t *testing.T) { svc := setupTestService(t) out, err := svc.ListAccessPolicies(context.Background(), &eks.ListAccessPoliciesInput{}, testAccountID) diff --git a/spinifex/handlers/eks/token_review.go b/spinifex/handlers/eks/token_review.go index e56489e4b..26a7afaa9 100644 --- a/spinifex/handlers/eks/token_review.go +++ b/spinifex/handlers/eks/token_review.go @@ -57,10 +57,44 @@ func Authenticate( Authenticated: true, Username: rec.KubernetesUsername, UID: uid, - Groups: rec.KubernetesGroups, + Groups: effectiveGroups(rec), } } +// effectiveGroups returns rec's static groups plus the Kubernetes group each +// cluster-scoped associated policy projects, deduplicated in first-seen order. +// A namespace-scoped or unrecognized association contributes nothing, so an +// older or wider record degrades safely instead of widening scope or panicking. +func effectiveGroups(rec *AccessEntryRecord) []string { + seen := make(map[string]struct{}, len(rec.KubernetesGroups)) + groups := make([]string, 0, len(rec.KubernetesGroups)) + add := func(g string) { + if g == "" { + return + } + if _, ok := seen[g]; ok { + return + } + seen[g] = struct{}{} + groups = append(groups, g) + } + for _, g := range rec.KubernetesGroups { + add(g) + } + for _, p := range rec.AssociatedPolicies { + if p.AccessScope.Type != accessScopeCluster { + continue + } + g, ok := accessPolicyGroups[p.PolicyARN] + if !ok { + slog.Debug("access policy projection: unrecognized policy ARN", "policy_arn", p.PolicyARN) + continue + } + add(g) + } + return groups +} + // ResolveTokenReview runs the TokenReview decision host-side, wiring the real // STS verify (NATS request to the awsgw-hosted verify subject) and AccessEntry // KV read into Authenticate. The eks-token-webhook reaches this through the diff --git a/spinifex/handlers/eks/token_review_test.go b/spinifex/handlers/eks/token_review_test.go index c3a562cac..c526369d7 100644 --- a/spinifex/handlers/eks/token_review_test.go +++ b/spinifex/handlers/eks/token_review_test.go @@ -107,6 +107,89 @@ func TestAuthenticate_FallsBackUIDToARN(t *testing.T) { assert.Equal(t, testARN, res.UID) } +func TestEffectiveGroups_EachPolicyProducesItsGroup(t *testing.T) { + cases := []struct { + policyARN string + want string + }{ + {"arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy", "system:masters"}, + {"arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminPolicy", "mulga:eks-admin"}, + {"arn:aws:eks::aws:cluster-access-policy/AmazonEKSEditPolicy", "mulga:eks-edit"}, + {"arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy", "mulga:eks-view"}, + } + for _, tc := range cases { + t.Run(tc.want, func(t *testing.T) { + rec := &AccessEntryRecord{ + AssociatedPolicies: []AssociatedAccessPolicy{ + {PolicyARN: tc.policyARN, AccessScope: AccessScope{Type: accessScopeCluster}}, + }, + } + assert.Equal(t, []string{tc.want}, effectiveGroups(rec)) + }) + } +} + +func TestEffectiveGroups_TwoAssociationsProduceBothGroups(t *testing.T) { + rec := &AccessEntryRecord{ + AssociatedPolicies: []AssociatedAccessPolicy{ + {PolicyARN: "arn:aws:eks::aws:cluster-access-policy/AmazonEKSEditPolicy", AccessScope: AccessScope{Type: accessScopeCluster}}, + {PolicyARN: "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy", AccessScope: AccessScope{Type: accessScopeCluster}}, + }, + } + assert.Equal(t, []string{"mulga:eks-edit", "mulga:eks-view"}, effectiveGroups(rec)) +} + +// A creator seeded system:masters directly (CreateCluster bootstrap) who is +// also associated to AmazonEKSClusterAdminPolicy must not see the group twice. +func TestEffectiveGroups_DedupesSeededAndAssociatedClusterAdmin(t *testing.T) { + rec := &AccessEntryRecord{ + KubernetesGroups: []string{"system:masters"}, + AssociatedPolicies: []AssociatedAccessPolicy{ + {PolicyARN: "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy", AccessScope: AccessScope{Type: accessScopeCluster}}, + }, + } + assert.Equal(t, []string{"system:masters"}, effectiveGroups(rec)) +} + +// Records written before namespace scope was rejected at association time may +// still carry one; it must not silently widen to a cluster-scope grant. +func TestEffectiveGroups_NamespaceScopeContributesNothing(t *testing.T) { + rec := &AccessEntryRecord{ + AssociatedPolicies: []AssociatedAccessPolicy{ + {PolicyARN: "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy", AccessScope: AccessScope{Type: accessScopeNamespace, Namespaces: []string{"team-a"}}}, + }, + } + assert.Empty(t, effectiveGroups(rec)) +} + +func TestEffectiveGroups_UnrecognizedPolicyDoesNotPanic(t *testing.T) { + rec := &AccessEntryRecord{ + AssociatedPolicies: []AssociatedAccessPolicy{ + {PolicyARN: "arn:aws:eks::aws:cluster-access-policy/MadeUpPolicy", AccessScope: AccessScope{Type: accessScopeCluster}}, + }, + } + assert.NotPanics(t, func() { effectiveGroups(rec) }) + assert.Empty(t, effectiveGroups(rec)) +} + +// End-to-end through Authenticate: an associated policy's group must reach the +// TokenReview result, not just the internal helper. +func TestAuthenticate_ProjectsAssociatedPolicyGroup(t *testing.T) { + lookup := func(arn string) (*AccessEntryRecord, error) { + return &AccessEntryRecord{ + KubernetesUsername: testARN, + AssociatedPolicies: []AssociatedAccessPolicy{ + {PolicyARN: "arn:aws:eks::aws:cluster-access-policy/AmazonEKSViewPolicy", AccessScope: AccessScope{Type: accessScopeCluster}}, + }, + }, nil + } + + res := Authenticate(validToken("https://sts.amazonaws.com/?Action=GetCallerIdentity"), okVerify, lookup) + + require.True(t, res.Authenticated) + assert.Equal(t, []string{"mulga:eks-view"}, res.Groups) +} + func TestResolveTokenReview_NilConn(t *testing.T) { _, err := ResolveTokenReview(context.Background(), nil, "111122223333", "alpha", "tok", time.Second) require.Error(t, err)