diff --git a/cmd/eks-webhook-cert/main.go b/cmd/eks-webhook-cert/main.go new file mode 100644 index 000000000..1004c37b6 --- /dev/null +++ b/cmd/eks-webhook-cert/main.go @@ -0,0 +1,147 @@ +// Command eks-webhook-cert mints, then reuses, a self-signed ECDSA-P256 serving +// certificate for an in-cluster admission webhook and prints its base64 PEM so +// the addon renderer can embed it as the webhook Secret + each caBundle. +// +// The eks-node AMI carries no openssl, so this is the Go equivalent of the +// upstream Helm chart's install-time genSelfSignedCert: one cert per cluster, +// cached on disk under -dir, stable across addon-sync ticks. The single cert is +// its own CA (IsCA + ServerAuth), so the same PEM serves as both the TLS leaf +// and the caBundle the apiserver trusts. +// +// Usage: +// +// eks-webhook-cert -dir /var/lib/spinifex-eks/webhook-certs/ \ +// -cn ..svc -dns ..svc,..svc.cluster.local +// +// Output (one line): \t\t +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/pem" + "flag" + "fmt" + "log/slog" + "math/big" + "os" + "path/filepath" + "strings" + "time" +) + +func main() { + dir := flag.String("dir", "", "directory to cache tls.crt/tls.key in") + cn := flag.String("cn", "", "certificate common name") + dnsCSV := flag.String("dns", "", "comma-separated DNS SANs") + flag.Parse() + + if *dir == "" || *cn == "" || *dnsCSV == "" { + fatal("eks-webhook-cert: -dir, -cn and -dns are required") + } + dnsNames := splitCSV(*dnsCSV) + if len(dnsNames) == 0 { + fatal("eks-webhook-cert: -dns lists no names") + } + + certPEM, keyPEM, err := ensureCert(*dir, *cn, dnsNames) + if err != nil { + fatal("eks-webhook-cert: " + err.Error()) + } + b := base64.StdEncoding.EncodeToString + fmt.Printf("%s\t%s\t%s\n", b(certPEM), b(certPEM), b(keyPEM)) +} + +// ensureCert returns the cached cert/key under dir, minting and persisting a +// fresh self-signed pair (for cn + dnsNames) when none is present or readable. +func ensureCert(dir, cn string, dnsNames []string) (certPEM, keyPEM []byte, err error) { + certPath := filepath.Join(dir, "tls.crt") + keyPath := filepath.Join(dir, "tls.key") + + if c, cerr := os.ReadFile(certPath); cerr == nil { + if k, kerr := os.ReadFile(keyPath); kerr == nil { + if _, perr := x509.ParseCertificate(decodeFirst(c)); perr == nil && len(k) > 0 { + return c, k, nil + } + } + } + + certPEM, keyPEM, err = generateSelfSigned(cn, dnsNames) + if err != nil { + return nil, nil, err + } + if err = os.MkdirAll(dir, 0o700); err != nil { + return nil, nil, fmt.Errorf("mkdir %s: %w", dir, err) + } + if err = os.WriteFile(certPath, certPEM, 0o644); err != nil { + return nil, nil, fmt.Errorf("write %s: %w", certPath, err) + } + if err = os.WriteFile(keyPath, keyPEM, 0o600); err != nil { + return nil, nil, fmt.Errorf("write %s: %w", keyPath, err) + } + return certPEM, keyPEM, nil +} + +// generateSelfSigned mints an ECDSA-P256 self-signed cert valid as both a CA and +// a server cert, with the given common name and DNS SANs. +func generateSelfSigned(cn string, dnsNames []string) (certPEM, keyPEM []byte, err error) { + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("generate key: %w", err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, nil, fmt.Errorf("serial: %w", err) + } + now := time.Now().UTC() + tmpl := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: cn}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.AddDate(10, 0, 0), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + DNSNames: dnsNames, + } + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv) + if err != nil { + return nil, nil, fmt.Errorf("create cert: %w", err) + } + keyDER, err := x509.MarshalECPrivateKey(priv) + if err != nil { + return nil, nil, fmt.Errorf("marshal key: %w", err) + } + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + return certPEM, keyPEM, nil +} + +// decodeFirst returns the DER bytes of the first PEM block, or nil. +func decodeFirst(pemBytes []byte) []byte { + blk, _ := pem.Decode(pemBytes) + if blk == nil { + return nil + } + return blk.Bytes +} + +func splitCSV(s string) []string { + var out []string + for p := range strings.SplitSeq(s, ",") { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} + +func fatal(msg string) { + slog.Error(msg) + os.Exit(1) +} diff --git a/cmd/eks-webhook-cert/main_test.go b/cmd/eks-webhook-cert/main_test.go new file mode 100644 index 000000000..e05d2001f --- /dev/null +++ b/cmd/eks-webhook-cert/main_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "crypto/x509" + "os" + "path/filepath" + "testing" +) + +const ( + testCN = "aws-load-balancer-webhook-service.kube-system.svc" + testDNS = "aws-load-balancer-webhook-service.kube-system.svc.cluster.local" +) + +func parseLeaf(t *testing.T, certPEM []byte) *x509.Certificate { + t.Helper() + leaf, err := x509.ParseCertificate(decodeFirst(certPEM)) + if err != nil { + t.Fatalf("parse cert: %v", err) + } + return leaf +} + +func TestEnsureCertMintsServingCert(t *testing.T) { + dir := t.TempDir() + certPEM, keyPEM, err := ensureCert(dir, testCN, []string{testCN, testDNS}) + if err != nil { + t.Fatalf("ensureCert: %v", err) + } + if len(keyPEM) == 0 { + t.Fatal("empty key PEM") + } + leaf := parseLeaf(t, certPEM) + + // Both webhook service DNS names must be present, or the apiserver's TLS + // dial to the webhook fails SAN verification. + for _, want := range []string{testCN, testDNS} { + if err := leaf.VerifyHostname(want); err != nil { + t.Errorf("SAN %q missing: %v", want, err) + } + } + if !leaf.IsCA { + t.Error("cert must be a CA so the same PEM serves as the webhook caBundle") + } + + // The single self-signed cert must verify against itself as a root — this is + // exactly what the apiserver does with caBundle == the serving cert. + roots := x509.NewCertPool() + roots.AddCert(leaf) + if _, err := leaf.Verify(x509.VerifyOptions{ + Roots: roots, + DNSName: testCN, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + t.Errorf("self-signed cert does not verify as its own CA: %v", err) + } +} + +func TestEnsureCertReusesCachedPair(t *testing.T) { + dir := t.TempDir() + cert1, key1, err := ensureCert(dir, testCN, []string{testCN}) + if err != nil { + t.Fatalf("first ensureCert: %v", err) + } + cert2, key2, err := ensureCert(dir, testCN, []string{testCN}) + if err != nil { + t.Fatalf("second ensureCert: %v", err) + } + if string(cert1) != string(cert2) || string(key1) != string(key2) { + t.Error("ensureCert regenerated instead of reusing the cached pair; a churning cert re-applies the addon every tick") + } + + // The private key must not be world-readable on disk. + info, err := os.Stat(filepath.Join(dir, "tls.key")) + if err != nil { + t.Fatalf("stat key: %v", err) + } + if info.Mode().Perm()&0o077 != 0 { + t.Errorf("tls.key mode %o is group/other-accessible", info.Mode().Perm()) + } +} diff --git a/scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/00-crds.yaml b/scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/00-crds.yaml new file mode 100644 index 000000000..152866ffc --- /dev/null +++ b/scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/00-crds.yaml @@ -0,0 +1,688 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app.kubernetes.io/name: aws-load-balancer-controller + name: ingressclassparams.elbv2.k8s.aws +spec: + group: elbv2.k8s.aws + names: + kind: IngressClassParams + listKind: IngressClassParamsList + plural: ingressclassparams + singular: ingressclassparams + scope: Cluster + versions: + - additionalPrinterColumns: + - description: The Ingress Group name + jsonPath: .spec.group.name + name: GROUP-NAME + type: string + - description: The AWS Load Balancer scheme + jsonPath: .spec.scheme + name: SCHEME + type: string + - description: The AWS Load Balancer ipAddressType + jsonPath: .spec.ipAddressType + name: IP-ADDRESS-TYPE + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: IngressClassParams is the Schema for the IngressClassParams API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: IngressClassParamsSpec defines the desired state of IngressClassParams + properties: + certificateArn: + description: CertificateArn specifies the ARN of the certificates + for all Ingresses that belong to IngressClass with this IngressClassParams. + items: + type: string + type: array + group: + description: Group defines the IngressGroup for all Ingresses that + belong to IngressClass with this IngressClassParams. + properties: + name: + description: Name is the name of IngressGroup. + type: string + required: + - name + type: object + inboundCIDRs: + description: InboundCIDRs specifies the CIDRs that are allowed to + access the Ingresses that belong to IngressClass with this IngressClassParams. + items: + type: string + type: array + ipAddressType: + description: IPAddressType defines the ip address type for all Ingresses + that belong to IngressClass with this IngressClassParams. + enum: + - ipv4 + - dualstack + - dualstack-without-public-ipv4 + type: string + listeners: + description: Listeners define a list of listeners with their protocol, + port and attributes. + items: + properties: + listenerAttributes: + description: The attributes of the listener + items: + description: Attributes defines custom attributes on resources. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + port: + description: The port of the listener + format: int32 + type: integer + protocol: + description: The protocol of the listener + type: string + type: object + type: array + loadBalancerAttributes: + description: LoadBalancerAttributes define the custom attributes to + LoadBalancers for all Ingress that that belong to IngressClass with + this IngressClassParams. + items: + description: Attributes defines custom attributes on resources. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + minimumLoadBalancerCapacity: + description: MinimumLoadBalancerCapacity define the capacity reservation + for LoadBalancers for all Ingress that belong to IngressClass with + this IngressClassParams. + properties: + capacityUnits: + description: The Capacity Units Value. + format: int32 + type: integer + required: + - capacityUnits + type: object + namespaceSelector: + description: |- + NamespaceSelector restrict the namespaces of Ingresses that are allowed to specify the IngressClass with this IngressClassParams. + * if absent or present but empty, it selects all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + scheme: + description: Scheme defines the scheme for all Ingresses that belong + to IngressClass with this IngressClassParams. + enum: + - internal + - internet-facing + type: string + sslPolicy: + description: SSLPolicy specifies the SSL Policy for all Ingresses + that belong to IngressClass with this IngressClassParams. + type: string + subnets: + description: Subnets defines the subnets for all Ingresses that belong + to IngressClass with this IngressClassParams. + properties: + ids: + description: IDs specify the resource IDs of subnets. Exactly + one of this or `tags` must be specified. + items: + description: SubnetID specifies a subnet ID. + pattern: subnet-[0-9a-f]+ + type: string + minItems: 1 + type: array + tags: + additionalProperties: + items: + type: string + type: array + description: |- + Tags specifies subnets in the load balancer's VPC where each + tag specified in the map key contains one of the values in the corresponding + value list. + Exactly one of this or `ids` must be specified. + type: object + type: object + tags: + description: Tags defines list of Tags on AWS resources provisioned + for Ingresses that belong to IngressClass with this IngressClassParams. + items: + description: Tag defines a AWS Tag on resources. + properties: + key: + description: The key of the tag. + type: string + value: + description: The value of the tag. + type: string + required: + - key + - value + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app.kubernetes.io/name: aws-load-balancer-controller + name: targetgroupbindings.elbv2.k8s.aws +spec: + group: elbv2.k8s.aws + names: + kind: TargetGroupBinding + listKind: TargetGroupBindingList + plural: targetgroupbindings + singular: targetgroupbinding + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The Kubernetes Service's name + jsonPath: .spec.serviceRef.name + name: SERVICE-NAME + type: string + - description: The Kubernetes Service's port + jsonPath: .spec.serviceRef.port + name: SERVICE-PORT + type: string + - description: The AWS TargetGroup's TargetType + jsonPath: .spec.targetType + name: TARGET-TYPE + type: string + - description: The AWS TargetGroup's Amazon Resource Name + jsonPath: .spec.targetGroupARN + name: ARN + priority: 1 + type: string + - description: The AWS TargetGroup's Name + jsonPath: .spec.targetGroupName + name: NAME + priority: 2 + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: TargetGroupBinding is the Schema for the TargetGroupBinding API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: TargetGroupBindingSpec defines the desired state of TargetGroupBinding + properties: + multiClusterTargetGroup: + description: MultiClusterTargetGroup Denotes if the TargetGroup is + shared among multiple clusters + type: boolean + networking: + description: networking provides the networking setup for ELBV2 LoadBalancer + to access targets in TargetGroup. + properties: + ingress: + description: List of ingress rules to allow ELBV2 LoadBalancer + to access targets in TargetGroup. + items: + properties: + from: + description: |- + List of peers which should be able to access the targets in TargetGroup. + At least one NetworkingPeer should be specified. + items: + description: NetworkingPeer defines the source/destination + peer for networking rules. + properties: + ipBlock: + description: |- + IPBlock defines an IPBlock peer. + If specified, none of the other fields can be set. + properties: + cidr: + description: |- + CIDR is the network CIDR. + Both IPV4 or IPV6 CIDR are accepted. + type: string + required: + - cidr + type: object + securityGroup: + description: |- + SecurityGroup defines a SecurityGroup peer. + If specified, none of the other fields can be set. + properties: + groupID: + description: GroupID is the EC2 SecurityGroupID. + type: string + required: + - groupID + type: object + type: object + type: array + ports: + description: |- + List of ports which should be made accessible on the targets in TargetGroup. + If ports is empty or unspecified, it defaults to all ports with TCP. + items: + properties: + port: + anyOf: + - type: integer + - type: string + description: |- + The port which traffic must match. + When NodePort endpoints(instance TargetType) is used, this must be a numerical port. + When Port endpoints(ip TargetType) is used, this can be either numerical or named port on pods. + if port is unspecified, it defaults to all ports. + x-kubernetes-int-or-string: true + protocol: + description: |- + The protocol which traffic must match. + If protocol is unspecified, it defaults to TCP. + enum: + - TCP + - UDP + type: string + type: object + type: array + required: + - from + - ports + type: object + type: array + type: object + serviceRef: + description: serviceRef is a reference to a Kubernetes Service and + ServicePort. + properties: + name: + description: Name is the name of the Service. + type: string + port: + anyOf: + - type: integer + - type: string + description: Port is the port of the ServicePort. + x-kubernetes-int-or-string: true + required: + - name + - port + type: object + targetGroupARN: + description: targetGroupARN is the Amazon Resource Name (ARN) for + the TargetGroup. + type: string + targetGroupName: + description: targetGroupName is the Name of the TargetGroup. + type: string + targetType: + description: targetType is the TargetType of TargetGroup. If unspecified, + it will be automatically inferred. + enum: + - instance + - ip + type: string + required: + - serviceRef + type: object + status: + description: TargetGroupBindingStatus defines the observed state of TargetGroupBinding + properties: + observedGeneration: + description: The generation observed by the TargetGroupBinding controller. + format: int64 + type: integer + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The Kubernetes Service's name + jsonPath: .spec.serviceRef.name + name: SERVICE-NAME + type: string + - description: The Kubernetes Service's port + jsonPath: .spec.serviceRef.port + name: SERVICE-PORT + type: string + - description: The AWS TargetGroup's TargetType + jsonPath: .spec.targetType + name: TARGET-TYPE + type: string + - description: The AWS TargetGroup's Amazon Resource Name + jsonPath: .spec.targetGroupARN + name: ARN + priority: 1 + type: string + - description: The AWS TargetGroup's Name + jsonPath: .spec.targetGroupName + name: NAME + priority: 2 + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: TargetGroupBinding is the Schema for the TargetGroupBinding API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: TargetGroupBindingSpec defines the desired state of TargetGroupBinding + properties: + ipAddressType: + description: ipAddressType specifies whether the target group is of + type IPv4 or IPv6. If unspecified, it will be automatically inferred. + enum: + - ipv4 + - ipv6 + type: string + multiClusterTargetGroup: + description: MultiClusterTargetGroup Denotes if the TargetGroup is + shared among multiple clusters + type: boolean + networking: + description: networking defines the networking rules to allow ELBV2 + LoadBalancer to access targets in TargetGroup. + properties: + ingress: + description: List of ingress rules to allow ELBV2 LoadBalancer + to access targets in TargetGroup. + items: + description: NetworkingIngressRule defines a particular set + of traffic that is allowed to access TargetGroup's targets. + properties: + from: + description: |- + List of peers which should be able to access the targets in TargetGroup. + At least one NetworkingPeer should be specified. + items: + description: NetworkingPeer defines the source/destination + peer for networking rules. + properties: + ipBlock: + description: |- + IPBlock defines an IPBlock peer. + If specified, none of the other fields can be set. + properties: + cidr: + description: |- + CIDR is the network CIDR. + Both IPV4 or IPV6 CIDR are accepted. + type: string + required: + - cidr + type: object + securityGroup: + description: |- + SecurityGroup defines a SecurityGroup peer. + If specified, none of the other fields can be set. + properties: + groupID: + description: GroupID is the EC2 SecurityGroupID. + type: string + required: + - groupID + type: object + type: object + type: array + ports: + description: |- + List of ports which should be made accessible on the targets in TargetGroup. + If ports is empty or unspecified, it defaults to all ports with TCP. + items: + description: NetworkingPort defines the port and protocol + for networking rules. + properties: + port: + anyOf: + - type: integer + - type: string + description: |- + The port which traffic must match. + When NodePort endpoints(instance TargetType) is used, this must be a numerical port. + When Port endpoints(ip TargetType) is used, this can be either numerical or named port on pods. + if port is unspecified, it defaults to all ports. + x-kubernetes-int-or-string: true + protocol: + description: |- + The protocol which traffic must match. + If protocol is unspecified, it defaults to TCP. + enum: + - TCP + - UDP + type: string + type: object + type: array + required: + - from + - ports + type: object + type: array + type: object + nodeSelector: + description: node selector for instance type target groups to only + register certain nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + serviceRef: + description: serviceRef is a reference to a Kubernetes Service and + ServicePort. + properties: + name: + description: Name is the name of the Service. + type: string + port: + anyOf: + - type: integer + - type: string + description: Port is the port of the ServicePort. + x-kubernetes-int-or-string: true + required: + - name + - port + type: object + targetGroupARN: + description: targetGroupARN is the Amazon Resource Name (ARN) for + the TargetGroup. + type: string + targetGroupName: + description: targetGroupName is the Name of the TargetGroup. + type: string + targetType: + description: targetType is the TargetType of TargetGroup. If unspecified, + it will be automatically inferred. + enum: + - instance + - ip + type: string + vpcID: + description: VpcID is the VPC of the TargetGroup. If unspecified, + it will be automatically inferred. + type: string + required: + - serviceRef + type: object + status: + description: TargetGroupBindingStatus defines the observed state of TargetGroupBinding + properties: + observedGeneration: + description: The generation observed by the TargetGroupBinding controller. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/10-aws-load-balancer-controller.yaml b/scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/10-aws-load-balancer-controller.yaml new file mode 100644 index 000000000..8602012a2 --- /dev/null +++ b/scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/10-aws-load-balancer-controller.yaml @@ -0,0 +1,552 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: aws-load-balancer-controller + namespace: kube-system + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: aws-load-balancer-controller + app.kubernetes.io/version: "v2.11.0" + annotations: + eks.amazonaws.com/role-arn: {{SERVICE_ACCOUNT_ROLE_ARN}} +automountServiceAccountToken: true +--- +# spinifex: node-minted webhook serving cert (render_addon mints a self-signed +# cert for the webhook service SANs; cert-manager is not present on the node). +apiVersion: v1 +kind: Secret +metadata: + name: aws-load-balancer-webhook-tls + namespace: kube-system + labels: + app.kubernetes.io/name: aws-load-balancer-controller +type: kubernetes.io/tls +data: + ca.crt: "{{WEBHOOK_CA_B64}}" + tls.crt: "{{WEBHOOK_TLS_CRT_B64}}" + tls.key: "{{WEBHOOK_TLS_KEY_B64}}" +--- +# spinifex: the AWS gateway CA, mounted as AWS_CA_BUNDLE so the web-identity +# credential bootstrap + elbv2/ec2/acm calls trust the gateway TLS endpoint. +apiVersion: v1 +kind: Secret +metadata: + name: spinifex-gateway-ca + namespace: kube-system + labels: + app.kubernetes.io/name: aws-load-balancer-controller +type: Opaque +data: + ca.pem: "{{GATEWAY_CA_PEM_B64}}" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: aws-load-balancer-controller + name: aws-load-balancer-controller-leader-election-role + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create +- apiGroups: + - "" + resourceNames: + - aws-load-balancer-controller-leader + resources: + - configmaps + verbs: + - get + - update + - patch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - coordination.k8s.io + resourceNames: + - aws-load-balancer-controller-leader + resources: + - leases + verbs: + - get + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: aws-load-balancer-controller + name: aws-load-balancer-controller-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - update +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods/status + verbs: + - patch + - update +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - update +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - elbv2.k8s.aws + resources: + - ingressclassparams + verbs: + - get + - list + - watch +- apiGroups: + - elbv2.k8s.aws + resources: + - targetgroupbindings + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - elbv2.k8s.aws + resources: + - targetgroupbindings/status + verbs: + - patch + - update +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - extensions + resources: + - ingresses/status + verbs: + - patch + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: aws-load-balancer-controller + name: aws-load-balancer-controller-leader-election-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: aws-load-balancer-controller-leader-election-role +subjects: +- kind: ServiceAccount + name: aws-load-balancer-controller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: aws-load-balancer-controller + name: aws-load-balancer-controller-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: aws-load-balancer-controller-role +subjects: +- kind: ServiceAccount + name: aws-load-balancer-controller + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: aws-load-balancer-controller + name: aws-load-balancer-webhook-service + namespace: kube-system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + app.kubernetes.io/component: controller + app.kubernetes.io/name: aws-load-balancer-controller +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: aws-load-balancer-controller + name: aws-load-balancer-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: aws-load-balancer-controller + template: + metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: aws-load-balancer-controller + spec: + containers: + - args: + - --cluster-name={{CLUSTER_NAME}} + - --ingress-class=alb + - --feature-gates=ALBSingleSubnet=true + - --aws-region={{AWS_REGION}} + - --aws-vpc-id={{AWS_VPC_ID}} + - --aws-api-endpoints=EC2={{GATEWAY_ENDPOINT}},Elastic Load Balancing v2={{GATEWAY_ENDPOINT}},ACM={{GATEWAY_ENDPOINT}},STS={{GATEWAY_ENDPOINT}} + - --enable-shield=false + - --enable-waf=false + - --enable-wafv2=false + env: +{{IRSA_ENV}} + image: public.ecr.aws/eks/aws-load-balancer-controller:v2.11.0 + livenessProbe: + failureThreshold: 2 + httpGet: + path: /healthz + port: 61779 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 10 + name: controller + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true +{{IRSA_VOLUME_MOUNT}} + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1337 + serviceAccountName: aws-load-balancer-controller + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: aws-load-balancer-webhook-tls +{{IRSA_VOLUME}} +--- +apiVersion: v1 +kind: List +metadata: + name: ingress-class +items: +- apiVersion: elbv2.k8s.aws/v1beta1 + kind: IngressClassParams + metadata: + name: alb + labels: + app.kubernetes.io/name: aws-load-balancer-controller + app.kubernetes.io/version: "v2.11.0" +{{ELB_INGRESS_PARAMS_SPEC}} +- apiVersion: networking.k8s.io/v1 + kind: IngressClass + metadata: + name: alb + labels: + app.kubernetes.io/name: aws-load-balancer-controller + app.kubernetes.io/version: "v2.11.0" + spec: + controller: ingress.k8s.aws/alb + parameters: + apiGroup: elbv2.k8s.aws + kind: IngressClassParams + name: alb +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: aws-load-balancer-controller + name: aws-load-balancer-webhook +webhooks: +- admissionReviewVersions: + - v1beta1 + clientConfig: + caBundle: "{{WEBHOOK_CA_B64}}" + service: + name: aws-load-balancer-webhook-service + namespace: kube-system + path: /mutate-v1-service + failurePolicy: Fail + name: mservice.elbv2.k8s.aws + objectSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: NotIn + values: + - aws-load-balancer-controller + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - services + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + caBundle: "{{WEBHOOK_CA_B64}}" + service: + name: aws-load-balancer-webhook-service + namespace: kube-system + path: /mutate-v1-pod + failurePolicy: Fail + name: mpod.elbv2.k8s.aws + namespaceSelector: + matchExpressions: + - key: elbv2.k8s.aws/pod-readiness-gate-inject + operator: In + values: + - enabled + objectSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: NotIn + values: + - aws-load-balancer-controller + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + caBundle: "{{WEBHOOK_CA_B64}}" + service: + name: aws-load-balancer-webhook-service + namespace: kube-system + path: /mutate-elbv2-k8s-aws-v1beta1-targetgroupbinding + failurePolicy: Fail + name: mtargetgroupbinding.elbv2.k8s.aws + rules: + - apiGroups: + - elbv2.k8s.aws + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - targetgroupbindings + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: aws-load-balancer-controller + name: aws-load-balancer-webhook +webhooks: +- admissionReviewVersions: + - v1beta1 + clientConfig: + caBundle: "{{WEBHOOK_CA_B64}}" + service: + name: aws-load-balancer-webhook-service + namespace: kube-system + path: /validate-elbv2-k8s-aws-v1beta1-ingressclassparams + failurePolicy: Fail + name: vingressclassparams.elbv2.k8s.aws + objectSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: NotIn + values: + - aws-load-balancer-controller + rules: + - apiGroups: + - elbv2.k8s.aws + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ingressclassparams + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + caBundle: "{{WEBHOOK_CA_B64}}" + service: + name: aws-load-balancer-webhook-service + namespace: kube-system + path: /validate-elbv2-k8s-aws-v1beta1-targetgroupbinding + failurePolicy: Fail + name: vtargetgroupbinding.elbv2.k8s.aws + rules: + - apiGroups: + - elbv2.k8s.aws + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - targetgroupbindings + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + caBundle: "{{WEBHOOK_CA_B64}}" + service: + name: aws-load-balancer-webhook-service + namespace: kube-system + path: /validate-networking-v1-ingress + failurePolicy: Fail + matchPolicy: Equivalent + name: vingress.elbv2.k8s.aws + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + sideEffects: None diff --git a/scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/webhook.conf b/scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/webhook.conf new file mode 100644 index 000000000..a251f6dce --- /dev/null +++ b/scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/webhook.conf @@ -0,0 +1,6 @@ +# Admission-webhook serving-cert SANs for this addon. render_addon mints a +# self-signed cert for these names (via eks-webhook-cert) and embeds it as the +# webhook Secret + every caBundle. Presence of this file is what opts a bundle +# into webhook cert-gen. +WEBHOOK_CN=aws-load-balancer-webhook-service.kube-system.svc +WEBHOOK_DNS=aws-load-balancer-webhook-service.kube-system.svc,aws-load-balancer-webhook-service.kube-system.svc.cluster.local diff --git a/scripts/images/eks-node/k3s-agent.initd b/scripts/images/eks-node/k3s-agent.initd index 4c7c51e79..45eeeda26 100755 --- a/scripts/images/eks-node/k3s-agent.initd +++ b/scripts/images/eks-node/k3s-agent.initd @@ -35,4 +35,27 @@ start_pre() { eerror "K3S_URL and K3S_TOKEN must be set in /etc/spinifex-eks/agent.env" return 1 fi + + # Stamp the node providerID as aws://// (EKS parity) so the + # AWS Load Balancer Controller can resolve nodes as EC2 instances for ip-mode + # target registration; k3s' default k3s:// providerID makes it reject + # the node. Instance-id + AZ come from IMDSv2 (token-gated GET). On any miss + # we leave the default providerID rather than block the agent from starting. + _imds=http://169.254.169.254/latest + _curl="curl -fsS --connect-timeout 3 --max-time 5" + _tok=$(${_curl} -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 120' \ + "${_imds}/api/token" 2>/dev/null) + if [ -n "${_tok}" ]; then + _iid=$(${_curl} -H "X-aws-ec2-metadata-token: ${_tok}" \ + "${_imds}/meta-data/instance-id" 2>/dev/null) + _az=$(${_curl} -H "X-aws-ec2-metadata-token: ${_tok}" \ + "${_imds}/meta-data/placement/availability-zone" 2>/dev/null) + if [ -n "${_iid}" ] && [ -n "${_az}" ]; then + command_args="agent --kubelet-arg=provider-id=aws:///${_az}/${_iid}" + else + ewarn "IMDS instance-id/az empty; keeping default providerID" + fi + else + ewarn "IMDS token request failed; keeping default providerID" + fi } diff --git a/scripts/images/eks-node/manifest.conf b/scripts/images/eks-node/manifest.conf index 13bc0bb2c..cd8eee972 100644 --- a/scripts/images/eks-node/manifest.conf +++ b/scripts/images/eks-node/manifest.conf @@ -18,9 +18,9 @@ APK_PACKAGES="ca-certificates curl jq iptables ip6tables conntrack-tools cgroup- # eks-gateway-publish relays the bootstrap envelopes + state reports to the AWS # gateway over SigV4 HTTPS (server role only). The k3s binary is downloaded + # SHA256-verified by setup.sh (network in chroot). -BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-publish ./cmd/eks-gateway-publish && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-fetch ./cmd/eks-gateway-fetch" +BUILD_COMMANDS="CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-token-webhook ./cmd/eks-token-webhook && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-publish ./cmd/eks-gateway-publish && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-gateway-fetch ./cmd/eks-gateway-fetch && CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -o bin/eks-webhook-cert ./cmd/eks-webhook-cert" -INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook bin/eks-gateway-publish:/usr/local/bin/eks-gateway-publish bin/eks-gateway-fetch:/usr/local/bin/eks-gateway-fetch" +INSTALL_BINARIES="bin/eks-token-webhook:/usr/local/bin/eks-token-webhook bin/eks-gateway-publish:/usr/local/bin/eks-gateway-publish bin/eks-gateway-fetch:/usr/local/bin/eks-gateway-fetch bin/eks-webhook-cert:/usr/local/bin/eks-webhook-cert" # All role services ship; none but the selector are enabled at bake time. # eks-node-role picks server vs agent at first boot and enables the rest. @@ -38,6 +38,9 @@ scripts/images/eks-node/mulga-eks-addon-sync.initd:/etc/init.d/mulga-eks-addon-s scripts/images/eks-node/mulga-eks-addon-sync.sh:/usr/local/sbin/mulga-eks-addon-sync \ scripts/images/eks-node/addons/spinifex-noop/0.1.0/noop.yaml:/usr/share/spinifex-eks/addons/spinifex-noop/0.1.0/noop.yaml \ scripts/images/eks-node/addons/argocd/3.0.23/argocd.yaml:/usr/share/spinifex-eks/addons/argocd/3.0.23/argocd.yaml \ +scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/00-crds.yaml:/usr/share/spinifex-eks/addons/aws-load-balancer-controller/2.11.0/00-crds.yaml \ +scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/10-aws-load-balancer-controller.yaml:/usr/share/spinifex-eks/addons/aws-load-balancer-controller/2.11.0/10-aws-load-balancer-controller.yaml \ +scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/webhook.conf:/usr/share/spinifex-eks/addons/aws-load-balancer-controller/2.11.0/webhook.conf \ scripts/images/eks-node/coredns/coredns-mulga.yaml:/usr/share/spinifex-eks/coredns/coredns-mulga.yaml \ scripts/images/eks-node/mulga-eks-etcd-snapshot.sh:/etc/periodic/daily/mulga-eks-etcd-snapshot" diff --git a/scripts/images/eks-node/mulga-eks-addon-sync.sh b/scripts/images/eks-node/mulga-eks-addon-sync.sh index 9a6d2b69f..ef36c6fff 100644 --- a/scripts/images/eks-node/mulga-eks-addon-sync.sh +++ b/scripts/images/eks-node/mulga-eks-addon-sync.sh @@ -26,6 +26,7 @@ set -a set +a : "${EKS_GATEWAY_URL:?}" +: "${EKS_ADDON_GATEWAY_URL:?}" : "${EKS_ACCESS_KEY:?}" : "${EKS_SECRET_KEY:?}" : "${EKS_ACCOUNT_ID:?}" @@ -40,6 +41,10 @@ export KUBECONFIG BAKED_DIR=/usr/share/spinifex-eks/addons DEPLOY_DIR=/var/lib/rancher/k3s/server/manifests RENDER_PREFIX=spinifex-addon- +# Per-cluster admission-webhook serving certs are minted once and cached here so +# the rendered manifest is byte-stable across sync ticks (a churning cert would +# re-apply the addon every tick). +WEBHOOK_CERT_DIR=/var/lib/spinifex-eks/webhook-certs # log mirrors to syslog (on-VM /var/log) and, best-effort, to the serial # console — the host captures ttyS0 to a per-instance log, so this is the only @@ -59,8 +64,10 @@ report() { } # render_addon ADDON VERSION ROLE_ARN — render the baked bundle into the -# auto-deploy dir, substituting the IRSA role ARN. Returns non-zero (and reports -# failed) when the baked bundle for ADDON/VERSION is absent. +# auto-deploy dir, substituting the IRSA role ARN plus the cluster/gateway +# placeholders an in-cluster AWS-SDK addon (e.g. the LB controller) needs to +# reach the gateway. Returns non-zero (and reports failed) when the baked bundle +# for ADDON/VERSION is absent. Placeholders not present in a bundle are no-ops. render_addon() { _addon=$1; _version=$2; _role=$3 _src="${BAKED_DIR}/${_addon}/${_version}" @@ -72,11 +79,125 @@ render_addon() { return 1 fi + # Gateway CA as single-line base64 for a kube Secret .data entry the addon + # mounts as AWS_CA_BUNDLE. tr -d so busybox base64 (no -w0) stays one line. + _ca_b64="" + if [ -n "${EKS_GATEWAY_CA:-}" ] && [ -f "${EKS_GATEWAY_CA}" ]; then + _ca_b64=$(base64 < "${EKS_GATEWAY_CA}" | tr -d '\n') + fi + + # Webhook serving cert (opt-in): a bundle ships webhook.conf with its service + # SANs when it runs an admission webhook. Mint/reuse a self-signed cert on the + # node (the AMI has no openssl) and embed it as the Secret + every caBundle. + _wh_ca=""; _wh_crt=""; _wh_key="" + if [ -f "${_src}/webhook.conf" ]; then + WEBHOOK_CN=""; WEBHOOK_DNS="" + # shellcheck disable=SC1090,SC1091 + . "${_src}/webhook.conf" + if [ -n "${WEBHOOK_CN}" ] && [ -n "${WEBHOOK_DNS}" ]; then + _wh_err="${_dst}.whcert.$$" + if ! _wh_out=$(eks-webhook-cert -dir "${WEBHOOK_CERT_DIR}/${_addon}" \ + -cn "${WEBHOOK_CN}" -dns "${WEBHOOK_DNS}" 2>"${_wh_err}"); then + log "addon ${_addon}: webhook cert gen failed: $(tr '\n' ' ' < "${_wh_err}")" + rm -f "${_wh_err}" + report failed "${_addon}" "${_version}" "webhook cert generation failed" + return 1 + fi + rm -f "${_wh_err}" + _wh_ca=$(printf '%s' "${_wh_out}" | cut -f1) + _wh_crt=$(printf '%s' "${_wh_out}" | cut -f2) + _wh_key=$(printf '%s' "${_wh_out}" | cut -f3) + fi + fi + + # Reusable IRSA (web-identity) injection. A bundle opts in by placing the + # {{IRSA_ENV}} / {{IRSA_VOLUME}} / {{IRSA_VOLUME_MOUNT}} markers on their own + # line; awk swaps each for the standard block, then the sed pass fills the + # nested role/region/gateway placeholders. Bundles without the markers are + # untouched. Blocks assume a Deployment pod spec (env 8sp, volumes 6sp, + # volumeMounts 8sp). The token (aud sts.amazonaws.com) + role ARN + gateway + # STS endpoint/CA are what AssumeRoleWithWebIdentity at the gateway needs. + # AWS_ENDPOINT_URL_STS points the web-identity credential STS client (built + # from the standard config, before per-service overrides) at the gateway. + # The LBC service clients (ec2/elbv2/acm) ignore the global AWS_ENDPOINT_URL + # because LBC installs its own per-client BaseEndpoint resolver; those are + # redirected via the bundle's --aws-api-endpoints flag, which keys on the + # SDK-v2 service IDs (EC2, "Elastic Load Balancing v2", ACM), not the legacy + # lowercase names. AWS_CA_BUNDLE makes every client trust the gateway CA. + _irsa_env=' - name: AWS_ROLE_ARN + value: "{{SERVICE_ACCOUNT_ROLE_ARN}}" + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/eks.amazonaws.com/serviceaccount/token + - name: AWS_STS_REGIONAL_ENDPOINTS + value: regional + - name: AWS_REGION + value: "{{AWS_REGION}}" + - name: AWS_DEFAULT_REGION + value: "{{AWS_REGION}}" + - name: AWS_ENDPOINT_URL_STS + value: "{{GATEWAY_ENDPOINT}}" + - name: AWS_CA_BUNDLE + value: /etc/spinifex/gateway-ca/ca.pem' + _irsa_vol=' - name: aws-iam-token + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: sts.amazonaws.com + expirationSeconds: 86400 + path: token + - name: spinifex-gateway-ca + secret: + defaultMode: 420 + secretName: spinifex-gateway-ca' + _irsa_mnt=' - name: aws-iam-token + mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + readOnly: true + - name: spinifex-gateway-ca + mountPath: /etc/spinifex/gateway-ca + readOnly: true' + + # IngressClassParams spec injection. EKS_ELB_SUBNET_IDS is the cluster's + # ELB-eligible subnets (CSV, deduped to one per AZ by the daemon). Inject them + # as explicit .spec.subnets.ids so every Ingress takes LBC's explicit-subnet + # path, the only one that honors ALBSingleSubnet; a single-AZ cluster otherwise + # fails reconcile. Empty value drops the marker line, leaving tag auto-discovery. + _icp_spec='' + if [ -n "${EKS_ELB_SUBNET_IDS:-}" ]; then + _icp_spec=' spec: + subnets: + ids:' + _oldifs=$IFS + IFS=',' + for _sn in ${EKS_ELB_SUBNET_IDS}; do + [ -n "${_sn}" ] || continue + _icp_spec="${_icp_spec} + - ${_sn}" + done + IFS=$_oldifs + fi + _tmp="${_dst}.tmp.$$" : > "${_tmp}" for _f in "${_src}"/*.yaml; do [ -e "${_f}" ] || continue - sed -e "s|{{SERVICE_ACCOUNT_ROLE_ARN}}|${_role}|g" "${_f}" >> "${_tmp}" + awk -v env="${_irsa_env}" -v vol="${_irsa_vol}" -v mnt="${_irsa_mnt}" -v icp="${_icp_spec}" ' + index($0, "{{IRSA_ENV}}") { print env; next } + index($0, "{{IRSA_VOLUME_MOUNT}}") { print mnt; next } + index($0, "{{IRSA_VOLUME}}") { print vol; next } + index($0, "{{ELB_INGRESS_PARAMS_SPEC}}") { if (icp != "") print icp; next } + { print } + ' "${_f}" \ + | sed -e "s|{{SERVICE_ACCOUNT_ROLE_ARN}}|${_role}|g" \ + -e "s|{{CLUSTER_NAME}}|${EKS_CLUSTER_NAME}|g" \ + -e "s|{{AWS_REGION}}|${EKS_REGION:-}|g" \ + -e "s|{{AWS_VPC_ID}}|${EKS_VPC_ID:-}|g" \ + -e "s|{{GATEWAY_ENDPOINT}}|${EKS_ADDON_GATEWAY_URL}|g" \ + -e "s|{{GATEWAY_CA_PEM_B64}}|${_ca_b64}|g" \ + -e "s|{{WEBHOOK_CA_B64}}|${_wh_ca}|g" \ + -e "s|{{WEBHOOK_TLS_CRT_B64}}|${_wh_crt}|g" \ + -e "s|{{WEBHOOK_TLS_KEY_B64}}|${_wh_key}|g" \ + >> "${_tmp}" printf '\n---\n' >> "${_tmp}" done # Idempotent: only replace the auto-deploy file when the rendered content diff --git a/spinifex/daemon/daemon_handlers_tag.go b/spinifex/daemon/daemon_handlers_tag.go index 2a20c6bca..8f468e9b3 100644 --- a/spinifex/daemon/daemon_handlers_tag.go +++ b/spinifex/daemon/daemon_handlers_tag.go @@ -1,15 +1,49 @@ package daemon -import "github.com/nats-io/nats.go" +import ( + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/nats-io/nats.go" +) func (d *Daemon) handleEC2CreateTags(msg *nats.Msg) { - handleNATSRequest(msg, d.tagsService.CreateTags) + handleNATSRequest(msg, d.createTags) } func (d *Daemon) handleEC2DeleteTags(msg *nats.Msg) { - handleNATSRequest(msg, d.tagsService.DeleteTags) + handleNATSRequest(msg, d.deleteTags) } func (d *Daemon) handleEC2DescribeTags(msg *nats.Msg) { handleNATSRequest(msg, d.tagsService.DescribeTags) } + +// createTags writes the generic tag store, then mirrors subnet/vpc tags into +// their owning records so tag-filtered describes (e.g. LBC subnet +// auto-discovery via DescribeSubnets) observe them. +func (d *Daemon) createTags(input *ec2.CreateTagsInput, accountID string) (*ec2.CreateTagsOutput, error) { + out, err := d.tagsService.CreateTags(input, accountID) + if err != nil { + return nil, err + } + if d.vpcService != nil { + if err := d.vpcService.ApplyRecordTags(input, accountID); err != nil { + return nil, err + } + } + return out, nil +} + +// deleteTags removes from the generic tag store, then mirrors the removal into +// the owning subnet/vpc records. +func (d *Daemon) deleteTags(input *ec2.DeleteTagsInput, accountID string) (*ec2.DeleteTagsOutput, error) { + out, err := d.tagsService.DeleteTags(input, accountID) + if err != nil { + return nil, err + } + if d.vpcService != nil { + if err := d.vpcService.RemoveRecordTags(input, accountID); err != nil { + return nil, err + } + } + return out, nil +} diff --git a/spinifex/daemon/daemon_test.go b/spinifex/daemon/daemon_test.go index 27006f0d3..14dfaad72 100644 --- a/spinifex/daemon/daemon_test.go +++ b/spinifex/daemon/daemon_test.go @@ -484,6 +484,18 @@ func TestHandleEC2DescribeInstanceTypes(t *testing.T) { initialCount := len(initialOutput.InstanceTypes) t.Logf("Initial supported instance types: %d", initialCount) + // Pin host capacity so the allocate below is independent of the + // runner's live headroom. Without this, readMemAvailableGB reads + // live /proc MemAvailable, which a box already running VMs can drive + // below a 2 vCPU type's footprint, failing allocate spuriously. + daemon.resourceMgr.mu.Lock() + daemon.resourceMgr.hostVCPU = 16 + daemon.resourceMgr.hostMemGB = 32.0 + daemon.resourceMgr.reservedVCPU = 0 + daemon.resourceMgr.reservedMem = 0 + daemon.resourceMgr.readMemAvailableGB = nil + daemon.resourceMgr.mu.Unlock() + schedulableMem := daemon.resourceMgr.hostMemGB - daemon.resourceMgr.reservedMem var instanceType2CPU *ec2.InstanceTypeInfo for _, it := range initialOutput.InstanceTypes { @@ -870,6 +882,18 @@ func TestCanAllocate_CountEdgeCases(t *testing.T) { require.NotNil(t, microType) require.NotNil(t, mediumType) + // Pin host capacity so the test is independent of the runner's + // schedulable headroom. A constrained box (e.g. 4 vCPU, 2 schedulable + // after reserve, more when EKS VMs hold capacity) cannot fit a medium + // and rm.allocate would fail with an unexpected insufficient-capacity error. + rm.mu.Lock() + rm.hostVCPU = 16 + rm.hostMemGB = 32.0 + rm.reservedVCPU = 0 + rm.reservedMem = 0 + rm.readMemAvailableGB = nil // accounting test: pin to synthetic host, not live /proc + rm.mu.Unlock() + initialMicro := rm.canAllocate(microType, 100) initialMedium := rm.canAllocate(mediumType, 100) diff --git a/spinifex/daemon/eks_adapter.go b/spinifex/daemon/eks_adapter.go index 806154a49..1e10b80c4 100644 --- a/spinifex/daemon/eks_adapter.go +++ b/spinifex/daemon/eks_adapter.go @@ -25,6 +25,17 @@ func (a *daemonEKSSubnetResolver) GetSubnetVPC(accountID, subnetID string) (stri return rec.VpcId, nil } +func (a *daemonEKSSubnetResolver) GetSubnetAZ(accountID, subnetID string) (string, error) { + if a.d == nil || a.d.vpcService == nil { + return "", errors.New("eks: VPC service not initialized") + } + rec, err := a.d.vpcService.GetSubnet(accountID, subnetID) + if err != nil { + return "", err + } + return rec.AvailabilityZone, nil +} + func (a *daemonEKSSubnetResolver) GetVPCCIDR(accountID, vpcID string) (string, error) { if a.d == nil || a.d.vpcService == nil { return "", errors.New("eks: VPC service not initialized") diff --git a/spinifex/gateway/ec2/zone/AvailabilityZone.go b/spinifex/gateway/ec2/zone/AvailabilityZone.go index 905be7bac..ec4fcb5e6 100644 --- a/spinifex/gateway/ec2/zone/AvailabilityZone.go +++ b/spinifex/gateway/ec2/zone/AvailabilityZone.go @@ -3,6 +3,7 @@ package gateway_ec2_zone import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" ) func DescribeAvailabilityZones(input *ec2.DescribeAvailabilityZonesInput, region string, az string) (output *ec2.DescribeAvailabilityZonesOutput, err error) { @@ -13,7 +14,7 @@ func DescribeAvailabilityZones(input *ec2.DescribeAvailabilityZonesInput, region OptInStatus: aws.String("opt-in-not-required"), RegionName: aws.String(region), ZoneName: aws.String(az), - ZoneId: aws.String("spinifexz1"), + ZoneId: aws.String(handlers_ec2_vpc.SingleZoneID), GroupName: aws.String(region), NetworkBorderGroup: aws.String(region), ZoneType: aws.String("availability-zone"), diff --git a/spinifex/handlers/ec2/vpc/service_impl.go b/spinifex/handlers/ec2/vpc/service_impl.go index db7d23a12..1d001da68 100644 --- a/spinifex/handlers/ec2/vpc/service_impl.go +++ b/spinifex/handlers/ec2/vpc/service_impl.go @@ -33,6 +33,11 @@ const ( KVBucketVPCsVersion = 1 KVBucketSubnetsVersion = 1 KVBucketVNICounterVersion = 1 + + // SingleZoneID is the only availability-zone id the gateway exposes. The + // deployment is single-AZ, so every subnet and every DescribeAvailabilityZones + // result reports it (AWS clients resolve AZ topology by zone-id, not name). + SingleZoneID = "spinifexz1" ) // VPCRecord represents a stored VPC. AZ is stamped at create time; empty AZ @@ -797,6 +802,116 @@ func (s *VPCServiceImpl) DescribeSubnets(input *ec2.DescribeSubnetsInput, accoun }, nil } +// ApplyRecordTags mirrors CreateTags into the owning subnet/vpc KV record so +// tag-filtered describes (DescribeSubnets/DescribeVpcs, e.g. LBC subnet +// auto-discovery) observe tags added after create. Resource types this service +// does not own are skipped; the generic tag store stays their record of truth. +func (s *VPCServiceImpl) ApplyRecordTags(input *ec2.CreateTagsInput, accountID string) error { + if input == nil { + return nil + } + merge := func(tags map[string]string) { + for _, t := range input.Tags { + if t.Key != nil && t.Value != nil { + tags[*t.Key] = *t.Value + } + } + } + for _, res := range input.Resources { + if res == nil { + continue + } + if err := s.updateRecordTags(accountID, *res, merge); err != nil { + return err + } + } + return nil +} + +// RemoveRecordTags mirrors DeleteTags into the owning subnet/vpc KV record. +// Empty input.Tags clears all tags; a tag with a value deletes only on a value +// match (AWS-faithful), a nil value deletes unconditionally. +func (s *VPCServiceImpl) RemoveRecordTags(input *ec2.DeleteTagsInput, accountID string) error { + if input == nil { + return nil + } + remove := func(tags map[string]string) { + if len(input.Tags) == 0 { + for k := range tags { + delete(tags, k) + } + return + } + for _, t := range input.Tags { + if t.Key == nil { + continue + } + if t.Value == nil { + delete(tags, *t.Key) + } else if cur, ok := tags[*t.Key]; ok && cur == *t.Value { + delete(tags, *t.Key) + } + } + } + for _, res := range input.Resources { + if res == nil { + continue + } + if err := s.updateRecordTags(accountID, *res, remove); err != nil { + return err + } + } + return nil +} + +// updateRecordTags applies mut to the tag map of the subnet- or vpc-scoped +// record identified by resourceID. Other resource ids are a no-op. +func (s *VPCServiceImpl) updateRecordTags(accountID, resourceID string, mut func(map[string]string)) error { + switch { + case strings.HasPrefix(resourceID, "subnet-"): + return updateKVRecordTags(s.subnetKV, accountID, resourceID, func(r *SubnetRecord) { + if r.Tags == nil { + r.Tags = map[string]string{} + } + mut(r.Tags) + }) + case strings.HasPrefix(resourceID, "vpc-"): + return updateKVRecordTags(s.vpcKV, accountID, resourceID, func(r *VPCRecord) { + if r.Tags == nil { + r.Tags = map[string]string{} + } + mut(r.Tags) + }) + } + return nil +} + +// updateKVRecordTags read-modify-writes a typed KV record, applying mut. A +// resource absent from this store is skipped (its tags live elsewhere). +func updateKVRecordTags[R any](kv nats.KeyValue, accountID, resourceID string, mut func(*R)) error { + key := utils.AccountKey(accountID, resourceID) + entry, err := kv.Get(key) + if err != nil { + if errors.Is(err, nats.ErrKeyNotFound) { + return nil + } + return errors.New(awserrors.ErrorServerInternal) + } + var rec R + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + return errors.New(awserrors.ErrorServerInternal) + } + mut(&rec) + data, err := json.Marshal(&rec) + if err != nil { + return errors.New(awserrors.ErrorServerInternal) + } + if _, err := kv.Put(key, data); err != nil { + return errors.New(awserrors.ErrorServerInternal) + } + return nil +} + // vpcMatchesFilters checks whether a VPCRecord satisfies all parsed filters. func vpcMatchesFilters(record *VPCRecord, accountID string, filters map[string][]string) bool { for name, values := range filters { @@ -902,6 +1017,7 @@ func (s *VPCServiceImpl) subnetRecordToEC2(record *SubnetRecord, availableIPs in VpcId: aws.String(record.VpcId), CidrBlock: aws.String(record.CidrBlock), AvailabilityZone: aws.String(record.AvailabilityZone), + AvailabilityZoneId: aws.String(SingleZoneID), State: aws.String(record.State), DefaultForAz: aws.Bool(record.IsDefault), AvailableIpAddressCount: aws.Int64(int64(availableIPs)), diff --git a/spinifex/handlers/ec2/vpc/service_impl_test.go b/spinifex/handlers/ec2/vpc/service_impl_test.go index 73c82512d..5b9b6ab0e 100644 --- a/spinifex/handlers/ec2/vpc/service_impl_test.go +++ b/spinifex/handlers/ec2/vpc/service_impl_test.go @@ -1742,3 +1742,107 @@ func TestGetSubnet_NotFound(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "subnet-missing") } + +// --- CreateTags write-through (mulga-siv-347) --- + +func TestApplyRecordTags_SubnetTagFilteredDescribe(t *testing.T) { + svc := setupTestVPCService(t) + vpcID := createTestVPC(t, svc, "10.0.0.0/16") + subnetID := createTestSubnet(t, svc, vpcID, "10.0.1.0/24") + + err := svc.ApplyRecordTags(&ec2.CreateTagsInput{ + Resources: []*string{aws.String(subnetID)}, + Tags: []*ec2.Tag{ + {Key: aws.String("kubernetes.io/role/elb"), Value: aws.String("1")}, + }, + }, testAccountID) + require.NoError(t, err) + + // DescribeSubnets must surface the tag... + out, err := svc.DescribeSubnets(&ec2.DescribeSubnetsInput{ + SubnetIds: []*string{aws.String(subnetID)}, + }, testAccountID) + require.NoError(t, err) + require.Len(t, out.Subnets, 1) + assert.Equal(t, "1", findTag(out.Subnets[0].Tags, "kubernetes.io/role/elb")) + + // ...and a tag: filter must match it (the LBC auto-discovery path). + filtered, err := svc.DescribeSubnets(&ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{ + {Name: aws.String("tag:kubernetes.io/role/elb"), Values: []*string{aws.String("1")}}, + }, + }, testAccountID) + require.NoError(t, err) + require.Len(t, filtered.Subnets, 1) + assert.Equal(t, subnetID, *filtered.Subnets[0].SubnetId) +} + +func TestApplyRecordTags_VpcMergePreservesRecord(t *testing.T) { + svc := setupTestVPCService(t) + vpcID := createTestVPC(t, svc, "10.1.0.0/16") + + err := svc.ApplyRecordTags(&ec2.CreateTagsInput{ + Resources: []*string{aws.String(vpcID)}, + Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("prod")}}, + }, testAccountID) + require.NoError(t, err) + + out, err := svc.DescribeVpcs(&ec2.DescribeVpcsInput{ + VpcIds: []*string{aws.String(vpcID)}, + }, testAccountID) + require.NoError(t, err) + require.Len(t, out.Vpcs, 1) + // Merge must not clobber the rest of the record. + assert.Equal(t, "10.1.0.0/16", *out.Vpcs[0].CidrBlock) + assert.Equal(t, "prod", findTag(out.Vpcs[0].Tags, "Name")) +} + +func TestRemoveRecordTags_Subnet(t *testing.T) { + svc := setupTestVPCService(t) + vpcID := createTestVPC(t, svc, "10.2.0.0/16") + subnetID := createTestSubnet(t, svc, vpcID, "10.2.1.0/24") + + require.NoError(t, svc.ApplyRecordTags(&ec2.CreateTagsInput{ + Resources: []*string{aws.String(subnetID)}, + Tags: []*ec2.Tag{ + {Key: aws.String("keep"), Value: aws.String("yes")}, + {Key: aws.String("drop"), Value: aws.String("v")}, + }, + }, testAccountID)) + + // Value-mismatched delete is a no-op; matched delete removes. + require.NoError(t, svc.RemoveRecordTags(&ec2.DeleteTagsInput{ + Resources: []*string{aws.String(subnetID)}, + Tags: []*ec2.Tag{ + {Key: aws.String("keep"), Value: aws.String("wrong")}, + {Key: aws.String("drop"), Value: aws.String("v")}, + }, + }, testAccountID)) + + out, err := svc.DescribeSubnets(&ec2.DescribeSubnetsInput{ + SubnetIds: []*string{aws.String(subnetID)}, + }, testAccountID) + require.NoError(t, err) + require.Len(t, out.Subnets, 1) + assert.Equal(t, "yes", findTag(out.Subnets[0].Tags, "keep")) + assert.Equal(t, "", findTag(out.Subnets[0].Tags, "drop")) +} + +func TestApplyRecordTags_UnknownResourceNoError(t *testing.T) { + svc := setupTestVPCService(t) + // Absent subnet + non-VPC-owned resource id: both skipped without error. + err := svc.ApplyRecordTags(&ec2.CreateTagsInput{ + Resources: []*string{aws.String("subnet-doesnotexist"), aws.String("i-instance")}, + Tags: []*ec2.Tag{{Key: aws.String("k"), Value: aws.String("v")}}, + }, testAccountID) + require.NoError(t, err) +} + +func findTag(tags []*ec2.Tag, key string) string { + for _, t := range tags { + if t.Key != nil && *t.Key == key { + return aws.StringValue(t.Value) + } + } + return "" +} diff --git a/spinifex/handlers/eks/addon_catalog.go b/spinifex/handlers/eks/addon_catalog.go index 53993e0c3..6cfd28589 100644 --- a/spinifex/handlers/eks/addon_catalog.go +++ b/spinifex/handlers/eks/addon_catalog.go @@ -19,7 +19,7 @@ type AddonSpec struct { var addonCatalog = buildAddonCatalog( newAddonSpec("aws-load-balancer-controller", true, "Provisions ELBv2 load balancers for Kubernetes Service/Ingress resources.", - "2.8.1"), + "2.11.0"), newAddonSpec("coredns", false, "Cluster DNS server.", "1.11.1"), diff --git a/spinifex/handlers/eks/delete_cluster_test.go b/spinifex/handlers/eks/delete_cluster_test.go index 05e303302..998b4d17e 100644 --- a/spinifex/handlers/eks/delete_cluster_test.go +++ b/spinifex/handlers/eks/delete_cluster_test.go @@ -22,6 +22,7 @@ var _ SubnetVPCResolver = (*fakeSubnetResolver)(nil) func (fakeSubnetResolver) GetSubnetVPC(_, _ string) (string, error) { return "vpc-aaa", nil } func (fakeSubnetResolver) GetVPCCIDR(_, _ string) (string, error) { return "10.0.0.0/16", nil } +func (fakeSubnetResolver) GetSubnetAZ(_, _ string) (string, error) { return "spinifexz1", nil } func deleteInput(name string) *eks.DeleteClusterInput { return &eks.DeleteClusterInput{Name: aws.String(name)} diff --git a/spinifex/handlers/eks/elb_subnets_test.go b/spinifex/handlers/eks/elb_subnets_test.go new file mode 100644 index 000000000..ffe3c58e6 --- /dev/null +++ b/spinifex/handlers/eks/elb_subnets_test.go @@ -0,0 +1,60 @@ +package handlers_eks + +import ( + "errors" + "reflect" + "testing" +) + +type azResolver struct { + az map[string]string + err map[string]error +} + +func (r azResolver) GetSubnetVPC(_, _ string) (string, error) { return "vpc-aaa", nil } +func (r azResolver) GetVPCCIDR(_, _ string) (string, error) { return "10.0.0.0/16", nil } +func (r azResolver) GetSubnetAZ(_, subnetID string) (string, error) { + if r.err != nil { + if e, ok := r.err[subnetID]; ok { + return "", e + } + } + return r.az[subnetID], nil +} + +func TestDedupSubnetsByAZ(t *testing.T) { + t.Run("single-AZ collapses to one subnet", func(t *testing.T) { + r := azResolver{az: map[string]string{"subnet-a": "z1", "subnet-b": "z1"}} + got := dedupSubnetsByAZ(r, "acct", []string{"subnet-a", "subnet-b"}) + if want := []string{"subnet-a"}; !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + }) + + t.Run("multi-AZ keeps one per AZ in order", func(t *testing.T) { + r := azResolver{az: map[string]string{ + "subnet-a": "z1", "subnet-b": "z2", "subnet-c": "z1", "subnet-d": "z2", + }} + got := dedupSubnetsByAZ(r, "acct", []string{"subnet-a", "subnet-b", "subnet-c", "subnet-d"}) + if want := []string{"subnet-a", "subnet-b"}; !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + }) + + t.Run("unresolved AZ is skipped", func(t *testing.T) { + r := azResolver{ + az: map[string]string{"subnet-b": "z2"}, + err: map[string]error{"subnet-a": errors.New("boom")}, + } + got := dedupSubnetsByAZ(r, "acct", []string{"subnet-a", "subnet-b"}) + if want := []string{"subnet-b"}; !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + }) + + t.Run("nil resolver yields nil", func(t *testing.T) { + if got := dedupSubnetsByAZ(nil, "acct", []string{"subnet-a"}); got != nil { + t.Fatalf("got %v, want nil", got) + } + }) +} diff --git a/spinifex/handlers/eks/k3s_server_vm.go b/spinifex/handlers/eks/k3s_server_vm.go index e16fa7189..283d7b260 100644 --- a/spinifex/handlers/eks/k3s_server_vm.go +++ b/spinifex/handlers/eks/k3s_server_vm.go @@ -99,6 +99,15 @@ type K3sServerInput struct { ClusterName string Region string SubnetID string + // VpcID is the cluster VPC; surfaced as EKS_VPC_ID so the in-cluster LB + // controller can pass --aws-vpc-id to the gateway elbv2/ec2 handlers. + VpcID string + // ELBSubnetIDs is the cluster's ELB-eligible subnets, deduped to one per AZ. + // Surfaced as EKS_ELB_SUBNET_IDS and injected into the alb IngressClassParams + // so every Ingress takes LBC's explicit-subnet path (the only path that honors + // the ALBSingleSubnet gate); tag auto-discovery never threads that gate, so a + // single-AZ cluster would otherwise dedup to 1<2 subnets and fail reconcile. + ELBSubnetIDs []string ControlPlaneSGID string NLBDNS string // EndpointIP is the NLB front-end IP added to the apiserver cert SANs for TLS. @@ -114,11 +123,15 @@ type K3sServerInput struct { // Gateway broker config: CP VM publishes via SigV4-signed HTTPS POST to AWSGW. // GatewayURL is the mgmt-reachable endpoint; AccessKey/SecretKey are system // SigV4 creds; GatewayCACert signs the gateway TLS cert. - GatewayURL string - AccessKey string - SecretKey string - GatewayCACert string - InstanceType string + GatewayURL string + // AddonGatewayURL is the customer-facing gateway endpoint baked into managed + // addon pod specs (EKS_ADDON_GATEWAY_URL). Those pods run on workers, which + // cannot reach the mgmt GatewayURL, so they target this public address. + AddonGatewayURL string + AccessKey string + SecretKey string + GatewayCACert string + InstanceType string // TargetNodeID pins the VM to a specific host for HA spread; empty = local node. TargetNodeID string // JoinToken is the shared k3s cluster token so HA servers join the etcd quorum. @@ -370,6 +383,8 @@ func validateK3sServerInput(in K3sServerInput) error { return errors.New("eks: K3sServerInput empty OIDCPublicKeyPEM") case in.GatewayURL == "": return errors.New("eks: K3sServerInput empty GatewayURL") + case in.AddonGatewayURL == "": + return errors.New("eks: K3sServerInput empty AddonGatewayURL") case in.AccessKey == "": return errors.New("eks: K3sServerInput empty AccessKey") case in.SecretKey == "": @@ -427,10 +442,13 @@ func buildK3sUserData(in K3sServerInput) string { envBody := strings.Join([]string{ "SPINIFEX_K3S_ROLE=" + role, "EKS_GATEWAY_URL=" + in.GatewayURL, + "EKS_ADDON_GATEWAY_URL=" + in.AddonGatewayURL, "EKS_GATEWAY_CA=" + k3sGatewayCAPath, "EKS_ACCESS_KEY=" + in.AccessKey, "EKS_SECRET_KEY=" + in.SecretKey, "EKS_REGION=" + in.Region, + "EKS_VPC_ID=" + in.VpcID, + "EKS_ELB_SUBNET_IDS=" + strings.Join(in.ELBSubnetIDs, ","), "EKS_ACCOUNT_ID=" + in.ClusterAccountID, "EKS_CLUSTER_NAME=" + in.ClusterName, "EKS_NLB_ENDPOINT=" + nlbEndpoint, @@ -455,6 +473,11 @@ func buildK3sUserData(in K3sServerInput) string { } configLines = append(configLines, "etcd-expose-metrics: true", + // Managed-CP: the apiserver VM sits in the system CP VPC with no route to + // the worker pod network. `cluster` routes apiserver->pod/service traffic + // (admission webhooks, aggregated APIs) through the agent's outbound tunnel; + // the default `agent` mode tunnels only kubelet, leaving webhooks unreachable. + "egress-selector-mode: cluster", // Prevent user workloads on the CP (EKS parity). k3s packaged addons tolerate CriticalAddonsOnly. "node-taint:", " - CriticalAddonsOnly=true:NoExecute", diff --git a/spinifex/handlers/eks/k3s_server_vm_test.go b/spinifex/handlers/eks/k3s_server_vm_test.go index 999e3a404..8957910f4 100644 --- a/spinifex/handlers/eks/k3s_server_vm_test.go +++ b/spinifex/handlers/eks/k3s_server_vm_test.go @@ -176,12 +176,14 @@ func validK3sInput() K3sServerInput { ClusterName: "alpha", Region: "us-east-1", SubnetID: "subnet-aaa", + VpcID: "vpc-aaa", ControlPlaneSGID: "sg-cp-aaa", NLBDNS: "eks-alpha-lb-001.us-east-1.elb.spinifex.local", OIDCIssuer: "https://oidc.spinifex.local/clusters/111122223333/alpha", OIDCPrivateKeyPEM: "-----BEGIN PRIVATE KEY-----\nFAKEKEY\n-----END PRIVATE KEY-----\n", OIDCPublicKeyPEM: "-----BEGIN PUBLIC KEY-----\nFAKEPUB\n-----END PUBLIC KEY-----\n", GatewayURL: "https://10.15.8.1:9999", + AddonGatewayURL: "https://192.168.1.33:9999", AccessKey: "AKIAEXAMPLE", SecretKey: "s3cr3t-key", GatewayCACert: "-----BEGIN CERTIFICATE-----\nFAKECA\n-----END CERTIFICATE-----\n", @@ -209,6 +211,7 @@ func TestLaunchK3sServerVM_EmptyInputsRejected(t *testing.T) { {"empty OIDCPrivateKeyPEM", mk(func(in *K3sServerInput) { in.OIDCPrivateKeyPEM = " \n " })}, {"empty OIDCPublicKeyPEM", mk(func(in *K3sServerInput) { in.OIDCPublicKeyPEM = " \n " })}, {"empty GatewayURL", mk(func(in *K3sServerInput) { in.GatewayURL = "" })}, + {"empty AddonGatewayURL", mk(func(in *K3sServerInput) { in.AddonGatewayURL = "" })}, {"empty AccessKey", mk(func(in *K3sServerInput) { in.AccessKey = "" })}, {"empty SecretKey", mk(func(in *K3sServerInput) { in.SecretKey = "" })}, {"empty GatewayCACert", mk(func(in *K3sServerInput) { in.GatewayCACert = " \n" })}, @@ -383,6 +386,12 @@ func TestBuildK3sUserData_AdvertiseAddressFallsBackToEndpoint(t *testing.T) { assert.Contains(t, ud, "advertise-address: 203.0.113.9") } +func TestBuildK3sUserData_EgressSelectorClusterMode(t *testing.T) { + ud := buildK3sUserData(validK3sInput()) + assert.Contains(t, ud, "egress-selector-mode: cluster", + "managed-CP apiserver must tunnel pod/service traffic (webhooks) through the agent") +} + func TestLaunchK3sServerVM_UserDataContainsAllArtifacts(t *testing.T) { vpc, inst, ami := &fakeK3sVPC{}, &fakeK3sInst{}, &fakeK3sAMI{} @@ -409,10 +418,12 @@ func TestLaunchK3sServerVM_UserDataContainsAllArtifacts(t *testing.T) { // services; without it the unified AMI boots into no role. assert.Contains(t, udata, "SPINIFEX_K3S_ROLE=server") assert.Contains(t, udata, "EKS_GATEWAY_URL=https://10.15.8.1:9999") + assert.Contains(t, udata, "EKS_ADDON_GATEWAY_URL=https://192.168.1.33:9999") assert.Contains(t, udata, "EKS_GATEWAY_CA="+k3sGatewayCAPath) assert.Contains(t, udata, "EKS_ACCESS_KEY=AKIAEXAMPLE") assert.Contains(t, udata, "EKS_SECRET_KEY=s3cr3t-key") assert.Contains(t, udata, "EKS_REGION=us-east-1") + assert.Contains(t, udata, "EKS_VPC_ID=vpc-aaa") // EKS_ACCOUNT_ID is the cluster-OWNER account (ClusterAccountID), not the // infra account the VM is launched under — the on-VM agents namespace their // bootstrap publish / state report / add-on fetch by it, so it must reach the diff --git a/spinifex/handlers/eks/service_impl.go b/spinifex/handlers/eks/service_impl.go index 5d63ef18d..f4fe521c2 100644 --- a/spinifex/handlers/eks/service_impl.go +++ b/spinifex/handlers/eks/service_impl.go @@ -29,6 +29,7 @@ import ( type SubnetVPCResolver interface { GetSubnetVPC(accountID, subnetID string) (vpcID string, err error) GetVPCCIDR(accountID, vpcID string) (cidr string, err error) + GetSubnetAZ(accountID, subnetID string) (az string, err error) } // EKSServiceDeps wires the external collaborators EKSServiceImpl needs. @@ -370,6 +371,32 @@ func (s *EKSServiceImpl) CreateCluster(input *eks.CreateClusterInput, accountID, return out, nil } +// dedupSubnetsByAZ resolves each subnet's AZ and keeps the first subnet per AZ, +// preserving input order. AWS ALB rejects two subnets in the same AZ, so the +// alb IngressClassParams must carry at most one per AZ; the ALBSingleSubnet gate +// then relaxes the minimum to one. A resolve error drops that subnet (and logs), +// leaving an empty result that falls back to LBC tag auto-discovery. +func dedupSubnetsByAZ(resolver SubnetVPCResolver, accountID string, subnetIDs []string) []string { + if resolver == nil { + return nil + } + seen := make(map[string]struct{}, len(subnetIDs)) + out := make([]string, 0, len(subnetIDs)) + for _, id := range subnetIDs { + az, err := resolver.GetSubnetAZ(accountID, id) + if err != nil || az == "" { + slog.Warn("dedupSubnetsByAZ: skip subnet, AZ unresolved", "subnet", id, "err", err) + continue + } + if _, ok := seen[az]; ok { + continue + } + seen[az] = struct{}{} + out = append(out, id) + } + return out +} + // clusterLaunchCtx carries the inputs the asynchronous control-plane launch // needs, captured at the end of CreateCluster's fast (claim) phase. type clusterLaunchCtx struct { @@ -553,12 +580,21 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) { return } + // Dedup the customer's ELB-eligible subnets to one per AZ for the alb + // IngressClassParams. LBC's tag auto-discovery never honors ALBSingleSubnet, so + // a single-AZ cluster collapses to 1<2 subnets and fails; the explicit-subnet + // path (driven by these IDs) is the only one that threads the gate. Best-effort: + // an unresolved AZ falls back to auto-discovery rather than risk a dup-AZ error. + elbSubnets := dedupSubnetsByAZ(s.deps.VPCSubnet, accountID, lc.subnetIDs) + cpNodes, spreadGroup, err := s.placeControlPlane(sysAcct, name, K3sServerInput{ AccountID: sysAcct, ClusterAccountID: accountID, ClusterName: name, Region: region, SubnetID: cpRefs.PrivateSubnetIDs[0], + VpcID: meta.ResourcesVpcConfig.VpcId, + ELBSubnetIDs: elbSubnets, ControlPlaneSGID: cpSG, NLBDNS: nlb.DNSName, EndpointIP: nlb.FrontendIP, @@ -567,6 +603,7 @@ func (s *EKSServiceImpl) launchClusterInfra(lc clusterLaunchCtx) { OIDCPrivateKeyPEM: privPEM, OIDCPublicKeyPEM: pubPEM, GatewayURL: s.deps.SystemGatewayURL, + AddonGatewayURL: s.deps.GatewayBaseURL, AccessKey: s.deps.SystemAccessKey, SecretKey: s.deps.SystemSecretKey, GatewayCACert: s.deps.GatewayCACert, diff --git a/spinifex/handlers/elbv2/service_impl.go b/spinifex/handlers/elbv2/service_impl.go index 50246d92e..9e57b5dbc 100644 --- a/spinifex/handlers/elbv2/service_impl.go +++ b/spinifex/handlers/elbv2/service_impl.go @@ -1136,12 +1136,7 @@ func (s *ELBv2ServiceImpl) createLoadBalancer(input *elbv2.CreateLoadBalancerInp return nil, errors.New(awserrors.ErrorServerInternal) } - var subnets []string - for _, sn := range input.Subnets { - if sn != nil { - subnets = append(subnets, *sn) - } - } + subnets := flattenSubnetIDs(input.Subnets, input.SubnetMappings) var securityGroups []string for _, sg := range input.SecurityGroups { @@ -2027,6 +2022,17 @@ func listenerActionFromSDK(a *elbv2.Action) ListenerAction { if a.TargetGroupArn != nil { action.TargetGroupArn = *a.TargetGroupArn } + // Forward actions may carry the target group via ForwardConfig (the modern + // weighted shape the LB controller emits) instead of the flat field. Flatten + // the first target group so the single-TG model resolves it. + if action.TargetGroupArn == "" && a.ForwardConfig != nil { + for _, tg := range a.ForwardConfig.TargetGroups { + if tg != nil && tg.TargetGroupArn != nil && *tg.TargetGroupArn != "" { + action.TargetGroupArn = *tg.TargetGroupArn + break + } + } + } if a.FixedResponseConfig != nil { fr := &FixedResponseAction{} if a.FixedResponseConfig.StatusCode != nil { @@ -2640,6 +2646,20 @@ func (s *ELBv2ServiceImpl) DescribeTags(input *elbv2.DescribeTagsInput, accountI found = true tags = r.Tags ownerAccount = r.AccountID + } else if lArn, isDefault := listenerArnFromDefaultRuleArn(arn); isDefault { + // Synthetic default rule: not stored, carries no tags. Resolve via + // its parent listener so a controller's post-create rule-tag sync + // gets an empty TagDescription instead of an error. + l, lErr := s.store.GetListenerByArn(lArn) + if lErr != nil { + slog.Error("DescribeTags: failed to get listener", "arn", lArn, "err", lErr) + return nil, errors.New(awserrors.ErrorServerInternal) + } + if l != nil { + found = true + tags = nil + ownerAccount = l.AccountID + } } } diff --git a/spinifex/handlers/elbv2/service_impl_lb_network.go b/spinifex/handlers/elbv2/service_impl_lb_network.go index 2e5314cc9..00445809f 100644 --- a/spinifex/handlers/elbv2/service_impl_lb_network.go +++ b/spinifex/handlers/elbv2/service_impl_lb_network.go @@ -239,6 +239,13 @@ func (s *ELBv2ServiceImpl) SetSubnets(input *elbv2.SetSubnetsInput, accountID st // desiredSubnetSet flattens Subnets and SubnetMappings into a deduplicated, ordered list. func desiredSubnetSet(input *elbv2.SetSubnetsInput) []string { + return flattenSubnetIDs(input.Subnets, input.SubnetMappings) +} + +// flattenSubnetIDs deduplicates the explicit Subnets list and the SubnetMappings +// list into one ordered subnet-ID slice. LBC supplies ALB subnets via +// SubnetMappings, so both sources must be honoured wherever subnets are read. +func flattenSubnetIDs(subnets []*string, mappings []*elbv2.SubnetMapping) []string { seen := make(map[string]bool) var out []string add := func(sn string) { @@ -247,12 +254,12 @@ func desiredSubnetSet(input *elbv2.SetSubnetsInput) []string { out = append(out, sn) } } - for _, sn := range input.Subnets { + for _, sn := range subnets { if sn != nil { add(*sn) } } - for _, m := range input.SubnetMappings { + for _, m := range mappings { if m != nil && m.SubnetId != nil { add(*m.SubnetId) } diff --git a/spinifex/handlers/elbv2/service_impl_rules.go b/spinifex/handlers/elbv2/service_impl_rules.go index 533488412..001391abf 100644 --- a/spinifex/handlers/elbv2/service_impl_rules.go +++ b/spinifex/handlers/elbv2/service_impl_rules.go @@ -242,7 +242,22 @@ func (s *ELBv2ServiceImpl) DescribeRules(input *elbv2.DescribeRulesInput, accoun slog.Error("DescribeRules: failed to get rule", "arn", *arnPtr, "err", err) return nil, errors.New(awserrors.ErrorServerInternal) } - if r == nil || r.AccountID != accountID { + if r == nil { + // Synthetic default rule: resolve by its parent listener. + if lArn, isDefault := listenerArnFromDefaultRuleArn(*arnPtr); isDefault { + l, lErr := s.store.GetListenerByArn(lArn) + if lErr != nil { + slog.Error("DescribeRules: failed to get listener", "arn", lArn, "err", lErr) + return nil, errors.New(awserrors.ErrorServerInternal) + } + if l != nil && l.AccountID == accountID { + out.Rules = append(out.Rules, defaultRuleFromListener(l)) + continue + } + } + return nil, errors.New(awserrors.ErrorELBv2RuleNotFound) + } + if r.AccountID != accountID { return nil, errors.New(awserrors.ErrorELBv2RuleNotFound) } out.Rules = append(out.Rules, ruleRecordToSDK(r)) @@ -597,6 +612,33 @@ func buildRuleArn(listenerArn, ruleID string) (string, error) { return before + ":listener-rule/" + after + "/" + ruleID, nil } +// defaultRuleArn returns the deterministic ARN of a listener's synthetic default +// rule. AWS gives the default rule its own ARN; an empty value makes a controller +// issue DescribeTags with no resource ARN and get MissingParameter back. +func defaultRuleArn(listenerArn string) (string, error) { + id := listenerArn[strings.LastIndex(listenerArn, "/")+1:] + return buildRuleArn(listenerArn, id) +} + +// listenerArnFromDefaultRuleArn reverses defaultRuleArn, returning the parent +// listener ARN only when ruleArn is exactly that listener's default-rule ARN. +// Stored or unknown rule ARNs return false. +func listenerArnFromDefaultRuleArn(ruleArn string) (string, bool) { + before, after, ok := strings.Cut(ruleArn, ":listener-rule/") + if !ok { + return "", false + } + slash := strings.LastIndex(after, "/") + if slash <= 0 { + return "", false + } + listenerArn := before + ":listener/" + after[:slash] + if want, err := defaultRuleArn(listenerArn); err != nil || want != ruleArn { + return "", false + } + return listenerArn, true +} + // ruleRecordToSDK converts a stored rule record to the AWS SDK shape. func ruleRecordToSDK(r *RuleRecord) *elbv2.Rule { rule := &elbv2.Rule{ @@ -654,6 +696,9 @@ func defaultRuleFromListener(l *ListenerRecord) *elbv2.Rule { Priority: aws.String("default"), IsDefault: aws.Bool(true), } + if arn, err := defaultRuleArn(l.ListenerArn); err == nil { + rule.RuleArn = aws.String(arn) + } for _, a := range l.DefaultActions { rule.Actions = append(rule.Actions, listenerActionToSDK(a)) } diff --git a/spinifex/handlers/elbv2/service_impl_rules_test.go b/spinifex/handlers/elbv2/service_impl_rules_test.go index 0a8a59220..ac9882243 100644 --- a/spinifex/handlers/elbv2/service_impl_rules_test.go +++ b/spinifex/handlers/elbv2/service_impl_rules_test.go @@ -70,6 +70,33 @@ func TestCreateRule_PathPattern(t *testing.T) { assert.Contains(t, *out.Rules[0].RuleArn, ":listener-rule/") } +// TestCreateRule_ForwardConfig guards the regression where a forward action that +// carries its target group via ForwardConfig (the shape the AWS Load Balancer +// Controller emits) was rejected with MissingParameter because only the flat +// TargetGroupArn field was read. +func TestCreateRule_ForwardConfig(t *testing.T) { + env := newRuleTestEnv(t, "cr-fwdcfg") + + out, err := env.svc.CreateRule(&elbv2.CreateRuleInput{ + ListenerArn: aws.String(env.listenerArn), + Priority: aws.Int64(1), + Conditions: []*elbv2.RuleCondition{ + {Field: aws.String("path-pattern"), Values: aws.StringSlice([]string{"/*"})}, + }, + Actions: []*elbv2.Action{ + {Type: aws.String("forward"), ForwardConfig: &elbv2.ForwardActionConfig{ + TargetGroups: []*elbv2.TargetGroupTuple{ + {TargetGroupArn: aws.String(env.tgAltArn)}, + }, + }}, + }, + }, testAccountID) + require.NoError(t, err) + require.Len(t, out.Rules, 1) + require.Len(t, out.Rules[0].Actions, 1) + assert.Equal(t, env.tgAltArn, *out.Rules[0].Actions[0].TargetGroupArn) +} + // TestDeleteLoadBalancer_CascadesRuleDeletion guards against the regression where // DeleteLoadBalancer called store.DeleteListener directly, bypassing rule cascade, // leaving a rule's target group pinned as ResourceInUse permanently. @@ -397,6 +424,44 @@ func TestDescribeRules_ByListener_SortedAndDefaultLast(t *testing.T) { assert.True(t, *out.Rules[3].IsDefault) } +// TestDefaultRule_HasArnAndIsTaggable guards against the regression where the +// synthetic default rule was returned with no ARN: the controller then issued +// DescribeTags with an empty resource and got MissingParameter, aborting the +// Ingress reconcile right after listener creation. +func TestDefaultRule_HasArnAndIsTaggable(t *testing.T) { + env := newRuleTestEnv(t, "defrule") + + rules, err := env.svc.DescribeRules(&elbv2.DescribeRulesInput{ListenerArn: aws.String(env.listenerArn)}, testAccountID) + require.NoError(t, err) + require.Len(t, rules.Rules, 1) + def := rules.Rules[0] + require.True(t, *def.IsDefault) + require.NotNil(t, def.RuleArn, "default rule must carry an ARN") + require.NotEmpty(t, *def.RuleArn) + assert.Contains(t, *def.RuleArn, ":listener-rule/") + + // DescribeTags on the default rule ARN must succeed with empty tags, not error. + tagsOut, err := env.svc.DescribeTags(&elbv2.DescribeTagsInput{ + ResourceArns: []*string{def.RuleArn}, + }, testAccountID) + require.NoError(t, err) + require.Len(t, tagsOut.TagDescriptions, 1) + assert.Equal(t, *def.RuleArn, *tagsOut.TagDescriptions[0].ResourceArn) + assert.Empty(t, tagsOut.TagDescriptions[0].Tags) + + // DescribeRules by the default rule ARN resolves the synthetic rule. + byArn, err := env.svc.DescribeRules(&elbv2.DescribeRulesInput{ + RuleArns: []*string{def.RuleArn}, + }, testAccountID) + require.NoError(t, err) + require.Len(t, byArn.Rules, 1) + assert.True(t, *byArn.Rules[0].IsDefault) + + // A non-default, unknown rule ARN under the same listener still 404s. + _, err = env.svc.DescribeTags(&elbv2.DescribeTagsInput{ResourceArns: []*string{aws.String(*def.RuleArn + "-bogus")}}, testAccountID) + require.Error(t, err) +} + func TestSetRulePriorities_Reorders(t *testing.T) { env := newRuleTestEnv(t, "spri") diff --git a/spinifex/handlers/elbv2/service_impl_vpc_test.go b/spinifex/handlers/elbv2/service_impl_vpc_test.go index 9d6352ff4..c076639ff 100644 --- a/spinifex/handlers/elbv2/service_impl_vpc_test.go +++ b/spinifex/handlers/elbv2/service_impl_vpc_test.go @@ -109,6 +109,38 @@ func TestCreateLoadBalancer_CreatesENIs(t *testing.T) { assert.True(t, foundTag, "ENI should have spinifex:managed-by=elbv2 tag") } +// TestCreateLoadBalancer_SubnetMappings_CreatesENIs guards the LBC path: the AWS +// Load Balancer Controller supplies ALB subnets via SubnetMappings, not the plain +// Subnets list. Ignoring SubnetMappings skips ENI allocation, so the data-plane VM +// never launches and the ALB is created but cannot serve traffic. +func TestCreateLoadBalancer_SubnetMappings_CreatesENIs(t *testing.T) { + svc, vpcSvc := setupTestServiceWithVPC(t) + + subnets, err := vpcSvc.DescribeSubnets(&ec2.DescribeSubnetsInput{}, testAccountID) + require.NoError(t, err) + require.NotEmpty(t, subnets.Subnets) + subnetID := *subnets.Subnets[0].SubnetId + + out, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{ + Name: aws.String("subnetmapping-alb"), + SubnetMappings: []*elbv2.SubnetMapping{ + {SubnetId: aws.String(subnetID)}, + }, + }, testAccountID) + require.NoError(t, err) + require.Len(t, out.LoadBalancers, 1) + lb := out.LoadBalancers[0] + + assert.NotEmpty(t, *lb.VpcId, "VpcId must be resolved from the SubnetMappings subnet") + require.Len(t, lb.AvailabilityZones, 1) + assert.Equal(t, subnetID, *lb.AvailabilityZones[0].SubnetId) + + eniDesc, err := vpcSvc.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{}, testAccountID) + require.NoError(t, err) + require.Len(t, eniDesc.NetworkInterfaces, 1, "an ENI must be created for the SubnetMappings subnet") + assert.Equal(t, subnetID, *eniDesc.NetworkInterfaces[0].SubnetId) +} + func TestCreateLoadBalancer_MultipleSubnets(t *testing.T) { svc, vpcSvc := setupTestServiceWithVPC(t) diff --git a/spinifex/vm/gc.go b/spinifex/vm/gc.go index 6a07d8b67..16179898a 100644 --- a/spinifex/vm/gc.go +++ b/spinifex/vm/gc.go @@ -57,9 +57,13 @@ func (g *GarbageCollector) WithLeaderElection(acquire func() (release func(), el // Start runs an immediate sweep, then one every GCInterval until ctx is done. // Call once per GC to avoid duplicating the goroutine. func (g *GarbageCollector) Start(ctx context.Context) { + // Read GCInterval on the caller's goroutine so tests that mutate the package + // var (set then restore in a defer) are ordered against this read; the + // spawned goroutine then only touches the captured local. + interval := GCInterval go func() { g.SweepOnce(ctx) - ticker := time.NewTicker(GCInterval) + ticker := time.NewTicker(interval) defer ticker.Stop() for { select {