mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-22 20:26:44 +00:00
refactor: extract the managed system-VPC builder out of EKS (#590)
This commit is contained in:
Julian Sommer
@@ -152,6 +152,11 @@ type NorthstarConfig struct {
|
||||
InternalDomain string `json:"InternalDomain" mapstructure:"internal_domain"`
|
||||
}
|
||||
|
||||
// RDSDefaultSystemVPCSupernet anchors the RDS system VPC address space at
|
||||
// 10.248.0.0/14, immediately below and disjoint from the EKS control-plane
|
||||
// supernet, so a name-hash collision can never place an RDS subnet in EKS space.
|
||||
const RDSDefaultSystemVPCSupernet = "10.248.0.0/14"
|
||||
|
||||
// ParseEndpoints splits a comma-separated OVSDB endpoint list (NB/SB RAFT
|
||||
// cluster) into individual endpoints, trimming whitespace and dropping empties.
|
||||
// A single endpoint yields a one-element slice; empty input yields nil. Both the
|
||||
|
||||
+73
-575
@@ -2,54 +2,77 @@ package handlers_eks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"log/slog"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
handlers_systemvpc "github.com/mulgadc/spinifex/spinifex/handlers/systemvpc"
|
||||
"github.com/mulgadc/spinifex/spinifex/tags"
|
||||
"github.com/mulgadc/spinifex/spinifex/utils"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// The managed control-plane VPC ("Set B") is the spinifex analogue of AWS EKS's
|
||||
// hidden managed-account VPC: CreateCluster builds it automatically, it is owned
|
||||
// by the system account (admin.SystemAccountID), and the customer never
|
||||
// provisions or sees it. The internet-facing NLB lives in its public subnet and
|
||||
// the k3s control-plane VM(s) in its private subnet(s); a NAT gateway gives the
|
||||
// private CP egress for image pulls. It is composed from the real EC2
|
||||
// VPC-family APIs (not direct OVN mutation), so per-subnet egress gating
|
||||
// (public 1000 reroute / NAT-GW reroute / private 1100 drop) is wired by the
|
||||
// existing topology subscribers off the route-table events those APIs publish.
|
||||
// The managed control-plane VPC is the spinifex analogue of AWS EKS's hidden
|
||||
// managed-account VPC: CreateCluster builds it, the system account owns it, and
|
||||
// the customer never sees it. The NLB lives in its public subnet, the k3s
|
||||
// control-plane VMs in its NAT-routed private subnets.
|
||||
//
|
||||
// The topology is the shared systemvpc builder; EKS supplies only the identity
|
||||
// — its own tag keys, so no other component's sweep reaches these resources —
|
||||
// and the address space.
|
||||
const (
|
||||
clusterEKSRoleCPVPC = "cp-vpc"
|
||||
clusterEKSRolePublicSubnet = "cp-public"
|
||||
clusterEKSRolePrivateSubnet = "cp-private"
|
||||
clusterEKSRolePublicRT = "cp-public-rt"
|
||||
clusterEKSRolePrivateRT = "cp-private-rt"
|
||||
clusterEKSRoleCPNatGW = "cp-natgw"
|
||||
// cpVPCRolePrefix namespaces the CP VPC's role tag values ("cp-vpc",
|
||||
// "cp-public", …). Baked into live deployments' tags: changing it orphans
|
||||
// every existing CP VPC from its teardown.
|
||||
cpVPCRolePrefix = "cp"
|
||||
|
||||
// cpVPCSupernet anchors the managed CP VPC address space at 10.252.0.0/14.
|
||||
// Each cluster gets a deterministic /22 carved from it: a public /24 + up to
|
||||
// three private /24s (HA 3-node etcd spread).
|
||||
cpVPCSupernet = "10.252.0.0/14"
|
||||
)
|
||||
|
||||
// cpVPCSupernetSecondOctet anchors the managed CP VPC address space at
|
||||
// 10.252.0.0/14 (10.252–10.255). Each cluster gets a deterministic /22 carved by
|
||||
// cpVPCCIDRs: a public /24 + up to three private /24s (HA 3-node etcd spread).
|
||||
// VPCs are L3-isolated (separate OVN logical routers) and egress via distinct
|
||||
// per-VPC external IPs, so a hash collision across clusters is non-fatal; a
|
||||
// proper managed-IPAM allocator is a follow-up.
|
||||
const cpVPCSupernetSecondOctet = 252
|
||||
|
||||
const maxCPPrivateSubnets = 3
|
||||
|
||||
// cpVPCPrivateSubnetCount is how many private CP subnets the managed VPC carves.
|
||||
// One today: the placement rule (CP ⇒ private subnet) is identical for single-
|
||||
// and multi-node clusters, so all control-plane VMs share the private subnet;
|
||||
// per-AZ private-subnet spread is a follow-up sizing concern, not a placement
|
||||
// fork.
|
||||
// One today: the placement rule is identical for single- and multi-node
|
||||
// clusters, so all control-plane VMs share the private subnet.
|
||||
const cpVPCPrivateSubnetCount = 1
|
||||
|
||||
// cpVPCRoles are the role-tag values the managed CP VPC's resources carry.
|
||||
var cpVPCRoles = handlers_systemvpc.Spec{RolePrefix: cpVPCRolePrefix}.Roles()
|
||||
|
||||
// CPVPCDeps and ManagedCPVPCRefs are the systemvpc types under the names EKS
|
||||
// callers (and ClusterMeta's projection) already use.
|
||||
type (
|
||||
CPVPCDeps = handlers_systemvpc.Deps
|
||||
ManagedCPVPCRefs = handlers_systemvpc.Refs
|
||||
)
|
||||
|
||||
// The provisioner surfaces the CP VPC composes from, under their EKS-local
|
||||
// names so the service deps and their fakes read against one vocabulary.
|
||||
type (
|
||||
vpcProvisioner = handlers_systemvpc.VPCProvisioner
|
||||
routeTableProvisioner = handlers_systemvpc.RouteTableProvisioner
|
||||
natGatewayProvisioner = handlers_systemvpc.NATGatewayProvisioner
|
||||
)
|
||||
|
||||
// cpVPCOwner is the tag identity every managed CP VPC resource carries. Keyed on
|
||||
// the EKS cluster tags, so the EKS teardown and billable reapers see exactly
|
||||
// these resources and no other component's.
|
||||
func cpVPCOwner(clusterName string) handlers_systemvpc.Owner {
|
||||
return handlers_systemvpc.Owner{
|
||||
Name: clusterName,
|
||||
ManagedBy: tags.ManagedByEKS,
|
||||
OwnerTagKey: clusterEKSClusterTagKey,
|
||||
RoleTagKey: clusterEKSRoleTagKey,
|
||||
}
|
||||
}
|
||||
|
||||
// cpVPCSpec is the full build spec for clusterName's managed control-plane VPC.
|
||||
func cpVPCSpec(clusterName, region string, privateCount int) handlers_systemvpc.Spec {
|
||||
return handlers_systemvpc.Spec{
|
||||
Owner: cpVPCOwner(clusterName),
|
||||
Region: region,
|
||||
RolePrefix: cpVPCRolePrefix,
|
||||
Supernet: cpVPCSupernet,
|
||||
PrivateSubnets: privateCount,
|
||||
}
|
||||
}
|
||||
|
||||
// managedCPVPCFromRefs projects the resolved refs onto the persisted ClusterMeta
|
||||
// shape used for teardown.
|
||||
func managedCPVPCFromRefs(r *ManagedCPVPCRefs) *ManagedCPVPC {
|
||||
@@ -69,52 +92,6 @@ func managedCPVPCFromRefs(r *ManagedCPVPCRefs) *ManagedCPVPC {
|
||||
}
|
||||
}
|
||||
|
||||
// vpcProvisioner is the narrow VPC + subnet surface the managed CP VPC needs.
|
||||
// The daemon adapts the concrete VPC service onto this.
|
||||
type vpcProvisioner interface {
|
||||
CreateVpc(ctx context.Context, input *ec2.CreateVpcInput, accountID string) (*ec2.CreateVpcOutput, error)
|
||||
DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInput, accountID string) (*ec2.DeleteVpcOutput, error)
|
||||
DescribeVpcs(ctx context.Context, input *ec2.DescribeVpcsInput, accountID string) (*ec2.DescribeVpcsOutput, error)
|
||||
CreateSubnet(ctx context.Context, input *ec2.CreateSubnetInput, accountID string) (*ec2.CreateSubnetOutput, error)
|
||||
DeleteSubnet(ctx context.Context, input *ec2.DeleteSubnetInput, accountID string) (*ec2.DeleteSubnetOutput, error)
|
||||
DescribeSubnets(ctx context.Context, input *ec2.DescribeSubnetsInput, accountID string) (*ec2.DescribeSubnetsOutput, error)
|
||||
}
|
||||
|
||||
// routeTableProvisioner is the narrow route-table surface the managed CP VPC
|
||||
// needs to express public (0/0→IGW) and private (0/0→NAT GW) egress.
|
||||
type routeTableProvisioner interface {
|
||||
CreateRouteTable(ctx context.Context, input *ec2.CreateRouteTableInput, accountID string) (*ec2.CreateRouteTableOutput, error)
|
||||
DeleteRouteTable(ctx context.Context, input *ec2.DeleteRouteTableInput, accountID string) (*ec2.DeleteRouteTableOutput, error)
|
||||
DescribeRouteTables(ctx context.Context, input *ec2.DescribeRouteTablesInput, accountID string) (*ec2.DescribeRouteTablesOutput, error)
|
||||
CreateRoute(ctx context.Context, input *ec2.CreateRouteInput, accountID string) (*ec2.CreateRouteOutput, error)
|
||||
AssociateRouteTable(ctx context.Context, input *ec2.AssociateRouteTableInput, accountID string) (*ec2.AssociateRouteTableOutput, error)
|
||||
DisassociateRouteTable(ctx context.Context, input *ec2.DisassociateRouteTableInput, accountID string) (*ec2.DisassociateRouteTableOutput, error)
|
||||
}
|
||||
|
||||
// natGatewayProvisioner is the narrow NAT-gateway surface the managed CP VPC
|
||||
// needs to give the private control-plane subnet egress.
|
||||
type natGatewayProvisioner interface {
|
||||
CreateNatGateway(ctx context.Context, input *ec2.CreateNatGatewayInput, accountID string) (*ec2.CreateNatGatewayOutput, error)
|
||||
DeleteNatGateway(ctx context.Context, input *ec2.DeleteNatGatewayInput, accountID string) (*ec2.DeleteNatGatewayOutput, error)
|
||||
DescribeNatGateways(ctx context.Context, input *ec2.DescribeNatGatewaysInput, accountID string) (*ec2.DescribeNatGatewaysOutput, error)
|
||||
}
|
||||
|
||||
// CPVPCDeps bundles the EC2-family collaborators EnsureClusterCPVPC composes the
|
||||
// managed control-plane VPC from. All calls run under the system account.
|
||||
type CPVPCDeps struct {
|
||||
VPC vpcProvisioner
|
||||
// SG is used only by teardown, to clear security groups left in the managed
|
||||
// CP VPC that the tag-driven sweep cannot see. Nil is tolerated.
|
||||
SG sgProvisioner
|
||||
IGW igwProvisioner
|
||||
RT routeTableProvisioner
|
||||
NGW natGatewayProvisioner
|
||||
EIP eipProvisioner
|
||||
// NATSConn republishes vpc.delete for OVN topology GC (see
|
||||
// gcClusterCPVPCTopology). Nil is tolerated (GC is skipped, never blocking).
|
||||
NATSConn *nats.Conn
|
||||
}
|
||||
|
||||
// cpVPCDeps adapts the service's EC2-family deps onto CPVPCDeps for the managed
|
||||
// control-plane VPC build + teardown.
|
||||
func (s *EKSServiceImpl) cpVPCDeps() CPVPCDeps {
|
||||
@@ -129,502 +106,23 @@ func (s *EKSServiceImpl) cpVPCDeps() CPVPCDeps {
|
||||
}
|
||||
}
|
||||
|
||||
// ManagedCPVPCRefs is the resolved set of managed CP VPC resource IDs + CIDRs.
|
||||
// EnsureClusterCPVPC returns it; the caller persists it into ClusterMeta for
|
||||
// placement (public/private subnet selection) and teardown.
|
||||
type ManagedCPVPCRefs struct {
|
||||
VpcID string
|
||||
VpcCIDR string
|
||||
IGWID string
|
||||
PublicSubnetID string
|
||||
PublicSubnetCIDR string
|
||||
PublicRouteTableID string
|
||||
PrivateSubnetIDs []string
|
||||
PrivateSubnetCIDRs []string
|
||||
PrivateRouteTableID string
|
||||
NatGatewayID string
|
||||
NatEIPAllocationID string
|
||||
NatEIPPublicIP string
|
||||
}
|
||||
|
||||
// cpVPCCIDRs derives the managed CP VPC CIDR + subnet CIDRs deterministically
|
||||
// from the cluster name. The /22 is carved from 10.252.0.0/14; subnet 0 (.0/24)
|
||||
// is public, subnets 1..maxCPPrivateSubnets are private.
|
||||
func cpVPCCIDRs(clusterName string, privateCount int) (vpcCIDR, publicCIDR string, privateCIDRs []string) {
|
||||
h := fnv.New32a()
|
||||
_, _ = h.Write([]byte(clusterName))
|
||||
idx := int(h.Sum32() % 256) // 256 /22 blocks in a /14
|
||||
|
||||
combined := idx * 4 // each /22 spans 4 third-octet values
|
||||
second := cpVPCSupernetSecondOctet + combined/256
|
||||
third := combined % 256
|
||||
|
||||
vpcCIDR = fmt.Sprintf("10.%d.%d.0/22", second, third)
|
||||
publicCIDR = fmt.Sprintf("10.%d.%d.0/24", second, third)
|
||||
if privateCount < 1 {
|
||||
privateCount = 1
|
||||
}
|
||||
if privateCount > maxCPPrivateSubnets {
|
||||
privateCount = maxCPPrivateSubnets
|
||||
}
|
||||
for k := 0; k < privateCount; k++ {
|
||||
privateCIDRs = append(privateCIDRs, fmt.Sprintf("10.%d.%d.0/24", second, third+1+k))
|
||||
}
|
||||
return vpcCIDR, publicCIDR, privateCIDRs
|
||||
}
|
||||
|
||||
// cpVPCAZ returns the AvailabilityZone name for subnet index k (cosmetic AWS
|
||||
// parity; spinifex spreads CP by host placement group, not AZ).
|
||||
func cpVPCAZ(region string, k int) string {
|
||||
return fmt.Sprintf("%s%c", region, 'a'+k)
|
||||
}
|
||||
|
||||
func cpVPCTagSpec(resourceType, clusterName, role string) []*ec2.TagSpecification {
|
||||
return []*ec2.TagSpecification{{
|
||||
ResourceType: aws.String(resourceType),
|
||||
Tags: []*ec2.Tag{
|
||||
{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)},
|
||||
{Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(clusterName)},
|
||||
{Key: aws.String(clusterEKSRoleTagKey), Value: aws.String(role)},
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
func cpClusterRoleFilters(clusterName, role string) []*ec2.Filter {
|
||||
return []*ec2.Filter{
|
||||
{Name: aws.String("tag:" + clusterEKSClusterTagKey), Values: aws.StringSlice([]string{clusterName})},
|
||||
{Name: aws.String("tag:" + clusterEKSRoleTagKey), Values: aws.StringSlice([]string{role})},
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureClusterCPVPC builds (idempotently) the managed control-plane VPC under
|
||||
// accountID (the system account) for clusterName: a VPC, an attached IGW, a
|
||||
// public subnet routed to the IGW, privateCount private subnet(s) routed to a
|
||||
// NAT gateway, and the NAT gateway itself. Each resource is describe-or-create
|
||||
// keyed on the cluster + role tags so a relaunch after partial failure converges
|
||||
// rather than duplicating. The route-table associations publish the topology
|
||||
// events the per-subnet egress subscribers consume, so the OVN policy split
|
||||
// (public 1000 / NAT-GW reroute / private 1100 drop) falls out without any
|
||||
// direct OVN mutation here.
|
||||
// accountID (the system account) for clusterName.
|
||||
func EnsureClusterCPVPC(ctx context.Context, deps CPVPCDeps, accountID, clusterName, region string, privateCount int) (*ManagedCPVPCRefs, error) {
|
||||
if clusterName == "" {
|
||||
return nil, errors.New("eks: EnsureClusterCPVPC empty cluster name")
|
||||
}
|
||||
vpcCIDR, publicCIDR, privateCIDRs := cpVPCCIDRs(clusterName, privateCount)
|
||||
refs := &ManagedCPVPCRefs{VpcCIDR: vpcCIDR, PublicSubnetCIDR: publicCIDR, PrivateSubnetCIDRs: privateCIDRs}
|
||||
|
||||
vpcID, err := ensureCPVPC(ctx, deps.VPC, accountID, clusterName, vpcCIDR)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs.VpcID = vpcID
|
||||
|
||||
if err := EnsureClusterIGW(ctx, deps.IGW, accountID, vpcID, clusterName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
igw, err := attachedVPCIGW(ctx, deps.IGW, accountID, vpcID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if igw == nil {
|
||||
return nil, fmt.Errorf("eks: cp vpc %s has no attached IGW after ensure", vpcID)
|
||||
}
|
||||
refs.IGWID = aws.StringValue(igw.InternetGatewayId)
|
||||
|
||||
pubSubnet, err := ensureCPSubnet(ctx, deps.VPC, accountID, clusterName, vpcID, publicCIDR, clusterEKSRolePublicSubnet, cpVPCAZ(region, 0))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs.PublicSubnetID = pubSubnet
|
||||
|
||||
for k, cidr := range privateCIDRs {
|
||||
priv, err := ensureCPSubnet(ctx, deps.VPC, accountID, clusterName, vpcID, cidr, clusterEKSRolePrivateSubnet, cpVPCAZ(region, k))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs.PrivateSubnetIDs = append(refs.PrivateSubnetIDs, priv)
|
||||
}
|
||||
|
||||
// Public route table: 0.0.0.0/0 → IGW, associated to the public subnet. The
|
||||
// association publishes vpc.add-igw-route → EnsureSubnetEgress (1000 reroute).
|
||||
pubRT, err := ensureCPRouteTable(ctx, deps.RT, accountID, clusterName, vpcID, clusterEKSRolePublicRT,
|
||||
&ec2.CreateRouteInput{DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String(refs.IGWID)},
|
||||
[]string{refs.PublicSubnetID})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs.PublicRouteTableID = pubRT
|
||||
|
||||
// NAT gateway in the public subnet, fed by a dedicated EIP. The private CP
|
||||
// subnets route 0.0.0.0/0 → this NAT GW for egress-only internet (image pulls).
|
||||
natID, allocID, natIP, err := ensureCPNatGateway(ctx, deps, accountID, clusterName, refs.PublicSubnetID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs.NatGatewayID = natID
|
||||
refs.NatEIPAllocationID = allocID
|
||||
refs.NatEIPPublicIP = natIP
|
||||
|
||||
// Private route table: 0.0.0.0/0 → NAT GW, associated to every private subnet.
|
||||
// The association publishes vpc.add-nat-gateway → EnsureNATGatewaySubnetEgress.
|
||||
privRT, err := ensureCPRouteTable(ctx, deps.RT, accountID, clusterName, vpcID, clusterEKSRolePrivateRT,
|
||||
&ec2.CreateRouteInput{DestinationCidrBlock: aws.String("0.0.0.0/0"), NatGatewayId: aws.String(natID)},
|
||||
refs.PrivateSubnetIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs.PrivateRouteTableID = privRT
|
||||
|
||||
slog.InfoContext(ctx, "EnsureClusterCPVPC: managed control-plane VPC ready",
|
||||
"cluster", clusterName, "vpc", refs.VpcID, "publicSubnet", refs.PublicSubnetID,
|
||||
"privateSubnets", refs.PrivateSubnetIDs, "natgw", refs.NatGatewayID)
|
||||
return refs, nil
|
||||
return handlers_systemvpc.Ensure(ctx, deps, cpVPCSpec(clusterName, region, privateCount), accountID)
|
||||
}
|
||||
|
||||
func ensureCPVPC(ctx context.Context, vpcp vpcProvisioner, accountID, clusterName, cidr string) (string, error) {
|
||||
out, err := vpcp.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{Filters: cpClusterRoleFilters(clusterName, clusterEKSRoleCPVPC)}, accountID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("eks: describe cp vpc for %s: %w", clusterName, err)
|
||||
}
|
||||
if out != nil && len(out.Vpcs) > 0 {
|
||||
return aws.StringValue(out.Vpcs[0].VpcId), nil
|
||||
}
|
||||
created, err := vpcp.CreateVpc(ctx, &ec2.CreateVpcInput{
|
||||
CidrBlock: aws.String(cidr),
|
||||
TagSpecifications: cpVPCTagSpec("vpc", clusterName, clusterEKSRoleCPVPC),
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("eks: create cp vpc for %s: %w", clusterName, err)
|
||||
}
|
||||
if created == nil || created.Vpc == nil || aws.StringValue(created.Vpc.VpcId) == "" {
|
||||
return "", fmt.Errorf("eks: create cp vpc for %s: empty vpc id", clusterName)
|
||||
}
|
||||
return aws.StringValue(created.Vpc.VpcId), nil
|
||||
}
|
||||
|
||||
func ensureCPSubnet(ctx context.Context, vpcp vpcProvisioner, accountID, clusterName, vpcID, cidr, role, az string) (string, error) {
|
||||
out, err := vpcp.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{Filters: append(
|
||||
cpClusterRoleFilters(clusterName, role),
|
||||
&ec2.Filter{Name: aws.String("cidr-block"), Values: aws.StringSlice([]string{cidr})},
|
||||
)}, accountID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("eks: describe cp subnet %s (%s): %w", cidr, role, err)
|
||||
}
|
||||
if out != nil && len(out.Subnets) > 0 {
|
||||
return aws.StringValue(out.Subnets[0].SubnetId), nil
|
||||
}
|
||||
created, err := vpcp.CreateSubnet(ctx, &ec2.CreateSubnetInput{
|
||||
VpcId: aws.String(vpcID),
|
||||
CidrBlock: aws.String(cidr),
|
||||
AvailabilityZone: aws.String(az),
|
||||
TagSpecifications: cpVPCTagSpec("subnet", clusterName, role),
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("eks: create cp subnet %s (%s): %w", cidr, role, err)
|
||||
}
|
||||
if created == nil || created.Subnet == nil || aws.StringValue(created.Subnet.SubnetId) == "" {
|
||||
return "", fmt.Errorf("eks: create cp subnet %s (%s): empty subnet id", cidr, role)
|
||||
}
|
||||
return aws.StringValue(created.Subnet.SubnetId), nil
|
||||
}
|
||||
|
||||
// ensureCPRouteTable describe-or-creates the role-tagged route table, installs
|
||||
// the default route, and associates it to subnetIDs. Idempotent: a re-run reuses
|
||||
// the existing table and its associations (AssociateRouteTable is a no-op for an
|
||||
// already-associated subnet at the service layer).
|
||||
func ensureCPRouteTable(ctx context.Context, rtp routeTableProvisioner, accountID, clusterName, vpcID, role string, route *ec2.CreateRouteInput, subnetIDs []string) (string, error) {
|
||||
rtID, fresh, err := describeOrCreateCPRouteTable(ctx, rtp, accountID, clusterName, vpcID, role)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if fresh {
|
||||
route.RouteTableId = aws.String(rtID)
|
||||
if _, err := rtp.CreateRoute(ctx, route, accountID); err != nil {
|
||||
return "", fmt.Errorf("eks: create cp route (%s) on %s: %w", role, rtID, err)
|
||||
}
|
||||
}
|
||||
for _, sn := range subnetIDs {
|
||||
if sn == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := rtp.AssociateRouteTable(ctx, &ec2.AssociateRouteTableInput{
|
||||
RouteTableId: aws.String(rtID),
|
||||
SubnetId: aws.String(sn),
|
||||
}, accountID); err != nil {
|
||||
return "", fmt.Errorf("eks: associate cp route table %s → subnet %s: %w", rtID, sn, err)
|
||||
}
|
||||
}
|
||||
return rtID, nil
|
||||
}
|
||||
|
||||
func describeOrCreateCPRouteTable(ctx context.Context, rtp routeTableProvisioner, accountID, clusterName, vpcID, role string) (rtID string, fresh bool, err error) {
|
||||
out, err := rtp.DescribeRouteTables(ctx, &ec2.DescribeRouteTablesInput{Filters: cpClusterRoleFilters(clusterName, role)}, accountID)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("eks: describe cp route table (%s): %w", role, err)
|
||||
}
|
||||
if out != nil && len(out.RouteTables) > 0 {
|
||||
return aws.StringValue(out.RouteTables[0].RouteTableId), false, nil
|
||||
}
|
||||
created, err := rtp.CreateRouteTable(ctx, &ec2.CreateRouteTableInput{
|
||||
VpcId: aws.String(vpcID),
|
||||
TagSpecifications: cpVPCTagSpec("route-table", clusterName, role),
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("eks: create cp route table (%s): %w", role, err)
|
||||
}
|
||||
if created == nil || created.RouteTable == nil || aws.StringValue(created.RouteTable.RouteTableId) == "" {
|
||||
return "", false, fmt.Errorf("eks: create cp route table (%s): empty id", role)
|
||||
}
|
||||
return aws.StringValue(created.RouteTable.RouteTableId), true, nil
|
||||
}
|
||||
|
||||
func ensureCPNatGateway(ctx context.Context, deps CPVPCDeps, accountID, clusterName, publicSubnetID string) (natID, allocID, publicIP string, err error) {
|
||||
out, err := deps.NGW.DescribeNatGateways(ctx, &ec2.DescribeNatGatewaysInput{Filter: append(
|
||||
cpClusterRoleFilters(clusterName, clusterEKSRoleCPNatGW),
|
||||
&ec2.Filter{Name: aws.String("state"), Values: aws.StringSlice([]string{"available", "pending"})},
|
||||
)}, accountID)
|
||||
if err != nil {
|
||||
return "", "", "", fmt.Errorf("eks: describe cp nat gateway for %s: %w", clusterName, err)
|
||||
}
|
||||
if out != nil && len(out.NatGateways) > 0 {
|
||||
ng := out.NatGateways[0]
|
||||
var alloc, ip string
|
||||
if len(ng.NatGatewayAddresses) > 0 {
|
||||
alloc = aws.StringValue(ng.NatGatewayAddresses[0].AllocationId)
|
||||
ip = aws.StringValue(ng.NatGatewayAddresses[0].PublicIp)
|
||||
}
|
||||
return aws.StringValue(ng.NatGatewayId), alloc, ip, nil
|
||||
}
|
||||
|
||||
eip, err := deps.EIP.AllocateAddress(ctx, &ec2.AllocateAddressInput{
|
||||
Domain: aws.String("vpc"),
|
||||
TagSpecifications: cpVPCTagSpec("elastic-ip", clusterName, clusterEKSRoleCPNatGW),
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return "", "", "", fmt.Errorf("eks: allocate cp nat gateway EIP for %s: %w", clusterName, err)
|
||||
}
|
||||
if eip == nil || aws.StringValue(eip.AllocationId) == "" {
|
||||
return "", "", "", fmt.Errorf("eks: allocate cp nat gateway EIP for %s: empty allocation", clusterName)
|
||||
}
|
||||
allocID = aws.StringValue(eip.AllocationId)
|
||||
publicIP = aws.StringValue(eip.PublicIp)
|
||||
|
||||
created, err := deps.NGW.CreateNatGateway(ctx, &ec2.CreateNatGatewayInput{
|
||||
SubnetId: aws.String(publicSubnetID),
|
||||
AllocationId: aws.String(allocID),
|
||||
TagSpecifications: cpVPCTagSpec("natgateway", clusterName, clusterEKSRoleCPNatGW),
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
// Release the orphaned EIP so a failed NAT-GW create does not leak it.
|
||||
if _, relErr := deps.EIP.ReleaseAddress(ctx, &ec2.ReleaseAddressInput{AllocationId: aws.String(allocID)}, accountID); relErr != nil {
|
||||
slog.WarnContext(ctx, "ensureCPNatGateway: failed to release EIP after NAT-GW create failure", "alloc", allocID, "err", relErr)
|
||||
}
|
||||
return "", "", "", fmt.Errorf("eks: create cp nat gateway for %s: %w", clusterName, err)
|
||||
}
|
||||
if created == nil || created.NatGateway == nil || aws.StringValue(created.NatGateway.NatGatewayId) == "" {
|
||||
return "", "", "", fmt.Errorf("eks: create cp nat gateway for %s: empty id", clusterName)
|
||||
}
|
||||
return aws.StringValue(created.NatGateway.NatGatewayId), allocID, publicIP, nil
|
||||
}
|
||||
|
||||
// DeleteClusterCPVPC tears down the managed control-plane VPC for clusterName in
|
||||
// dependency order: NAT gateway (+ its EIP) → route tables → subnets → IGW →
|
||||
// VPC. Tag-driven (not meta-driven) so a partial create still converges to a
|
||||
// clean delete. Best-effort: every step is attempted and failures are logged;
|
||||
// the final VPC delete failure is returned so the caller can surface a leak.
|
||||
// Safe to call when nothing was provisioned (all describes return empty).
|
||||
// DeleteClusterCPVPC tears down the managed control-plane VPC for clusterName.
|
||||
//
|
||||
// knownRefs is the caller's last-persisted cp-vpc state record (ClusterMeta's
|
||||
// ManagedCPVPC), used only as an OVN-GC target fallback when the tag-indexed
|
||||
// EC2 VPC is already gone — the live describe can no longer name it, but the
|
||||
// record still can. Pass nil when no such record exists (e.g. the billable
|
||||
// reaper, which acts once the cluster meta itself is gone).
|
||||
// knownRefs is the caller's last-persisted cp-vpc record, used only as an OVN-GC
|
||||
// fallback when the tag-indexed EC2 VPC is already gone. Pass nil when no such
|
||||
// record exists, e.g. the billable reaper acting after the cluster meta.
|
||||
func DeleteClusterCPVPC(ctx context.Context, deps CPVPCDeps, accountID, clusterName string, knownRefs *ManagedCPVPC) error {
|
||||
if clusterName == "" {
|
||||
return errors.New("eks: DeleteClusterCPVPC empty cluster name")
|
||||
gcFallback := ""
|
||||
if knownRefs != nil {
|
||||
gcFallback = knownRefs.VpcId
|
||||
}
|
||||
|
||||
// 1. Route tables: disassociate every subnet, then delete. Disassociation
|
||||
// publishes the egress-reroute removal for each subnet. Route tables go first
|
||||
// so the NAT gateway's live route refs are gone before its delete — the NAT
|
||||
// GW delete is guarded against live route forwards (rule #3), so deleting it
|
||||
// while the private RT still routes 0.0.0.0/0 to it would fail (and strand
|
||||
// its billable EIP).
|
||||
for _, role := range []string{clusterEKSRolePublicRT, clusterEKSRolePrivateRT} {
|
||||
rtOut, err := deps.RT.DescribeRouteTables(ctx, &ec2.DescribeRouteTablesInput{Filters: cpClusterRoleFilters(clusterName, role)}, accountID)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterCPVPC: describe route tables failed", "role", role, "err", err)
|
||||
continue
|
||||
}
|
||||
if rtOut == nil {
|
||||
continue
|
||||
}
|
||||
for _, rt := range rtOut.RouteTables {
|
||||
rtID := aws.StringValue(rt.RouteTableId)
|
||||
for _, assoc := range rt.Associations {
|
||||
if aws.BoolValue(assoc.Main) {
|
||||
continue
|
||||
}
|
||||
if aID := aws.StringValue(assoc.RouteTableAssociationId); aID != "" {
|
||||
if _, err := deps.RT.DisassociateRouteTable(ctx, &ec2.DisassociateRouteTableInput{AssociationId: aws.String(aID)}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterCPVPC: disassociate route table failed", "assoc", aID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := deps.RT.DeleteRouteTable(ctx, &ec2.DeleteRouteTableInput{RouteTableId: aws.String(rtID)}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterCPVPC: delete route table failed", "rt", rtID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. NAT gateway + its EIP. DeleteNatGateway publishes the SNAT removal and
|
||||
// disassociates the EIP; release follows so the billable address is reclaimed.
|
||||
if ngOut, err := deps.NGW.DescribeNatGateways(ctx, &ec2.DescribeNatGatewaysInput{
|
||||
Filter: cpClusterRoleFilters(clusterName, clusterEKSRoleCPNatGW),
|
||||
}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterCPVPC: describe NAT gateways failed", "cluster", clusterName, "err", err)
|
||||
} else if ngOut != nil {
|
||||
for _, ng := range ngOut.NatGateways {
|
||||
if state := aws.StringValue(ng.State); state == "deleted" || state == "deleting" {
|
||||
continue
|
||||
}
|
||||
ngID := aws.StringValue(ng.NatGatewayId)
|
||||
if _, err := deps.NGW.DeleteNatGateway(ctx, &ec2.DeleteNatGatewayInput{NatGatewayId: aws.String(ngID)}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterCPVPC: delete NAT gateway failed", "natgw", ngID, "err", err)
|
||||
}
|
||||
for _, addr := range ng.NatGatewayAddresses {
|
||||
if alloc := aws.StringValue(addr.AllocationId); alloc != "" {
|
||||
if _, err := deps.EIP.ReleaseAddress(ctx, &ec2.ReleaseAddressInput{AllocationId: aws.String(alloc)}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterCPVPC: release NAT EIP failed", "alloc", alloc, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Subnets (public + private).
|
||||
for _, role := range []string{clusterEKSRolePublicSubnet, clusterEKSRolePrivateSubnet} {
|
||||
snOut, err := deps.VPC.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{Filters: cpClusterRoleFilters(clusterName, role)}, accountID)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterCPVPC: describe subnets failed", "role", role, "err", err)
|
||||
continue
|
||||
}
|
||||
if snOut == nil {
|
||||
continue
|
||||
}
|
||||
for _, sn := range snOut.Subnets {
|
||||
snID := aws.StringValue(sn.SubnetId)
|
||||
if _, err := deps.VPC.DeleteSubnet(ctx, &ec2.DeleteSubnetInput{SubnetId: aws.String(snID)}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterCPVPC: delete subnet failed", "subnet", snID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. VPC + its IGW. Resolve the VPC first so DeleteClusterIGW can detach.
|
||||
vpcOut, err := deps.VPC.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{Filters: cpClusterRoleFilters(clusterName, clusterEKSRoleCPVPC)}, accountID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("eks: describe cp vpc for delete (%s): %w", clusterName, err)
|
||||
}
|
||||
|
||||
// The tag-indexed VPC is already gone — a prior re-drive completed the EC2
|
||||
// delete, or it was reclaimed out-of-band. Idempotent success: nothing left
|
||||
// to delete here. Its OVN logical router/subnets/egress-drop policies may
|
||||
// still be orphaned if that prior delete's fire-and-forget vpc.delete event
|
||||
// never reached vpcd (e.g. NATS was down), so GC still runs below against
|
||||
// knownRefs — the only surviving reference to its identity.
|
||||
if vpcOut == nil || len(vpcOut.Vpcs) == 0 {
|
||||
gcClusterCPVPCTopology(ctx, deps.NATSConn, clusterName, cpVPCGCTarget(knownRefs))
|
||||
return nil
|
||||
}
|
||||
vpcID := aws.StringValue(vpcOut.Vpcs[0].VpcId)
|
||||
|
||||
if err := DeleteClusterIGW(ctx, deps.IGW, accountID, vpcID, clusterName); err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterCPVPC: delete IGW failed", "vpc", vpcID, "err", err)
|
||||
}
|
||||
|
||||
// Sweep residual subnets by VPC identity, not by role tag.
|
||||
//
|
||||
// The tagged sweep above only finds subnets that survived long enough to be
|
||||
// tagged. A create that fails partway — the first control-plane server
|
||||
// failing to launch, say — leaves untagged subnets behind, which the tagged
|
||||
// sweep cannot see. DeleteVpc then rejects with DependencyViolation, and
|
||||
// because teardown is re-driven on a timer the cluster sits in DELETING
|
||||
// forever, retrying the same doomed delete every couple of minutes. The
|
||||
// cluster name stays unusable and terraform's delete wait can never pass.
|
||||
if snOut, err := deps.VPC.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{
|
||||
Filters: []*ec2.Filter{{Name: aws.String("vpc-id"), Values: aws.StringSlice([]string{vpcID})}},
|
||||
}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterCPVPC: describe residual subnets failed", "vpc", vpcID, "err", err)
|
||||
} else if snOut != nil {
|
||||
for _, sn := range snOut.Subnets {
|
||||
snID := aws.StringValue(sn.SubnetId)
|
||||
slog.InfoContext(ctx, "DeleteClusterCPVPC: removing residual subnet the tagged sweep missed",
|
||||
"cluster", clusterName, "vpc", vpcID, "subnet", snID)
|
||||
if _, err := deps.VPC.DeleteSubnet(ctx, &ec2.DeleteSubnetInput{SubnetId: aws.String(snID)}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterCPVPC: delete residual subnet failed", "subnet", snID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Same for security groups: DeleteVpc rejects any non-default SG, and the
|
||||
// tagged CP-SG teardown misses ones a partial create never tagged.
|
||||
if deps.SG != nil {
|
||||
if sgOut, err := deps.SG.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{
|
||||
Filters: []*ec2.Filter{{Name: aws.String("vpc-id"), Values: aws.StringSlice([]string{vpcID})}},
|
||||
}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterCPVPC: describe residual SGs failed", "vpc", vpcID, "err", err)
|
||||
} else if sgOut != nil {
|
||||
for _, sg := range sgOut.SecurityGroups {
|
||||
// The default SG goes with the VPC cascade and cannot be
|
||||
// deleted on its own; it is not what blocks DeleteVpc.
|
||||
if aws.StringValue(sg.GroupName) == "default" {
|
||||
continue
|
||||
}
|
||||
sgID := aws.StringValue(sg.GroupId)
|
||||
slog.InfoContext(ctx, "DeleteClusterCPVPC: removing residual SG the tagged sweep missed",
|
||||
"cluster", clusterName, "vpc", vpcID, "sg", sgID)
|
||||
if _, err := deps.SG.DeleteSecurityGroup(ctx, &ec2.DeleteSecurityGroupInput{GroupId: aws.String(sgID)}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterCPVPC: delete residual SG failed", "sg", sgID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := deps.VPC.DeleteVpc(ctx, &ec2.DeleteVpcInput{VpcId: aws.String(vpcID)}, accountID); err != nil && !awserrors.IsNotFound(err) {
|
||||
return fmt.Errorf("eks: delete cp vpc %s: %w", vpcID, err)
|
||||
}
|
||||
gcClusterCPVPCTopology(ctx, deps.NATSConn, clusterName, vpcID)
|
||||
slog.InfoContext(ctx, "DeleteClusterCPVPC: managed control-plane VPC removed", "cluster", clusterName, "vpc", vpcID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// cpVPCGCTarget resolves the best-known VpcId for OVN GC when the tag-indexed
|
||||
// EC2 record is already gone: the persisted (internal cp-vpc state record)
|
||||
// VpcId, or "" if none was ever recorded (nil knownRefs).
|
||||
func cpVPCGCTarget(knownRefs *ManagedCPVPC) string {
|
||||
if knownRefs == nil {
|
||||
return ""
|
||||
}
|
||||
return knownRefs.VpcId
|
||||
}
|
||||
|
||||
// gcClusterCPVPCTopology republishes vpc.delete for vpcID so vpcd's topology
|
||||
// manager gets another chance to remove the CP VPC's OVN logical router,
|
||||
// subnets, DHCP options, and SubnetEgressPriorityDrop policies (owned by the
|
||||
// router row, so OVSDB cascades their removal with it). That cleanup normally
|
||||
// runs only as a side effect of a *live* DeleteVpc KV mutation — a re-drive
|
||||
// against an already-gone VPC does not mutate the KV again, so a lost or
|
||||
// never-delivered event would otherwise orphan the OVN state forever.
|
||||
// Best-effort and idempotent: a nil conn or empty vpcID is a no-op, and vpcd
|
||||
// tolerates re-deleting an already-absent router.
|
||||
func gcClusterCPVPCTopology(ctx context.Context, nc *nats.Conn, clusterName, vpcID string) {
|
||||
if nc == nil || vpcID == "" {
|
||||
return
|
||||
}
|
||||
utils.PublishEvent(nc, "vpc.delete", struct {
|
||||
VpcId string `json:"vpc_id"`
|
||||
CidrBlock string `json:"cidr_block"`
|
||||
VNI int64 `json:"vni"`
|
||||
}{VpcId: vpcID})
|
||||
slog.InfoContext(ctx, "DeleteClusterCPVPC: republished vpc.delete for OVN GC", "cluster", clusterName, "vpc", vpcID)
|
||||
// Region and private-subnet count do not affect teardown: every lookup is a
|
||||
// describe-by-tag, and the CIDRs are never re-derived.
|
||||
return handlers_systemvpc.Delete(ctx, deps, cpVPCSpec(clusterName, "", cpVPCPrivateSubnetCount), accountID, gcFallback)
|
||||
}
|
||||
|
||||
@@ -4,17 +4,22 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
"github.com/mulgadc/spinifex/spinifex/config"
|
||||
handlers_systemvpc "github.com/mulgadc/spinifex/spinifex/handlers/systemvpc"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Fakes for the managed control-plane VPC ("Set B") collaborators. They model
|
||||
// describe-or-create idempotency by storing each resource with the tag map its
|
||||
// create spec carried, so a re-describe under the same cluster+role filters
|
||||
// returns the existing resource rather than a duplicate.
|
||||
// Fakes for the managed control-plane VPC collaborators. They model
|
||||
// describe-or-create idempotency by storing each resource with its create tags,
|
||||
// so a re-describe under the same filters returns it rather than a duplicate.
|
||||
|
||||
var (
|
||||
_ vpcProvisioner = (*fakeVPCProvisioner)(nil)
|
||||
@@ -397,3 +402,36 @@ func (f *fakeNatGatewayProvisioner) DeleteNatGateway(_ context.Context, input *e
|
||||
}
|
||||
return &ec2.DeleteNatGatewayOutput{}, nil
|
||||
}
|
||||
|
||||
// --- Address-space ownership ---
|
||||
|
||||
// TestCPVPCTagValuesAreFrozen pins the values live CP VPCs already carry. Every
|
||||
// other test in the package seeds its fake and filters through cpVPCRoles, so
|
||||
// only a literal assertion catches a change to the prefix or the supernet — and
|
||||
// a change to either orphans every deployed cluster from its own teardown,
|
||||
// stranding a billable NAT gateway and hanging DeleteCluster in DELETING.
|
||||
func TestCPVPCTagValuesAreFrozen(t *testing.T) {
|
||||
assert.Equal(t, handlers_systemvpc.Roles{
|
||||
VPC: "cp-vpc",
|
||||
PublicSubnet: "cp-public",
|
||||
PrivateSubnet: "cp-private",
|
||||
PublicRT: "cp-public-rt",
|
||||
PrivateRT: "cp-private-rt",
|
||||
NatGW: "cp-natgw",
|
||||
}, cpVPCRoles, "these role tags are baked into live deployments; DeleteClusterCPVPC and EKSBillableReaper find resources by them")
|
||||
assert.Equal(t, "10.252.0.0/14", cpVPCSupernet, "existing clusters' CIDRs are hashed out of this block")
|
||||
}
|
||||
|
||||
func TestCPVPCSupernetIsDisjointFromOtherComponents(t *testing.T) {
|
||||
supernet, err := netip.ParsePrefix(cpVPCSupernet)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, supernet.Masked(), supernet, "a supernet with host bits set is rejected by the builder")
|
||||
assert.Equal(t, 14, supernet.Bits(), "the builder carves a /22 out of a /14 and requires that shape")
|
||||
|
||||
// Every other component that builds a system VPC out of the same builder
|
||||
// gets its own block, so an operator can tell from an address which
|
||||
// component owns it.
|
||||
rds := netip.MustParsePrefix(config.RDSDefaultSystemVPCSupernet)
|
||||
assert.False(t, supernet.Overlaps(rds),
|
||||
"the EKS control-plane space %s must not overlap the RDS system VPC space %s", supernet, rds)
|
||||
}
|
||||
|
||||
@@ -190,14 +190,14 @@ func TestRLC5_DeleteClusterCPVPCReleasesNATGWEIPAfterRoutes(t *testing.T) {
|
||||
f.ngw.routeGuard = f.rt
|
||||
f.ngw.gws = []*fakeCPNatGateway{{
|
||||
id: "nat-cp",
|
||||
tags: map[string]string{clusterEKSClusterTagKey: "alpha", clusterEKSRoleTagKey: clusterEKSRoleCPNatGW},
|
||||
tags: map[string]string{clusterEKSClusterTagKey: "alpha", clusterEKSRoleTagKey: cpVPCRoles.NatGW},
|
||||
state: "available",
|
||||
addrs: []*ec2.NatGatewayAddress{{AllocationId: aws.String("eipalloc-cp")}},
|
||||
}}
|
||||
f.rt.tables = []*fakeCPRouteTable{{
|
||||
id: "rtb-cp",
|
||||
vpc: "vpc-cp",
|
||||
tags: map[string]string{clusterEKSClusterTagKey: "alpha", clusterEKSRoleTagKey: clusterEKSRolePrivateRT},
|
||||
tags: map[string]string{clusterEKSClusterTagKey: "alpha", clusterEKSRoleTagKey: cpVPCRoles.PrivateRT},
|
||||
}}
|
||||
|
||||
require.NoError(t, DeleteClusterCPVPC(context.Background(), f.svc.cpVPCDeps(), testAccountID, "alpha", nil))
|
||||
@@ -218,7 +218,7 @@ func TestRLC1_DeleteClusterCPVPCToleratesAbsentVPC(t *testing.T) {
|
||||
cpVPC := []*fakeCPVPC{{
|
||||
id: "vpc-cp",
|
||||
cidr: "10.0.0.0/16",
|
||||
tags: map[string]string{clusterEKSClusterTagKey: "alpha", clusterEKSRoleTagKey: clusterEKSRoleCPVPC},
|
||||
tags: map[string]string{clusterEKSClusterTagKey: "alpha", clusterEKSRoleTagKey: cpVPCRoles.VPC},
|
||||
}}
|
||||
|
||||
t.Run("absent VPC (NotFound) converges to success", func(t *testing.T) {
|
||||
@@ -503,7 +503,7 @@ func TestDeleteClusterCPVPC_OVNGCSelection(t *testing.T) {
|
||||
f.vpcMgr.vpcs = []*fakeCPVPC{{
|
||||
id: "vpc-cp-live",
|
||||
cidr: "10.0.0.0/16",
|
||||
tags: map[string]string{clusterEKSClusterTagKey: "alpha", clusterEKSRoleTagKey: clusterEKSRoleCPVPC},
|
||||
tags: map[string]string{clusterEKSClusterTagKey: "alpha", clusterEKSRoleTagKey: cpVPCRoles.VPC},
|
||||
}}
|
||||
nextVPCID := subscribeVPCDelete(t, f.svc.deps.NATSConn)
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ func TestRLC5_EKSBillableReaperTerminatesOrphanCPVM(t *testing.T) {
|
||||
}
|
||||
f.ngw.gws = []*fakeCPNatGateway{{
|
||||
id: "nat-orphan",
|
||||
tags: map[string]string{clusterEKSClusterTagKey: "gone-cluster", clusterEKSRoleTagKey: clusterEKSRoleCPNatGW},
|
||||
tags: map[string]string{clusterEKSClusterTagKey: "gone-cluster", clusterEKSRoleTagKey: cpVPCRoles.NatGW},
|
||||
state: "available",
|
||||
addrs: []*ec2.NatGatewayAddress{{AllocationId: aws.String("eipalloc-orphan")}},
|
||||
}}
|
||||
|
||||
+12
-123
@@ -3,141 +3,30 @@ package handlers_eks
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
"github.com/mulgadc/spinifex/spinifex/tags"
|
||||
handlers_systemvpc "github.com/mulgadc/spinifex/spinifex/handlers/systemvpc"
|
||||
)
|
||||
|
||||
// igwProvisioner is the narrow Internet Gateway surface the cluster IGW helpers
|
||||
// need. An internet-facing cluster endpoint allocates an external front-end IP
|
||||
// and an OVN dnat_and_snat for it, but the external switch + localnet + gateway
|
||||
// LRP that make that IP answerable on the physical wire are built only when an
|
||||
// IGW is attached to the VPC. The control-plane NLB is a system instance — no
|
||||
// customer provisions the VPC's IGW for it — so EKS ensures one itself. The
|
||||
// daemon adapts the concrete IGW service onto this.
|
||||
type igwProvisioner interface {
|
||||
DescribeInternetGateways(ctx context.Context, input *ec2.DescribeInternetGatewaysInput, accountID string) (*ec2.DescribeInternetGatewaysOutput, error)
|
||||
CreateInternetGateway(ctx context.Context, input *ec2.CreateInternetGatewayInput, accountID string) (*ec2.CreateInternetGatewayOutput, error)
|
||||
AttachInternetGateway(ctx context.Context, input *ec2.AttachInternetGatewayInput, accountID string) (*ec2.AttachInternetGatewayOutput, error)
|
||||
DetachInternetGateway(ctx context.Context, input *ec2.DetachInternetGatewayInput, accountID string) (*ec2.DetachInternetGatewayOutput, error)
|
||||
DeleteInternetGateway(ctx context.Context, input *ec2.DeleteInternetGatewayInput, accountID string) (*ec2.DeleteInternetGatewayOutput, error)
|
||||
}
|
||||
// igwProvisioner is the Internet Gateway surface the cluster IGW helpers need,
|
||||
// under its EKS-local name. The daemon adapts the concrete IGW service onto it.
|
||||
type igwProvisioner = handlers_systemvpc.IGWProvisioner
|
||||
|
||||
// EnsureClusterIGW guarantees vpcID has an attached Internet Gateway so an
|
||||
// internet-facing cluster endpoint is reachable on the external network.
|
||||
// Idempotent and non-destructive: if the VPC already has an attached IGW
|
||||
// (customer-provisioned or from a prior launch) it is reused as-is and left
|
||||
// untagged; only when none exists is a cluster-owned IGW created and attached.
|
||||
// internet-facing cluster endpoint is reachable. An already-attached IGW is
|
||||
// reused as-is; only when none exists is a cluster-owned one created.
|
||||
func EnsureClusterIGW(ctx context.Context, igwp igwProvisioner, accountID, vpcID, clusterName string) error {
|
||||
if vpcID == "" {
|
||||
return errors.New("eks: EnsureClusterIGW empty vpc id")
|
||||
}
|
||||
if clusterName == "" {
|
||||
return errors.New("eks: EnsureClusterIGW empty cluster name")
|
||||
}
|
||||
|
||||
existing, err := attachedVPCIGW(ctx, igwp, accountID, vpcID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if existing != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
out, err := igwp.CreateInternetGateway(ctx, &ec2.CreateInternetGatewayInput{
|
||||
TagSpecifications: []*ec2.TagSpecification{{
|
||||
ResourceType: aws.String("internet-gateway"),
|
||||
Tags: []*ec2.Tag{
|
||||
{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)},
|
||||
{Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(clusterName)},
|
||||
},
|
||||
}},
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("eks: create cluster IGW for vpc %s: %w", vpcID, err)
|
||||
}
|
||||
if out == nil || out.InternetGateway == nil || aws.StringValue(out.InternetGateway.InternetGatewayId) == "" {
|
||||
return fmt.Errorf("eks: create cluster IGW for vpc %s: empty gateway id", vpcID)
|
||||
}
|
||||
igwID := aws.StringValue(out.InternetGateway.InternetGatewayId)
|
||||
|
||||
if _, err := igwp.AttachInternetGateway(ctx, &ec2.AttachInternetGatewayInput{
|
||||
InternetGatewayId: aws.String(igwID),
|
||||
VpcId: aws.String(vpcID),
|
||||
}, accountID); err != nil {
|
||||
return fmt.Errorf("eks: attach cluster IGW %s to vpc %s: %w", igwID, vpcID, err)
|
||||
}
|
||||
slog.InfoContext(ctx, "EnsureClusterIGW: attached cluster IGW", "igw", igwID, "vpc", vpcID, "cluster", clusterName)
|
||||
return nil
|
||||
return handlers_systemvpc.EnsureIGW(ctx, igwp, cpVPCOwner(clusterName), accountID, vpcID)
|
||||
}
|
||||
|
||||
// DeleteClusterIGW detaches and deletes the cluster-owned IGW attached to vpcID.
|
||||
// Best-effort and ownership-scoped: it only removes an IGW that carries this
|
||||
// cluster's managed-by tag, so a customer-provisioned IGW reused by
|
||||
// EnsureClusterIGW is never deleted.
|
||||
// Best-effort and ownership-scoped: a customer-provisioned IGW that
|
||||
// EnsureClusterIGW reused is never deleted.
|
||||
func DeleteClusterIGW(ctx context.Context, igwp igwProvisioner, accountID, vpcID, clusterName string) error {
|
||||
if vpcID == "" || clusterName == "" {
|
||||
return errors.New("eks: DeleteClusterIGW empty vpc id or cluster name")
|
||||
if clusterName == "" {
|
||||
return errors.New("eks: DeleteClusterIGW empty cluster name")
|
||||
}
|
||||
|
||||
igw, err := attachedVPCIGW(ctx, igwp, accountID, vpcID)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterIGW: IGW lookup failed", "vpc", vpcID, "err", err)
|
||||
return nil
|
||||
}
|
||||
if igw == nil || !ownedByCluster(igw, clusterName) {
|
||||
return nil
|
||||
}
|
||||
igwID := aws.StringValue(igw.InternetGatewayId)
|
||||
|
||||
if _, err := igwp.DetachInternetGateway(ctx, &ec2.DetachInternetGatewayInput{
|
||||
InternetGatewayId: aws.String(igwID),
|
||||
VpcId: aws.String(vpcID),
|
||||
}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "DeleteClusterIGW: detach failed", "igw", igwID, "vpc", vpcID, "err", err)
|
||||
return nil
|
||||
}
|
||||
if _, err := igwp.DeleteInternetGateway(ctx, &ec2.DeleteInternetGatewayInput{
|
||||
InternetGatewayId: aws.String(igwID),
|
||||
}, accountID); err != nil && !awserrors.IsNotFound(err) {
|
||||
slog.WarnContext(ctx, "DeleteClusterIGW: delete failed", "igw", igwID, "err", err)
|
||||
return nil
|
||||
}
|
||||
slog.InfoContext(ctx, "DeleteClusterIGW: removed cluster IGW", "igw", igwID, "vpc", vpcID, "cluster", clusterName)
|
||||
return nil
|
||||
}
|
||||
|
||||
// attachedVPCIGW returns the Internet Gateway attached to vpcID, or nil if none.
|
||||
func attachedVPCIGW(ctx context.Context, igwp igwProvisioner, accountID, vpcID string) (*ec2.InternetGateway, error) {
|
||||
out, err := igwp.DescribeInternetGateways(ctx, &ec2.DescribeInternetGatewaysInput{
|
||||
Filters: []*ec2.Filter{{
|
||||
Name: aws.String("attachment.vpc-id"),
|
||||
Values: aws.StringSlice([]string{vpcID}),
|
||||
}},
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("eks: describe IGWs for vpc %s: %w", vpcID, err)
|
||||
}
|
||||
if out == nil || len(out.InternetGateways) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return out.InternetGateways[0], nil
|
||||
}
|
||||
|
||||
// ownedByCluster reports whether igw carries this cluster's EKS managed-by tags.
|
||||
func ownedByCluster(igw *ec2.InternetGateway, clusterName string) bool {
|
||||
var managed, named bool
|
||||
for _, t := range igw.Tags {
|
||||
switch aws.StringValue(t.Key) {
|
||||
case tags.ManagedByKey:
|
||||
managed = aws.StringValue(t.Value) == tags.ManagedByEKS
|
||||
case clusterEKSClusterTagKey:
|
||||
named = aws.StringValue(t.Value) == clusterName
|
||||
}
|
||||
}
|
||||
return managed && named
|
||||
return handlers_systemvpc.DeleteIGW(ctx, igwp, cpVPCOwner(clusterName), accountID, vpcID)
|
||||
}
|
||||
|
||||
@@ -114,7 +114,10 @@ func TestEnsureClusterIGW_FreshCreatesAndAttaches(t *testing.T) {
|
||||
|
||||
igw := f.gateways["igw-fake0001"]
|
||||
require.NotNil(t, igw)
|
||||
assert.True(t, ownedByCluster(igw, "demo"), "created IGW must carry cluster ownership tags")
|
||||
assert.ElementsMatch(t, []*ec2.Tag{
|
||||
{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)},
|
||||
{Key: aws.String(clusterEKSClusterTagKey), Value: aws.String("demo")},
|
||||
}, igw.Tags, "created IGW must carry cluster ownership tags, so teardown can tell it from a customer's")
|
||||
}
|
||||
|
||||
func TestEnsureClusterIGW_ExistingIsReusedNotRecreated(t *testing.T) {
|
||||
@@ -171,5 +174,5 @@ func TestEnsureClusterIGW_CreateErrorPropagates(t *testing.T) {
|
||||
f.createErr = errors.New("boom")
|
||||
err := EnsureClusterIGW(context.Background(), f, "acct", "vpc-1", "demo")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "create cluster IGW")
|
||||
assert.Contains(t, err.Error(), "create IGW")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package handlers_systemvpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
"github.com/mulgadc/spinifex/spinifex/tags"
|
||||
)
|
||||
|
||||
// IGWProvisioner is the narrow Internet Gateway surface the system VPC needs.
|
||||
// The external switch, localnet and gateway LRP that make a front-end IP
|
||||
// answerable on the wire are built only once an IGW is attached, and a system
|
||||
// VPC has no customer to provision one.
|
||||
type IGWProvisioner interface {
|
||||
DescribeInternetGateways(ctx context.Context, input *ec2.DescribeInternetGatewaysInput, accountID string) (*ec2.DescribeInternetGatewaysOutput, error)
|
||||
CreateInternetGateway(ctx context.Context, input *ec2.CreateInternetGatewayInput, accountID string) (*ec2.CreateInternetGatewayOutput, error)
|
||||
AttachInternetGateway(ctx context.Context, input *ec2.AttachInternetGatewayInput, accountID string) (*ec2.AttachInternetGatewayOutput, error)
|
||||
DetachInternetGateway(ctx context.Context, input *ec2.DetachInternetGatewayInput, accountID string) (*ec2.DetachInternetGatewayOutput, error)
|
||||
DeleteInternetGateway(ctx context.Context, input *ec2.DeleteInternetGatewayInput, accountID string) (*ec2.DeleteInternetGatewayOutput, error)
|
||||
}
|
||||
|
||||
// EnsureIGW guarantees vpcID has an attached Internet Gateway. An IGW already
|
||||
// attached is reused as-is and left untagged; only when none exists is an
|
||||
// owner-tagged one created.
|
||||
//
|
||||
// Exported so a component needing an IGW on a customer VPC reaches it through
|
||||
// here rather than duplicating the reuse rule.
|
||||
func EnsureIGW(ctx context.Context, igwp IGWProvisioner, owner Owner, accountID, vpcID string) error {
|
||||
if vpcID == "" {
|
||||
return errors.New("systemvpc: EnsureIGW empty vpc id")
|
||||
}
|
||||
if owner.Name == "" {
|
||||
return errors.New("systemvpc: EnsureIGW empty owner name")
|
||||
}
|
||||
|
||||
existing, err := AttachedIGW(ctx, igwp, accountID, vpcID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if existing != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
out, err := igwp.CreateInternetGateway(ctx, &ec2.CreateInternetGatewayInput{
|
||||
TagSpecifications: []*ec2.TagSpecification{{
|
||||
ResourceType: aws.String("internet-gateway"),
|
||||
Tags: []*ec2.Tag{
|
||||
{Key: aws.String(tags.ManagedByKey), Value: aws.String(owner.ManagedBy)},
|
||||
{Key: aws.String(owner.OwnerTagKey), Value: aws.String(owner.Name)},
|
||||
},
|
||||
}},
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("systemvpc: create IGW for vpc %s: %w", vpcID, err)
|
||||
}
|
||||
if out == nil || out.InternetGateway == nil || aws.StringValue(out.InternetGateway.InternetGatewayId) == "" {
|
||||
return fmt.Errorf("systemvpc: create IGW for vpc %s: empty gateway id", vpcID)
|
||||
}
|
||||
igwID := aws.StringValue(out.InternetGateway.InternetGatewayId)
|
||||
|
||||
if _, err := igwp.AttachInternetGateway(ctx, &ec2.AttachInternetGatewayInput{
|
||||
InternetGatewayId: aws.String(igwID),
|
||||
VpcId: aws.String(vpcID),
|
||||
}, accountID); err != nil {
|
||||
return fmt.Errorf("systemvpc: attach IGW %s to vpc %s: %w", igwID, vpcID, err)
|
||||
}
|
||||
slog.InfoContext(ctx, "systemvpc: attached IGW", "igw", igwID, "vpc", vpcID, "owner", owner.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteIGW detaches and deletes the owner's IGW attached to vpcID. Best-effort
|
||||
// and ownership-scoped: it only removes an IGW carrying this owner's tags, so a
|
||||
// customer-provisioned IGW that EnsureIGW reused is never deleted.
|
||||
func DeleteIGW(ctx context.Context, igwp IGWProvisioner, owner Owner, accountID, vpcID string) error {
|
||||
if vpcID == "" || owner.Name == "" {
|
||||
return errors.New("systemvpc: DeleteIGW empty vpc id or owner name")
|
||||
}
|
||||
|
||||
igw, err := AttachedIGW(ctx, igwp, accountID, vpcID)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc DeleteIGW: IGW lookup failed", "vpc", vpcID, "err", err)
|
||||
return nil
|
||||
}
|
||||
if igw == nil || !ownedBy(igw, owner) {
|
||||
return nil
|
||||
}
|
||||
igwID := aws.StringValue(igw.InternetGatewayId)
|
||||
|
||||
if _, err := igwp.DetachInternetGateway(ctx, &ec2.DetachInternetGatewayInput{
|
||||
InternetGatewayId: aws.String(igwID),
|
||||
VpcId: aws.String(vpcID),
|
||||
}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc DeleteIGW: detach failed", "igw", igwID, "vpc", vpcID, "err", err)
|
||||
return nil
|
||||
}
|
||||
if _, err := igwp.DeleteInternetGateway(ctx, &ec2.DeleteInternetGatewayInput{
|
||||
InternetGatewayId: aws.String(igwID),
|
||||
}, accountID); err != nil && !awserrors.IsNotFound(err) {
|
||||
slog.WarnContext(ctx, "systemvpc DeleteIGW: delete failed", "igw", igwID, "err", err)
|
||||
return nil
|
||||
}
|
||||
slog.InfoContext(ctx, "systemvpc DeleteIGW: removed IGW", "igw", igwID, "vpc", vpcID, "owner", owner.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// AttachedIGW returns the Internet Gateway attached to vpcID, or nil if none.
|
||||
func AttachedIGW(ctx context.Context, igwp IGWProvisioner, accountID, vpcID string) (*ec2.InternetGateway, error) {
|
||||
out, err := igwp.DescribeInternetGateways(ctx, &ec2.DescribeInternetGatewaysInput{
|
||||
Filters: []*ec2.Filter{{
|
||||
Name: aws.String("attachment.vpc-id"),
|
||||
Values: aws.StringSlice([]string{vpcID}),
|
||||
}},
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("systemvpc: describe IGWs for vpc %s: %w", vpcID, err)
|
||||
}
|
||||
if out == nil || len(out.InternetGateways) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return out.InternetGateways[0], nil
|
||||
}
|
||||
|
||||
// ownedBy reports whether igw carries this owner's managed-by + name tags.
|
||||
func ownedBy(igw *ec2.InternetGateway, owner Owner) bool {
|
||||
var managed, named bool
|
||||
for _, t := range igw.Tags {
|
||||
switch aws.StringValue(t.Key) {
|
||||
case tags.ManagedByKey:
|
||||
managed = aws.StringValue(t.Value) == owner.ManagedBy
|
||||
case owner.OwnerTagKey:
|
||||
named = aws.StringValue(t.Value) == owner.Name
|
||||
}
|
||||
}
|
||||
return managed && named
|
||||
}
|
||||
@@ -0,0 +1,688 @@
|
||||
// Package handlers_systemvpc builds the managed VPCs Spinifex platform
|
||||
// components run their own VMs in — the analogue of AWS's hidden managed-account
|
||||
// VPCs. A component asks for one by name; it is owned by the system account and
|
||||
// carries a public subnet routed to an IGW plus NAT-routed private subnets.
|
||||
//
|
||||
// The topology is composed from the real EC2 VPC-family APIs rather than direct
|
||||
// OVN mutation, so per-subnet egress gating is wired by the existing topology
|
||||
// subscribers off the route-table events those APIs publish.
|
||||
//
|
||||
// Every lookup is a describe-or-create keyed on the owner + role tags a Spec
|
||||
// carries, so two components never see each other's resources and a relaunch
|
||||
// after partial failure converges rather than duplicating.
|
||||
package handlers_systemvpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
"github.com/mulgadc/spinifex/spinifex/tags"
|
||||
"github.com/mulgadc/spinifex/spinifex/utils"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// supernetBits is the prefix length every component's supernet must carry. A /14
|
||||
// holds the 256 /22 blocks nameHash indexes into.
|
||||
const supernetBits = 14
|
||||
|
||||
// maxPrivateSubnets caps the private subnets carved per VPC, bounded by the /22:
|
||||
// one public /24 plus three private /24s.
|
||||
const maxPrivateSubnets = 3
|
||||
|
||||
// Owner is the tag identity stamped on every resource in a managed system VPC.
|
||||
// It is what makes one component's tag-driven sweeps and orphan reapers blind to
|
||||
// another's resources, so no two components may share an OwnerTagKey.
|
||||
type Owner struct {
|
||||
// Name keys this particular VPC: it is the OwnerTagKey value and the seed
|
||||
// the address space is hashed from, so it must be stable for the VPC's life.
|
||||
Name string
|
||||
// ManagedBy is the tags.ManagedBy* value identifying the owning component.
|
||||
ManagedBy string
|
||||
// OwnerTagKey holds Name; RoleTagKey holds the per-resource role.
|
||||
OwnerTagKey string
|
||||
RoleTagKey string
|
||||
}
|
||||
|
||||
// Spec is an Owner plus the addressing and shape the VPC is built with. The same
|
||||
// Spec must be passed to Ensure and Delete: a Spec differing on Name, either tag
|
||||
// key or the supernet cannot find what a previous call created.
|
||||
type Spec struct {
|
||||
Owner
|
||||
|
||||
// Region names the cosmetic AvailabilityZone each subnet reports.
|
||||
Region string
|
||||
// RolePrefix namespaces the role tag values ("cp" → "cp-vpc", "cp-public"),
|
||||
// so the role tag stays readable when a deployment holds several components'
|
||||
// system VPCs.
|
||||
RolePrefix string
|
||||
// Supernet is the IPv4 /14 this VPC's /22 is carved from. Components must
|
||||
// not share one: a collision is survivable, but leaves an operator unable to
|
||||
// tell from an address which component owns it.
|
||||
Supernet string
|
||||
// PrivateSubnets is how many private subnets to carve, clamped to 1..3.
|
||||
PrivateSubnets int
|
||||
}
|
||||
|
||||
// Roles are the role-tag values a Spec stamps on the resources it builds.
|
||||
type Roles struct {
|
||||
VPC string
|
||||
PublicSubnet string
|
||||
PrivateSubnet string
|
||||
PublicRT string
|
||||
PrivateRT string
|
||||
NatGW string
|
||||
}
|
||||
|
||||
// Roles returns the spec's namespaced role-tag values. Exported so a caller's
|
||||
// own teardown or test can filter on the same values Ensure wrote.
|
||||
func (s Spec) Roles() Roles {
|
||||
return Roles{
|
||||
VPC: s.RolePrefix + "-vpc",
|
||||
PublicSubnet: s.RolePrefix + "-public",
|
||||
PrivateSubnet: s.RolePrefix + "-private",
|
||||
PublicRT: s.RolePrefix + "-public-rt",
|
||||
PrivateRT: s.RolePrefix + "-private-rt",
|
||||
NatGW: s.RolePrefix + "-natgw",
|
||||
}
|
||||
}
|
||||
|
||||
// VPCProvisioner is the narrow VPC + subnet surface a system VPC needs.
|
||||
// The daemon adapts the concrete VPC service onto this.
|
||||
type VPCProvisioner interface {
|
||||
CreateVpc(ctx context.Context, input *ec2.CreateVpcInput, accountID string) (*ec2.CreateVpcOutput, error)
|
||||
DeleteVpc(ctx context.Context, input *ec2.DeleteVpcInput, accountID string) (*ec2.DeleteVpcOutput, error)
|
||||
DescribeVpcs(ctx context.Context, input *ec2.DescribeVpcsInput, accountID string) (*ec2.DescribeVpcsOutput, error)
|
||||
CreateSubnet(ctx context.Context, input *ec2.CreateSubnetInput, accountID string) (*ec2.CreateSubnetOutput, error)
|
||||
DeleteSubnet(ctx context.Context, input *ec2.DeleteSubnetInput, accountID string) (*ec2.DeleteSubnetOutput, error)
|
||||
DescribeSubnets(ctx context.Context, input *ec2.DescribeSubnetsInput, accountID string) (*ec2.DescribeSubnetsOutput, error)
|
||||
}
|
||||
|
||||
// RouteTableProvisioner is the narrow route-table surface needed to express
|
||||
// public (0/0→IGW) and private (0/0→NAT GW) egress.
|
||||
type RouteTableProvisioner interface {
|
||||
CreateRouteTable(ctx context.Context, input *ec2.CreateRouteTableInput, accountID string) (*ec2.CreateRouteTableOutput, error)
|
||||
DeleteRouteTable(ctx context.Context, input *ec2.DeleteRouteTableInput, accountID string) (*ec2.DeleteRouteTableOutput, error)
|
||||
DescribeRouteTables(ctx context.Context, input *ec2.DescribeRouteTablesInput, accountID string) (*ec2.DescribeRouteTablesOutput, error)
|
||||
CreateRoute(ctx context.Context, input *ec2.CreateRouteInput, accountID string) (*ec2.CreateRouteOutput, error)
|
||||
AssociateRouteTable(ctx context.Context, input *ec2.AssociateRouteTableInput, accountID string) (*ec2.AssociateRouteTableOutput, error)
|
||||
DisassociateRouteTable(ctx context.Context, input *ec2.DisassociateRouteTableInput, accountID string) (*ec2.DisassociateRouteTableOutput, error)
|
||||
}
|
||||
|
||||
// NATGatewayProvisioner is the narrow NAT-gateway surface that gives the private
|
||||
// subnets egress.
|
||||
type NATGatewayProvisioner interface {
|
||||
CreateNatGateway(ctx context.Context, input *ec2.CreateNatGatewayInput, accountID string) (*ec2.CreateNatGatewayOutput, error)
|
||||
DeleteNatGateway(ctx context.Context, input *ec2.DeleteNatGatewayInput, accountID string) (*ec2.DeleteNatGatewayOutput, error)
|
||||
DescribeNatGateways(ctx context.Context, input *ec2.DescribeNatGatewaysInput, accountID string) (*ec2.DescribeNatGatewaysOutput, error)
|
||||
}
|
||||
|
||||
// EIPProvisioner is the narrow EIP surface backing the NAT gateway's address.
|
||||
type EIPProvisioner interface {
|
||||
AllocateAddress(ctx context.Context, input *ec2.AllocateAddressInput, accountID string) (*ec2.AllocateAddressOutput, error)
|
||||
ReleaseAddress(ctx context.Context, input *ec2.ReleaseAddressInput, accountID string) (*ec2.ReleaseAddressOutput, error)
|
||||
}
|
||||
|
||||
// SGProvisioner is the security-group surface Delete uses to clear groups left
|
||||
// in the VPC that the tag-driven sweep cannot see.
|
||||
type SGProvisioner interface {
|
||||
DescribeSecurityGroups(ctx context.Context, input *ec2.DescribeSecurityGroupsInput, accountID string) (*ec2.DescribeSecurityGroupsOutput, error)
|
||||
DeleteSecurityGroup(ctx context.Context, input *ec2.DeleteSecurityGroupInput, accountID string) (*ec2.DeleteSecurityGroupOutput, error)
|
||||
}
|
||||
|
||||
// Deps bundles the EC2-family collaborators a system VPC is composed from. All
|
||||
// calls run under the accountID passed to Ensure/Delete (the system account).
|
||||
type Deps struct {
|
||||
VPC VPCProvisioner
|
||||
// SG is used only by Delete. Nil is tolerated.
|
||||
SG SGProvisioner
|
||||
IGW IGWProvisioner
|
||||
RT RouteTableProvisioner
|
||||
NGW NATGatewayProvisioner
|
||||
EIP EIPProvisioner
|
||||
// NATSConn republishes vpc.delete for OVN topology GC (see gcTopology).
|
||||
// Nil is tolerated (GC is skipped, never blocking).
|
||||
NATSConn *nats.Conn
|
||||
}
|
||||
|
||||
// Refs is the resolved set of system VPC resource IDs + CIDRs. Ensure returns
|
||||
// it; the caller persists it for placement (public/private subnet selection)
|
||||
// and teardown.
|
||||
type Refs struct {
|
||||
VpcID string
|
||||
VpcCIDR string
|
||||
IGWID string
|
||||
PublicSubnetID string
|
||||
PublicSubnetCIDR string
|
||||
PrivateRouteTableID string
|
||||
PublicRouteTableID string
|
||||
PrivateSubnetIDs []string
|
||||
PrivateSubnetCIDRs []string
|
||||
NatGatewayID string
|
||||
NatEIPAllocationID string
|
||||
NatEIPPublicIP string
|
||||
}
|
||||
|
||||
// cidrs derives the VPC CIDR + subnet CIDRs deterministically from the spec's
|
||||
// name. The /22 is carved from the spec's /14 supernet; subnet 0 is public,
|
||||
// subnets 1..PrivateSubnets are private. A hash collision is non-fatal.
|
||||
func (s Spec) cidrs() (vpcCIDR, publicCIDR string, privateCIDRs []string, err error) {
|
||||
base, err := s.supernetBase()
|
||||
if err != nil {
|
||||
return "", "", nil, err
|
||||
}
|
||||
|
||||
h := fnv.New32a()
|
||||
_, _ = h.Write([]byte(s.Name))
|
||||
idx := int(h.Sum32() % 256) // 256 /22 blocks in a /14
|
||||
|
||||
combined := idx * 4 // each /22 spans 4 third-octet values
|
||||
second := int(base[1]) + combined/256
|
||||
third := combined % 256
|
||||
|
||||
vpcCIDR = fmt.Sprintf("%d.%d.%d.0/22", base[0], second, third)
|
||||
publicCIDR = fmt.Sprintf("%d.%d.%d.0/24", base[0], second, third)
|
||||
privateCount := min(max(s.PrivateSubnets, 1), maxPrivateSubnets)
|
||||
for k := range privateCount {
|
||||
privateCIDRs = append(privateCIDRs, fmt.Sprintf("%d.%d.%d.0/24", base[0], second, third+1+k))
|
||||
}
|
||||
return vpcCIDR, publicCIDR, privateCIDRs, nil
|
||||
}
|
||||
|
||||
// supernetBase validates the spec's supernet and returns its network address.
|
||||
// The /14 is required so the 256-block hash index cannot walk past the space
|
||||
// the component was allotted.
|
||||
func (s Spec) supernetBase() ([4]byte, error) {
|
||||
prefix, err := netip.ParsePrefix(s.Supernet)
|
||||
if err != nil {
|
||||
return [4]byte{}, fmt.Errorf("systemvpc: %s: parse supernet %q: %w", s.Name, s.Supernet, err)
|
||||
}
|
||||
if !prefix.Addr().Is4() || prefix.Bits() != supernetBits {
|
||||
return [4]byte{}, fmt.Errorf("systemvpc: %s: supernet %q must be an IPv4 /%d", s.Name, s.Supernet, supernetBits)
|
||||
}
|
||||
if prefix.Masked() != prefix {
|
||||
return [4]byte{}, fmt.Errorf("systemvpc: %s: supernet %q has host bits set", s.Name, s.Supernet)
|
||||
}
|
||||
return prefix.Addr().As4(), nil
|
||||
}
|
||||
|
||||
// az returns the AvailabilityZone name for subnet index k (cosmetic AWS parity;
|
||||
// spinifex spreads placement by host placement group, not AZ).
|
||||
func (s Spec) az(k int) string {
|
||||
return fmt.Sprintf("%s%c", s.Region, 'a'+k)
|
||||
}
|
||||
|
||||
// tagSpec builds the owner + role tag set stamped on a created resource.
|
||||
func (s Spec) tagSpec(resourceType, role string) []*ec2.TagSpecification {
|
||||
return []*ec2.TagSpecification{{
|
||||
ResourceType: aws.String(resourceType),
|
||||
Tags: []*ec2.Tag{
|
||||
{Key: aws.String(tags.ManagedByKey), Value: aws.String(s.ManagedBy)},
|
||||
{Key: aws.String(s.OwnerTagKey), Value: aws.String(s.Name)},
|
||||
{Key: aws.String(s.RoleTagKey), Value: aws.String(role)},
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
// roleFilters is the describe-side counterpart of tagSpec: it matches only the
|
||||
// resources this owner created for this role.
|
||||
func (s Spec) roleFilters(role string) []*ec2.Filter {
|
||||
return []*ec2.Filter{
|
||||
{Name: aws.String("tag:" + s.OwnerTagKey), Values: aws.StringSlice([]string{s.Name})},
|
||||
{Name: aws.String("tag:" + s.RoleTagKey), Values: aws.StringSlice([]string{role})},
|
||||
}
|
||||
}
|
||||
|
||||
// validate rejects a Spec that could not address or tag its resources. Checked
|
||||
// up front so a half-specified component fails before it creates anything.
|
||||
func (s Spec) validate() error {
|
||||
switch {
|
||||
case s.Name == "":
|
||||
return errors.New("systemvpc: empty name")
|
||||
case s.ManagedBy == "":
|
||||
return fmt.Errorf("systemvpc: %s: empty managed-by value", s.Name)
|
||||
case s.OwnerTagKey == "" || s.RoleTagKey == "":
|
||||
return fmt.Errorf("systemvpc: %s: empty owner or role tag key", s.Name)
|
||||
case s.RolePrefix == "":
|
||||
return fmt.Errorf("systemvpc: %s: empty role prefix", s.Name)
|
||||
}
|
||||
_, err := s.supernetBase()
|
||||
return err
|
||||
}
|
||||
|
||||
// Ensure idempotently builds the managed system VPC described by spec: a VPC, an
|
||||
// attached IGW, an IGW-routed public subnet, NAT-routed private subnets, and the
|
||||
// NAT gateway itself. Each resource is describe-or-create on the role tags.
|
||||
//
|
||||
// The route-table associations publish the topology events the per-subnet egress
|
||||
// subscribers consume, so the OVN policy split falls out without any direct OVN
|
||||
// mutation here.
|
||||
func Ensure(ctx context.Context, deps Deps, spec Spec, accountID string) (*Refs, error) {
|
||||
if err := spec.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
vpcCIDR, publicCIDR, privateCIDRs, err := spec.cidrs()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
roles := spec.Roles()
|
||||
refs := &Refs{VpcCIDR: vpcCIDR, PublicSubnetCIDR: publicCIDR, PrivateSubnetCIDRs: privateCIDRs}
|
||||
|
||||
vpcID, err := ensureVPC(ctx, deps.VPC, spec, accountID, vpcCIDR)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs.VpcID = vpcID
|
||||
|
||||
if err := EnsureIGW(ctx, deps.IGW, spec.Owner, accountID, vpcID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
igw, err := AttachedIGW(ctx, deps.IGW, accountID, vpcID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if igw == nil {
|
||||
return nil, fmt.Errorf("systemvpc: %s: vpc %s has no attached IGW after ensure", spec.Name, vpcID)
|
||||
}
|
||||
refs.IGWID = aws.StringValue(igw.InternetGatewayId)
|
||||
|
||||
pubSubnet, err := ensureSubnet(ctx, deps.VPC, spec, accountID, vpcID, publicCIDR, roles.PublicSubnet, spec.az(0))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs.PublicSubnetID = pubSubnet
|
||||
|
||||
for k, cidr := range privateCIDRs {
|
||||
priv, err := ensureSubnet(ctx, deps.VPC, spec, accountID, vpcID, cidr, roles.PrivateSubnet, spec.az(k))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs.PrivateSubnetIDs = append(refs.PrivateSubnetIDs, priv)
|
||||
}
|
||||
|
||||
// Public route table: 0.0.0.0/0 → IGW, associated to the public subnet. The
|
||||
// association publishes vpc.add-igw-route → EnsureSubnetEgress (1000 reroute).
|
||||
pubRT, err := ensureRouteTable(ctx, deps.RT, spec, accountID, vpcID, roles.PublicRT,
|
||||
&ec2.CreateRouteInput{DestinationCidrBlock: aws.String("0.0.0.0/0"), GatewayId: aws.String(refs.IGWID)},
|
||||
[]string{refs.PublicSubnetID})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs.PublicRouteTableID = pubRT
|
||||
|
||||
// NAT gateway in the public subnet, fed by a dedicated EIP. The private
|
||||
// subnets route 0.0.0.0/0 → this NAT GW for egress-only internet.
|
||||
natID, allocID, natIP, err := ensureNatGateway(ctx, deps, spec, accountID, refs.PublicSubnetID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs.NatGatewayID = natID
|
||||
refs.NatEIPAllocationID = allocID
|
||||
refs.NatEIPPublicIP = natIP
|
||||
|
||||
// Private route table: 0.0.0.0/0 → NAT GW, associated to every private subnet.
|
||||
// The association publishes vpc.add-nat-gateway → EnsureNATGatewaySubnetEgress.
|
||||
privRT, err := ensureRouteTable(ctx, deps.RT, spec, accountID, vpcID, roles.PrivateRT,
|
||||
&ec2.CreateRouteInput{DestinationCidrBlock: aws.String("0.0.0.0/0"), NatGatewayId: aws.String(natID)},
|
||||
refs.PrivateSubnetIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs.PrivateRouteTableID = privRT
|
||||
|
||||
slog.InfoContext(ctx, "systemvpc: managed system VPC ready",
|
||||
"name", spec.Name, "managedBy", spec.ManagedBy, "vpc", refs.VpcID,
|
||||
"publicSubnet", refs.PublicSubnetID, "privateSubnets", refs.PrivateSubnetIDs,
|
||||
"natgw", refs.NatGatewayID)
|
||||
return refs, nil
|
||||
}
|
||||
|
||||
func ensureVPC(ctx context.Context, vpcp VPCProvisioner, spec Spec, accountID, cidr string) (string, error) {
|
||||
role := spec.Roles().VPC
|
||||
out, err := vpcp.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{Filters: spec.roleFilters(role)}, accountID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("systemvpc: describe vpc for %s: %w", spec.Name, err)
|
||||
}
|
||||
if out != nil && len(out.Vpcs) > 0 {
|
||||
return aws.StringValue(out.Vpcs[0].VpcId), nil
|
||||
}
|
||||
created, err := vpcp.CreateVpc(ctx, &ec2.CreateVpcInput{
|
||||
CidrBlock: aws.String(cidr),
|
||||
TagSpecifications: spec.tagSpec("vpc", role),
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("systemvpc: create vpc for %s: %w", spec.Name, err)
|
||||
}
|
||||
if created == nil || created.Vpc == nil || aws.StringValue(created.Vpc.VpcId) == "" {
|
||||
return "", fmt.Errorf("systemvpc: create vpc for %s: empty vpc id", spec.Name)
|
||||
}
|
||||
return aws.StringValue(created.Vpc.VpcId), nil
|
||||
}
|
||||
|
||||
func ensureSubnet(ctx context.Context, vpcp VPCProvisioner, spec Spec, accountID, vpcID, cidr, role, az string) (string, error) {
|
||||
out, err := vpcp.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{Filters: append(
|
||||
spec.roleFilters(role),
|
||||
&ec2.Filter{Name: aws.String("cidr-block"), Values: aws.StringSlice([]string{cidr})},
|
||||
)}, accountID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("systemvpc: describe subnet %s (%s): %w", cidr, role, err)
|
||||
}
|
||||
if out != nil && len(out.Subnets) > 0 {
|
||||
return aws.StringValue(out.Subnets[0].SubnetId), nil
|
||||
}
|
||||
created, err := vpcp.CreateSubnet(ctx, &ec2.CreateSubnetInput{
|
||||
VpcId: aws.String(vpcID),
|
||||
CidrBlock: aws.String(cidr),
|
||||
AvailabilityZone: aws.String(az),
|
||||
TagSpecifications: spec.tagSpec("subnet", role),
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("systemvpc: create subnet %s (%s): %w", cidr, role, err)
|
||||
}
|
||||
if created == nil || created.Subnet == nil || aws.StringValue(created.Subnet.SubnetId) == "" {
|
||||
return "", fmt.Errorf("systemvpc: create subnet %s (%s): empty subnet id", cidr, role)
|
||||
}
|
||||
return aws.StringValue(created.Subnet.SubnetId), nil
|
||||
}
|
||||
|
||||
// ensureRouteTable describe-or-creates the role-tagged route table, installs the
|
||||
// default route, and associates it to subnetIDs. Idempotent: a re-run reuses the
|
||||
// existing table and re-drives the associations.
|
||||
func ensureRouteTable(ctx context.Context, rtp RouteTableProvisioner, spec Spec, accountID, vpcID, role string, route *ec2.CreateRouteInput, subnetIDs []string) (string, error) {
|
||||
rt, fresh, err := describeOrCreateRouteTable(ctx, rtp, spec, accountID, vpcID, role)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
rtID := aws.StringValue(rt.RouteTableId)
|
||||
if fresh {
|
||||
route.RouteTableId = aws.String(rtID)
|
||||
if _, err := rtp.CreateRoute(ctx, route, accountID); err != nil {
|
||||
return "", fmt.Errorf("systemvpc: create route (%s) on %s: %w", role, rtID, err)
|
||||
}
|
||||
}
|
||||
for _, sn := range subnetIDs {
|
||||
if sn == "" {
|
||||
continue
|
||||
}
|
||||
_, err := rtp.AssociateRouteTable(ctx, &ec2.AssociateRouteTableInput{
|
||||
RouteTableId: aws.String(rtID),
|
||||
SubnetId: aws.String(sn),
|
||||
}, accountID)
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
// AssociateRouteTable rejects a subnet holding an explicit association on
|
||||
// any table in the VPC. Tolerate that only when the association is this
|
||||
// table's — a subnet parked on someone else's table has no egress, and for
|
||||
// a shared system VPC every Ensure after the first takes this path.
|
||||
if !awserrors.IsErrorCode(err, awserrors.ErrorResourceAlreadyAssociated) || !associatedWith(rt, sn) {
|
||||
return "", fmt.Errorf("systemvpc: associate route table %s → subnet %s: %w", rtID, sn, err)
|
||||
}
|
||||
}
|
||||
return rtID, nil
|
||||
}
|
||||
|
||||
// associatedWith reports whether subnetID already holds an explicit association
|
||||
// on rt, as of the describe ensureRouteTable adopted it from.
|
||||
func associatedWith(rt *ec2.RouteTable, subnetID string) bool {
|
||||
for _, assoc := range rt.Associations {
|
||||
if aws.StringValue(assoc.SubnetId) == subnetID && !aws.BoolValue(assoc.Main) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// describeOrCreateRouteTable returns the role-tagged route table, adopting an
|
||||
// existing one where possible. The table is returned whole rather than by ID so
|
||||
// the caller can read the associations the describe already carried.
|
||||
func describeOrCreateRouteTable(ctx context.Context, rtp RouteTableProvisioner, spec Spec, accountID, vpcID, role string) (rt *ec2.RouteTable, fresh bool, err error) {
|
||||
out, err := rtp.DescribeRouteTables(ctx, &ec2.DescribeRouteTablesInput{Filters: spec.roleFilters(role)}, accountID)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("systemvpc: describe route table (%s): %w", role, err)
|
||||
}
|
||||
if out != nil && len(out.RouteTables) > 0 {
|
||||
return out.RouteTables[0], false, nil
|
||||
}
|
||||
created, err := rtp.CreateRouteTable(ctx, &ec2.CreateRouteTableInput{
|
||||
VpcId: aws.String(vpcID),
|
||||
TagSpecifications: spec.tagSpec("route-table", role),
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("systemvpc: create route table (%s): %w", role, err)
|
||||
}
|
||||
if created == nil || created.RouteTable == nil || aws.StringValue(created.RouteTable.RouteTableId) == "" {
|
||||
return nil, false, fmt.Errorf("systemvpc: create route table (%s): empty id", role)
|
||||
}
|
||||
return created.RouteTable, true, nil
|
||||
}
|
||||
|
||||
func ensureNatGateway(ctx context.Context, deps Deps, spec Spec, accountID, publicSubnetID string) (natID, allocID, publicIP string, err error) {
|
||||
role := spec.Roles().NatGW
|
||||
out, err := deps.NGW.DescribeNatGateways(ctx, &ec2.DescribeNatGatewaysInput{Filter: append(
|
||||
spec.roleFilters(role),
|
||||
&ec2.Filter{Name: aws.String("state"), Values: aws.StringSlice([]string{"available", "pending"})},
|
||||
)}, accountID)
|
||||
if err != nil {
|
||||
return "", "", "", fmt.Errorf("systemvpc: describe nat gateway for %s: %w", spec.Name, err)
|
||||
}
|
||||
if out != nil && len(out.NatGateways) > 0 {
|
||||
ng := out.NatGateways[0]
|
||||
var alloc, ip string
|
||||
if len(ng.NatGatewayAddresses) > 0 {
|
||||
alloc = aws.StringValue(ng.NatGatewayAddresses[0].AllocationId)
|
||||
ip = aws.StringValue(ng.NatGatewayAddresses[0].PublicIp)
|
||||
}
|
||||
return aws.StringValue(ng.NatGatewayId), alloc, ip, nil
|
||||
}
|
||||
|
||||
eip, err := deps.EIP.AllocateAddress(ctx, &ec2.AllocateAddressInput{
|
||||
Domain: aws.String("vpc"),
|
||||
TagSpecifications: spec.tagSpec("elastic-ip", role),
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
return "", "", "", fmt.Errorf("systemvpc: allocate nat gateway EIP for %s: %w", spec.Name, err)
|
||||
}
|
||||
if eip == nil || aws.StringValue(eip.AllocationId) == "" {
|
||||
return "", "", "", fmt.Errorf("systemvpc: allocate nat gateway EIP for %s: empty allocation", spec.Name)
|
||||
}
|
||||
allocID = aws.StringValue(eip.AllocationId)
|
||||
publicIP = aws.StringValue(eip.PublicIp)
|
||||
|
||||
created, err := deps.NGW.CreateNatGateway(ctx, &ec2.CreateNatGatewayInput{
|
||||
SubnetId: aws.String(publicSubnetID),
|
||||
AllocationId: aws.String(allocID),
|
||||
TagSpecifications: spec.tagSpec("natgateway", role),
|
||||
}, accountID)
|
||||
if err != nil {
|
||||
// Release the orphaned EIP so a failed NAT-GW create does not leak it.
|
||||
if _, relErr := deps.EIP.ReleaseAddress(ctx, &ec2.ReleaseAddressInput{AllocationId: aws.String(allocID)}, accountID); relErr != nil {
|
||||
slog.WarnContext(ctx, "systemvpc: failed to release EIP after NAT-GW create failure", "alloc", allocID, "err", relErr)
|
||||
}
|
||||
return "", "", "", fmt.Errorf("systemvpc: create nat gateway for %s: %w", spec.Name, err)
|
||||
}
|
||||
if created == nil || created.NatGateway == nil || aws.StringValue(created.NatGateway.NatGatewayId) == "" {
|
||||
return "", "", "", fmt.Errorf("systemvpc: create nat gateway for %s: empty id", spec.Name)
|
||||
}
|
||||
return aws.StringValue(created.NatGateway.NatGatewayId), allocID, publicIP, nil
|
||||
}
|
||||
|
||||
// Delete tears down the managed system VPC in dependency order: NAT gateway
|
||||
// (+ its EIP) → route tables → subnets → IGW → VPC. Tag-driven, so a partial
|
||||
// create still converges, and best-effort apart from the final VPC delete.
|
||||
//
|
||||
// gcFallbackVpcID is the caller's last-persisted VpcId, used as an OVN-GC target
|
||||
// when the tag-indexed EC2 VPC is already gone. Pass "" when no such record
|
||||
// exists, e.g. an orphan reaper acting after the owning record.
|
||||
func Delete(ctx context.Context, deps Deps, spec Spec, accountID, gcFallbackVpcID string) error {
|
||||
if err := spec.validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
roles := spec.Roles()
|
||||
|
||||
// 1. Route tables: disassociate every subnet, then delete. They go first so
|
||||
// the NAT gateway's live route refs are gone before its own delete, which is
|
||||
// guarded against live forwards and would otherwise strand its billable EIP.
|
||||
for _, role := range []string{roles.PublicRT, roles.PrivateRT} {
|
||||
rtOut, err := deps.RT.DescribeRouteTables(ctx, &ec2.DescribeRouteTablesInput{Filters: spec.roleFilters(role)}, accountID)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc Delete: describe route tables failed", "role", role, "err", err)
|
||||
continue
|
||||
}
|
||||
if rtOut == nil {
|
||||
continue
|
||||
}
|
||||
for _, rt := range rtOut.RouteTables {
|
||||
rtID := aws.StringValue(rt.RouteTableId)
|
||||
for _, assoc := range rt.Associations {
|
||||
if aws.BoolValue(assoc.Main) {
|
||||
continue
|
||||
}
|
||||
if aID := aws.StringValue(assoc.RouteTableAssociationId); aID != "" {
|
||||
if _, err := deps.RT.DisassociateRouteTable(ctx, &ec2.DisassociateRouteTableInput{AssociationId: aws.String(aID)}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc Delete: disassociate route table failed", "assoc", aID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := deps.RT.DeleteRouteTable(ctx, &ec2.DeleteRouteTableInput{RouteTableId: aws.String(rtID)}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc Delete: delete route table failed", "rt", rtID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. NAT gateway + its EIP. DeleteNatGateway publishes the SNAT removal and
|
||||
// disassociates the EIP; release follows so the billable address is reclaimed.
|
||||
if ngOut, err := deps.NGW.DescribeNatGateways(ctx, &ec2.DescribeNatGatewaysInput{
|
||||
Filter: spec.roleFilters(roles.NatGW),
|
||||
}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc Delete: describe NAT gateways failed", "name", spec.Name, "err", err)
|
||||
} else if ngOut != nil {
|
||||
for _, ng := range ngOut.NatGateways {
|
||||
if state := aws.StringValue(ng.State); state == "deleted" || state == "deleting" {
|
||||
continue
|
||||
}
|
||||
ngID := aws.StringValue(ng.NatGatewayId)
|
||||
if _, err := deps.NGW.DeleteNatGateway(ctx, &ec2.DeleteNatGatewayInput{NatGatewayId: aws.String(ngID)}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc Delete: delete NAT gateway failed", "natgw", ngID, "err", err)
|
||||
}
|
||||
for _, addr := range ng.NatGatewayAddresses {
|
||||
if alloc := aws.StringValue(addr.AllocationId); alloc != "" {
|
||||
if _, err := deps.EIP.ReleaseAddress(ctx, &ec2.ReleaseAddressInput{AllocationId: aws.String(alloc)}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc Delete: release NAT EIP failed", "alloc", alloc, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Subnets (public + private).
|
||||
for _, role := range []string{roles.PublicSubnet, roles.PrivateSubnet} {
|
||||
snOut, err := deps.VPC.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{Filters: spec.roleFilters(role)}, accountID)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc Delete: describe subnets failed", "role", role, "err", err)
|
||||
continue
|
||||
}
|
||||
if snOut == nil {
|
||||
continue
|
||||
}
|
||||
for _, sn := range snOut.Subnets {
|
||||
snID := aws.StringValue(sn.SubnetId)
|
||||
if _, err := deps.VPC.DeleteSubnet(ctx, &ec2.DeleteSubnetInput{SubnetId: aws.String(snID)}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc Delete: delete subnet failed", "subnet", snID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. VPC + its IGW. Resolve the VPC first so DeleteIGW can detach.
|
||||
vpcOut, err := deps.VPC.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{Filters: spec.roleFilters(roles.VPC)}, accountID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("systemvpc: describe vpc for delete (%s): %w", spec.Name, err)
|
||||
}
|
||||
|
||||
// The tag-indexed VPC is already gone, so this is an idempotent success. Its
|
||||
// OVN state may still be orphaned if that delete's fire-and-forget vpc.delete
|
||||
// never reached vpcd, so GC runs against gcFallbackVpcID.
|
||||
if vpcOut == nil || len(vpcOut.Vpcs) == 0 {
|
||||
gcTopology(ctx, deps.NATSConn, spec.Name, gcFallbackVpcID)
|
||||
return nil
|
||||
}
|
||||
vpcID := aws.StringValue(vpcOut.Vpcs[0].VpcId)
|
||||
|
||||
if err := DeleteIGW(ctx, deps.IGW, spec.Owner, accountID, vpcID); err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc Delete: delete IGW failed", "vpc", vpcID, "err", err)
|
||||
}
|
||||
|
||||
// Sweep residual subnets by VPC identity, not by role tag: a create that
|
||||
// failed before tagging leaves subnets the tagged sweep cannot see, and
|
||||
// DeleteVpc then rejects with DependencyViolation on every re-drive.
|
||||
if snOut, err := deps.VPC.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{
|
||||
Filters: []*ec2.Filter{{Name: aws.String("vpc-id"), Values: aws.StringSlice([]string{vpcID})}},
|
||||
}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc Delete: describe residual subnets failed", "vpc", vpcID, "err", err)
|
||||
} else if snOut != nil {
|
||||
for _, sn := range snOut.Subnets {
|
||||
snID := aws.StringValue(sn.SubnetId)
|
||||
slog.InfoContext(ctx, "systemvpc Delete: removing residual subnet the tagged sweep missed",
|
||||
"name", spec.Name, "vpc", vpcID, "subnet", snID)
|
||||
if _, err := deps.VPC.DeleteSubnet(ctx, &ec2.DeleteSubnetInput{SubnetId: aws.String(snID)}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc Delete: delete residual subnet failed", "subnet", snID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Same for security groups: DeleteVpc rejects any non-default SG, and the
|
||||
// tagged teardown misses ones a partial create never tagged.
|
||||
if deps.SG != nil {
|
||||
if sgOut, err := deps.SG.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{
|
||||
Filters: []*ec2.Filter{{Name: aws.String("vpc-id"), Values: aws.StringSlice([]string{vpcID})}},
|
||||
}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc Delete: describe residual SGs failed", "vpc", vpcID, "err", err)
|
||||
} else if sgOut != nil {
|
||||
for _, sg := range sgOut.SecurityGroups {
|
||||
// The default SG goes with the VPC cascade and cannot be
|
||||
// deleted on its own; it is not what blocks DeleteVpc.
|
||||
if aws.StringValue(sg.GroupName) == "default" {
|
||||
continue
|
||||
}
|
||||
sgID := aws.StringValue(sg.GroupId)
|
||||
slog.InfoContext(ctx, "systemvpc Delete: removing residual SG the tagged sweep missed",
|
||||
"name", spec.Name, "vpc", vpcID, "sg", sgID)
|
||||
if _, err := deps.SG.DeleteSecurityGroup(ctx, &ec2.DeleteSecurityGroupInput{GroupId: aws.String(sgID)}, accountID); err != nil {
|
||||
slog.WarnContext(ctx, "systemvpc Delete: delete residual SG failed", "sg", sgID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := deps.VPC.DeleteVpc(ctx, &ec2.DeleteVpcInput{VpcId: aws.String(vpcID)}, accountID); err != nil && !awserrors.IsNotFound(err) {
|
||||
return fmt.Errorf("systemvpc: delete vpc %s: %w", vpcID, err)
|
||||
}
|
||||
gcTopology(ctx, deps.NATSConn, spec.Name, vpcID)
|
||||
slog.InfoContext(ctx, "systemvpc Delete: managed system VPC removed", "name", spec.Name, "vpc", vpcID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// gcTopology republishes vpc.delete for vpcID so vpcd's topology manager gets
|
||||
// another chance to remove the VPC's OVN logical router and everything OVSDB
|
||||
// cascades with it.
|
||||
//
|
||||
// That cleanup normally runs only off a live DeleteVpc KV mutation, so without
|
||||
// this a lost event would orphan the OVN state forever. Best-effort and
|
||||
// idempotent: vpcd tolerates re-deleting an absent router.
|
||||
func gcTopology(ctx context.Context, nc *nats.Conn, name, vpcID string) {
|
||||
if nc == nil || vpcID == "" {
|
||||
return
|
||||
}
|
||||
utils.PublishEvent(nc, "vpc.delete", struct {
|
||||
VpcId string `json:"vpc_id"`
|
||||
CidrBlock string `json:"cidr_block"`
|
||||
VNI int64 `json:"vni"`
|
||||
}{VpcId: vpcID})
|
||||
slog.InfoContext(ctx, "systemvpc Delete: republished vpc.delete for OVN GC", "name", name, "vpc", vpcID)
|
||||
}
|
||||
@@ -0,0 +1,715 @@
|
||||
package handlers_systemvpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/spinifex/awserrors"
|
||||
"github.com/mulgadc/spinifex/spinifex/tags"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeEC2 is an in-memory stand-in for the whole EC2 VPC family: resources
|
||||
// remember their creation tags and describes honour this package's filters. It
|
||||
// counts creates so idempotency is observable, and records the calling account.
|
||||
type fakeEC2 struct {
|
||||
mu sync.Mutex
|
||||
seq int
|
||||
accs map[string]int
|
||||
|
||||
vpcs map[string]*ec2.Vpc
|
||||
subnets map[string]*ec2.Subnet
|
||||
rts map[string]*ec2.RouteTable
|
||||
ngws map[string]*ec2.NatGateway
|
||||
igws map[string]*ec2.InternetGateway
|
||||
sgs map[string]*ec2.SecurityGroup
|
||||
eips map[string]string
|
||||
|
||||
// routes records the default route installed on each route table, keyed by
|
||||
// route-table id, as "<target-kind>:<target-id>".
|
||||
routes map[string]string
|
||||
|
||||
creates map[string]int
|
||||
}
|
||||
|
||||
var (
|
||||
_ VPCProvisioner = (*fakeEC2)(nil)
|
||||
_ RouteTableProvisioner = (*fakeEC2)(nil)
|
||||
_ NATGatewayProvisioner = (*fakeEC2)(nil)
|
||||
_ EIPProvisioner = (*fakeEC2)(nil)
|
||||
_ IGWProvisioner = (*fakeEC2)(nil)
|
||||
_ SGProvisioner = (*fakeEC2)(nil)
|
||||
)
|
||||
|
||||
func newFakeEC2() *fakeEC2 {
|
||||
return &fakeEC2{
|
||||
accs: map[string]int{},
|
||||
vpcs: map[string]*ec2.Vpc{},
|
||||
subnets: map[string]*ec2.Subnet{},
|
||||
rts: map[string]*ec2.RouteTable{},
|
||||
ngws: map[string]*ec2.NatGateway{},
|
||||
igws: map[string]*ec2.InternetGateway{},
|
||||
sgs: map[string]*ec2.SecurityGroup{},
|
||||
eips: map[string]string{},
|
||||
routes: map[string]string{},
|
||||
creates: map[string]int{},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeEC2) deps() Deps {
|
||||
return Deps{VPC: f, SG: f, IGW: f, RT: f, NGW: f, EIP: f}
|
||||
}
|
||||
|
||||
// id mints a unique AWS-shaped resource id and records the create + account.
|
||||
func (f *fakeEC2) id(op, prefix, accountID string) string {
|
||||
f.seq++
|
||||
f.creates[op]++
|
||||
f.accs[accountID]++
|
||||
return fmt.Sprintf("%s-%04d", prefix, f.seq)
|
||||
}
|
||||
|
||||
// tagsFrom flattens a TagSpecification list into the tag set stored on the
|
||||
// created resource.
|
||||
func tagsFrom(specs []*ec2.TagSpecification) []*ec2.Tag {
|
||||
if len(specs) == 0 {
|
||||
return nil
|
||||
}
|
||||
return specs[0].Tags
|
||||
}
|
||||
|
||||
// matches applies EC2 filter semantics: every filter must match at least one of
|
||||
// the resource's values for that filter's name. "tag:<key>" reads the resource's
|
||||
// tags; anything else reads attrs.
|
||||
func matches(filters []*ec2.Filter, resTags []*ec2.Tag, attrs map[string]string) bool {
|
||||
for _, f := range filters {
|
||||
name := aws.StringValue(f.Name)
|
||||
var have []string
|
||||
if key, ok := strings.CutPrefix(name, "tag:"); ok {
|
||||
for _, t := range resTags {
|
||||
if aws.StringValue(t.Key) == key {
|
||||
have = append(have, aws.StringValue(t.Value))
|
||||
}
|
||||
}
|
||||
} else if v, ok := attrs[name]; ok {
|
||||
have = []string{v}
|
||||
}
|
||||
|
||||
var hit bool
|
||||
for _, want := range aws.StringValueSlice(f.Values) {
|
||||
for _, got := range have {
|
||||
if got == want {
|
||||
hit = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !hit {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (f *fakeEC2) CreateVpc(_ context.Context, in *ec2.CreateVpcInput, accountID string) (*ec2.CreateVpcOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
vpc := &ec2.Vpc{
|
||||
VpcId: aws.String(f.id("vpc", "vpc", accountID)),
|
||||
CidrBlock: in.CidrBlock,
|
||||
Tags: tagsFrom(in.TagSpecifications),
|
||||
}
|
||||
f.vpcs[aws.StringValue(vpc.VpcId)] = vpc
|
||||
return &ec2.CreateVpcOutput{Vpc: vpc}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) DeleteVpc(_ context.Context, in *ec2.DeleteVpcInput, _ string) (*ec2.DeleteVpcOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.vpcs, aws.StringValue(in.VpcId))
|
||||
return &ec2.DeleteVpcOutput{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) DescribeVpcs(_ context.Context, in *ec2.DescribeVpcsInput, _ string) (*ec2.DescribeVpcsOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := &ec2.DescribeVpcsOutput{}
|
||||
for _, v := range f.vpcs {
|
||||
if matches(in.Filters, v.Tags, map[string]string{"vpc-id": aws.StringValue(v.VpcId)}) {
|
||||
out.Vpcs = append(out.Vpcs, v)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) CreateSubnet(_ context.Context, in *ec2.CreateSubnetInput, accountID string) (*ec2.CreateSubnetOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
sn := &ec2.Subnet{
|
||||
SubnetId: aws.String(f.id("subnet", "subnet", accountID)),
|
||||
VpcId: in.VpcId,
|
||||
CidrBlock: in.CidrBlock,
|
||||
AvailabilityZone: in.AvailabilityZone,
|
||||
Tags: tagsFrom(in.TagSpecifications),
|
||||
}
|
||||
f.subnets[aws.StringValue(sn.SubnetId)] = sn
|
||||
return &ec2.CreateSubnetOutput{Subnet: sn}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) DeleteSubnet(_ context.Context, in *ec2.DeleteSubnetInput, _ string) (*ec2.DeleteSubnetOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.subnets, aws.StringValue(in.SubnetId))
|
||||
return &ec2.DeleteSubnetOutput{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) DescribeSubnets(_ context.Context, in *ec2.DescribeSubnetsInput, _ string) (*ec2.DescribeSubnetsOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := &ec2.DescribeSubnetsOutput{}
|
||||
for _, sn := range f.subnets {
|
||||
if matches(in.Filters, sn.Tags, map[string]string{
|
||||
"cidr-block": aws.StringValue(sn.CidrBlock),
|
||||
"vpc-id": aws.StringValue(sn.VpcId),
|
||||
}) {
|
||||
out.Subnets = append(out.Subnets, sn)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) CreateRouteTable(_ context.Context, in *ec2.CreateRouteTableInput, accountID string) (*ec2.CreateRouteTableOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
rt := &ec2.RouteTable{
|
||||
RouteTableId: aws.String(f.id("route-table", "rtb", accountID)),
|
||||
VpcId: in.VpcId,
|
||||
Tags: tagsFrom(in.TagSpecifications),
|
||||
}
|
||||
f.rts[aws.StringValue(rt.RouteTableId)] = rt
|
||||
return &ec2.CreateRouteTableOutput{RouteTable: rt}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) DeleteRouteTable(_ context.Context, in *ec2.DeleteRouteTableInput, _ string) (*ec2.DeleteRouteTableOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.rts, aws.StringValue(in.RouteTableId))
|
||||
return &ec2.DeleteRouteTableOutput{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) DescribeRouteTables(_ context.Context, in *ec2.DescribeRouteTablesInput, _ string) (*ec2.DescribeRouteTablesOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := &ec2.DescribeRouteTablesOutput{}
|
||||
for _, rt := range f.rts {
|
||||
if matches(in.Filters, rt.Tags, map[string]string{"vpc-id": aws.StringValue(rt.VpcId)}) {
|
||||
out.RouteTables = append(out.RouteTables, rt)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) CreateRoute(_ context.Context, in *ec2.CreateRouteInput, accountID string) (*ec2.CreateRouteOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.creates["route"]++
|
||||
f.accs[accountID]++
|
||||
target := "gateway:" + aws.StringValue(in.GatewayId)
|
||||
if in.NatGatewayId != nil {
|
||||
target = "natgw:" + aws.StringValue(in.NatGatewayId)
|
||||
}
|
||||
f.routes[aws.StringValue(in.RouteTableId)] = aws.StringValue(in.DestinationCidrBlock) + "->" + target
|
||||
return &ec2.CreateRouteOutput{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) AssociateRouteTable(_ context.Context, in *ec2.AssociateRouteTableInput, accountID string) (*ec2.AssociateRouteTableOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
rt, ok := f.rts[aws.StringValue(in.RouteTableId)]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no such route table %s", aws.StringValue(in.RouteTableId))
|
||||
}
|
||||
// The real service is AWS-faithful and rejects a subnet that already carries
|
||||
// an explicit association, on any table in the VPC. Ensure has to absorb that
|
||||
// itself, so the fake must reproduce it rather than quietly succeeding.
|
||||
for _, table := range f.rts {
|
||||
for _, a := range table.Associations {
|
||||
if aws.StringValue(a.SubnetId) == aws.StringValue(in.SubnetId) {
|
||||
return nil, errors.New(awserrors.ErrorResourceAlreadyAssociated)
|
||||
}
|
||||
}
|
||||
}
|
||||
assocID := f.id("assoc", "rtbassoc", accountID)
|
||||
rt.Associations = append(rt.Associations, &ec2.RouteTableAssociation{
|
||||
RouteTableAssociationId: aws.String(assocID),
|
||||
RouteTableId: in.RouteTableId,
|
||||
SubnetId: in.SubnetId,
|
||||
})
|
||||
return &ec2.AssociateRouteTableOutput{AssociationId: aws.String(assocID)}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) DisassociateRouteTable(_ context.Context, in *ec2.DisassociateRouteTableInput, _ string) (*ec2.DisassociateRouteTableOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for _, rt := range f.rts {
|
||||
kept := rt.Associations[:0]
|
||||
for _, a := range rt.Associations {
|
||||
if aws.StringValue(a.RouteTableAssociationId) != aws.StringValue(in.AssociationId) {
|
||||
kept = append(kept, a)
|
||||
}
|
||||
}
|
||||
rt.Associations = kept
|
||||
}
|
||||
return &ec2.DisassociateRouteTableOutput{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) CreateNatGateway(_ context.Context, in *ec2.CreateNatGatewayInput, accountID string) (*ec2.CreateNatGatewayOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
ng := &ec2.NatGateway{
|
||||
NatGatewayId: aws.String(f.id("natgw", "nat", accountID)),
|
||||
SubnetId: in.SubnetId,
|
||||
State: aws.String("available"),
|
||||
Tags: tagsFrom(in.TagSpecifications),
|
||||
NatGatewayAddresses: []*ec2.NatGatewayAddress{{
|
||||
AllocationId: in.AllocationId,
|
||||
PublicIp: aws.String(f.eips[aws.StringValue(in.AllocationId)]),
|
||||
}},
|
||||
}
|
||||
f.ngws[aws.StringValue(ng.NatGatewayId)] = ng
|
||||
return &ec2.CreateNatGatewayOutput{NatGateway: ng}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) DeleteNatGateway(_ context.Context, in *ec2.DeleteNatGatewayInput, _ string) (*ec2.DeleteNatGatewayOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if ng, ok := f.ngws[aws.StringValue(in.NatGatewayId)]; ok {
|
||||
ng.State = aws.String("deleted")
|
||||
}
|
||||
return &ec2.DeleteNatGatewayOutput{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) DescribeNatGateways(_ context.Context, in *ec2.DescribeNatGatewaysInput, _ string) (*ec2.DescribeNatGatewaysOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := &ec2.DescribeNatGatewaysOutput{}
|
||||
for _, ng := range f.ngws {
|
||||
if matches(in.Filter, ng.Tags, map[string]string{"state": aws.StringValue(ng.State)}) {
|
||||
out.NatGateways = append(out.NatGateways, ng)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) AllocateAddress(_ context.Context, _ *ec2.AllocateAddressInput, accountID string) (*ec2.AllocateAddressOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
alloc := f.id("eip", "eipalloc", accountID)
|
||||
ip := fmt.Sprintf("198.51.100.%d", len(f.eips)+1)
|
||||
f.eips[alloc] = ip
|
||||
return &ec2.AllocateAddressOutput{AllocationId: aws.String(alloc), PublicIp: aws.String(ip)}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) ReleaseAddress(_ context.Context, in *ec2.ReleaseAddressInput, _ string) (*ec2.ReleaseAddressOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.eips, aws.StringValue(in.AllocationId))
|
||||
return &ec2.ReleaseAddressOutput{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) CreateInternetGateway(_ context.Context, in *ec2.CreateInternetGatewayInput, accountID string) (*ec2.CreateInternetGatewayOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
igw := &ec2.InternetGateway{
|
||||
InternetGatewayId: aws.String(f.id("igw", "igw", accountID)),
|
||||
Tags: tagsFrom(in.TagSpecifications),
|
||||
}
|
||||
f.igws[aws.StringValue(igw.InternetGatewayId)] = igw
|
||||
return &ec2.CreateInternetGatewayOutput{InternetGateway: igw}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) AttachInternetGateway(_ context.Context, in *ec2.AttachInternetGatewayInput, _ string) (*ec2.AttachInternetGatewayOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
igw, ok := f.igws[aws.StringValue(in.InternetGatewayId)]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no such igw %s", aws.StringValue(in.InternetGatewayId))
|
||||
}
|
||||
igw.Attachments = []*ec2.InternetGatewayAttachment{{VpcId: in.VpcId, State: aws.String("available")}}
|
||||
return &ec2.AttachInternetGatewayOutput{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) DetachInternetGateway(_ context.Context, in *ec2.DetachInternetGatewayInput, _ string) (*ec2.DetachInternetGatewayOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if igw, ok := f.igws[aws.StringValue(in.InternetGatewayId)]; ok {
|
||||
igw.Attachments = nil
|
||||
}
|
||||
return &ec2.DetachInternetGatewayOutput{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) DeleteInternetGateway(_ context.Context, in *ec2.DeleteInternetGatewayInput, _ string) (*ec2.DeleteInternetGatewayOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.igws, aws.StringValue(in.InternetGatewayId))
|
||||
return &ec2.DeleteInternetGatewayOutput{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) DescribeInternetGateways(_ context.Context, in *ec2.DescribeInternetGatewaysInput, _ string) (*ec2.DescribeInternetGatewaysOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := &ec2.DescribeInternetGatewaysOutput{}
|
||||
for _, igw := range f.igws {
|
||||
var vpcID string
|
||||
if len(igw.Attachments) > 0 {
|
||||
vpcID = aws.StringValue(igw.Attachments[0].VpcId)
|
||||
}
|
||||
if matches(in.Filters, igw.Tags, map[string]string{"attachment.vpc-id": vpcID}) {
|
||||
out.InternetGateways = append(out.InternetGateways, igw)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) DescribeSecurityGroups(_ context.Context, in *ec2.DescribeSecurityGroupsInput, _ string) (*ec2.DescribeSecurityGroupsOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := &ec2.DescribeSecurityGroupsOutput{}
|
||||
for _, sg := range f.sgs {
|
||||
if matches(in.Filters, sg.Tags, map[string]string{"vpc-id": aws.StringValue(sg.VpcId)}) {
|
||||
out.SecurityGroups = append(out.SecurityGroups, sg)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeEC2) DeleteSecurityGroup(_ context.Context, in *ec2.DeleteSecurityGroupInput, _ string) (*ec2.DeleteSecurityGroupOutput, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.sgs, aws.StringValue(in.GroupId))
|
||||
return &ec2.DeleteSecurityGroupOutput{}, nil
|
||||
}
|
||||
|
||||
// testSpec is a two-private-subnet spec for an arbitrary component, distinct in
|
||||
// every tag key from the second owner used by the isolation test.
|
||||
func testSpec(name string) Spec {
|
||||
return Spec{
|
||||
Owner: Owner{
|
||||
Name: name,
|
||||
ManagedBy: tags.ManagedByEKS,
|
||||
OwnerTagKey: "spinifex:test-owner",
|
||||
RoleTagKey: "spinifex:test-role",
|
||||
},
|
||||
Region: "ap-southeast-2",
|
||||
RolePrefix: "tst",
|
||||
Supernet: "10.252.0.0/14",
|
||||
PrivateSubnets: 2,
|
||||
}
|
||||
}
|
||||
|
||||
// tagValue reads one tag off a resource's tag set.
|
||||
func tagValue(resTags []*ec2.Tag, key string) string {
|
||||
for _, t := range resTags {
|
||||
if aws.StringValue(t.Key) == key {
|
||||
return aws.StringValue(t.Value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestSpecRolesAreNamespacedByPrefix(t *testing.T) {
|
||||
roles := Spec{RolePrefix: "rds"}.Roles()
|
||||
assert.Equal(t, Roles{
|
||||
VPC: "rds-vpc",
|
||||
PublicSubnet: "rds-public",
|
||||
PrivateSubnet: "rds-private",
|
||||
PublicRT: "rds-public-rt",
|
||||
PrivateRT: "rds-private-rt",
|
||||
NatGW: "rds-natgw",
|
||||
}, roles, "role values must be prefix-namespaced, so two components' system VPCs stay distinguishable in one deployment")
|
||||
}
|
||||
|
||||
func TestSpecCIDRsAreDeterministicAndCarved(t *testing.T) {
|
||||
spec := testSpec("cp-demo")
|
||||
|
||||
vpcCIDR, publicCIDR, privateCIDRs, err := spec.cidrs()
|
||||
require.NoError(t, err)
|
||||
|
||||
again, _, againPrivate, err := spec.cidrs()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, vpcCIDR, again, "the address space is derived from the name, so it must survive a process restart unchanged")
|
||||
assert.Equal(t, privateCIDRs, againPrivate)
|
||||
|
||||
assert.True(t, strings.HasSuffix(vpcCIDR, "/22"), "each system VPC gets a /22 out of the supernet, got %s", vpcCIDR)
|
||||
assert.Equal(t, strings.TrimSuffix(vpcCIDR, "/22")+"/24", publicCIDR, "the public subnet is the /22's first /24")
|
||||
assert.Len(t, privateCIDRs, 2, "PrivateSubnets is honoured")
|
||||
assert.NotContains(t, privateCIDRs, publicCIDR, "the private subnets must not overlap the public one")
|
||||
|
||||
// The whole /22 is inside the supernet: with a /14 base of 10.252, the
|
||||
// second octet can only run 252..255.
|
||||
assert.True(t, strings.HasPrefix(vpcCIDR, "10.25"), "the /22 must stay inside the 10.252.0.0/14 supernet, got %s", vpcCIDR)
|
||||
|
||||
other, _, _, err := testSpec("cp-other").cidrs()
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, vpcCIDR, other, "different names must hash to different blocks")
|
||||
}
|
||||
|
||||
func TestSpecPrivateSubnetCountIsClamped(t *testing.T) {
|
||||
spec := testSpec("clamp")
|
||||
|
||||
spec.PrivateSubnets = 0
|
||||
_, _, none, err := spec.cidrs()
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, none, 1, "an unset count still yields the one private subnet system VMs are placed in")
|
||||
|
||||
spec.PrivateSubnets = 99
|
||||
_, _, many, err := spec.cidrs()
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, many, maxPrivateSubnets, "the count is bounded by the /24s the /22 has room for")
|
||||
}
|
||||
|
||||
func TestSpecValidateRejectsUnusableSpecs(t *testing.T) {
|
||||
for name, mutate := range map[string]func(*Spec){
|
||||
"no name": func(s *Spec) { s.Name = "" },
|
||||
"no managed-by": func(s *Spec) { s.ManagedBy = "" },
|
||||
"no owner tag": func(s *Spec) { s.OwnerTagKey = "" },
|
||||
"no role tag": func(s *Spec) { s.RoleTagKey = "" },
|
||||
"no role prefix": func(s *Spec) { s.RolePrefix = "" },
|
||||
"bad supernet": func(s *Spec) { s.Supernet = "not-a-cidr" },
|
||||
// A supernet that is not a /14 would let the 256-block hash index walk
|
||||
// out of the component's allotted space and into a neighbour's.
|
||||
"wrong prefix length": func(s *Spec) { s.Supernet = "10.252.0.0/16" },
|
||||
"host bits set": func(s *Spec) { s.Supernet = "10.252.0.1/14" },
|
||||
"ipv6 supernet": func(s *Spec) { s.Supernet = "fd00::/14" },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
spec := testSpec("valid")
|
||||
mutate(&spec)
|
||||
assert.Error(t, spec.validate())
|
||||
|
||||
// Nothing may be created for a spec that cannot address or tag itself.
|
||||
f := newFakeEC2()
|
||||
_, err := Ensure(t.Context(), f.deps(), spec, "000000000000")
|
||||
assert.Error(t, err)
|
||||
assert.Empty(t, f.creates, "a rejected spec must not have created anything")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureBuildsTheFullTopology(t *testing.T) {
|
||||
f := newFakeEC2()
|
||||
spec := testSpec("cp-demo")
|
||||
|
||||
refs, err := Ensure(t.Context(), f.deps(), spec, "000000000000")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotEmpty(t, refs.VpcID)
|
||||
assert.NotEmpty(t, refs.IGWID)
|
||||
assert.NotEmpty(t, refs.PublicSubnetID)
|
||||
assert.Len(t, refs.PrivateSubnetIDs, 2)
|
||||
assert.NotEmpty(t, refs.NatGatewayID)
|
||||
assert.Equal(t, f.eips[refs.NatEIPAllocationID], refs.NatEIPPublicIP, "the reported NAT address must be the EIP that was allocated")
|
||||
|
||||
// Public egress is via the IGW, private egress via the NAT gateway. These
|
||||
// two routes are what the OVN topology subscribers key the per-subnet egress
|
||||
// policy off, so they are the point of building the VPC through the EC2 APIs.
|
||||
assert.Equal(t, "0.0.0.0/0->gateway:"+refs.IGWID, f.routes[refs.PublicRouteTableID])
|
||||
assert.Equal(t, "0.0.0.0/0->natgw:"+refs.NatGatewayID, f.routes[refs.PrivateRouteTableID])
|
||||
|
||||
privRT := f.rts[refs.PrivateRouteTableID]
|
||||
require.Len(t, privRT.Associations, 2, "every private subnet must be associated, or a VM placed there has no egress")
|
||||
|
||||
// Ownership: the tags are the only handle a later Ensure, Delete or reaper
|
||||
// has on these resources.
|
||||
vpc := f.vpcs[refs.VpcID]
|
||||
assert.Equal(t, tags.ManagedByEKS, tagValue(vpc.Tags, tags.ManagedByKey))
|
||||
assert.Equal(t, "cp-demo", tagValue(vpc.Tags, spec.OwnerTagKey))
|
||||
assert.Equal(t, spec.Roles().VPC, tagValue(vpc.Tags, spec.RoleTagKey))
|
||||
assert.Equal(t, refs.VpcCIDR, aws.StringValue(vpc.CidrBlock))
|
||||
|
||||
assert.Equal(t, spec.az(0), aws.StringValue(f.subnets[refs.PublicSubnetID].AvailabilityZone))
|
||||
|
||||
// Everything was created under the account Ensure was given. A resource that
|
||||
// landed in another account could not be found again by any later describe,
|
||||
// which all run as the system account.
|
||||
assert.Equal(t, []string{"000000000000"}, slices.Collect(maps.Keys(f.accs)))
|
||||
}
|
||||
|
||||
func TestEnsureIsIdempotent(t *testing.T) {
|
||||
f := newFakeEC2()
|
||||
spec := testSpec("cp-demo")
|
||||
|
||||
first, err := Ensure(t.Context(), f.deps(), spec, "000000000000")
|
||||
require.NoError(t, err)
|
||||
createsAfterFirst := maps.Clone(f.creates)
|
||||
|
||||
second, err := Ensure(t.Context(), f.deps(), spec, "000000000000")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, first, second, "a re-run must converge on the same resources rather than build a parallel set")
|
||||
assert.Equal(t, createsAfterFirst, f.creates, "a re-run must create nothing")
|
||||
assert.Len(t, f.rts[first.PrivateRouteTableID].Associations, 2, "re-associating must not duplicate associations")
|
||||
}
|
||||
|
||||
func TestEnsureRejectsASubnetParkedOnAnotherRouteTable(t *testing.T) {
|
||||
f := newFakeEC2()
|
||||
spec := testSpec("cp-demo")
|
||||
|
||||
first, err := Ensure(t.Context(), f.deps(), spec, "000000000000")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Move a private subnet onto the public table, as an operator or a competing
|
||||
// component sharing this VPC could. AssociateRouteTable reports the subnet as
|
||||
// already associated either way, so tolerating that blindly would report the
|
||||
// VPC ready while the subnet has no route to the NAT gateway.
|
||||
privRT, pubRT := f.rts[first.PrivateRouteTableID], f.rts[first.PublicRouteTableID]
|
||||
moved := privRT.Associations[0]
|
||||
privRT.Associations = privRT.Associations[1:]
|
||||
pubRT.Associations = append(pubRT.Associations, moved)
|
||||
|
||||
_, err = Ensure(t.Context(), f.deps(), spec, "000000000000")
|
||||
require.Error(t, err, "a private subnet on someone else's route table must fail loudly, not be reported ready without egress")
|
||||
assert.ErrorContains(t, err, aws.StringValue(moved.SubnetId))
|
||||
}
|
||||
|
||||
func TestEnsureConvergesAfterPartialFailure(t *testing.T) {
|
||||
f := newFakeEC2()
|
||||
spec := testSpec("cp-demo")
|
||||
|
||||
// A create that died after the VPC and its IGW: the second call must adopt
|
||||
// them by tag rather than build a second VPC in a second address block.
|
||||
vpcID, err := ensureVPC(t.Context(), f, spec, "000000000000", "10.253.0.0/22")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, EnsureIGW(t.Context(), f, spec.Owner, "000000000000", vpcID))
|
||||
|
||||
refs, err := Ensure(t.Context(), f.deps(), spec, "000000000000")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, vpcID, refs.VpcID, "the half-built VPC must be adopted, not duplicated")
|
||||
assert.Equal(t, 1, f.creates["vpc"])
|
||||
assert.Equal(t, 1, f.creates["igw"])
|
||||
}
|
||||
|
||||
func TestEnsureOwnersDoNotSeeEachOther(t *testing.T) {
|
||||
f := newFakeEC2()
|
||||
|
||||
eksSpec := testSpec("cp-demo")
|
||||
rdsSpec := Spec{
|
||||
Owner: Owner{
|
||||
Name: "rds-system-ap-southeast-2",
|
||||
ManagedBy: tags.ManagedByRDS,
|
||||
OwnerTagKey: "spinifex:rds-system-vpc",
|
||||
RoleTagKey: "spinifex:rds-role",
|
||||
},
|
||||
Region: "ap-southeast-2",
|
||||
RolePrefix: "rds",
|
||||
Supernet: "10.248.0.0/14",
|
||||
PrivateSubnets: 1,
|
||||
}
|
||||
|
||||
eksRefs, err := Ensure(t.Context(), f.deps(), eksSpec, "000000000000")
|
||||
require.NoError(t, err)
|
||||
rdsRefs, err := Ensure(t.Context(), f.deps(), rdsSpec, "000000000000")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotEqual(t, eksRefs.VpcID, rdsRefs.VpcID, "distinct owners must get distinct VPCs")
|
||||
assert.NotEqual(t, eksRefs.NatGatewayID, rdsRefs.NatGatewayID)
|
||||
assert.False(t, strings.HasPrefix(rdsRefs.VpcCIDR, "10.252."), "the components' supernets must not overlap, or an address cannot name its owner")
|
||||
|
||||
// One owner's teardown must leave the other's topology untouched — this is
|
||||
// the property that makes a shared builder safe for two components.
|
||||
require.NoError(t, Delete(t.Context(), f.deps(), eksSpec, "000000000000", eksRefs.VpcID))
|
||||
|
||||
assert.NotContains(t, f.vpcs, eksRefs.VpcID)
|
||||
assert.Contains(t, f.vpcs, rdsRefs.VpcID, "deleting the EKS system VPC must not reach the RDS one")
|
||||
assert.Contains(t, f.subnets, rdsRefs.PrivateSubnetIDs[0])
|
||||
assert.Contains(t, f.rts, rdsRefs.PrivateRouteTableID)
|
||||
assert.Contains(t, f.igws, rdsRefs.IGWID)
|
||||
assert.Equal(t, "available", aws.StringValue(f.ngws[rdsRefs.NatGatewayID].State))
|
||||
|
||||
// And the deleted owner's own re-Ensure rebuilds from scratch.
|
||||
rebuilt, err := Ensure(t.Context(), f.deps(), eksSpec, "000000000000")
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, eksRefs.VpcID, rebuilt.VpcID)
|
||||
}
|
||||
|
||||
func TestDeleteReclaimsTheBillableAddress(t *testing.T) {
|
||||
f := newFakeEC2()
|
||||
spec := testSpec("cp-demo")
|
||||
|
||||
refs, err := Ensure(t.Context(), f.deps(), spec, "000000000000")
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, f.eips, refs.NatEIPAllocationID)
|
||||
|
||||
require.NoError(t, Delete(t.Context(), f.deps(), spec, "000000000000", refs.VpcID))
|
||||
|
||||
assert.Equal(t, "deleted", aws.StringValue(f.ngws[refs.NatGatewayID].State))
|
||||
assert.NotContains(t, f.eips, refs.NatEIPAllocationID, "an EIP left allocated after teardown bills forever")
|
||||
assert.Empty(t, f.rts, "route tables must go before the NAT gateway they forward to")
|
||||
assert.Empty(t, f.subnets)
|
||||
assert.Empty(t, f.vpcs)
|
||||
}
|
||||
|
||||
func TestDeleteOnNothingProvisionedSucceeds(t *testing.T) {
|
||||
f := newFakeEC2()
|
||||
// Teardown is re-driven on a timer, so a delete after a successful delete —
|
||||
// or one that never had anything to delete — must be a quiet success.
|
||||
assert.NoError(t, Delete(t.Context(), f.deps(), testSpec("cp-demo"), "000000000000", ""))
|
||||
}
|
||||
|
||||
func TestEnsureReleasesTheEIPWhenTheNATGatewayFails(t *testing.T) {
|
||||
f := newFakeEC2()
|
||||
deps := f.deps()
|
||||
deps.NGW = failingNGW{f}
|
||||
|
||||
_, err := Ensure(t.Context(), deps, testSpec("cp-demo"), "000000000000")
|
||||
require.Error(t, err)
|
||||
assert.Empty(t, f.eips, "an EIP allocated for a NAT gateway that never came up must not be stranded")
|
||||
}
|
||||
|
||||
// failingNGW passes describes through to the fake but refuses to create, which
|
||||
// is the window in which an already-allocated EIP can be orphaned.
|
||||
type failingNGW struct{ *fakeEC2 }
|
||||
|
||||
var _ NATGatewayProvisioner = failingNGW{}
|
||||
|
||||
func (failingNGW) CreateNatGateway(context.Context, *ec2.CreateNatGatewayInput, string) (*ec2.CreateNatGatewayOutput, error) {
|
||||
return nil, fmt.Errorf("nat gateway capacity exhausted")
|
||||
}
|
||||
|
||||
func TestEnsureIGWReusesAnExistingGatewayUntagged(t *testing.T) {
|
||||
f := newFakeEC2()
|
||||
owner := testSpec("cp-demo").Owner
|
||||
|
||||
// A customer-provisioned IGW already on the VPC: adopted as-is, never
|
||||
// retagged, so DeleteIGW later declines to remove somebody else's gateway.
|
||||
created, err := f.CreateInternetGateway(t.Context(), &ec2.CreateInternetGatewayInput{}, "000000000000")
|
||||
require.NoError(t, err)
|
||||
igwID := aws.StringValue(created.InternetGateway.InternetGatewayId)
|
||||
_, err = f.AttachInternetGateway(t.Context(), &ec2.AttachInternetGatewayInput{
|
||||
InternetGatewayId: aws.String(igwID),
|
||||
VpcId: aws.String("vpc-customer"),
|
||||
}, "000000000000")
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, EnsureIGW(t.Context(), f, owner, "000000000000", "vpc-customer"))
|
||||
assert.Equal(t, 1, f.creates["igw"], "an attached IGW must be reused, not supplemented")
|
||||
|
||||
require.NoError(t, DeleteIGW(t.Context(), f, owner, "000000000000", "vpc-customer"))
|
||||
assert.Contains(t, f.igws, igwID, "an IGW this owner did not create must survive its teardown")
|
||||
}
|
||||
|
||||
func TestEnsureIGWRejectsAnUnidentifiedRequest(t *testing.T) {
|
||||
f := newFakeEC2()
|
||||
owner := testSpec("cp-demo").Owner
|
||||
|
||||
assert.Error(t, EnsureIGW(t.Context(), f, owner, "000000000000", ""))
|
||||
assert.Error(t, EnsureIGW(t.Context(), f, Owner{}, "000000000000", "vpc-1"))
|
||||
assert.Empty(t, f.creates, "an unidentified IGW could never be found again to delete")
|
||||
}
|
||||
@@ -20,6 +20,12 @@ const (
|
||||
// so this value is not added to IsSystemManaged.
|
||||
ManagedByECS = "ecs"
|
||||
|
||||
// ManagedByRDS identifies RDS-owned resources (DB engine VMs, the
|
||||
// rds-postgres AMI, the shared RDS system VPC and its ENIs). Unlike
|
||||
// ManagedByECS the VMs themselves carry it, marking them system-owned to the
|
||||
// UI's listings and the platform's own sweeps.
|
||||
ManagedByRDS = "rds"
|
||||
|
||||
// LBARNKey stores the parent LB ARN on ELBv2-managed ENIs.
|
||||
LBARNKey = "spinifex:lb-arn"
|
||||
|
||||
@@ -39,5 +45,5 @@ const (
|
||||
// VMs bind a system.TerminateInstance.{id} subject so a cluster-wide teardown
|
||||
// invoked on any node can route a terminate to the owning node.
|
||||
func IsSystemManaged(managedBy string) bool {
|
||||
return managedBy == ManagedByELBv2 || managedBy == ManagedByEKS
|
||||
return managedBy == ManagedByELBv2 || managedBy == ManagedByEKS || managedBy == ManagedByRDS
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package tags
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestIsSystemManaged(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
managedBy string
|
||||
want bool
|
||||
why string
|
||||
}{
|
||||
"elbv2": {ManagedByELBv2, true, "HAProxy VMs are platform-owned"},
|
||||
"eks": {ManagedByEKS, true, "K3s control-plane VMs are platform-owned"},
|
||||
"rds": {ManagedByRDS, true, "DB engine VMs are platform-owned and live in the system account"},
|
||||
// Container instances launched from the ECS node AMI stay customer-owned,
|
||||
// so only the AMI carries the tag — never a VM.
|
||||
"ecs": {ManagedByECS, false, "ECS container instances are customer-owned"},
|
||||
"customer": {"", false, "an untagged instance is a customer instance"},
|
||||
"unknown": {"redshift", false, "an unrecognised component is not system-managed"},
|
||||
}
|
||||
for name, tc := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
// A system VM that reports false never binds its
|
||||
// system.TerminateInstance.{id} subject, so a teardown invoked on a
|
||||
// non-owning node has no responder.
|
||||
assert.Equal(t, tc.want, IsSystemManaged(tc.managedBy), tc.why)
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user