mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 09:01:31 +00:00
fix(vpcd): EnsureSubnet must EnsureVPC first to avoid NATS race
vpc.create-vpc and vpc.create-subnet are independent NATS events with no inter-event ordering guarantee. Tenant bootstrap on iam.account.created publishes both within <1ms, so the subnet handler's CreateLogicalRouterPort can race the parent VPC's CreateLogicalRouter and fail with "logical router not found". The cascade fails CreateNetworkInterface which fails RunInstances which surfaces as InsufficientInstanceCapacity to the SDK — masking the underlying topology race. handleSubnetCreate now calls EnsureVPC(VPCID-only) before EnsureSubnet. EnsureVPC short-circuits on existing LR (idempotent), so the race-winning vpc.create-vpc handler's full CIDR/VNI ExternalIDs are preserved; if subnet wins the race the LR is created with bare ExternalIDs (cosmetic) and the follow-on vpc.create-vpc short-circuits. Adds TestIntegration_SubnetBeforeVPC regression covering both orderings. Closes mulga-siv-133.
This commit is contained in:
@@ -542,3 +542,62 @@ func TestIntegration_IGWErrorPaths(t *testing.T) {
|
||||
InternetGatewayId: aws.String(igwID),
|
||||
}, testIGWAccountID)
|
||||
}
|
||||
|
||||
// TestIntegration_SubnetBeforeVPC asserts that vpc.create-subnet does not
|
||||
// fail when it arrives before the matching vpc.create-vpc (mulga-siv-133).
|
||||
// The subnet handler must idempotently EnsureVPC so the parent logical
|
||||
// router is in place by the time CreateLogicalRouterPort runs.
|
||||
func TestIntegration_SubnetBeforeVPC(t *testing.T) {
|
||||
_, nc := startTestJetStreamNATS(t)
|
||||
mock := NewMockOVNClient()
|
||||
_ = mock.Connect(context.Background())
|
||||
ctx := context.Background()
|
||||
|
||||
topo := NewTopologyHandler(mock)
|
||||
subs, err := topo.Subscribe(nc)
|
||||
if err != nil {
|
||||
t.Fatalf("subscribe: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
for _, s := range subs {
|
||||
_ = s.Unsubscribe()
|
||||
}
|
||||
}()
|
||||
|
||||
// Subnet event arrives first — no prior vpc.create-vpc.
|
||||
subEvt := SubnetEvent{SubnetId: "subnet-race", VpcId: "vpc-race", CidrBlock: "10.0.1.0/24"}
|
||||
subData, _ := json.Marshal(subEvt)
|
||||
resp, err := nc.Request(TopicSubnetCreate, subData, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("vpc.create-subnet (race): %v", err)
|
||||
}
|
||||
assertSuccess(t, resp, "subnet-before-vpc create")
|
||||
|
||||
// Router was created defensively.
|
||||
router, err := mock.GetLogicalRouter(ctx, "vpc-vpc-race")
|
||||
if err != nil {
|
||||
t.Fatalf("expected VPC router after subnet pre-create: %v", err)
|
||||
}
|
||||
// Subnet topology landed.
|
||||
if _, err := mock.GetLogicalSwitch(ctx, "subnet-subnet-race"); err != nil {
|
||||
t.Fatalf("expected subnet switch: %v", err)
|
||||
}
|
||||
if len(router.Ports) != 1 {
|
||||
t.Errorf("router ports = %d, want 1", len(router.Ports))
|
||||
}
|
||||
|
||||
// Now publish the (delayed) vpc.create-vpc with full metadata. EnsureVPC
|
||||
// must short-circuit on the existing LR — no duplicate, no error.
|
||||
vpcEvt := VPCEvent{VpcId: "vpc-race", CidrBlock: "10.0.0.0/16", VNI: 4242}
|
||||
vpcData, _ := json.Marshal(vpcEvt)
|
||||
resp, err = nc.Request(TopicVPCCreate, vpcData, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("vpc.create (late): %v", err)
|
||||
}
|
||||
assertSuccess(t, resp, "late VPC create")
|
||||
|
||||
routerList, _ := mock.ListLogicalRouters(ctx)
|
||||
if len(routerList) != 1 {
|
||||
t.Errorf("routers = %d, want 1 (duplicate after race-loser create)", len(routerList))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -554,7 +554,18 @@ func (h *TopologyHandler) handleSubnetCreate(msg *nats.Msg) {
|
||||
respond(msg, err)
|
||||
return
|
||||
}
|
||||
if err := h.EnsureSubnet(context.Background(), topology.SubnetSpec{
|
||||
ctx := context.Background()
|
||||
// vpc.create-vpc and vpc.create-subnet are independent NATS events with no
|
||||
// inter-event ordering guarantee. Tenant bootstrap publishes both within
|
||||
// <1ms, so EnsureSubnet can race the parent VPC's CreateLogicalRouter and
|
||||
// fail with "logical router not found". EnsureVPC is idempotent on the
|
||||
// router name (mulga-siv-133).
|
||||
if err := h.EnsureVPC(ctx, topology.VPCSpec{VPCID: evt.VpcId}); err != nil {
|
||||
slog.Error("vpcd: EnsureVPC (subnet pre-create) failed", "vpc_id", evt.VpcId, "err", err)
|
||||
respond(msg, err)
|
||||
return
|
||||
}
|
||||
if err := h.EnsureSubnet(ctx, topology.SubnetSpec{
|
||||
SubnetID: evt.SubnetId,
|
||||
VPCID: evt.VpcId,
|
||||
CIDR: cidr,
|
||||
|
||||
Reference in New Issue
Block a user