From 1ce9981c1a262295e22f8e4ed0bc5aa30bc4e7e8 Mon Sep 17 00:00:00 2001 From: joshsiv-mulga Date: Sun, 7 Jun 2026 08:49:37 +1000 Subject: [PATCH] fix(elbv2): internal single-node LB agent dials br-mgmt IP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Internal-scheme LBs on single-node have no dedicated mgmt-bind IP on br-mgmt, so resolveMgmtRoute falls back to routing the agent to the advertised WAN IP via the mgmt NIC. That advertised IP lives on a different bridge than the agent's mgmt NIC, and the cross-bridge return path never delivers — the agent never heartbeats and the LB hangs in provisioning (normal single-node dodges this entirely via dev_networking slirp at 10.0.2.2; bare-metal external-mode is the first real exercise of this path). Point the agent at the br-mgmt host IP, which shares its mgmt L2 — no cross-bridge hop — mirroring the multi-node datapath. Requires AWSGW to listen on br-mgmt (bind 0.0.0.0); the server cert already SANs that IP. --- spinifex/handlers/elbv2/arn_test.go | 26 ++++++++++++++++++++- spinifex/handlers/elbv2/service_impl.go | 30 +++++++++++++++++++++---- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/spinifex/handlers/elbv2/arn_test.go b/spinifex/handlers/elbv2/arn_test.go index cea5ca05a..7bc1ef696 100644 --- a/spinifex/handlers/elbv2/arn_test.go +++ b/spinifex/handlers/elbv2/arn_test.go @@ -48,7 +48,7 @@ func TestBuildLBAgentEnv(t *testing.T) { SystemSecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", region: "ap-southeast-2", } - env := svc.buildLBAgentEnv("lb-abc123") + env := svc.buildLBAgentEnv("lb-abc123", SchemeInternetFacing) lines := strings.Split(strings.TrimRight(env, "\n"), "\n") kvs := make(map[string]string, len(lines)) @@ -65,6 +65,30 @@ func TestBuildLBAgentEnv(t *testing.T) { assert.Equal(t, "ap-southeast-2", kvs["LB_REGION"]) } +// TestAgentGatewayURL covers the internal-scheme br-mgmt override: on single-node +// (MgmtRouteTarget empty) internal LBs must dial the br-mgmt host IP, while +// internet-facing LBs and the multi-node case keep GatewayURL untouched. +func TestAgentGatewayURL(t *testing.T) { + base := func() *ELBv2ServiceImpl { + return &ELBv2ServiceImpl{GatewayURL: "https://192.168.0.140:9999", MgmtBridgeIP: "10.15.8.1"} + } + + // Internal + single-node fallback (MgmtRouteTarget empty) → br-mgmt IP. + assert.Equal(t, "https://10.15.8.1:9999", base().agentGatewayURL(SchemeInternal)) + + // Internet-facing → unchanged. + assert.Equal(t, "https://192.168.0.140:9999", base().agentGatewayURL(SchemeInternetFacing)) + + // Multi-node internal (MgmtRouteTarget set) → unchanged. + svc := base() + svc.MgmtRouteTarget = "10.20.0.5" + assert.Equal(t, "https://192.168.0.140:9999", svc.agentGatewayURL(SchemeInternal)) + + // Internal but no br-mgmt → unchanged (nothing to fall back to). + svc = &ELBv2ServiceImpl{GatewayURL: "https://192.168.0.140:9999"} + assert.Equal(t, "https://192.168.0.140:9999", svc.agentGatewayURL(SchemeInternal)) +} + // TestSubnetCIDRForIP and TestSubnetGatewayIP cover the CIDR helpers. func TestSubnetCIDRForIP(t *testing.T) { assert.Equal(t, "10.0.1.5/24", subnetCIDRForIP("10.0.1.5", "10.0.1.0/24")) diff --git a/spinifex/handlers/elbv2/service_impl.go b/spinifex/handlers/elbv2/service_impl.go index 37f968d85..22ed66dcb 100644 --- a/spinifex/handlers/elbv2/service_impl.go +++ b/spinifex/handlers/elbv2/service_impl.go @@ -8,6 +8,7 @@ import ( "fmt" "log/slog" "net" + "net/url" "regexp" "slices" "sort" @@ -280,9 +281,30 @@ func (s *ELBv2ServiceImpl) getSystemInstanceType() string { // buildLBAgentEnv returns the KEY=value blob written to /etc/conf.d/lb-agent // via fw_cfg on the direct-boot path. -func (s *ELBv2ServiceImpl) buildLBAgentEnv(lbID string) string { +func (s *ELBv2ServiceImpl) buildLBAgentEnv(lbID, scheme string) string { return fmt.Sprintf("LB_LB_ID=%s\nLB_GATEWAY_URL=%s\nLB_ACCESS_KEY=%s\nLB_SECRET_KEY=%s\nLB_REGION=%s\n", - lbID, s.GatewayURL, s.SystemAccessKey, s.SystemSecretKey, s.region) + lbID, s.agentGatewayURL(scheme), s.SystemAccessKey, s.SystemSecretKey, s.region) +} + +// agentGatewayURL returns the AWSGW URL the lb-agent dials. Internal-scheme LBs +// on single-node have no dedicated mgmt-bind IP on br-mgmt (MgmtRouteTarget is +// empty), so the advertised IP in GatewayURL lives on a different bridge than +// the agent's mgmt NIC and the cross-bridge return path is unreliable. Target +// the br-mgmt host IP instead — it shares the agent's mgmt L2, mirroring the +// multi-node datapath where the agent reaches AWSGW over br-mgmt directly. This +// requires AWSGW to listen on br-mgmt (bind 0.0.0.0); the server cert already +// carries the br-mgmt IP as a SAN. +func (s *ELBv2ServiceImpl) agentGatewayURL(scheme string) string { + if scheme == SchemeInternal && s.MgmtBridgeIP != "" && s.MgmtRouteTarget == "" && s.GatewayURL != "" { + if u, err := url.Parse(s.GatewayURL); err == nil { + port := u.Port() + if port == "" { + port = "9999" + } + return "https://" + net.JoinHostPort(s.MgmtBridgeIP, port) + } + } + return s.GatewayURL } // subnetCIDRForIP computes "ip/prefixlen" given a host IP and the subnet's @@ -518,7 +540,7 @@ func (s *ELBv2ServiceImpl) RebuildSystemInstanceInput(ctx RecoveryContext) (*Sys Scheme: lb.Scheme, AccountID: lb.AccountID, NICs: nics, - LBAgentEnv: s.buildLBAgentEnv(lb.LoadBalancerID), + LBAgentEnv: s.buildLBAgentEnv(lb.LoadBalancerID, lb.Scheme), CACert: s.CACert, }, nil } @@ -1169,7 +1191,7 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp Scheme: scheme, AccountID: accountID, NICs: nics, - LBAgentEnv: s.buildLBAgentEnv(lbID), + LBAgentEnv: s.buildLBAgentEnv(lbID, scheme), CACert: s.CACert, } }