fix(elbv2): internal single-node LB agent dials br-mgmt IP

Internal-scheme LBs on single-node have no dedicated mgmt-bind IP on
br-mgmt, so resolveMgmtRoute falls back to routing the agent to the
advertised WAN IP via the mgmt NIC. That advertised IP lives on a
different bridge than the agent's mgmt NIC, and the cross-bridge return
path never delivers — the agent never heartbeats and the LB hangs in
provisioning (normal single-node dodges this entirely via dev_networking
slirp at 10.0.2.2; bare-metal external-mode is the first real exercise of
this path).

Point the agent at the br-mgmt host IP, which shares its mgmt L2 — no
cross-bridge hop — mirroring the multi-node datapath. Requires AWSGW to
listen on br-mgmt (bind 0.0.0.0); the server cert already SANs that IP.
This commit is contained in:
joshsiv-mulga
2026-06-07 08:49:37 +10:00
parent 63fd8b9f66
commit 1ce9981c1a
2 changed files with 51 additions and 5 deletions
+25 -1
View File
@@ -48,7 +48,7 @@ func TestBuildLBAgentEnv(t *testing.T) {
SystemSecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
region: "ap-southeast-2",
}
env := svc.buildLBAgentEnv("lb-abc123")
env := svc.buildLBAgentEnv("lb-abc123", SchemeInternetFacing)
lines := strings.Split(strings.TrimRight(env, "\n"), "\n")
kvs := make(map[string]string, len(lines))
@@ -65,6 +65,30 @@ func TestBuildLBAgentEnv(t *testing.T) {
assert.Equal(t, "ap-southeast-2", kvs["LB_REGION"])
}
// TestAgentGatewayURL covers the internal-scheme br-mgmt override: on single-node
// (MgmtRouteTarget empty) internal LBs must dial the br-mgmt host IP, while
// internet-facing LBs and the multi-node case keep GatewayURL untouched.
func TestAgentGatewayURL(t *testing.T) {
base := func() *ELBv2ServiceImpl {
return &ELBv2ServiceImpl{GatewayURL: "https://192.168.0.140:9999", MgmtBridgeIP: "10.15.8.1"}
}
// Internal + single-node fallback (MgmtRouteTarget empty) → br-mgmt IP.
assert.Equal(t, "https://10.15.8.1:9999", base().agentGatewayURL(SchemeInternal))
// Internet-facing → unchanged.
assert.Equal(t, "https://192.168.0.140:9999", base().agentGatewayURL(SchemeInternetFacing))
// Multi-node internal (MgmtRouteTarget set) → unchanged.
svc := base()
svc.MgmtRouteTarget = "10.20.0.5"
assert.Equal(t, "https://192.168.0.140:9999", svc.agentGatewayURL(SchemeInternal))
// Internal but no br-mgmt → unchanged (nothing to fall back to).
svc = &ELBv2ServiceImpl{GatewayURL: "https://192.168.0.140:9999"}
assert.Equal(t, "https://192.168.0.140:9999", svc.agentGatewayURL(SchemeInternal))
}
// TestSubnetCIDRForIP and TestSubnetGatewayIP cover the CIDR helpers.
func TestSubnetCIDRForIP(t *testing.T) {
assert.Equal(t, "10.0.1.5/24", subnetCIDRForIP("10.0.1.5", "10.0.1.0/24"))
+26 -4
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"log/slog"
"net"
"net/url"
"regexp"
"slices"
"sort"
@@ -280,9 +281,30 @@ func (s *ELBv2ServiceImpl) getSystemInstanceType() string {
// buildLBAgentEnv returns the KEY=value blob written to /etc/conf.d/lb-agent
// via fw_cfg on the direct-boot path.
func (s *ELBv2ServiceImpl) buildLBAgentEnv(lbID string) string {
func (s *ELBv2ServiceImpl) buildLBAgentEnv(lbID, scheme string) string {
return fmt.Sprintf("LB_LB_ID=%s\nLB_GATEWAY_URL=%s\nLB_ACCESS_KEY=%s\nLB_SECRET_KEY=%s\nLB_REGION=%s\n",
lbID, s.GatewayURL, s.SystemAccessKey, s.SystemSecretKey, s.region)
lbID, s.agentGatewayURL(scheme), s.SystemAccessKey, s.SystemSecretKey, s.region)
}
// agentGatewayURL returns the AWSGW URL the lb-agent dials. Internal-scheme LBs
// on single-node have no dedicated mgmt-bind IP on br-mgmt (MgmtRouteTarget is
// empty), so the advertised IP in GatewayURL lives on a different bridge than
// the agent's mgmt NIC and the cross-bridge return path is unreliable. Target
// the br-mgmt host IP instead — it shares the agent's mgmt L2, mirroring the
// multi-node datapath where the agent reaches AWSGW over br-mgmt directly. This
// requires AWSGW to listen on br-mgmt (bind 0.0.0.0); the server cert already
// carries the br-mgmt IP as a SAN.
func (s *ELBv2ServiceImpl) agentGatewayURL(scheme string) string {
if scheme == SchemeInternal && s.MgmtBridgeIP != "" && s.MgmtRouteTarget == "" && s.GatewayURL != "" {
if u, err := url.Parse(s.GatewayURL); err == nil {
port := u.Port()
if port == "" {
port = "9999"
}
return "https://" + net.JoinHostPort(s.MgmtBridgeIP, port)
}
}
return s.GatewayURL
}
// subnetCIDRForIP computes "ip/prefixlen" given a host IP and the subnet's
@@ -518,7 +540,7 @@ func (s *ELBv2ServiceImpl) RebuildSystemInstanceInput(ctx RecoveryContext) (*Sys
Scheme: lb.Scheme,
AccountID: lb.AccountID,
NICs: nics,
LBAgentEnv: s.buildLBAgentEnv(lb.LoadBalancerID),
LBAgentEnv: s.buildLBAgentEnv(lb.LoadBalancerID, lb.Scheme),
CACert: s.CACert,
}, nil
}
@@ -1169,7 +1191,7 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
Scheme: scheme,
AccountID: accountID,
NICs: nics,
LBAgentEnv: s.buildLBAgentEnv(lbID),
LBAgentEnv: s.buildLBAgentEnv(lbID, scheme),
CACert: s.CACert,
}
}