mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 10:52:04 +00:00
fix(elbv2): internal single-node LB agent dials br-mgmt IP
Internal-scheme LBs on single-node have no dedicated mgmt-bind IP on br-mgmt, so resolveMgmtRoute falls back to routing the agent to the advertised WAN IP via the mgmt NIC. That advertised IP lives on a different bridge than the agent's mgmt NIC, and the cross-bridge return path never delivers — the agent never heartbeats and the LB hangs in provisioning (normal single-node dodges this entirely via dev_networking slirp at 10.0.2.2; bare-metal external-mode is the first real exercise of this path). Point the agent at the br-mgmt host IP, which shares its mgmt L2 — no cross-bridge hop — mirroring the multi-node datapath. Requires AWSGW to listen on br-mgmt (bind 0.0.0.0); the server cert already SANs that IP.
This commit is contained in:
@@ -48,7 +48,7 @@ func TestBuildLBAgentEnv(t *testing.T) {
|
||||
SystemSecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
region: "ap-southeast-2",
|
||||
}
|
||||
env := svc.buildLBAgentEnv("lb-abc123")
|
||||
env := svc.buildLBAgentEnv("lb-abc123", SchemeInternetFacing)
|
||||
|
||||
lines := strings.Split(strings.TrimRight(env, "\n"), "\n")
|
||||
kvs := make(map[string]string, len(lines))
|
||||
@@ -65,6 +65,30 @@ func TestBuildLBAgentEnv(t *testing.T) {
|
||||
assert.Equal(t, "ap-southeast-2", kvs["LB_REGION"])
|
||||
}
|
||||
|
||||
// TestAgentGatewayURL covers the internal-scheme br-mgmt override: on single-node
|
||||
// (MgmtRouteTarget empty) internal LBs must dial the br-mgmt host IP, while
|
||||
// internet-facing LBs and the multi-node case keep GatewayURL untouched.
|
||||
func TestAgentGatewayURL(t *testing.T) {
|
||||
base := func() *ELBv2ServiceImpl {
|
||||
return &ELBv2ServiceImpl{GatewayURL: "https://192.168.0.140:9999", MgmtBridgeIP: "10.15.8.1"}
|
||||
}
|
||||
|
||||
// Internal + single-node fallback (MgmtRouteTarget empty) → br-mgmt IP.
|
||||
assert.Equal(t, "https://10.15.8.1:9999", base().agentGatewayURL(SchemeInternal))
|
||||
|
||||
// Internet-facing → unchanged.
|
||||
assert.Equal(t, "https://192.168.0.140:9999", base().agentGatewayURL(SchemeInternetFacing))
|
||||
|
||||
// Multi-node internal (MgmtRouteTarget set) → unchanged.
|
||||
svc := base()
|
||||
svc.MgmtRouteTarget = "10.20.0.5"
|
||||
assert.Equal(t, "https://192.168.0.140:9999", svc.agentGatewayURL(SchemeInternal))
|
||||
|
||||
// Internal but no br-mgmt → unchanged (nothing to fall back to).
|
||||
svc = &ELBv2ServiceImpl{GatewayURL: "https://192.168.0.140:9999"}
|
||||
assert.Equal(t, "https://192.168.0.140:9999", svc.agentGatewayURL(SchemeInternal))
|
||||
}
|
||||
|
||||
// TestSubnetCIDRForIP and TestSubnetGatewayIP cover the CIDR helpers.
|
||||
func TestSubnetCIDRForIP(t *testing.T) {
|
||||
assert.Equal(t, "10.0.1.5/24", subnetCIDRForIP("10.0.1.5", "10.0.1.0/24"))
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -280,9 +281,30 @@ func (s *ELBv2ServiceImpl) getSystemInstanceType() string {
|
||||
|
||||
// buildLBAgentEnv returns the KEY=value blob written to /etc/conf.d/lb-agent
|
||||
// via fw_cfg on the direct-boot path.
|
||||
func (s *ELBv2ServiceImpl) buildLBAgentEnv(lbID string) string {
|
||||
func (s *ELBv2ServiceImpl) buildLBAgentEnv(lbID, scheme string) string {
|
||||
return fmt.Sprintf("LB_LB_ID=%s\nLB_GATEWAY_URL=%s\nLB_ACCESS_KEY=%s\nLB_SECRET_KEY=%s\nLB_REGION=%s\n",
|
||||
lbID, s.GatewayURL, s.SystemAccessKey, s.SystemSecretKey, s.region)
|
||||
lbID, s.agentGatewayURL(scheme), s.SystemAccessKey, s.SystemSecretKey, s.region)
|
||||
}
|
||||
|
||||
// agentGatewayURL returns the AWSGW URL the lb-agent dials. Internal-scheme LBs
|
||||
// on single-node have no dedicated mgmt-bind IP on br-mgmt (MgmtRouteTarget is
|
||||
// empty), so the advertised IP in GatewayURL lives on a different bridge than
|
||||
// the agent's mgmt NIC and the cross-bridge return path is unreliable. Target
|
||||
// the br-mgmt host IP instead — it shares the agent's mgmt L2, mirroring the
|
||||
// multi-node datapath where the agent reaches AWSGW over br-mgmt directly. This
|
||||
// requires AWSGW to listen on br-mgmt (bind 0.0.0.0); the server cert already
|
||||
// carries the br-mgmt IP as a SAN.
|
||||
func (s *ELBv2ServiceImpl) agentGatewayURL(scheme string) string {
|
||||
if scheme == SchemeInternal && s.MgmtBridgeIP != "" && s.MgmtRouteTarget == "" && s.GatewayURL != "" {
|
||||
if u, err := url.Parse(s.GatewayURL); err == nil {
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
port = "9999"
|
||||
}
|
||||
return "https://" + net.JoinHostPort(s.MgmtBridgeIP, port)
|
||||
}
|
||||
}
|
||||
return s.GatewayURL
|
||||
}
|
||||
|
||||
// subnetCIDRForIP computes "ip/prefixlen" given a host IP and the subnet's
|
||||
@@ -518,7 +540,7 @@ func (s *ELBv2ServiceImpl) RebuildSystemInstanceInput(ctx RecoveryContext) (*Sys
|
||||
Scheme: lb.Scheme,
|
||||
AccountID: lb.AccountID,
|
||||
NICs: nics,
|
||||
LBAgentEnv: s.buildLBAgentEnv(lb.LoadBalancerID),
|
||||
LBAgentEnv: s.buildLBAgentEnv(lb.LoadBalancerID, lb.Scheme),
|
||||
CACert: s.CACert,
|
||||
}, nil
|
||||
}
|
||||
@@ -1169,7 +1191,7 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
|
||||
Scheme: scheme,
|
||||
AccountID: accountID,
|
||||
NICs: nics,
|
||||
LBAgentEnv: s.buildLBAgentEnv(lbID),
|
||||
LBAgentEnv: s.buildLBAgentEnv(lbID, scheme),
|
||||
CACert: s.CACert,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user