From 407a67f8b1bc071608141e0052ecea9e63d2078e Mon Sep 17 00:00:00 2001 From: joshsiv-mulga Date: Thu, 21 May 2026 14:15:19 +1000 Subject: [PATCH] feat(network/host): L0 HostWiring interface + physical/veth impls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 deliverable for mulga-siv-125.2. Introduces network/host (L0 of ADR-0006): UplinkMode enum, Wiring interface, and Physical/Veth implementations with injectable Runner + InterfaceReader for testability. Not yet wired into daemon/vpcd — Phase 2 replaces daemon.SetupComputeNode and vpcd's resolveExternalCIDR with calls through this package. --- spinifex/network/host/bridges.go | 43 +++++ spinifex/network/host/host.go | 115 +++++++++++ spinifex/network/host/host_test.go | 299 +++++++++++++++++++++++++++++ spinifex/network/host/physical.go | 97 ++++++++++ spinifex/network/host/run.go | 93 +++++++++ spinifex/network/host/veth.go | 109 +++++++++++ 6 files changed, 756 insertions(+) create mode 100644 spinifex/network/host/bridges.go create mode 100644 spinifex/network/host/host.go create mode 100644 spinifex/network/host/host_test.go create mode 100644 spinifex/network/host/physical.go create mode 100644 spinifex/network/host/run.go create mode 100644 spinifex/network/host/veth.go diff --git a/spinifex/network/host/bridges.go b/spinifex/network/host/bridges.go new file mode 100644 index 000000000..f16cf2a9b --- /dev/null +++ b/spinifex/network/host/bridges.go @@ -0,0 +1,43 @@ +package host + +import ( + "context" + "fmt" +) + +// ensureBridges idempotently creates br-int and uplinkBridge with the OVS +// settings ovn-controller requires: +// - br-int: fail-mode=secure (preserve flows across ovn-controller restart) +// and other-config:disable-in-band=true (no OVS-installed flows). These +// are the same settings daemon.SetupComputeNode applies today; the +// responsibility moves here in Phase 2. +// - uplinkBridge: created if missing. ovn-bridge-mappings (managed by +// setup-ovn.sh) targets it; the localnet port LSPs will reference it +// once L5 lands. +// +// Shared between Physical and Veth implementations because both modes +// require identical OVS bridge state — only the uplink port wiring differs. +func ensureBridges(ctx context.Context, r Runner, uplinkBridge string) error { + if uplinkBridge == "" { + return fmt.Errorf("host: ensureBridges UplinkBridge required") + } + if _, err := r.Run(ctx, "ovs-vsctl", "--may-exist", "add-br", "br-int"); err != nil { + return fmt.Errorf("create br-int: %w", err) + } + if _, err := r.Run(ctx, "ovs-vsctl", "set", "Bridge", "br-int", "fail-mode=secure"); err != nil { + return fmt.Errorf("set br-int fail-mode: %w", err) + } + if _, err := r.Run(ctx, "ovs-vsctl", "set", "Bridge", "br-int", "other-config:disable-in-band=true"); err != nil { + return fmt.Errorf("set br-int disable-in-band: %w", err) + } + if _, err := r.Run(ctx, "ip", "link", "set", "br-int", "up"); err != nil { + return fmt.Errorf("bring up br-int: %w", err) + } + if _, err := r.Run(ctx, "ovs-vsctl", "--may-exist", "add-br", uplinkBridge); err != nil { + return fmt.Errorf("create %s: %w", uplinkBridge, err) + } + if _, err := r.Run(ctx, "ip", "link", "set", uplinkBridge, "up"); err != nil { + return fmt.Errorf("bring up %s: %w", uplinkBridge, err) + } + return nil +} diff --git a/spinifex/network/host/host.go b/spinifex/network/host/host.go new file mode 100644 index 000000000..d54d17e0d --- /dev/null +++ b/spinifex/network/host/host.go @@ -0,0 +1,115 @@ +// Package host is L0 of the spinifex network stack: kernel/OVS host wiring. +// It owns the host's OVS bridges (br-int, br-ext), the uplink port that +// attaches br-ext to the WAN, and the external CIDR readable off the WAN +// bridge. Bridge mode (physical vs veth) is resolved here and is invisible +// above L0; layers L1+ receive UplinkMode as an init-time enum. +// +// See docs/development/feature/spinifex-network-redesign.md §5 and +// docs/development/proposals/az-local-architecture/0006-spinifex-network-layer-contract.md +// (S2, S3) for the full contract. +package host + +import ( + "context" + "fmt" + "net" + "net/netip" +) + +// UplinkMode names the physical wiring strategy that attaches br-ext to the +// WAN. The two supported values map to former vpcd bridge modes "direct" and +// "veth"; macvlan was removed in Phase 0. +type UplinkMode int + +const ( + // UplinkModeUnknown is the zero value; never returned by a configured + // Wiring implementation. + UplinkModeUnknown UplinkMode = iota + + // UplinkModePhysical enslaves the WAN NIC directly to br-ext. Distributed + // NAT is available — each compute node's gateway LRP can SNAT locally + // using its own uplink MAC/IP. + UplinkModePhysical + + // UplinkModeVeth bridges a Linux bridge (e.g. br-wan, holding the OS- + // assigned WAN IP) to br-ext via a veth pair. Centralised NAT is used — + // only one chassis owns the gateway LRP, and that chassis owns the + // uplink IP. + UplinkModeVeth +) + +// String returns the canonical name used in logs and config diagnostics. +func (m UplinkMode) String() string { + switch m { + case UplinkModePhysical: + return "physical" + case UplinkModeVeth: + return "veth" + default: + return "unknown" + } +} + +// Wiring owns the host's L0 network state. Implementations are constructed +// once at vpcd startup (Phase 2) and consulted by L2/L5 for uplink CIDR and +// by L3 for NAT distribution mode (via UplinkMode at init time). +// +// Every method is idempotent: a second call with the same host state is a +// no-op. EnsureBridges and EnsureUplinkPort may be called before any other +// method and may be safely re-invoked on reconcile. +type Wiring interface { + // EnsureBridges ensures br-int and br-ext exist with the OVS settings + // required by ovn-controller (br-int: fail-mode=secure, disable-in-band). + // br-int is the OVN integration bridge; br-ext is the external bridge + // that ovn-bridge-mappings targets and that the uplink port attaches to. + EnsureBridges(ctx context.Context) error + + // EnsureUplinkPort ensures the uplink port (physical NIC or veth-wan-ovs) + // is attached to br-ext and returns its MAC address. The MAC is used by + // L2 when programming the gateway LRP so that the LRP shares L2 identity + // with the uplink, satisfying upstream switch MAC learning. + EnsureUplinkPort(ctx context.Context) (net.HardwareAddr, error) + + // UplinkMode returns the configured mode. L3 reads this once at startup + // to decide NAT distribution (distributed for physical, centralised for + // veth); it never re-reads at runtime (ADR-0006 S3). + UplinkMode() UplinkMode + + // ExternalCIDR returns the IPv4 prefix assigned to this node's uplink + // bridge by the OS network stack (netplan static config or + // systemd-networkd DHCP). Only valid after EnsureUplinkPort has returned + // successfully — that call blocks the boot race where vpcd starts before + // the WAN address is assigned. + ExternalCIDR(ctx context.Context) (netip.Prefix, error) +} + +// Runner executes a privileged host command (ovs-vsctl, ip) and returns +// combined stdout/stderr. It exists so tests can substitute a stub without +// invoking sudo or mutating the host's OVS conf.db. The production +// implementation is execRunner in run.go. +type Runner interface { + Run(ctx context.Context, name string, args ...string) ([]byte, error) +} + +// InterfaceReader observes kernel network state (interface IP, link master). +// Split from Runner because the production reader uses net.InterfaceByName +// and /sys/class/net rather than shelling out; testability is the same but +// the implementations are unrelated. +type InterfaceReader interface { + // BridgeCIDR returns the first IPv4 prefix assigned to the named link, + // or an error when the link has no IPv4 address yet (boot race). + BridgeCIDR(name string) (netip.Prefix, error) + + // LinkMAC returns the hardware address of the named link. + LinkMAC(name string) (net.HardwareAddr, error) + + // LinkMaster returns the master link name (the bridge a port is + // enslaved to), or "" when the link has no master. + LinkMaster(name string) (string, error) +} + +// ErrNoUplinkAddr is returned by ExternalCIDR when the WAN bridge has no +// IPv4 address yet. Callers (Phase 2's startup path) retry with a bounded +// timeout to absorb the boot race between vpcd start and OS DHCP/netplan +// completion. +var ErrNoUplinkAddr = fmt.Errorf("uplink bridge has no IPv4 address") diff --git a/spinifex/network/host/host_test.go b/spinifex/network/host/host_test.go new file mode 100644 index 000000000..824c4f569 --- /dev/null +++ b/spinifex/network/host/host_test.go @@ -0,0 +1,299 @@ +package host + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "slices" + "strings" + "sync" + "testing" +) + +// stubRunner records every command invocation and returns scripted output. +// Each invocation pops the first matching script entry by prefix; unmatched +// invocations fail the test so missing setup is loud rather than silent. +type stubRunner struct { + mu sync.Mutex + calls []string + resp map[string]stubResp +} + +type stubResp struct { + out []byte + err error +} + +func newStubRunner() *stubRunner { return &stubRunner{resp: map[string]stubResp{}} } + +// expect maps a " " prefix to a response. Used because +// ovs-vsctl flags vary across calls but the binary + verb identify intent. +func (s *stubRunner) expect(prefix string, out []byte, err error) { + s.resp[prefix] = stubResp{out: out, err: err} +} + +func (s *stubRunner) Run(_ context.Context, name string, args ...string) ([]byte, error) { + s.mu.Lock() + defer s.mu.Unlock() + joined := strings.Join(append([]string{name}, args...), " ") + s.calls = append(s.calls, joined) + for prefix, r := range s.resp { + if strings.HasPrefix(joined, prefix) { + return r.out, r.err + } + } + return nil, fmt.Errorf("stubRunner: unexpected command %q", joined) +} + +func (s *stubRunner) called(prefix string) bool { + s.mu.Lock() + defer s.mu.Unlock() + for _, c := range s.calls { + if strings.HasPrefix(c, prefix) { + return true + } + } + return false +} + +// stubReader returns canned interface state without touching the kernel. +type stubReader struct { + cidrs map[string]netip.Prefix + macs map[string]net.HardwareAddr + master map[string]string + errs map[string]error +} + +func newStubReader() *stubReader { + return &stubReader{ + cidrs: map[string]netip.Prefix{}, + macs: map[string]net.HardwareAddr{}, + master: map[string]string{}, + errs: map[string]error{}, + } +} + +func (r *stubReader) BridgeCIDR(name string) (netip.Prefix, error) { + if err, ok := r.errs["cidr:"+name]; ok { + return netip.Prefix{}, err + } + if p, ok := r.cidrs[name]; ok { + return p, nil + } + return netip.Prefix{}, fmt.Errorf("%w: %q", ErrNoUplinkAddr, name) +} + +func (r *stubReader) LinkMAC(name string) (net.HardwareAddr, error) { + if err, ok := r.errs["mac:"+name]; ok { + return nil, err + } + if m, ok := r.macs[name]; ok { + return m, nil + } + return nil, fmt.Errorf("no MAC for %q", name) +} + +func (r *stubReader) LinkMaster(name string) (string, error) { + if err, ok := r.errs["master:"+name]; ok { + return "", err + } + return r.master[name], nil +} + +func mustMAC(t *testing.T, s string) net.HardwareAddr { + t.Helper() + mac, err := net.ParseMAC(s) + if err != nil { + t.Fatalf("parse MAC %q: %v", s, err) + } + return mac +} + +func TestUplinkModeString(t *testing.T) { + cases := []struct { + m UplinkMode + want string + }{ + {UplinkModePhysical, "physical"}, + {UplinkModeVeth, "veth"}, + {UplinkModeUnknown, "unknown"}, + {UplinkMode(99), "unknown"}, + } + for _, c := range cases { + if got := c.m.String(); got != c.want { + t.Errorf("UplinkMode(%d).String() = %q, want %q", c.m, got, c.want) + } + } +} + +func TestPhysical_EnsureUplinkPort(t *testing.T) { + r := newStubRunner() + r.expect("ovs-vsctl port-to-br enp0s3", []byte("br-ext\n"), nil) + rd := newStubReader() + rd.macs["enp0s3"] = mustMAC(t, "aa:bb:cc:dd:ee:ff") + + p := &Physical{ + ExternalInterface: "enp0s3", + UplinkBridge: "br-ext", + Runner: r, + Reader: rd, + } + mac, err := p.EnsureUplinkPort(context.Background()) + if err != nil { + t.Fatalf("EnsureUplinkPort: %v", err) + } + if mac.String() != "aa:bb:cc:dd:ee:ff" { + t.Errorf("got MAC %q, want aa:bb:cc:dd:ee:ff", mac) + } + if p.UplinkMode() != UplinkModePhysical { + t.Errorf("UplinkMode = %v, want physical", p.UplinkMode()) + } +} + +func TestPhysical_EnsureUplinkPort_WrongBridge(t *testing.T) { + r := newStubRunner() + r.expect("ovs-vsctl port-to-br enp0s3", []byte("br-other\n"), nil) + p := &Physical{ + ExternalInterface: "enp0s3", + UplinkBridge: "br-ext", + Runner: r, + Reader: newStubReader(), + } + _, err := p.EnsureUplinkPort(context.Background()) + if err == nil || !strings.Contains(err.Error(), "br-other") { + t.Fatalf("expected bridge mismatch error, got: %v", err) + } +} + +func TestPhysical_ExternalCIDR(t *testing.T) { + rd := newStubReader() + want := netip.MustParsePrefix("192.0.2.5/24") + rd.cidrs["br-ext"] = want + p := &Physical{UplinkBridge: "br-ext", Reader: rd} + got, err := p.ExternalCIDR(context.Background()) + if err != nil { + t.Fatalf("ExternalCIDR: %v", err) + } + if got != want { + t.Errorf("got %v, want %v", got, want) + } +} + +func TestPhysical_ExternalCIDR_NoAddr(t *testing.T) { + p := &Physical{UplinkBridge: "br-ext", Reader: newStubReader()} + _, err := p.ExternalCIDR(context.Background()) + if !errors.Is(err, ErrNoUplinkAddr) { + t.Fatalf("expected ErrNoUplinkAddr, got: %v", err) + } +} + +func TestVeth_EnsureUplinkPort(t *testing.T) { + r := newStubRunner() + r.expect("ovs-vsctl port-to-br veth-wan-ovs", []byte("br-ext\n"), nil) + rd := newStubReader() + rd.macs[VethOVSEnd] = mustMAC(t, "02:11:22:33:44:55") + rd.master[VethLinuxEnd] = "br-wan" + + v := &Veth{ + LinuxBridge: "br-wan", + UplinkBridge: "br-ext", + Runner: r, + Reader: rd, + } + mac, err := v.EnsureUplinkPort(context.Background()) + if err != nil { + t.Fatalf("EnsureUplinkPort: %v", err) + } + if mac.String() != "02:11:22:33:44:55" { + t.Errorf("got MAC %q", mac) + } + if v.UplinkMode() != UplinkModeVeth { + t.Errorf("UplinkMode = %v, want veth", v.UplinkMode()) + } +} + +func TestVeth_EnsureUplinkPort_WrongMaster(t *testing.T) { + r := newStubRunner() + r.expect("ovs-vsctl port-to-br veth-wan-ovs", []byte("br-ext\n"), nil) + rd := newStubReader() + rd.macs[VethOVSEnd] = mustMAC(t, "02:11:22:33:44:55") + rd.master[VethLinuxEnd] = "br-other" + + v := &Veth{LinuxBridge: "br-wan", UplinkBridge: "br-ext", Runner: r, Reader: rd} + _, err := v.EnsureUplinkPort(context.Background()) + if err == nil || !strings.Contains(err.Error(), "br-other") { + t.Fatalf("expected master mismatch error, got: %v", err) + } +} + +func TestVeth_ExternalCIDR(t *testing.T) { + rd := newStubReader() + want := netip.MustParsePrefix("10.0.0.42/24") + rd.cidrs["br-wan"] = want + v := &Veth{LinuxBridge: "br-wan", UplinkBridge: "br-ext", Reader: rd} + got, err := v.ExternalCIDR(context.Background()) + if err != nil { + t.Fatalf("ExternalCIDR: %v", err) + } + if got != want { + t.Errorf("got %v, want %v", got, want) + } +} + +func TestEnsureBridges_RunsExpectedOps(t *testing.T) { + r := newStubRunner() + for _, prefix := range []string{ + "ovs-vsctl --may-exist add-br br-int", + "ovs-vsctl set Bridge br-int fail-mode=secure", + "ovs-vsctl set Bridge br-int other-config:disable-in-band=true", + "ip link set br-int up", + "ovs-vsctl --may-exist add-br br-ext", + "ip link set br-ext up", + } { + r.expect(prefix, nil, nil) + } + p := &Physical{ExternalInterface: "enp0s3", UplinkBridge: "br-ext", Runner: r, Reader: newStubReader()} + if err := p.EnsureBridges(context.Background()); err != nil { + t.Fatalf("EnsureBridges: %v", err) + } + for _, want := range []string{ + "ovs-vsctl --may-exist add-br br-int", + "ovs-vsctl set Bridge br-int fail-mode=secure", + "ovs-vsctl --may-exist add-br br-ext", + } { + if !r.called(want) { + t.Errorf("missing call %q; got %v", want, r.calls) + } + } +} + +func TestEnsureBridges_FailureSurfacesContext(t *testing.T) { + r := newStubRunner() + r.expect("ovs-vsctl --may-exist add-br br-int", []byte("permission denied"), errors.New("exit 1")) + p := &Physical{ExternalInterface: "enp0s3", UplinkBridge: "br-ext", Runner: r} + err := p.EnsureBridges(context.Background()) + if err == nil || !strings.Contains(err.Error(), "create br-int") { + t.Fatalf("expected create br-int wrap, got: %v", err) + } +} + +// Compile-time witnesses that the interface assertions hold; explicit so +// `go vet` flags any future drift without depending on test execution. +var ( + _ Wiring = (*Physical)(nil) + _ Wiring = (*Veth)(nil) + _ Runner = execRunner{} + _ InterfaceReader = kernelReader{} +) + +// Compile-time check that the supported UplinkModes form a stable set. +// Slices are stack-only and never read; the test exists to fail compilation +// if a constant is removed. +func TestUplinkModes_ConstantSet(t *testing.T) { + modes := []UplinkMode{UplinkModeUnknown, UplinkModePhysical, UplinkModeVeth} + if !slices.Contains(modes, UplinkModePhysical) || !slices.Contains(modes, UplinkModeVeth) { + t.Fatal("expected physical and veth in supported mode set") + } +} diff --git a/spinifex/network/host/physical.go b/spinifex/network/host/physical.go new file mode 100644 index 000000000..47ae3b4ea --- /dev/null +++ b/spinifex/network/host/physical.go @@ -0,0 +1,97 @@ +package host + +import ( + "context" + "fmt" + "log/slog" + "net" + "net/netip" + "strings" +) + +// Physical implements Wiring for the "WAN NIC enslaved to br-ext" model +// (former vpcd BridgeModeDirect). The gateway LRP uses the NIC's MAC and +// the OS-assigned IP on the bridge; NAT can be distributed because each +// chassis owns its own uplink. +type Physical struct { + // ExternalInterface is the WAN NIC name (e.g. "enp0s3") that must be an + // OVS port on UplinkBridge. Required; an empty value is a misconfig. + ExternalInterface string + + // UplinkBridge is the OVS bridge that holds the WAN NIC and carries the + // IPv4 uplink address. Conventionally "br-ext"; configurable for legacy + // deployments where the bridge holding the IP differs from the OVN + // external bridge. + UplinkBridge string + + // Runner executes ovs-vsctl / ip; nil falls back to NewExecRunner. + Runner Runner + + // Reader observes kernel link state; nil falls back to NewKernelReader. + Reader InterfaceReader +} + +var _ Wiring = (*Physical)(nil) + +func (p *Physical) runner() Runner { + if p.Runner != nil { + return p.Runner + } + return NewExecRunner() +} + +func (p *Physical) reader() InterfaceReader { + if p.Reader != nil { + return p.Reader + } + return NewKernelReader() +} + +// EnsureBridges creates br-int and br-ext idempotently with the OVS settings +// ovn-controller expects. setup-ovn.sh historically does this; Phase 2 will +// route the daemon through here so the responsibility is in one place. +func (p *Physical) EnsureBridges(ctx context.Context) error { + return ensureBridges(ctx, p.runner(), p.UplinkBridge) +} + +// EnsureUplinkPort verifies the WAN NIC is an OVS port on UplinkBridge and +// returns its MAC. The MAC is the L2 identity the gateway LRP must share so +// upstream switch CAM stays consistent across LRP MAC writes. +func (p *Physical) EnsureUplinkPort(ctx context.Context) (net.HardwareAddr, error) { + if p.ExternalInterface == "" { + return nil, fmt.Errorf("host.Physical: ExternalInterface required") + } + if p.UplinkBridge == "" { + return nil, fmt.Errorf("host.Physical: UplinkBridge required") + } + + out, err := p.runner().Run(ctx, "ovs-vsctl", "port-to-br", p.ExternalInterface) + if err != nil { + return nil, fmt.Errorf("port-to-br %q: %w", p.ExternalInterface, err) + } + br := strings.TrimSpace(string(out)) + if br != p.UplinkBridge { + return nil, fmt.Errorf("host.Physical: %q on bridge %q, expected %q", p.ExternalInterface, br, p.UplinkBridge) + } + + mac, err := p.reader().LinkMAC(p.ExternalInterface) + if err != nil { + return nil, fmt.Errorf("read MAC for %q: %w", p.ExternalInterface, err) + } + slog.Info("host.Physical: uplink verified", "iface", p.ExternalInterface, "bridge", br, "mac", mac.String()) + return mac, nil +} + +// UplinkMode returns UplinkModePhysical (distributed NAT eligible). +func (p *Physical) UplinkMode() UplinkMode { return UplinkModePhysical } + +// ExternalCIDR returns the IPv4 prefix on UplinkBridge. The OS network stack +// assigns it (netplan static or systemd-networkd DHCP) before vpcd starts in +// steady state; ErrNoUplinkAddr means the boot race is still active and the +// caller should retry. +func (p *Physical) ExternalCIDR(_ context.Context) (netip.Prefix, error) { + if p.UplinkBridge == "" { + return netip.Prefix{}, fmt.Errorf("host.Physical: UplinkBridge required") + } + return p.reader().BridgeCIDR(p.UplinkBridge) +} diff --git a/spinifex/network/host/run.go b/spinifex/network/host/run.go new file mode 100644 index 000000000..24a2b4761 --- /dev/null +++ b/spinifex/network/host/run.go @@ -0,0 +1,93 @@ +package host + +import ( + "context" + "fmt" + "net" + "net/netip" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// execRunner runs the privileged host command directly when uid 0, otherwise +// via sudo. Mirrors daemon.sudoCommand to preserve dev-environment behaviour +// (OVS/ip require CAP_NET_ADMIN; production runs as root, but dev VMs do not). +type execRunner struct{} + +var _ Runner = execRunner{} + +// NewExecRunner returns the default Runner used by the production Wiring +// implementations. Tests inject their own Runner instead. +func NewExecRunner() Runner { return execRunner{} } + +func (execRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) { + var cmd *exec.Cmd + if os.Getuid() == 0 { + cmd = exec.CommandContext(ctx, name, args...) + } else { + cmd = exec.CommandContext(ctx, "sudo", append([]string{name}, args...)...) + } + out, err := cmd.CombinedOutput() + if err != nil { + return out, fmt.Errorf("%s %s: %s: %w", name, strings.Join(args, " "), strings.TrimSpace(string(out)), err) + } + return out, nil +} + +// kernelReader satisfies InterfaceReader by reading from the live kernel +// (net.InterfaceByName, /sys/class/net). Tests use a stub instead. +type kernelReader struct{} + +var _ InterfaceReader = kernelReader{} + +// NewKernelReader returns the default InterfaceReader. +func NewKernelReader() InterfaceReader { return kernelReader{} } + +func (kernelReader) BridgeCIDR(name string) (netip.Prefix, error) { + iface, err := net.InterfaceByName(name) + if err != nil { + return netip.Prefix{}, fmt.Errorf("interface %q: %w", name, err) + } + addrs, err := iface.Addrs() + if err != nil { + return netip.Prefix{}, fmt.Errorf("addrs %q: %w", name, err) + } + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok { + continue + } + v4 := ipnet.IP.To4() + if v4 == nil { + continue + } + ones, _ := ipnet.Mask.Size() + addr, _ := netip.AddrFromSlice(v4) + return netip.PrefixFrom(addr, ones), nil + } + return netip.Prefix{}, fmt.Errorf("%w: %q", ErrNoUplinkAddr, name) +} + +func (kernelReader) LinkMAC(name string) (net.HardwareAddr, error) { + iface, err := net.InterfaceByName(name) + if err != nil { + return nil, fmt.Errorf("interface %q: %w", name, err) + } + if len(iface.HardwareAddr) == 0 { + return nil, fmt.Errorf("interface %q: no hardware address", name) + } + return iface.HardwareAddr, nil +} + +func (kernelReader) LinkMaster(name string) (string, error) { + target, err := os.Readlink(filepath.Join("/sys/class/net", name, "master")) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", err + } + return filepath.Base(target), nil +} diff --git a/spinifex/network/host/veth.go b/spinifex/network/host/veth.go new file mode 100644 index 000000000..a5db82baf --- /dev/null +++ b/spinifex/network/host/veth.go @@ -0,0 +1,109 @@ +package host + +import ( + "context" + "fmt" + "log/slog" + "net" + "net/netip" + "strings" +) + +// Default veth pair endpoint names. veth-wan-ovs is the OVS-side endpoint +// (attached to UplinkBridge / br-ext); veth-wan-br is the Linux-side +// endpoint enslaved to the Linux bridge that holds the WAN IP. +const ( + VethOVSEnd = "veth-wan-ovs" + VethLinuxEnd = "veth-wan-br" +) + +// Veth implements Wiring for the "Linux bridge bridged into OVS via veth" +// model (former vpcd BridgeModeVeth). The OS-managed Linux bridge holds the +// IPv4 uplink; OVS sees a veth peer. NAT is centralised because only one +// chassis owns the bridge IP. +type Veth struct { + // LinuxBridge is the Linux bridge holding the OS-assigned WAN IPv4 + // (e.g. "br-wan"). veth-wan-br must be enslaved to it. + LinuxBridge string + + // UplinkBridge is the OVS bridge that ovn-bridge-mappings targets and + // that veth-wan-ovs attaches to. Conventionally "br-ext". + UplinkBridge string + + Runner Runner + Reader InterfaceReader +} + +var _ Wiring = (*Veth)(nil) + +func (v *Veth) runner() Runner { + if v.Runner != nil { + return v.Runner + } + return NewExecRunner() +} + +func (v *Veth) reader() InterfaceReader { + if v.Reader != nil { + return v.Reader + } + return NewKernelReader() +} + +// EnsureBridges ensures br-int and UplinkBridge exist. The veth pair itself +// is provisioned by spinifex-veth-wan.service (a oneshot systemd unit); this +// method only verifies the OVS bridges that the veth peers will attach to. +func (v *Veth) EnsureBridges(ctx context.Context) error { + return ensureBridges(ctx, v.runner(), v.UplinkBridge) +} + +// EnsureUplinkPort verifies that the veth pair is wired as expected +// (veth-wan-ovs → UplinkBridge, veth-wan-br → LinuxBridge) and returns the +// MAC of the OVS-side endpoint. The gateway LRP uses this MAC so that frames +// egressing OVS carry the same source-L2 the upstream router learned. +func (v *Veth) EnsureUplinkPort(ctx context.Context) (net.HardwareAddr, error) { + if v.LinuxBridge == "" { + return nil, fmt.Errorf("host.Veth: LinuxBridge required") + } + if v.UplinkBridge == "" { + return nil, fmt.Errorf("host.Veth: UplinkBridge required") + } + + out, err := v.runner().Run(ctx, "ovs-vsctl", "port-to-br", VethOVSEnd) + if err != nil { + return nil, fmt.Errorf("port-to-br %q: not on OVS — is spinifex-veth-wan.service running? %w", VethOVSEnd, err) + } + br := strings.TrimSpace(string(out)) + if br != v.UplinkBridge { + return nil, fmt.Errorf("host.Veth: %q on bridge %q, expected %q", VethOVSEnd, br, v.UplinkBridge) + } + + master, err := v.reader().LinkMaster(VethLinuxEnd) + if err != nil { + return nil, fmt.Errorf("host.Veth: read master for %q: %w", VethLinuxEnd, err) + } + if master != v.LinuxBridge { + return nil, fmt.Errorf("host.Veth: %q master is %q, expected %q", VethLinuxEnd, master, v.LinuxBridge) + } + + mac, err := v.reader().LinkMAC(VethOVSEnd) + if err != nil { + return nil, fmt.Errorf("read MAC for %q: %w", VethOVSEnd, err) + } + slog.Info("host.Veth: uplink verified", + "ovs_end", VethOVSEnd, "linux_end", VethLinuxEnd, + "linux_bridge", v.LinuxBridge, "uplink_bridge", v.UplinkBridge, "mac", mac.String()) + return mac, nil +} + +// UplinkMode returns UplinkModeVeth (centralised NAT). +func (v *Veth) UplinkMode() UplinkMode { return UplinkModeVeth } + +// ExternalCIDR returns the IPv4 prefix on LinuxBridge — the OS network +// stack (netplan or networkd) owns this address; Spinifex never assigns it. +func (v *Veth) ExternalCIDR(_ context.Context) (netip.Prefix, error) { + if v.LinuxBridge == "" { + return netip.Prefix{}, fmt.Errorf("host.Veth: LinuxBridge required") + } + return v.reader().BridgeCIDR(v.LinuxBridge) +}