mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
feat(network/host): L0 HostWiring interface + physical/veth impls
Phase 1 deliverable for mulga-siv-125.2. Introduces network/host (L0 of ADR-0006): UplinkMode enum, Wiring interface, and Physical/Veth implementations with injectable Runner + InterfaceReader for testability. Not yet wired into daemon/vpcd — Phase 2 replaces daemon.SetupComputeNode and vpcd's resolveExternalCIDR with calls through this package.
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
package host
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// ensureBridges idempotently creates br-int and uplinkBridge with the OVS
|
||||
// settings ovn-controller requires:
|
||||
// - br-int: fail-mode=secure (preserve flows across ovn-controller restart)
|
||||
// and other-config:disable-in-band=true (no OVS-installed flows). These
|
||||
// are the same settings daemon.SetupComputeNode applies today; the
|
||||
// responsibility moves here in Phase 2.
|
||||
// - uplinkBridge: created if missing. ovn-bridge-mappings (managed by
|
||||
// setup-ovn.sh) targets it; the localnet port LSPs will reference it
|
||||
// once L5 lands.
|
||||
//
|
||||
// Shared between Physical and Veth implementations because both modes
|
||||
// require identical OVS bridge state — only the uplink port wiring differs.
|
||||
func ensureBridges(ctx context.Context, r Runner, uplinkBridge string) error {
|
||||
if uplinkBridge == "" {
|
||||
return fmt.Errorf("host: ensureBridges UplinkBridge required")
|
||||
}
|
||||
if _, err := r.Run(ctx, "ovs-vsctl", "--may-exist", "add-br", "br-int"); err != nil {
|
||||
return fmt.Errorf("create br-int: %w", err)
|
||||
}
|
||||
if _, err := r.Run(ctx, "ovs-vsctl", "set", "Bridge", "br-int", "fail-mode=secure"); err != nil {
|
||||
return fmt.Errorf("set br-int fail-mode: %w", err)
|
||||
}
|
||||
if _, err := r.Run(ctx, "ovs-vsctl", "set", "Bridge", "br-int", "other-config:disable-in-band=true"); err != nil {
|
||||
return fmt.Errorf("set br-int disable-in-band: %w", err)
|
||||
}
|
||||
if _, err := r.Run(ctx, "ip", "link", "set", "br-int", "up"); err != nil {
|
||||
return fmt.Errorf("bring up br-int: %w", err)
|
||||
}
|
||||
if _, err := r.Run(ctx, "ovs-vsctl", "--may-exist", "add-br", uplinkBridge); err != nil {
|
||||
return fmt.Errorf("create %s: %w", uplinkBridge, err)
|
||||
}
|
||||
if _, err := r.Run(ctx, "ip", "link", "set", uplinkBridge, "up"); err != nil {
|
||||
return fmt.Errorf("bring up %s: %w", uplinkBridge, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// Package host is L0 of the spinifex network stack: kernel/OVS host wiring.
|
||||
// It owns the host's OVS bridges (br-int, br-ext), the uplink port that
|
||||
// attaches br-ext to the WAN, and the external CIDR readable off the WAN
|
||||
// bridge. Bridge mode (physical vs veth) is resolved here and is invisible
|
||||
// above L0; layers L1+ receive UplinkMode as an init-time enum.
|
||||
//
|
||||
// See docs/development/feature/spinifex-network-redesign.md §5 and
|
||||
// docs/development/proposals/az-local-architecture/0006-spinifex-network-layer-contract.md
|
||||
// (S2, S3) for the full contract.
|
||||
package host
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
)
|
||||
|
||||
// UplinkMode names the physical wiring strategy that attaches br-ext to the
|
||||
// WAN. The two supported values map to former vpcd bridge modes "direct" and
|
||||
// "veth"; macvlan was removed in Phase 0.
|
||||
type UplinkMode int
|
||||
|
||||
const (
|
||||
// UplinkModeUnknown is the zero value; never returned by a configured
|
||||
// Wiring implementation.
|
||||
UplinkModeUnknown UplinkMode = iota
|
||||
|
||||
// UplinkModePhysical enslaves the WAN NIC directly to br-ext. Distributed
|
||||
// NAT is available — each compute node's gateway LRP can SNAT locally
|
||||
// using its own uplink MAC/IP.
|
||||
UplinkModePhysical
|
||||
|
||||
// UplinkModeVeth bridges a Linux bridge (e.g. br-wan, holding the OS-
|
||||
// assigned WAN IP) to br-ext via a veth pair. Centralised NAT is used —
|
||||
// only one chassis owns the gateway LRP, and that chassis owns the
|
||||
// uplink IP.
|
||||
UplinkModeVeth
|
||||
)
|
||||
|
||||
// String returns the canonical name used in logs and config diagnostics.
|
||||
func (m UplinkMode) String() string {
|
||||
switch m {
|
||||
case UplinkModePhysical:
|
||||
return "physical"
|
||||
case UplinkModeVeth:
|
||||
return "veth"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// Wiring owns the host's L0 network state. Implementations are constructed
|
||||
// once at vpcd startup (Phase 2) and consulted by L2/L5 for uplink CIDR and
|
||||
// by L3 for NAT distribution mode (via UplinkMode at init time).
|
||||
//
|
||||
// Every method is idempotent: a second call with the same host state is a
|
||||
// no-op. EnsureBridges and EnsureUplinkPort may be called before any other
|
||||
// method and may be safely re-invoked on reconcile.
|
||||
type Wiring interface {
|
||||
// EnsureBridges ensures br-int and br-ext exist with the OVS settings
|
||||
// required by ovn-controller (br-int: fail-mode=secure, disable-in-band).
|
||||
// br-int is the OVN integration bridge; br-ext is the external bridge
|
||||
// that ovn-bridge-mappings targets and that the uplink port attaches to.
|
||||
EnsureBridges(ctx context.Context) error
|
||||
|
||||
// EnsureUplinkPort ensures the uplink port (physical NIC or veth-wan-ovs)
|
||||
// is attached to br-ext and returns its MAC address. The MAC is used by
|
||||
// L2 when programming the gateway LRP so that the LRP shares L2 identity
|
||||
// with the uplink, satisfying upstream switch MAC learning.
|
||||
EnsureUplinkPort(ctx context.Context) (net.HardwareAddr, error)
|
||||
|
||||
// UplinkMode returns the configured mode. L3 reads this once at startup
|
||||
// to decide NAT distribution (distributed for physical, centralised for
|
||||
// veth); it never re-reads at runtime (ADR-0006 S3).
|
||||
UplinkMode() UplinkMode
|
||||
|
||||
// ExternalCIDR returns the IPv4 prefix assigned to this node's uplink
|
||||
// bridge by the OS network stack (netplan static config or
|
||||
// systemd-networkd DHCP). Only valid after EnsureUplinkPort has returned
|
||||
// successfully — that call blocks the boot race where vpcd starts before
|
||||
// the WAN address is assigned.
|
||||
ExternalCIDR(ctx context.Context) (netip.Prefix, error)
|
||||
}
|
||||
|
||||
// Runner executes a privileged host command (ovs-vsctl, ip) and returns
|
||||
// combined stdout/stderr. It exists so tests can substitute a stub without
|
||||
// invoking sudo or mutating the host's OVS conf.db. The production
|
||||
// implementation is execRunner in run.go.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, name string, args ...string) ([]byte, error)
|
||||
}
|
||||
|
||||
// InterfaceReader observes kernel network state (interface IP, link master).
|
||||
// Split from Runner because the production reader uses net.InterfaceByName
|
||||
// and /sys/class/net rather than shelling out; testability is the same but
|
||||
// the implementations are unrelated.
|
||||
type InterfaceReader interface {
|
||||
// BridgeCIDR returns the first IPv4 prefix assigned to the named link,
|
||||
// or an error when the link has no IPv4 address yet (boot race).
|
||||
BridgeCIDR(name string) (netip.Prefix, error)
|
||||
|
||||
// LinkMAC returns the hardware address of the named link.
|
||||
LinkMAC(name string) (net.HardwareAddr, error)
|
||||
|
||||
// LinkMaster returns the master link name (the bridge a port is
|
||||
// enslaved to), or "" when the link has no master.
|
||||
LinkMaster(name string) (string, error)
|
||||
}
|
||||
|
||||
// ErrNoUplinkAddr is returned by ExternalCIDR when the WAN bridge has no
|
||||
// IPv4 address yet. Callers (Phase 2's startup path) retry with a bounded
|
||||
// timeout to absorb the boot race between vpcd start and OS DHCP/netplan
|
||||
// completion.
|
||||
var ErrNoUplinkAddr = fmt.Errorf("uplink bridge has no IPv4 address")
|
||||
@@ -0,0 +1,299 @@
|
||||
package host
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// stubRunner records every command invocation and returns scripted output.
|
||||
// Each invocation pops the first matching script entry by prefix; unmatched
|
||||
// invocations fail the test so missing setup is loud rather than silent.
|
||||
type stubRunner struct {
|
||||
mu sync.Mutex
|
||||
calls []string
|
||||
resp map[string]stubResp
|
||||
}
|
||||
|
||||
type stubResp struct {
|
||||
out []byte
|
||||
err error
|
||||
}
|
||||
|
||||
func newStubRunner() *stubRunner { return &stubRunner{resp: map[string]stubResp{}} }
|
||||
|
||||
// expect maps a "<binary> <first-arg>" prefix to a response. Used because
|
||||
// ovs-vsctl flags vary across calls but the binary + verb identify intent.
|
||||
func (s *stubRunner) expect(prefix string, out []byte, err error) {
|
||||
s.resp[prefix] = stubResp{out: out, err: err}
|
||||
}
|
||||
|
||||
func (s *stubRunner) Run(_ context.Context, name string, args ...string) ([]byte, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
joined := strings.Join(append([]string{name}, args...), " ")
|
||||
s.calls = append(s.calls, joined)
|
||||
for prefix, r := range s.resp {
|
||||
if strings.HasPrefix(joined, prefix) {
|
||||
return r.out, r.err
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("stubRunner: unexpected command %q", joined)
|
||||
}
|
||||
|
||||
func (s *stubRunner) called(prefix string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, c := range s.calls {
|
||||
if strings.HasPrefix(c, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// stubReader returns canned interface state without touching the kernel.
|
||||
type stubReader struct {
|
||||
cidrs map[string]netip.Prefix
|
||||
macs map[string]net.HardwareAddr
|
||||
master map[string]string
|
||||
errs map[string]error
|
||||
}
|
||||
|
||||
func newStubReader() *stubReader {
|
||||
return &stubReader{
|
||||
cidrs: map[string]netip.Prefix{},
|
||||
macs: map[string]net.HardwareAddr{},
|
||||
master: map[string]string{},
|
||||
errs: map[string]error{},
|
||||
}
|
||||
}
|
||||
|
||||
func (r *stubReader) BridgeCIDR(name string) (netip.Prefix, error) {
|
||||
if err, ok := r.errs["cidr:"+name]; ok {
|
||||
return netip.Prefix{}, err
|
||||
}
|
||||
if p, ok := r.cidrs[name]; ok {
|
||||
return p, nil
|
||||
}
|
||||
return netip.Prefix{}, fmt.Errorf("%w: %q", ErrNoUplinkAddr, name)
|
||||
}
|
||||
|
||||
func (r *stubReader) LinkMAC(name string) (net.HardwareAddr, error) {
|
||||
if err, ok := r.errs["mac:"+name]; ok {
|
||||
return nil, err
|
||||
}
|
||||
if m, ok := r.macs[name]; ok {
|
||||
return m, nil
|
||||
}
|
||||
return nil, fmt.Errorf("no MAC for %q", name)
|
||||
}
|
||||
|
||||
func (r *stubReader) LinkMaster(name string) (string, error) {
|
||||
if err, ok := r.errs["master:"+name]; ok {
|
||||
return "", err
|
||||
}
|
||||
return r.master[name], nil
|
||||
}
|
||||
|
||||
func mustMAC(t *testing.T, s string) net.HardwareAddr {
|
||||
t.Helper()
|
||||
mac, err := net.ParseMAC(s)
|
||||
if err != nil {
|
||||
t.Fatalf("parse MAC %q: %v", s, err)
|
||||
}
|
||||
return mac
|
||||
}
|
||||
|
||||
func TestUplinkModeString(t *testing.T) {
|
||||
cases := []struct {
|
||||
m UplinkMode
|
||||
want string
|
||||
}{
|
||||
{UplinkModePhysical, "physical"},
|
||||
{UplinkModeVeth, "veth"},
|
||||
{UplinkModeUnknown, "unknown"},
|
||||
{UplinkMode(99), "unknown"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.m.String(); got != c.want {
|
||||
t.Errorf("UplinkMode(%d).String() = %q, want %q", c.m, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhysical_EnsureUplinkPort(t *testing.T) {
|
||||
r := newStubRunner()
|
||||
r.expect("ovs-vsctl port-to-br enp0s3", []byte("br-ext\n"), nil)
|
||||
rd := newStubReader()
|
||||
rd.macs["enp0s3"] = mustMAC(t, "aa:bb:cc:dd:ee:ff")
|
||||
|
||||
p := &Physical{
|
||||
ExternalInterface: "enp0s3",
|
||||
UplinkBridge: "br-ext",
|
||||
Runner: r,
|
||||
Reader: rd,
|
||||
}
|
||||
mac, err := p.EnsureUplinkPort(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureUplinkPort: %v", err)
|
||||
}
|
||||
if mac.String() != "aa:bb:cc:dd:ee:ff" {
|
||||
t.Errorf("got MAC %q, want aa:bb:cc:dd:ee:ff", mac)
|
||||
}
|
||||
if p.UplinkMode() != UplinkModePhysical {
|
||||
t.Errorf("UplinkMode = %v, want physical", p.UplinkMode())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhysical_EnsureUplinkPort_WrongBridge(t *testing.T) {
|
||||
r := newStubRunner()
|
||||
r.expect("ovs-vsctl port-to-br enp0s3", []byte("br-other\n"), nil)
|
||||
p := &Physical{
|
||||
ExternalInterface: "enp0s3",
|
||||
UplinkBridge: "br-ext",
|
||||
Runner: r,
|
||||
Reader: newStubReader(),
|
||||
}
|
||||
_, err := p.EnsureUplinkPort(context.Background())
|
||||
if err == nil || !strings.Contains(err.Error(), "br-other") {
|
||||
t.Fatalf("expected bridge mismatch error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhysical_ExternalCIDR(t *testing.T) {
|
||||
rd := newStubReader()
|
||||
want := netip.MustParsePrefix("192.0.2.5/24")
|
||||
rd.cidrs["br-ext"] = want
|
||||
p := &Physical{UplinkBridge: "br-ext", Reader: rd}
|
||||
got, err := p.ExternalCIDR(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ExternalCIDR: %v", err)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhysical_ExternalCIDR_NoAddr(t *testing.T) {
|
||||
p := &Physical{UplinkBridge: "br-ext", Reader: newStubReader()}
|
||||
_, err := p.ExternalCIDR(context.Background())
|
||||
if !errors.Is(err, ErrNoUplinkAddr) {
|
||||
t.Fatalf("expected ErrNoUplinkAddr, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVeth_EnsureUplinkPort(t *testing.T) {
|
||||
r := newStubRunner()
|
||||
r.expect("ovs-vsctl port-to-br veth-wan-ovs", []byte("br-ext\n"), nil)
|
||||
rd := newStubReader()
|
||||
rd.macs[VethOVSEnd] = mustMAC(t, "02:11:22:33:44:55")
|
||||
rd.master[VethLinuxEnd] = "br-wan"
|
||||
|
||||
v := &Veth{
|
||||
LinuxBridge: "br-wan",
|
||||
UplinkBridge: "br-ext",
|
||||
Runner: r,
|
||||
Reader: rd,
|
||||
}
|
||||
mac, err := v.EnsureUplinkPort(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("EnsureUplinkPort: %v", err)
|
||||
}
|
||||
if mac.String() != "02:11:22:33:44:55" {
|
||||
t.Errorf("got MAC %q", mac)
|
||||
}
|
||||
if v.UplinkMode() != UplinkModeVeth {
|
||||
t.Errorf("UplinkMode = %v, want veth", v.UplinkMode())
|
||||
}
|
||||
}
|
||||
|
||||
func TestVeth_EnsureUplinkPort_WrongMaster(t *testing.T) {
|
||||
r := newStubRunner()
|
||||
r.expect("ovs-vsctl port-to-br veth-wan-ovs", []byte("br-ext\n"), nil)
|
||||
rd := newStubReader()
|
||||
rd.macs[VethOVSEnd] = mustMAC(t, "02:11:22:33:44:55")
|
||||
rd.master[VethLinuxEnd] = "br-other"
|
||||
|
||||
v := &Veth{LinuxBridge: "br-wan", UplinkBridge: "br-ext", Runner: r, Reader: rd}
|
||||
_, err := v.EnsureUplinkPort(context.Background())
|
||||
if err == nil || !strings.Contains(err.Error(), "br-other") {
|
||||
t.Fatalf("expected master mismatch error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVeth_ExternalCIDR(t *testing.T) {
|
||||
rd := newStubReader()
|
||||
want := netip.MustParsePrefix("10.0.0.42/24")
|
||||
rd.cidrs["br-wan"] = want
|
||||
v := &Veth{LinuxBridge: "br-wan", UplinkBridge: "br-ext", Reader: rd}
|
||||
got, err := v.ExternalCIDR(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ExternalCIDR: %v", err)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureBridges_RunsExpectedOps(t *testing.T) {
|
||||
r := newStubRunner()
|
||||
for _, prefix := range []string{
|
||||
"ovs-vsctl --may-exist add-br br-int",
|
||||
"ovs-vsctl set Bridge br-int fail-mode=secure",
|
||||
"ovs-vsctl set Bridge br-int other-config:disable-in-band=true",
|
||||
"ip link set br-int up",
|
||||
"ovs-vsctl --may-exist add-br br-ext",
|
||||
"ip link set br-ext up",
|
||||
} {
|
||||
r.expect(prefix, nil, nil)
|
||||
}
|
||||
p := &Physical{ExternalInterface: "enp0s3", UplinkBridge: "br-ext", Runner: r, Reader: newStubReader()}
|
||||
if err := p.EnsureBridges(context.Background()); err != nil {
|
||||
t.Fatalf("EnsureBridges: %v", err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"ovs-vsctl --may-exist add-br br-int",
|
||||
"ovs-vsctl set Bridge br-int fail-mode=secure",
|
||||
"ovs-vsctl --may-exist add-br br-ext",
|
||||
} {
|
||||
if !r.called(want) {
|
||||
t.Errorf("missing call %q; got %v", want, r.calls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureBridges_FailureSurfacesContext(t *testing.T) {
|
||||
r := newStubRunner()
|
||||
r.expect("ovs-vsctl --may-exist add-br br-int", []byte("permission denied"), errors.New("exit 1"))
|
||||
p := &Physical{ExternalInterface: "enp0s3", UplinkBridge: "br-ext", Runner: r}
|
||||
err := p.EnsureBridges(context.Background())
|
||||
if err == nil || !strings.Contains(err.Error(), "create br-int") {
|
||||
t.Fatalf("expected create br-int wrap, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Compile-time witnesses that the interface assertions hold; explicit so
|
||||
// `go vet` flags any future drift without depending on test execution.
|
||||
var (
|
||||
_ Wiring = (*Physical)(nil)
|
||||
_ Wiring = (*Veth)(nil)
|
||||
_ Runner = execRunner{}
|
||||
_ InterfaceReader = kernelReader{}
|
||||
)
|
||||
|
||||
// Compile-time check that the supported UplinkModes form a stable set.
|
||||
// Slices are stack-only and never read; the test exists to fail compilation
|
||||
// if a constant is removed.
|
||||
func TestUplinkModes_ConstantSet(t *testing.T) {
|
||||
modes := []UplinkMode{UplinkModeUnknown, UplinkModePhysical, UplinkModeVeth}
|
||||
if !slices.Contains(modes, UplinkModePhysical) || !slices.Contains(modes, UplinkModeVeth) {
|
||||
t.Fatal("expected physical and veth in supported mode set")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package host
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/netip"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Physical implements Wiring for the "WAN NIC enslaved to br-ext" model
|
||||
// (former vpcd BridgeModeDirect). The gateway LRP uses the NIC's MAC and
|
||||
// the OS-assigned IP on the bridge; NAT can be distributed because each
|
||||
// chassis owns its own uplink.
|
||||
type Physical struct {
|
||||
// ExternalInterface is the WAN NIC name (e.g. "enp0s3") that must be an
|
||||
// OVS port on UplinkBridge. Required; an empty value is a misconfig.
|
||||
ExternalInterface string
|
||||
|
||||
// UplinkBridge is the OVS bridge that holds the WAN NIC and carries the
|
||||
// IPv4 uplink address. Conventionally "br-ext"; configurable for legacy
|
||||
// deployments where the bridge holding the IP differs from the OVN
|
||||
// external bridge.
|
||||
UplinkBridge string
|
||||
|
||||
// Runner executes ovs-vsctl / ip; nil falls back to NewExecRunner.
|
||||
Runner Runner
|
||||
|
||||
// Reader observes kernel link state; nil falls back to NewKernelReader.
|
||||
Reader InterfaceReader
|
||||
}
|
||||
|
||||
var _ Wiring = (*Physical)(nil)
|
||||
|
||||
func (p *Physical) runner() Runner {
|
||||
if p.Runner != nil {
|
||||
return p.Runner
|
||||
}
|
||||
return NewExecRunner()
|
||||
}
|
||||
|
||||
func (p *Physical) reader() InterfaceReader {
|
||||
if p.Reader != nil {
|
||||
return p.Reader
|
||||
}
|
||||
return NewKernelReader()
|
||||
}
|
||||
|
||||
// EnsureBridges creates br-int and br-ext idempotently with the OVS settings
|
||||
// ovn-controller expects. setup-ovn.sh historically does this; Phase 2 will
|
||||
// route the daemon through here so the responsibility is in one place.
|
||||
func (p *Physical) EnsureBridges(ctx context.Context) error {
|
||||
return ensureBridges(ctx, p.runner(), p.UplinkBridge)
|
||||
}
|
||||
|
||||
// EnsureUplinkPort verifies the WAN NIC is an OVS port on UplinkBridge and
|
||||
// returns its MAC. The MAC is the L2 identity the gateway LRP must share so
|
||||
// upstream switch CAM stays consistent across LRP MAC writes.
|
||||
func (p *Physical) EnsureUplinkPort(ctx context.Context) (net.HardwareAddr, error) {
|
||||
if p.ExternalInterface == "" {
|
||||
return nil, fmt.Errorf("host.Physical: ExternalInterface required")
|
||||
}
|
||||
if p.UplinkBridge == "" {
|
||||
return nil, fmt.Errorf("host.Physical: UplinkBridge required")
|
||||
}
|
||||
|
||||
out, err := p.runner().Run(ctx, "ovs-vsctl", "port-to-br", p.ExternalInterface)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("port-to-br %q: %w", p.ExternalInterface, err)
|
||||
}
|
||||
br := strings.TrimSpace(string(out))
|
||||
if br != p.UplinkBridge {
|
||||
return nil, fmt.Errorf("host.Physical: %q on bridge %q, expected %q", p.ExternalInterface, br, p.UplinkBridge)
|
||||
}
|
||||
|
||||
mac, err := p.reader().LinkMAC(p.ExternalInterface)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read MAC for %q: %w", p.ExternalInterface, err)
|
||||
}
|
||||
slog.Info("host.Physical: uplink verified", "iface", p.ExternalInterface, "bridge", br, "mac", mac.String())
|
||||
return mac, nil
|
||||
}
|
||||
|
||||
// UplinkMode returns UplinkModePhysical (distributed NAT eligible).
|
||||
func (p *Physical) UplinkMode() UplinkMode { return UplinkModePhysical }
|
||||
|
||||
// ExternalCIDR returns the IPv4 prefix on UplinkBridge. The OS network stack
|
||||
// assigns it (netplan static or systemd-networkd DHCP) before vpcd starts in
|
||||
// steady state; ErrNoUplinkAddr means the boot race is still active and the
|
||||
// caller should retry.
|
||||
func (p *Physical) ExternalCIDR(_ context.Context) (netip.Prefix, error) {
|
||||
if p.UplinkBridge == "" {
|
||||
return netip.Prefix{}, fmt.Errorf("host.Physical: UplinkBridge required")
|
||||
}
|
||||
return p.reader().BridgeCIDR(p.UplinkBridge)
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package host
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// execRunner runs the privileged host command directly when uid 0, otherwise
|
||||
// via sudo. Mirrors daemon.sudoCommand to preserve dev-environment behaviour
|
||||
// (OVS/ip require CAP_NET_ADMIN; production runs as root, but dev VMs do not).
|
||||
type execRunner struct{}
|
||||
|
||||
var _ Runner = execRunner{}
|
||||
|
||||
// NewExecRunner returns the default Runner used by the production Wiring
|
||||
// implementations. Tests inject their own Runner instead.
|
||||
func NewExecRunner() Runner { return execRunner{} }
|
||||
|
||||
func (execRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||||
var cmd *exec.Cmd
|
||||
if os.Getuid() == 0 {
|
||||
cmd = exec.CommandContext(ctx, name, args...)
|
||||
} else {
|
||||
cmd = exec.CommandContext(ctx, "sudo", append([]string{name}, args...)...)
|
||||
}
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("%s %s: %s: %w", name, strings.Join(args, " "), strings.TrimSpace(string(out)), err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// kernelReader satisfies InterfaceReader by reading from the live kernel
|
||||
// (net.InterfaceByName, /sys/class/net). Tests use a stub instead.
|
||||
type kernelReader struct{}
|
||||
|
||||
var _ InterfaceReader = kernelReader{}
|
||||
|
||||
// NewKernelReader returns the default InterfaceReader.
|
||||
func NewKernelReader() InterfaceReader { return kernelReader{} }
|
||||
|
||||
func (kernelReader) BridgeCIDR(name string) (netip.Prefix, error) {
|
||||
iface, err := net.InterfaceByName(name)
|
||||
if err != nil {
|
||||
return netip.Prefix{}, fmt.Errorf("interface %q: %w", name, err)
|
||||
}
|
||||
addrs, err := iface.Addrs()
|
||||
if err != nil {
|
||||
return netip.Prefix{}, fmt.Errorf("addrs %q: %w", name, err)
|
||||
}
|
||||
for _, a := range addrs {
|
||||
ipnet, ok := a.(*net.IPNet)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
v4 := ipnet.IP.To4()
|
||||
if v4 == nil {
|
||||
continue
|
||||
}
|
||||
ones, _ := ipnet.Mask.Size()
|
||||
addr, _ := netip.AddrFromSlice(v4)
|
||||
return netip.PrefixFrom(addr, ones), nil
|
||||
}
|
||||
return netip.Prefix{}, fmt.Errorf("%w: %q", ErrNoUplinkAddr, name)
|
||||
}
|
||||
|
||||
func (kernelReader) LinkMAC(name string) (net.HardwareAddr, error) {
|
||||
iface, err := net.InterfaceByName(name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("interface %q: %w", name, err)
|
||||
}
|
||||
if len(iface.HardwareAddr) == 0 {
|
||||
return nil, fmt.Errorf("interface %q: no hardware address", name)
|
||||
}
|
||||
return iface.HardwareAddr, nil
|
||||
}
|
||||
|
||||
func (kernelReader) LinkMaster(name string) (string, error) {
|
||||
target, err := os.Readlink(filepath.Join("/sys/class/net", name, "master"))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return "", nil
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
return filepath.Base(target), nil
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package host
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/netip"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Default veth pair endpoint names. veth-wan-ovs is the OVS-side endpoint
|
||||
// (attached to UplinkBridge / br-ext); veth-wan-br is the Linux-side
|
||||
// endpoint enslaved to the Linux bridge that holds the WAN IP.
|
||||
const (
|
||||
VethOVSEnd = "veth-wan-ovs"
|
||||
VethLinuxEnd = "veth-wan-br"
|
||||
)
|
||||
|
||||
// Veth implements Wiring for the "Linux bridge bridged into OVS via veth"
|
||||
// model (former vpcd BridgeModeVeth). The OS-managed Linux bridge holds the
|
||||
// IPv4 uplink; OVS sees a veth peer. NAT is centralised because only one
|
||||
// chassis owns the bridge IP.
|
||||
type Veth struct {
|
||||
// LinuxBridge is the Linux bridge holding the OS-assigned WAN IPv4
|
||||
// (e.g. "br-wan"). veth-wan-br must be enslaved to it.
|
||||
LinuxBridge string
|
||||
|
||||
// UplinkBridge is the OVS bridge that ovn-bridge-mappings targets and
|
||||
// that veth-wan-ovs attaches to. Conventionally "br-ext".
|
||||
UplinkBridge string
|
||||
|
||||
Runner Runner
|
||||
Reader InterfaceReader
|
||||
}
|
||||
|
||||
var _ Wiring = (*Veth)(nil)
|
||||
|
||||
func (v *Veth) runner() Runner {
|
||||
if v.Runner != nil {
|
||||
return v.Runner
|
||||
}
|
||||
return NewExecRunner()
|
||||
}
|
||||
|
||||
func (v *Veth) reader() InterfaceReader {
|
||||
if v.Reader != nil {
|
||||
return v.Reader
|
||||
}
|
||||
return NewKernelReader()
|
||||
}
|
||||
|
||||
// EnsureBridges ensures br-int and UplinkBridge exist. The veth pair itself
|
||||
// is provisioned by spinifex-veth-wan.service (a oneshot systemd unit); this
|
||||
// method only verifies the OVS bridges that the veth peers will attach to.
|
||||
func (v *Veth) EnsureBridges(ctx context.Context) error {
|
||||
return ensureBridges(ctx, v.runner(), v.UplinkBridge)
|
||||
}
|
||||
|
||||
// EnsureUplinkPort verifies that the veth pair is wired as expected
|
||||
// (veth-wan-ovs → UplinkBridge, veth-wan-br → LinuxBridge) and returns the
|
||||
// MAC of the OVS-side endpoint. The gateway LRP uses this MAC so that frames
|
||||
// egressing OVS carry the same source-L2 the upstream router learned.
|
||||
func (v *Veth) EnsureUplinkPort(ctx context.Context) (net.HardwareAddr, error) {
|
||||
if v.LinuxBridge == "" {
|
||||
return nil, fmt.Errorf("host.Veth: LinuxBridge required")
|
||||
}
|
||||
if v.UplinkBridge == "" {
|
||||
return nil, fmt.Errorf("host.Veth: UplinkBridge required")
|
||||
}
|
||||
|
||||
out, err := v.runner().Run(ctx, "ovs-vsctl", "port-to-br", VethOVSEnd)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("port-to-br %q: not on OVS — is spinifex-veth-wan.service running? %w", VethOVSEnd, err)
|
||||
}
|
||||
br := strings.TrimSpace(string(out))
|
||||
if br != v.UplinkBridge {
|
||||
return nil, fmt.Errorf("host.Veth: %q on bridge %q, expected %q", VethOVSEnd, br, v.UplinkBridge)
|
||||
}
|
||||
|
||||
master, err := v.reader().LinkMaster(VethLinuxEnd)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("host.Veth: read master for %q: %w", VethLinuxEnd, err)
|
||||
}
|
||||
if master != v.LinuxBridge {
|
||||
return nil, fmt.Errorf("host.Veth: %q master is %q, expected %q", VethLinuxEnd, master, v.LinuxBridge)
|
||||
}
|
||||
|
||||
mac, err := v.reader().LinkMAC(VethOVSEnd)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read MAC for %q: %w", VethOVSEnd, err)
|
||||
}
|
||||
slog.Info("host.Veth: uplink verified",
|
||||
"ovs_end", VethOVSEnd, "linux_end", VethLinuxEnd,
|
||||
"linux_bridge", v.LinuxBridge, "uplink_bridge", v.UplinkBridge, "mac", mac.String())
|
||||
return mac, nil
|
||||
}
|
||||
|
||||
// UplinkMode returns UplinkModeVeth (centralised NAT).
|
||||
func (v *Veth) UplinkMode() UplinkMode { return UplinkModeVeth }
|
||||
|
||||
// ExternalCIDR returns the IPv4 prefix on LinuxBridge — the OS network
|
||||
// stack (netplan or networkd) owns this address; Spinifex never assigns it.
|
||||
func (v *Veth) ExternalCIDR(_ context.Context) (netip.Prefix, error) {
|
||||
if v.LinuxBridge == "" {
|
||||
return netip.Prefix{}, fmt.Errorf("host.Veth: LinuxBridge required")
|
||||
}
|
||||
return v.reader().BridgeCIDR(v.LinuxBridge)
|
||||
}
|
||||
Reference in New Issue
Block a user