From 7ddcac53bb92588b1dbe99d84eacd809673587fb Mon Sep 17 00:00:00 2001 From: Ben Duncan <5178243+benduncan@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:45:26 -0500 Subject: [PATCH] feat: block outbound SMTP egress via platform OVN ACL (#1001) * feat: block outbound SMTP egress via platform OVN ACL Add a platform egress ACL that drops outbound TCP to 25/465/587 for public destinations on every guest, matching AWS's default outbound-SMTP block. The control is global by default with an operator-held per-VPC exemption; tenants cannot open the ports via a security group because the block sits above the tenant-allow priority band. Private destinations are always exempt. Config exposes blocked_ports_wan (nil defaults to [25,465,587], [] disables) and egress_block_exempt_vpcs. Adds unit tests for the ACL builder, the SG manager wiring and the config resolution, plus an e2e that digs gmail.com MX and confirms a 5s SMTP connect is blocked while other WAN egress works. * docs: note the multi-node sync + restart requirement for the egress block --- cmd/spinifex/cmd/service.go | 2 + cmd/spinifex/cmd/templates/spinifex.toml | 7 + docs/compute/vpc-networking/README.md | 65 +++++++++ spinifex/config/config.go | 24 ++++ spinifex/config/config_test.go | 13 ++ spinifex/network/policy/acl.go | 42 ++++++ spinifex/network/policy/acl_test.go | 28 ++++ spinifex/network/policy/sg.go | 28 +++- spinifex/network/policy/sg_test.go | 58 +++++++- spinifex/network/reconcile/apply_test.go | 8 +- .../network/reconcile/scenario_live_test.go | 2 +- .../network/subscribers/subscribers_test.go | 6 +- spinifex/vpcd/vpcd.go | 15 ++- tests/e2e/single/smtpblock_test.go | 127 ++++++++++++++++++ tests/e2e/single/tests_test.go | 7 + tests/fixtures/vpcd/vpcd.go | 2 +- 16 files changed, 415 insertions(+), 19 deletions(-) create mode 100644 tests/e2e/single/smtpblock_test.go diff --git a/cmd/spinifex/cmd/service.go b/cmd/spinifex/cmd/service.go index 50de4edb0..3a72b6b05 100644 --- a/cmd/spinifex/cmd/service.go +++ b/cmd/spinifex/cmd/service.go @@ -1028,6 +1028,8 @@ var vpcdStartCmd = &cobra.Command{ NATExemptCIDRs: clusterConfig.Network.NATExemptCIDRs, IPSecEnabled: clusterConfig.Network.IPSecEnabled, UnderlayMTU: clusterConfig.Network.UnderlayMTU, + BlockedWANPorts: clusterConfig.Network.ResolvedBlockedWANPorts(), + EgressBlockExemptVPCs: clusterConfig.Network.EgressBlockExemptVPCs, }) if err != nil { return fmt.Errorf("create vpcd service: %w", err) diff --git a/cmd/spinifex/cmd/templates/spinifex.toml b/cmd/spinifex/cmd/templates/spinifex.toml index 542e81c9f..3f51ce68e 100644 --- a/cmd/spinifex/cmd/templates/spinifex.toml +++ b/cmd/spinifex/cmd/templates/spinifex.toml @@ -14,6 +14,13 @@ internal_suffix = "spinifex.internal" [network] ipsec_enabled = {{.IPSecEnabled}} +# Outbound TCP ports blocked for guest egress to public destinations, like AWS's +# default outbound-SMTP block. Omit the key for the default [25, 465, 587]; set +# [] to disable. Private destinations are always exempt. +# blocked_ports_wan = [25, 465, 587] +# VPC IDs exempted from the block — the operator workaround for a tenant that +# legitimately needs to send mail, until per-account exceptions exist. +# egress_block_exempt_vpcs = ["vpc-..."] {{- if .ExternalMode}} external_mode = "{{.ExternalMode}}" {{- range .Pools}} diff --git a/docs/compute/vpc-networking/README.md b/docs/compute/vpc-networking/README.md index ecdb22712..6528f4b08 100644 --- a/docs/compute/vpc-networking/README.md +++ b/docs/compute/vpc-networking/README.md @@ -795,6 +795,71 @@ aws ec2 run-instances --image-id $AMI --instance-type t3.small \ Rule changes take effect immediately — no instance restart needed. +## Platform Default Egress Restrictions (Outbound SMTP) + +Like AWS, Spinifex blocks outbound SMTP from instances **by default** so a +compromised guest cannot turn a fresh account into a spam relay. Connections to +the mail ports are dropped at the OVS datapath before they reach the wire: + +| Port | Protocol | Purpose | +| --- | --- | --- | +| 25 | TCP | SMTP relay | +| 465 | TCP | SMTP over implicit TLS | +| 587 | TCP | SMTP submission | + +This is a **platform default, not a security-group rule**. It is enforced as an +OVN egress ACL on every guest's port group at a priority **above** tenant SG +allows, so a tenant **cannot** open these ports by adding a security-group rule — +matching AWS, where lifting the block is an operator action, not a tenant one. +Only **public** destinations are blocked; mail to private ranges (`10/8`, +`172.16/12`, `192.168/16`, `100.64/10`, link-local) is exempt, so an in-VPC or +on-prem relay still works. Dropped attempts are logged for abuse triage. + +It sits alongside the security-group ACLs and the host firewall (the `nft` +policy that scopes the node's own ports) as a third datapath control — this one +applies to guest egress specifically. + +**Operator controls** (cluster-wide, in `spinifex.toml`): + +```toml +[network] +# Omit for the default [25, 465, 587]; set [] to disable entirely. +blocked_ports_wan = [25, 465, 587] + +# Workaround to let a specific tenant/VPC send mail, until per-account +# exceptions exist: list the VPC IDs to exempt from the block. +egress_block_exempt_vpcs = ["vpc-0abc123..."] +``` + +Exempting a VPC removes the block for **all** guests in that VPC, so scope it +narrowly. Add every VPC ID you want exempt to the one list — +`egress_block_exempt_vpcs = ["vpc-a", "vpc-b", ...]`; the match is by VPC ID, so +one entry covers every security group in that VPC. + +### Multi-node clusters — keep the value identical on every node, and restart vpcd + +`[network]` is a cluster-wide *setting*, but it physically lives in each node's +own `spinifex.toml`, and there is no live distribution of it. Two properties of +the current implementation make the operator responsible for consistency: + +- **One node writes the ACLs.** SG reconcile runs on a single CAS-elected vpcd + leader, which programs the shared OVN northbound DB. Whichever node holds the + lease is the one whose `blocked_ports_wan` / `egress_block_exempt_vpcs` is in + force. Leadership moves on restart or crash, so if the node configs disagree, + the effective policy **changes when the leader changes** and the drift pass + flaps the ACLs between the two states. Edit the value **identically on every + node**. +- **It is read at vpcd startup, not hot-reloaded.** The policy is built once when + vpcd starts; editing the TOML does nothing until vpcd restarts. Deploy the + config change to all nodes and restart vpcd cluster-wide (take the target down + and confirm no `spx` process survives — a selective single-service restart is + not reliable on the shared binary). + +This is the current implementation and is deliberately minimal. A future revision +will move the exemption into shared cluster state (so a single edit propagates +and cannot drift between nodes) and make it a per-account control rather than a +per-VPC operator edit; until then, the two rules above are load-bearing. + ## The Instance-to-Host Plane (Metadata and VPC DNS) Security groups govern traffic **between instances** and **to the outside**. There diff --git a/spinifex/config/config.go b/spinifex/config/config.go index 47a881bc5..3c3f74188 100644 --- a/spinifex/config/config.go +++ b/spinifex/config/config.go @@ -84,6 +84,30 @@ type NetworkConfig struct { // NATExemptCIDRs are extra destinations that skip routed-mode SNAT (added // to the transit /24 in the spinifex_nat_exempt set). nat mode only. NATExemptCIDRs []string `mapstructure:"nat_exempt_cidrs"` + // BlockedPortsWAN are TCP destination ports blocked for guest egress to + // public destinations, like AWS's default outbound-SMTP block. Three-state: + // nil (key absent) uses DefaultBlockedWANPorts; an empty list disables the + // block; a list sets it. Private destinations are always exempt. See + // ResolvedBlockedWANPorts. + BlockedPortsWAN *[]int `mapstructure:"blocked_ports_wan"` + // EgressBlockExemptVPCs are VPC IDs exempted from the WAN egress block — the + // operator workaround for a tenant with a legitimate need until per-account + // exceptions exist. + EgressBlockExemptVPCs []string `mapstructure:"egress_block_exempt_vpcs"` +} + +// DefaultBlockedWANPorts mirrors AWS's out-of-the-box outbound mail block: +// SMTP (25), SMTPS (465) and submission (587). +var DefaultBlockedWANPorts = []int{25, 465, 587} + +// ResolvedBlockedWANPorts returns the configured WAN egress block ports: the +// AWS-parity default when the key is absent (nil), an empty slice when it is +// explicitly set to [] (disabled), or the configured list. +func (c NetworkConfig) ResolvedBlockedWANPorts() []int { + if c.BlockedPortsWAN == nil { + return DefaultBlockedWANPorts + } + return *c.BlockedPortsWAN } // BootstrapConfig holds the default VPC infrastructure IDs written by admin init. diff --git a/spinifex/config/config_test.go b/spinifex/config/config_test.go index 72f94b3fe..cee2cd0f2 100644 --- a/spinifex/config/config_test.go +++ b/spinifex/config/config_test.go @@ -1115,3 +1115,16 @@ cache_size_mb = -1 require.Error(t, err) assert.Contains(t, err.Error(), "cache_size_mb") } + +func TestResolvedBlockedWANPorts(t *testing.T) { + // Key absent (nil) → AWS-parity default. + assert.Equal(t, DefaultBlockedWANPorts, NetworkConfig{}.ResolvedBlockedWANPorts()) + + // Explicit empty list → disabled. + empty := []int{} + assert.Empty(t, NetworkConfig{BlockedPortsWAN: &empty}.ResolvedBlockedWANPorts()) + + // Explicit list → verbatim. + custom := []int{25, 2525} + assert.Equal(t, custom, NetworkConfig{BlockedPortsWAN: &custom}.ResolvedBlockedWANPorts()) +} diff --git a/spinifex/network/policy/acl.go b/spinifex/network/policy/acl.go index 7c29406d4..34ec5b0cf 100644 --- a/spinifex/network/policy/acl.go +++ b/spinifex/network/policy/acl.go @@ -2,6 +2,7 @@ package policy import ( "fmt" + "strconv" "strings" "github.com/mulgadc/spinifex/spinifex/network/ovn" @@ -18,6 +19,11 @@ const ( // can't write a 255.255.255.255 rule. ACLPriorityAllowDHCP = 1050 + // ACLPriorityPlatformEgressBlock: platform WAN egress block (AWS-parity + // outbound SMTP). Above tenant allows so a tenant SG cannot open it, below + // DHCP so lease traffic is untouched. Not tenant-writable. + ACLPriorityPlatformEgressBlock = 1049 + // ACLPriorityTenantAllow: tenant ingress/egress allows. Not logged. ACLPriorityTenantAllow = 1000 @@ -61,6 +67,42 @@ func InfrastructureACLs(portGroupName string) []ovn.ACLSpec { } } +// wanEgressBlockPrivateDsts are the destinations the WAN egress block exempts, +// so intra-VPC and on-prem traffic (e.g. a private mail relay) is never dropped +// — only public destinations are blocked. +var wanEgressBlockPrivateDsts = []string{ + "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "100.64.0.0/10", "169.254.0.0/16", +} + +// BlockedWANEgressACL drops guest egress to the given TCP destination ports for +// public destinations only — matching AWS, which blocks outbound SMTP out of the +// box so a compromised guest cannot become a spam relay. Logged for abuse +// triage. Returns ok=false when ports is empty, so a caller can skip it. +func BlockedWANEgressACL(portGroupName string, ports []int) (ovn.ACLSpec, bool) { + if len(ports) == 0 { + return ovn.ACLSpec{}, false + } + portStrs := make([]string, len(ports)) + for i, p := range ports { + portStrs[i] = strconv.Itoa(p) + } + match := fmt.Sprintf( + "inport == @%s && ip4 && ip4.dst != {%s} && tcp && tcp.dst == {%s}", + portGroupName, + strings.Join(wanEgressBlockPrivateDsts, ", "), + strings.Join(portStrs, ", "), + ) + return ovn.ACLSpec{ + Direction: "from-lport", + Priority: ACLPriorityPlatformEgressBlock, + Match: match, + Action: "drop", + Name: portGroupName + "-block-wan-egress", + Log: true, + Severity: denyACLSeverity, + }, true +} + // RuleACLSpecs builds priority-1000 allow ACLs with "allow-related" action. func RuleACLSpecs(portGroupName string, ingress, egress []Rule) ([]ovn.ACLSpec, error) { specs := make([]ovn.ACLSpec, 0, len(ingress)+len(egress)) diff --git a/spinifex/network/policy/acl_test.go b/spinifex/network/policy/acl_test.go index 084979c72..e07962939 100644 --- a/spinifex/network/policy/acl_test.go +++ b/spinifex/network/policy/acl_test.go @@ -210,3 +210,31 @@ func TestRuleACLSpecs_PriorityAndAction(t *testing.T) { assert.Equal(t, "allow-related", specs[1].Action) } } + +func TestBlockedWANEgressACL_MatchAndPriority(t *testing.T) { + spec, ok := BlockedWANEgressACL("sg_test", []int{25, 465, 587}) + assert.True(t, ok) + assert.Equal(t, "from-lport", spec.Direction) + assert.Equal(t, "drop", spec.Action) + assert.True(t, spec.Log) + assert.Equal(t, denyACLSeverity, spec.Severity) + assert.Equal(t, "sg_test-block-wan-egress", spec.Name) + assert.Contains(t, spec.Match, "inport == @sg_test") + assert.Contains(t, spec.Match, "tcp.dst == {25, 465, 587}") + assert.Contains(t, spec.Match, "ip4.dst != {10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 100.64.0.0/10, 169.254.0.0/16}") +} + +// TestBlockedWANEgressACL_Priority pins the band: above tenant allows so a +// tenant SG cannot open the port, below DHCP so lease traffic is untouched. +func TestBlockedWANEgressACL_Priority(t *testing.T) { + spec, _ := BlockedWANEgressACL("sg_test", []int{25}) + assert.Greater(t, spec.Priority, ACLPriorityTenantAllow) + assert.Less(t, spec.Priority, ACLPriorityAllowDHCP) +} + +func TestBlockedWANEgressACL_EmptyDisables(t *testing.T) { + _, ok := BlockedWANEgressACL("sg_test", nil) + assert.False(t, ok) + _, ok = BlockedWANEgressACL("sg_test", []int{}) + assert.False(t, ok) +} diff --git a/spinifex/network/policy/sg.go b/spinifex/network/policy/sg.go index 0b92177e6..e3008352c 100644 --- a/spinifex/network/policy/sg.go +++ b/spinifex/network/policy/sg.go @@ -32,15 +32,30 @@ type SecurityGroupManager interface { DeleteSG(ctx context.Context, groupID string) error } +// EgressPolicy is the platform egress-abuse policy applied to every guest port +// group regardless of the tenant's security groups. The zero value blocks +// nothing. +type EgressPolicy struct { + // BlockedWANPorts are TCP destination ports dropped to public destinations + // (AWS-parity outbound SMTP block). Empty disables the block. + BlockedWANPorts []int + // ExemptVPCs are VPC IDs the block is not applied to — the operator + // workaround for a tenant with a legitimate need until per-account + // exceptions exist. + ExemptVPCs map[string]bool +} + type sgManager struct { - ovn ovn.Client + ovn ovn.Client + egress EgressPolicy } var _ SecurityGroupManager = (*sgManager)(nil) // NewSecurityGroupManager constructs a SecurityGroupManager backed by client. -func NewSecurityGroupManager(client ovn.Client) SecurityGroupManager { - return &sgManager{ovn: client} +// egress carries the platform WAN egress block applied to every guest. +func NewSecurityGroupManager(client ovn.Client, egress EgressPolicy) SecurityGroupManager { + return &sgManager{ovn: client, egress: egress} } func (m *sgManager) EnsureSG(ctx context.Context, sg SGSpec) error { @@ -68,6 +83,13 @@ func (m *sgManager) applyACLs(ctx context.Context, sg SGSpec) error { return fmt.Errorf("build ACLs for %s: %w", pg, err) } specs := append(InfrastructureACLs(pg), ruleSpecs...) + // Platform WAN egress block: applied to every guest unless its VPC is + // exempted, at a priority a tenant SG allow cannot override. + if !m.egress.ExemptVPCs[sg.VPCID] { + if block, ok := BlockedWANEgressACL(pg, m.egress.BlockedWANPorts); ok { + specs = append(specs, block) + } + } if err := m.ovn.ReplaceACLs(ctx, pg, specs); err != nil { return fmt.Errorf("replace ACLs on %s: %w", pg, err) } diff --git a/spinifex/network/policy/sg_test.go b/spinifex/network/policy/sg_test.go index 592b4bead..dbe6276f1 100644 --- a/spinifex/network/policy/sg_test.go +++ b/spinifex/network/policy/sg_test.go @@ -22,7 +22,7 @@ func newSGMockWithPG(t *testing.T, sgID string) (*mock.Client, string) { func TestSGManager_EnsureSG_AppliesInfraPlusTenantACLs(t *testing.T) { ctx := context.Background() m, pg := newSGMockWithPG(t, "sg-web") - sg := NewSecurityGroupManager(m) + sg := NewSecurityGroupManager(m, EgressPolicy{}) spec := SGSpec{ GroupID: "sg-web", @@ -46,7 +46,7 @@ func TestSGManager_EnsureSG_AppliesInfraPlusTenantACLs(t *testing.T) { func TestSGManager_UpdateSG_ReplacesACLSet(t *testing.T) { ctx := context.Background() m, pg := newSGMockWithPG(t, "sg-web") - sg := NewSecurityGroupManager(m) + sg := NewSecurityGroupManager(m, EgressPolicy{}) require.NoError(t, sg.EnsureSG(ctx, SGSpec{ GroupID: "sg-web", @@ -70,7 +70,7 @@ func TestSGManager_UpdateSG_ReplacesACLSet(t *testing.T) { func TestSGManager_DeleteSG_ClearsACLsKeepsPortGroup(t *testing.T) { ctx := context.Background() m, pg := newSGMockWithPG(t, "sg-web") - sg := NewSecurityGroupManager(m) + sg := NewSecurityGroupManager(m, EgressPolicy{}) require.NoError(t, sg.EnsureSG(ctx, SGSpec{ GroupID: "sg-web", @@ -88,7 +88,7 @@ func TestSGManager_DeleteSG_ClearsACLsKeepsPortGroup(t *testing.T) { func TestSGManager_EnsureSG_MissingPortGroup_ReturnsError(t *testing.T) { ctx := context.Background() m := mock.New() - sg := NewSecurityGroupManager(m) + sg := NewSecurityGroupManager(m, EgressPolicy{}) err := sg.EnsureSG(ctx, SGSpec{GroupID: "sg-missing"}) require.Error(t, err) @@ -100,7 +100,7 @@ func TestSGManager_EnsureSG_MissingPortGroup_ReturnsError(t *testing.T) { func TestSGManager_UpdateSG_AtomicReplace(t *testing.T) { ctx := context.Background() m, pg := newSGMockWithPG(t, "sg-web") - sg := NewSecurityGroupManager(m) + sg := NewSecurityGroupManager(m, EgressPolicy{}) require.NoError(t, sg.EnsureSG(ctx, SGSpec{ GroupID: "sg-web", @@ -116,10 +116,56 @@ func TestSGManager_UpdateSG_AtomicReplace(t *testing.T) { assert.NotEmpty(t, m.PortGroups[pg].ACLs, "PG ACLs must never be empty after UpdateSG") } +func hasACLNamed(m *mock.Client, name string) bool { + for _, a := range m.ACLs { + if a.Name != nil && *a.Name == name { + return true + } + } + return false +} + +func TestSGManager_EnsureSG_AppliesWANEgressBlock(t *testing.T) { + ctx := context.Background() + m, pg := newSGMockWithPG(t, "sg-web") + sg := NewSecurityGroupManager(m, EgressPolicy{BlockedWANPorts: []int{25, 465, 587}}) + + require.NoError(t, sg.EnsureSG(ctx, SGSpec{ + GroupID: "sg-web", VPCID: "vpc-1", + EgressRules: []Rule{{IPProtocol: "-1", CIDR: "0.0.0.0/0"}}, + })) + // 6 infra + 1 tenant egress + 1 WAN block = 8. + assert.Len(t, m.PortGroups[pg].ACLs, 8) + assert.True(t, hasACLNamed(m, pg+"-block-wan-egress")) +} + +func TestSGManager_EnsureSG_NoBlockWhenPolicyEmpty(t *testing.T) { + ctx := context.Background() + m, pg := newSGMockWithPG(t, "sg-web") + sg := NewSecurityGroupManager(m, EgressPolicy{}) + + require.NoError(t, sg.EnsureSG(ctx, SGSpec{GroupID: "sg-web", VPCID: "vpc-1"})) + assert.Len(t, m.PortGroups[pg].ACLs, 6) // infra only + assert.False(t, hasACLNamed(m, pg+"-block-wan-egress")) +} + +func TestSGManager_EnsureSG_ExemptVPCOmitsBlock(t *testing.T) { + ctx := context.Background() + m, pg := newSGMockWithPG(t, "sg-web") + sg := NewSecurityGroupManager(m, EgressPolicy{ + BlockedWANPorts: []int{25}, + ExemptVPCs: map[string]bool{"vpc-exempt": true}, + }) + + require.NoError(t, sg.EnsureSG(ctx, SGSpec{GroupID: "sg-web", VPCID: "vpc-exempt"})) + assert.Len(t, m.PortGroups[pg].ACLs, 6) // infra only, block skipped + assert.False(t, hasACLNamed(m, pg+"-block-wan-egress")) +} + func TestSGManager_EnsureSG_Idempotent(t *testing.T) { ctx := context.Background() m, pg := newSGMockWithPG(t, "sg-web") - sg := NewSecurityGroupManager(m) + sg := NewSecurityGroupManager(m, EgressPolicy{}) spec := SGSpec{ GroupID: "sg-web", diff --git a/spinifex/network/reconcile/apply_test.go b/spinifex/network/reconcile/apply_test.go index dbeb33e22..e3ae46a32 100644 --- a/spinifex/network/reconcile/apply_test.go +++ b/spinifex/network/reconcile/apply_test.go @@ -21,7 +21,7 @@ import ( func newTestReconciler(t *testing.T) (*reconciler, *mock.Client) { t.Helper() m := mock.New() - sg := policy.NewSecurityGroupManager(m) + sg := policy.NewSecurityGroupManager(m, policy.EgressPolicy{}) nat, err := policy.NewNATManager(m, policy.NATModeDistributed) if err != nil { t.Fatalf("NewNATManager: %v", err) @@ -291,7 +291,7 @@ func TestReconcile_ApplyOnlyKeepsOrphanLSP(t *testing.T) { func TestReconcile_ChassisRebindOnExistingIGW(t *testing.T) { m := mock.New() - sg := policy.NewSecurityGroupManager(m) + sg := policy.NewSecurityGroupManager(m, policy.EgressPolicy{}) nat, _ := policy.NewNATManager(m, policy.NATModeDistributed) routes := policy.NewRouteManager(m) igw, _ := external.NewIGWManager(external.IGWManagerConfig{ @@ -356,7 +356,7 @@ func TestReconcile_ChassisRebindOnExistingIGW(t *testing.T) { func TestReconcile_GatewayClaimChecksChassisRedirectPort(t *testing.T) { withFastClaimBounds(t) m := mock.New() - sg := policy.NewSecurityGroupManager(m) + sg := policy.NewSecurityGroupManager(m, policy.EgressPolicy{}) nat, _ := policy.NewNATManager(m, policy.NATModeDistributed) routes := policy.NewRouteManager(m) igw, _ := external.NewIGWManager(external.IGWManagerConfig{ @@ -406,7 +406,7 @@ func TestReconcile_GatewayClaimChecksChassisRedirectPort(t *testing.T) { func TestReconcile_IGWAttachWhenTopologyMissing(t *testing.T) { m := mock.New() - sg := policy.NewSecurityGroupManager(m) + sg := policy.NewSecurityGroupManager(m, policy.EgressPolicy{}) nat, _ := policy.NewNATManager(m, policy.NATModeDistributed) routes := policy.NewRouteManager(m) igw, _ := external.NewIGWManager(external.IGWManagerConfig{ diff --git a/spinifex/network/reconcile/scenario_live_test.go b/spinifex/network/reconcile/scenario_live_test.go index dae205c8c..2c0eae658 100644 --- a/spinifex/network/reconcile/scenario_live_test.go +++ b/spinifex/network/reconcile/scenario_live_test.go @@ -46,7 +46,7 @@ func newLiveReconciler(t *testing.T) (*reconciler, ovn.Client) { } t.Cleanup(cli.Close) - sg := policy.NewSecurityGroupManager(cli) + sg := policy.NewSecurityGroupManager(cli, policy.EgressPolicy{}) nat, err := policy.NewNATManager(cli, policy.NATModeDistributed) if err != nil { t.Fatalf("NewNATManager: %v", err) diff --git a/spinifex/network/subscribers/subscribers_test.go b/spinifex/network/subscribers/subscribers_test.go index 19b420b1f..614dfe74e 100644 --- a/spinifex/network/subscribers/subscribers_test.go +++ b/spinifex/network/subscribers/subscribers_test.go @@ -29,7 +29,7 @@ func newTestSubscriber(t *testing.T) (*Subscriber, *mock.Client) { _ = m.Connect(context.Background()) topo := topology.NewLiveManager(m) - sg := policy.NewSecurityGroupManager(m) + sg := policy.NewSecurityGroupManager(m, policy.EgressPolicy{}) nat, err := policy.NewNATManager(m, policy.NATModeDistributed) if err != nil { t.Fatalf("NewNATManager: %v", err) @@ -66,7 +66,7 @@ func TestNew_RejectsMissingDeps(t *testing.T) { name string cfg Config }{ - {"no topology", Config{SG: policy.NewSecurityGroupManager(mock.New())}}, + {"no topology", Config{SG: policy.NewSecurityGroupManager(mock.New(), policy.EgressPolicy{})}}, {"no sg", Config{Topology: topology.NewLiveManager(mock.New())}}, } for _, c := range cases { @@ -607,7 +607,7 @@ func newTestSubscriberWithMAC(t *testing.T, mac MACBindingFlusher) (*Subscriber, _ = m.Connect(context.Background()) topo := topology.NewLiveManager(m) - sg := policy.NewSecurityGroupManager(m) + sg := policy.NewSecurityGroupManager(m, policy.EgressPolicy{}) nat, err := policy.NewNATManager(m, policy.NATModeDistributed) if err != nil { t.Fatalf("NewNATManager: %v", err) diff --git a/spinifex/vpcd/vpcd.go b/spinifex/vpcd/vpcd.go index f7b9a9c60..1acd8d721 100644 --- a/spinifex/vpcd/vpcd.go +++ b/spinifex/vpcd/vpcd.go @@ -161,6 +161,12 @@ type Config struct { // UnderlayMTU is the fabric MTU between nodes; the advertised guest MTU is // derived from it. UnderlayMTU int + // BlockedWANPorts are TCP destination ports dropped for guest egress to + // public destinations (AWS-parity outbound-SMTP block). Empty disables it. + BlockedWANPorts []int + // EgressBlockExemptVPCs are VPC IDs the WAN egress block skips — the + // operator workaround for a tenant with a legitimate need. + EgressBlockExemptVPCs []string } // Service implements the Spinifex service interface for vpcd. @@ -496,7 +502,14 @@ func launchService(cfg *Config) error { igwPool, publicPool := selectExternalPools(cfg.ExternalMode, cfg.ExternalPools) - sgMgr := policy.NewSecurityGroupManager(liveClient) + egressPolicy := policy.EgressPolicy{BlockedWANPorts: cfg.BlockedWANPorts} + if len(cfg.EgressBlockExemptVPCs) > 0 { + egressPolicy.ExemptVPCs = make(map[string]bool, len(cfg.EgressBlockExemptVPCs)) + for _, id := range cfg.EgressBlockExemptVPCs { + egressPolicy.ExemptVPCs[id] = true + } + } + sgMgr := policy.NewSecurityGroupManager(liveClient, egressPolicy) natOpts := []policy.Option{ policy.WithFlowsBarrier(flowsBarrier), policy.WithNeighFlusher(neighFlusher(wanBridge)), diff --git a/tests/e2e/single/smtpblock_test.go b/tests/e2e/single/smtpblock_test.go new file mode 100644 index 000000000..696689831 --- /dev/null +++ b/tests/e2e/single/smtpblock_test.go @@ -0,0 +1,127 @@ +//go:build e2e + +package single + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// wanBlockedSMTPPorts mirrors config.DefaultBlockedWANPorts: the outbound +// SMTP ports the platform egress ACL drops to public destinations, matching +// AWS's default outbound-25 block. Kept as a literal here so the e2e asserts +// the deployed default rather than importing and trusting the same constant. +var wanBlockedSMTPPorts = []int{25, 465, 587} + +// guestConnectProbe returns a guest-side shell one-liner that attempts a TCP +// connect to host:port with a hard timeout and prints OPEN or BLOCKED. A +// dropped SYN (the ACL's behaviour) makes the connect hang until timeout kills +// it; a RST would fail fast — either way the port is not OPEN. The timeout is +// the caller's budget: the user asked for a 5s SMTP probe. +func guestConnectProbe(host string, port, timeoutSec int) string { + return fmt.Sprintf( + "timeout %d bash -c 'exec 3<>/dev/tcp/%s/%d' >/dev/null 2>&1 && echo OPEN || echo BLOCKED", + timeoutSec, host, port) +} + +// runWANEgressSMTPBlock boots one public guest and proves the platform egress +// ACL drops outbound SMTP (25/465/587) to a public destination while ordinary +// WAN egress still works — the AWS-parity default. It resolves gmail.com's MX +// the way the user asked ("dig gmail.com mx; connect to one of the servers via +// smtp, 5 sec timeout, confirm it does not work"), then probes each blocked +// port against that real mail exchanger. +// +// Stage order and gating: +// - Setup (dedicated SG + one public guest + SSH ingress) is an +// unconditional prerequisite; failure is fatal in the ordinary sense. +// - Control proves the guest has working WAN egress at all: a resolvable +// public name and an open connect on a non-blocked port (443). Without it +// a total egress outage would masquerade as a working SMTP block, so its +// failure aborts the SMTP assertions rather than let them pass vacuously. +// - SMTPBlocked is the assertion under test and depends on Control having +// proven egress is otherwise alive. +func runWANEgressSMTPBlock(t *testing.T, fix *Fixture) { + if !fix.PoolMode { + t.Skip("WAN egress SMTP block scenario requires pool-mode networking (real WAN egress)") + } + harness.Phase(t, "Single — Platform egress ACL blocks outbound SMTP (25/465/587) to public destinations, AWS parity") + + vpcID, _, subnetID := harness.DiscoverDefaultVPC(t, fix.AWS) + instType, _ := needInstanceTypeArch(t, fix) + keyName, keyPath := needKeyPair(t, fix) + ami := needAMI(t, fix) + + sgID := harness.EnsureSG(t, fix.Harness, vpcID, "smtpblock-sg") + harness.Detail(t, "vpc", vpcID, "subnet", subnetID, "sg", sgID) + + instanceID := launchBaselineInstance(t, fix, ami, instType, keyName, subnetID, []string{sgID}) + pubIP := instancePublicIP(t, fix, instanceID) + harness.Detail(t, "instance", instanceID, "public_ip", pubIP) + + harness.Step(t, "authorizing tcp/22 ingress, expecting reachability") + harness.AuthorizeSSHIngress(t, fix.AWS, sgID) + require.Truef(t, trySSHReady(pubIP, 22, keyPath, sshReadyBudget), + "tcp/22 to %s never became reachable after authorizing ingress", pubIP) + tgt := harness.SSHTarget{User: "ubuntu", Host: pubIP, Port: 22, KeyPath: keyPath} + + // Resolve gmail.com's MX from the guest, exactly the "dig gmail.com mx" + // step, and fall back to the well-known exchanger if the guest image + // carries no dig. The ACL matches on destination port, not host, so any + // public IP would do; a real MX just makes the probe honest. + var mxHost string + harness.Step(t, "dig gmail.com MX from guest") + harness.EventuallyErr(t, func() error { + out, err := sshCapture(tgt, "dig +short gmail.com MX 2>/dev/null | sort -n | "+ + "awk '{print $2}' | sed 's/\\.$//' | head -1") + if err != nil { + return fmt.Errorf("ssh dig: %w (out=%q)", err, out) + } + if h := strings.TrimSpace(out); h != "" { + mxHost = h + return nil + } + return fmt.Errorf("dig returned no MX (out=%q)", strings.TrimSpace(out)) + }, 90*time.Second, 5*time.Second) + if mxHost == "" { + mxHost = "gmail-smtp-in.l.google.com" + } + harness.Detail(t, "gmail_mx", mxHost) + + // Control: egress is alive. The guest must resolve a public name and open + // a non-blocked port (443) — otherwise a broken WAN path, not the ACL, + // explains the SMTP failures below. + controlOK := t.Run("Control", func(t *testing.T) { + harness.RequireDNSEnabled(t, fix.Env) + harness.Step(t, "resolve gmail.com A via guest resolver") + res, err := sshCapture(tgt, "getent ahostsv4 gmail.com") + require.NoErrorf(t, err, "guest failed to resolve gmail.com — WAN DNS path is broken\n%s", res) + require.Regexpf(t, `\d{1,3}(\.\d{1,3}){3}`, res, + "resolve gmail.com returned no IPv4 — resolver path is broken\n%s", res) + + harness.Step(t, "connect gmail.com:443 (non-blocked port must be OPEN)") + out, err := sshCapture(tgt, guestConnectProbe("gmail.com", 443, 10)) + require.NoErrorf(t, err, "ssh connect probe 443\n%s", out) + require.Equalf(t, "OPEN", strings.TrimSpace(out), + "gmail.com:443 was not reachable — WAN egress is broken, cannot attribute an SMTP failure to the ACL\n%s", out) + }) + if !controlOK { + t.Fatalf("Control stage failed; skipping SMTP assertions that would pass vacuously under a dead WAN path") + } + + t.Run("SMTPBlocked", func(t *testing.T) { + for _, port := range wanBlockedSMTPPorts { + harness.Step(t, "connect %s:%d (blocked SMTP port, 5s timeout, expect BLOCKED)", mxHost, port) + out, err := sshCapture(tgt, guestConnectProbe(mxHost, port, 5)) + require.NoErrorf(t, err, "ssh connect probe %s:%d\n%s", mxHost, port, out) + assert.Equalf(t, "BLOCKED", strings.TrimSpace(out), + "outbound SMTP to %s:%d was OPEN — the platform egress ACL is not dropping it\n%s", + mxHost, port, out) + } + }) +} diff --git a/tests/e2e/single/tests_test.go b/tests/e2e/single/tests_test.go index 714ce918b..87e23611c 100644 --- a/tests/e2e/single/tests_test.go +++ b/tests/e2e/single/tests_test.go @@ -56,6 +56,13 @@ func TestVPCEgressPaths(t *testing.T) { runVPCEgressPaths(t, requireSingleNodeFixture(t)) } +// TestWANEgressSMTPBlock boots one public guest and asserts the platform +// egress ACL drops outbound SMTP (25/465/587) to a public destination while +// ordinary WAN egress still works — the AWS-parity default block. +func TestWANEgressSMTPBlock(t *testing.T) { + runWANEgressSMTPBlock(t, requireSingleNodeFixture(t)) +} + func TestInstanceClusterStats(t *testing.T) { runInstanceClusterStats(t, requireSingleNodeFixture(t)) } diff --git a/tests/fixtures/vpcd/vpcd.go b/tests/fixtures/vpcd/vpcd.go index d00ae3be8..b8b52dda6 100644 --- a/tests/fixtures/vpcd/vpcd.go +++ b/tests/fixtures/vpcd/vpcd.go @@ -61,7 +61,7 @@ func Start(t testing.TB) *Fixture { } t.Cleanup(cli.Close) - sg := policy.NewSecurityGroupManager(cli) + sg := policy.NewSecurityGroupManager(cli, policy.EgressPolicy{}) nat, err := policy.NewNATManager(cli, policy.NATModeDistributed) if err != nil { t.Fatalf("vpcd fixture: NewNATManager: %v", err)