From b6d01370dd68a7b198713c2fdddab0d453872d35 Mon Sep 17 00:00:00 2001 From: Ben Duncan <5178243+benduncan@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:48:16 -0500 Subject: [PATCH] fix: render northstar's recursion allowlist so a node is not an open resolver (#993) * fix: render northstar's recursion allowlist so a node is not an open resolver northstar.toml is written with allow_recursion = false and an allow_recursion_from list built from cluster membership, so a freshly formed node serves recursion to the cluster and refuses it to the internet. The list is the node's own bind and advertise addresses plus every peer's host, with any port stripped. Both local addresses matter: a node's resolv.conf points at its own advertised address, so the query arrives from that public IP over lo rather than from 127.0.0.1, and an allowlist of loopback alone would stop every node resolving internal names. Guests are covered without appearing in the list. vpcd's per-tap DNS shim relays guest queries to a node's :5300 from a node address, so trusting the nodes trusts every guest behind them and no guest CIDR is needed. The template comment carries the caveats for allow_recursion = true, since the cost of setting it is remote from the setting: an open resolver is found by scanners within hours and gets the node's whole address range blocklisted, which is an outage for every tenant on it. The list is rendered at formation, so a node added later is not in its peers' files. Regenerating it live from membership, as the daemon already does for peers.nft, is follow-on work tracked on mulga-t3mub. * chore: pin northstar to the recursion ACL and atomic zone reload The recursion allowlist this branch renders is enforced by northstar's RecursionPolicy, which lands in dev at a0e741e. CI resolves northstar from go.mod rather than the workspace, so the build failed on NewRecursionPolicy being undefined until this pin moved. The AWS SDK bumps come with it. --- cmd/spinifex/cmd/admin.go | 35 ++++++++++++++ cmd/spinifex/cmd/admin_test.go | 58 +++++++++++++++++++++++ cmd/spinifex/cmd/templates/northstar.toml | 22 +++++++++ go.mod | 8 ++-- go.sum | 8 ++++ spinifex/admin/admin.go | 7 +++ 6 files changed, 134 insertions(+), 4 deletions(-) diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index b1218204f..27913cd8c 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -2844,9 +2844,44 @@ func northstarFromFormation(creds *formation.SharedCredentials, dirs configDirs) }, filepath.Join(dirs.Northstar, "northstar.toml") } +// resolverAllowFrom returns the addresses permitted to recurse against this +// node's northstar: every cluster node, on whichever address its peers dial. +// +// Guests are covered without appearing here. vpcd's per-tap DNS shim relays +// guest queries to a node's :5300 from a node address, so trusting the nodes +// trusts every guest behind them, and no guest CIDR is needed. +func resolverAllowFrom(settings admin.ConfigSettings) []string { + seen := make(map[string]bool) + var addrs []string + add := func(ip string) { + ip = strings.TrimSpace(ip) + if ip == "" || ip == "0.0.0.0" || seen[ip] { + return + } + seen[ip] = true + addrs = append(addrs, ip) + } + + // Both local addresses: a node resolving through its own advertised address + // is seen arriving from that address over lo, not from 127.0.0.1. + add(settings.AdvertiseIP) + add(settings.BindIP) + add(settings.ClusterBindIP) + for _, peer := range settings.RemoteNodes { + host := peer.Host + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + add(host) + } + return addrs +} + // generateAndWriteConfigs renders the standard config files (spinifex.toml, // awsgw.toml, nats.conf, and optionally predastore.toml) from templates. func generateAndWriteConfigs(dirs configDirs, spinifexTomlPath string, settings admin.ConfigSettings, skipPredastore bool) error { + settings.ResolverAllowFrom = resolverAllowFrom(settings) + // A one-sided pair must disable Northstar wholesale. Rendering only the // public stanza or only the secret file advertises a resolver that cannot run. northstarEnabled := settings.NorthstarAccessKey != "" && settings.NorthstarSecretKey != "" diff --git a/cmd/spinifex/cmd/admin_test.go b/cmd/spinifex/cmd/admin_test.go index f3158f9d6..e095eab35 100644 --- a/cmd/spinifex/cmd/admin_test.go +++ b/cmd/spinifex/cmd/admin_test.go @@ -4,11 +4,14 @@ import ( "bytes" "errors" "fmt" + "net/netip" "os" "path/filepath" "strings" "testing" + "github.com/mulgadc/northstar/pkg/backend" + nsconfig "github.com/mulgadc/northstar/pkg/config" "github.com/mulgadc/spinifex/spinifex/admin" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/formation" @@ -1358,3 +1361,58 @@ func TestJoinRetryable(t *testing.T) { }) } } + +// The rendered resolver must refuse the internet and still serve the cluster. +// Both halves matter: an open resolver is the bug, and a resolver that refuses +// its own nodes silently stops internal names resolving. +func TestGenerateAndWriteConfigs_NorthstarRecursionIsClosedButServesTheCluster(t *testing.T) { + dirs, err := createConfigSubdirs(t.TempDir()) + require.NoError(t, err) + nsPath := filepath.Join(dirs.Northstar, "northstar.toml") + + settings := multinodeNorthstarSettings(nsPath) + settings.BindIP = "10.2.0.2" + settings.AdvertiseIP = "72.52.77.228" + settings.RemoteNodes = []admin.RemoteNode{ + {Name: "node2", Host: "72.52.77.229", NorthstarConfigPath: nsPath}, + {Name: "node3", Host: "72.52.77.230:4432", NorthstarConfigPath: nsPath}, + } + + require.NoError(t, generateAndWriteConfigs(dirs, filepath.Join(t.TempDir(), "spinifex.toml"), settings, true)) + + raw, err := os.ReadFile(nsPath) + require.NoError(t, err) + rendered := string(raw) + + assert.Contains(t, rendered, "allow_recursion = false", + "a node bound to a public address must not be an open resolver") + + // The node's own advertised address: resolv.conf points there, so the query + // arrives from it over lo rather than from 127.0.0.1. + assert.Contains(t, rendered, `"72.52.77.228"`) + assert.Contains(t, rendered, `"10.2.0.2"`) + // Peers, with any port stripped. + assert.Contains(t, rendered, `"72.52.77.229"`) + assert.Contains(t, rendered, `"72.52.77.230"`) + assert.NotContains(t, rendered, "72.52.77.230:4432") + + // It must parse as the config northstar will actually load. + cfg, err := nsconfig.LoadServerConfig(nsPath) + require.NoError(t, err) + assert.False(t, cfg.Upstream.AllowRecursion) + + prefixes, err := cfg.Upstream.ParseAllowRecursionFrom() + require.NoError(t, err) + policy := backend.NewRecursionPolicy(cfg.Upstream.AllowRecursion, prefixes) + + // The cluster resolves. + assert.True(t, policy.Allows(netip.MustParseAddr("72.52.77.228")), "node must resolve through itself") + assert.True(t, policy.Allows(netip.MustParseAddr("72.52.77.229")), "peer node must resolve") + assert.True(t, policy.Allows(netip.MustParseAddr("127.0.0.1")), "loopback is always allowed") + // Guests reach northstar via vpcd's shim, which relays from a node address, + // so a guest query is seen as one of the addresses asserted above. + + // The internet does not. + assert.False(t, policy.Allows(netip.MustParseAddr("198.51.100.7"))) + assert.False(t, policy.Allows(netip.MustParseAddr("8.8.8.8"))) +} diff --git a/cmd/spinifex/cmd/templates/northstar.toml b/cmd/spinifex/cmd/templates/northstar.toml index 85c742fc1..2774b82f2 100644 --- a/cmd/spinifex/cmd/templates/northstar.toml +++ b/cmd/spinifex/cmd/templates/northstar.toml @@ -38,3 +38,25 @@ secret_key = "{{.NorthstarSecretKey}}" # Forwarders for non-authoritative (recursive) queries. Empty => REFUSED. [upstream] nameservers = [{{range $i, $s := .PoolDNSServers}}{{if $i}}, {{end}}"{{$s}}:53"{{end}}] + +# Who may have a query forwarded to those nameservers. Authoritative answers are +# never gated by this — a public authoritative server must answer everyone. +# +# allow_recursion = true serves recursion to ANY client, including the whole +# internet, because :53 above is bound to the node's public address. Do not set +# it on a node reachable from the internet. An open resolver is found by scanners +# within hours and is then used as a DDoS reflector, which gets the node's whole +# address range blocklisted — an outage for every tenant on it, and slow to undo. +# It is here for an isolated or lab deployment where the DNS port is not exposed. +# +# Trusted clients do not need it. List them in allow_recursion_from instead, +# which is unaffected by allow_recursion and is how the cluster resolves: +# - Cluster nodes resolve through northstar (resolv.conf points at the node's +# own advertised address, so the query arrives from that public IP over lo, +# not from 127.0.0.1). +# - EC2 instances reach vpcd's per-tap DNS shim on 169.254.169.253, which +# relays to northstar's :5300 from a node address — so listing the nodes +# covers every guest, and guest addresses never appear here. +# Loopback is always allowed and does not need listing. +allow_recursion = false +allow_recursion_from = [{{range $i, $s := .ResolverAllowFrom}}{{if $i}}, {{end}}"{{$s}}"{{end}}] diff --git a/go.mod b/go.mod index f2bb96fac..8b8277ff6 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/klauspost/cpuid/v2 v2.4.0 github.com/miekg/dns v1.1.73 github.com/mulgadc/bluebottle v1.18.1-0.20260903003911-3d26c3a8def8 - github.com/mulgadc/northstar v1.18.0 + github.com/mulgadc/northstar v1.18.1-0.20260903173023-a0e741e2831f github.com/mulgadc/predastore v1.18.1-0.20260831064140-ff6a1dc42441 github.com/mulgadc/viperblock v1.18.0 github.com/nats-io/nats-server/v2 v2.14.5 @@ -63,10 +63,10 @@ require ( github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.10.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.41 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.108.0 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.6.0 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.34.0 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0 // indirect diff --git a/go.sum b/go.sum index 171413819..a7a083398 100644 --- a/go.sum +++ b/go.sum @@ -564,6 +564,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30 h1:5437eMoOwqqQpZn2XJy74mlDCuPYL81texMT3mXqgtU= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30/go.mod h1:xfu2m3dOpvW8lj98wQYa8V9ku/Rta59hsbireGzhh3A= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.10.0 h1:U8/A0RRBaEspzH1uul3JHLbypXwEGUkRkvoT9f0ATcM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.10.0/go.mod h1:UELStX5KwtJNtQxa+UuF8dc3z4UYc40e8yHYJSozNwY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2/go.mod h1:Ru7vg1iQ7cR4i7SZ/JTLYN9kaXtbL69UdgG0OQWQxW0= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= @@ -571,8 +573,12 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40 h1:gr3Fw1cx github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40/go.mod h1:8z/9CmfnQhiuXD7Ykbcg4a/whSWsniE0ODSx9uwVzfk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38 h1:gX8B8y3Ho30B1LPxefDKMi/HZqWEb47U9ogs3DtSG0M= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38/go.mod h1:l5WblZlcmGPe4/O7JY2HO25Z+xqTBvyfTyFbRMf8gYw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.41 h1:Q9DIKDuJix/oJnQxFpQ26L0EwVa/YNo4k2kbktrjQjE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.41/go.mod h1:x+TuqkOIG1SZS0+yN54sExGA9ZpjhPO6vPdYnpTFX1M= github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2 h1:GNU0/xtPEXMKilJZ/a8BedeuQnvu+Usi6qVm9EFfncc= github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2/go.mod h1:4jYWUecEsQtE73jPl7p3jrbYXH5ffcR4gegyCygagfg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.108.0 h1:Yp+x5PKXEmoqHsgP/pAkBy5Tyq1UlXAzM0OInh0vxWw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.108.0/go.mod h1:locV6DtXyp7Xzr2BG6jtsbeBi3YAWJ/CY4xUThYmIwQ= github.com/aws/aws-sdk-go-v2/service/signin v1.6.0 h1:agcr0j8YeFEzdXNo17Rg9MbbjLRjrimabwNtji4e+lU= github.com/aws/aws-sdk-go-v2/service/signin v1.6.0/go.mod h1:qU5PxgQ4JiUOOMotzfO3+5oUda5W+8JDVKyLQqlrJik= github.com/aws/aws-sdk-go-v2/service/sso v1.12.3/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw= @@ -1331,6 +1337,8 @@ github.com/mulgadc/bluebottle v1.18.1-0.20260903003911-3d26c3a8def8 h1:eTOKjTcXm github.com/mulgadc/bluebottle v1.18.1-0.20260903003911-3d26c3a8def8/go.mod h1:XBIMI+G2s3wuJNiEdYtlGDvCacw82OXEw50RIQhowYg= github.com/mulgadc/northstar v1.18.0 h1:ksRjIKE6qtrw/ZoYaJh8JRBh+xdnuDw8FsmFLFCiXbg= github.com/mulgadc/northstar v1.18.0/go.mod h1:z6iol77R1vsIQyIBc1JQboUPhSJ6eG4bipzfbZZbLEM= +github.com/mulgadc/northstar v1.18.1-0.20260903173023-a0e741e2831f h1:HXr5KpqRiueZrZPCAb2hBqx10QNFou9Nwu+YXVoQBsM= +github.com/mulgadc/northstar v1.18.1-0.20260903173023-a0e741e2831f/go.mod h1:/bALSgdZACw8+ANIg3lfdKcEgvwHJ1gqpY9e2TUpyn4= github.com/mulgadc/predastore v1.18.1-0.20260831064140-ff6a1dc42441 h1:5t+iuKpnRWXkKOmh5lLBoziBaWD+H71edWO/O4ceMp8= github.com/mulgadc/predastore v1.18.1-0.20260831064140-ff6a1dc42441/go.mod h1:ioGP2kHWYpW/opboAqlqQ5C2Jv+a1/z4txtenw7K7Ws= github.com/mulgadc/viperblock v1.18.0 h1:u6yu5lPfZ+RSmfzlg1OXI3whlRBnGp8t685xUzzxlns= diff --git a/spinifex/admin/admin.go b/spinifex/admin/admin.go index 514eeb2ce..c404af296 100644 --- a/spinifex/admin/admin.go +++ b/spinifex/admin/admin.go @@ -138,6 +138,13 @@ type ConfigSettings struct { // PoolDNSServers are the host-detected upstream DNS servers northstar forwards // recursive queries to, rendered into northstar.toml's [recursion] nameservers. PoolDNSServers []string + + // ResolverAllowFrom are the clients allowed to recurse, rendered into + // northstar.toml's [upstream].allow_recursion_from. Cluster node addresses: + // nodes resolve through northstar directly, and vpcd's per-tap guest DNS + // shim relays guest queries from a node address, so this covers both. + // Everyone else gets authoritative answers only. + ResolverAllowFrom []string } // PoolData is one [[network.external_pools]] block rendered into spinifex.toml.