diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index b1218204f..27913cd8c 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -2844,9 +2844,44 @@ func northstarFromFormation(creds *formation.SharedCredentials, dirs configDirs) }, filepath.Join(dirs.Northstar, "northstar.toml") } +// resolverAllowFrom returns the addresses permitted to recurse against this +// node's northstar: every cluster node, on whichever address its peers dial. +// +// Guests are covered without appearing here. vpcd's per-tap DNS shim relays +// guest queries to a node's :5300 from a node address, so trusting the nodes +// trusts every guest behind them, and no guest CIDR is needed. +func resolverAllowFrom(settings admin.ConfigSettings) []string { + seen := make(map[string]bool) + var addrs []string + add := func(ip string) { + ip = strings.TrimSpace(ip) + if ip == "" || ip == "0.0.0.0" || seen[ip] { + return + } + seen[ip] = true + addrs = append(addrs, ip) + } + + // Both local addresses: a node resolving through its own advertised address + // is seen arriving from that address over lo, not from 127.0.0.1. + add(settings.AdvertiseIP) + add(settings.BindIP) + add(settings.ClusterBindIP) + for _, peer := range settings.RemoteNodes { + host := peer.Host + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + add(host) + } + return addrs +} + // generateAndWriteConfigs renders the standard config files (spinifex.toml, // awsgw.toml, nats.conf, and optionally predastore.toml) from templates. func generateAndWriteConfigs(dirs configDirs, spinifexTomlPath string, settings admin.ConfigSettings, skipPredastore bool) error { + settings.ResolverAllowFrom = resolverAllowFrom(settings) + // A one-sided pair must disable Northstar wholesale. Rendering only the // public stanza or only the secret file advertises a resolver that cannot run. northstarEnabled := settings.NorthstarAccessKey != "" && settings.NorthstarSecretKey != "" diff --git a/cmd/spinifex/cmd/admin_test.go b/cmd/spinifex/cmd/admin_test.go index f3158f9d6..e095eab35 100644 --- a/cmd/spinifex/cmd/admin_test.go +++ b/cmd/spinifex/cmd/admin_test.go @@ -4,11 +4,14 @@ import ( "bytes" "errors" "fmt" + "net/netip" "os" "path/filepath" "strings" "testing" + "github.com/mulgadc/northstar/pkg/backend" + nsconfig "github.com/mulgadc/northstar/pkg/config" "github.com/mulgadc/spinifex/spinifex/admin" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/formation" @@ -1358,3 +1361,58 @@ func TestJoinRetryable(t *testing.T) { }) } } + +// The rendered resolver must refuse the internet and still serve the cluster. +// Both halves matter: an open resolver is the bug, and a resolver that refuses +// its own nodes silently stops internal names resolving. +func TestGenerateAndWriteConfigs_NorthstarRecursionIsClosedButServesTheCluster(t *testing.T) { + dirs, err := createConfigSubdirs(t.TempDir()) + require.NoError(t, err) + nsPath := filepath.Join(dirs.Northstar, "northstar.toml") + + settings := multinodeNorthstarSettings(nsPath) + settings.BindIP = "10.2.0.2" + settings.AdvertiseIP = "72.52.77.228" + settings.RemoteNodes = []admin.RemoteNode{ + {Name: "node2", Host: "72.52.77.229", NorthstarConfigPath: nsPath}, + {Name: "node3", Host: "72.52.77.230:4432", NorthstarConfigPath: nsPath}, + } + + require.NoError(t, generateAndWriteConfigs(dirs, filepath.Join(t.TempDir(), "spinifex.toml"), settings, true)) + + raw, err := os.ReadFile(nsPath) + require.NoError(t, err) + rendered := string(raw) + + assert.Contains(t, rendered, "allow_recursion = false", + "a node bound to a public address must not be an open resolver") + + // The node's own advertised address: resolv.conf points there, so the query + // arrives from it over lo rather than from 127.0.0.1. + assert.Contains(t, rendered, `"72.52.77.228"`) + assert.Contains(t, rendered, `"10.2.0.2"`) + // Peers, with any port stripped. + assert.Contains(t, rendered, `"72.52.77.229"`) + assert.Contains(t, rendered, `"72.52.77.230"`) + assert.NotContains(t, rendered, "72.52.77.230:4432") + + // It must parse as the config northstar will actually load. + cfg, err := nsconfig.LoadServerConfig(nsPath) + require.NoError(t, err) + assert.False(t, cfg.Upstream.AllowRecursion) + + prefixes, err := cfg.Upstream.ParseAllowRecursionFrom() + require.NoError(t, err) + policy := backend.NewRecursionPolicy(cfg.Upstream.AllowRecursion, prefixes) + + // The cluster resolves. + assert.True(t, policy.Allows(netip.MustParseAddr("72.52.77.228")), "node must resolve through itself") + assert.True(t, policy.Allows(netip.MustParseAddr("72.52.77.229")), "peer node must resolve") + assert.True(t, policy.Allows(netip.MustParseAddr("127.0.0.1")), "loopback is always allowed") + // Guests reach northstar via vpcd's shim, which relays from a node address, + // so a guest query is seen as one of the addresses asserted above. + + // The internet does not. + assert.False(t, policy.Allows(netip.MustParseAddr("198.51.100.7"))) + assert.False(t, policy.Allows(netip.MustParseAddr("8.8.8.8"))) +} diff --git a/cmd/spinifex/cmd/templates/northstar.toml b/cmd/spinifex/cmd/templates/northstar.toml index 85c742fc1..2774b82f2 100644 --- a/cmd/spinifex/cmd/templates/northstar.toml +++ b/cmd/spinifex/cmd/templates/northstar.toml @@ -38,3 +38,25 @@ secret_key = "{{.NorthstarSecretKey}}" # Forwarders for non-authoritative (recursive) queries. Empty => REFUSED. [upstream] nameservers = [{{range $i, $s := .PoolDNSServers}}{{if $i}}, {{end}}"{{$s}}:53"{{end}}] + +# Who may have a query forwarded to those nameservers. Authoritative answers are +# never gated by this — a public authoritative server must answer everyone. +# +# allow_recursion = true serves recursion to ANY client, including the whole +# internet, because :53 above is bound to the node's public address. Do not set +# it on a node reachable from the internet. An open resolver is found by scanners +# within hours and is then used as a DDoS reflector, which gets the node's whole +# address range blocklisted — an outage for every tenant on it, and slow to undo. +# It is here for an isolated or lab deployment where the DNS port is not exposed. +# +# Trusted clients do not need it. List them in allow_recursion_from instead, +# which is unaffected by allow_recursion and is how the cluster resolves: +# - Cluster nodes resolve through northstar (resolv.conf points at the node's +# own advertised address, so the query arrives from that public IP over lo, +# not from 127.0.0.1). +# - EC2 instances reach vpcd's per-tap DNS shim on 169.254.169.253, which +# relays to northstar's :5300 from a node address — so listing the nodes +# covers every guest, and guest addresses never appear here. +# Loopback is always allowed and does not need listing. +allow_recursion = false +allow_recursion_from = [{{range $i, $s := .ResolverAllowFrom}}{{if $i}}, {{end}}"{{$s}}"{{end}}] diff --git a/go.mod b/go.mod index f2bb96fac..8b8277ff6 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/klauspost/cpuid/v2 v2.4.0 github.com/miekg/dns v1.1.73 github.com/mulgadc/bluebottle v1.18.1-0.20260903003911-3d26c3a8def8 - github.com/mulgadc/northstar v1.18.0 + github.com/mulgadc/northstar v1.18.1-0.20260903173023-a0e741e2831f github.com/mulgadc/predastore v1.18.1-0.20260831064140-ff6a1dc42441 github.com/mulgadc/viperblock v1.18.0 github.com/nats-io/nats-server/v2 v2.14.5 @@ -63,10 +63,10 @@ require ( github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.10.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.41 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.108.0 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.6.0 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.34.0 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0 // indirect diff --git a/go.sum b/go.sum index 171413819..a7a083398 100644 --- a/go.sum +++ b/go.sum @@ -564,6 +564,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30 h1:5437eMoOwqqQpZn2XJy74mlDCuPYL81texMT3mXqgtU= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30/go.mod h1:xfu2m3dOpvW8lj98wQYa8V9ku/Rta59hsbireGzhh3A= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.10.0 h1:U8/A0RRBaEspzH1uul3JHLbypXwEGUkRkvoT9f0ATcM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.10.0/go.mod h1:UELStX5KwtJNtQxa+UuF8dc3z4UYc40e8yHYJSozNwY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2/go.mod h1:Ru7vg1iQ7cR4i7SZ/JTLYN9kaXtbL69UdgG0OQWQxW0= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= @@ -571,8 +573,12 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40 h1:gr3Fw1cx github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40/go.mod h1:8z/9CmfnQhiuXD7Ykbcg4a/whSWsniE0ODSx9uwVzfk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38 h1:gX8B8y3Ho30B1LPxefDKMi/HZqWEb47U9ogs3DtSG0M= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38/go.mod h1:l5WblZlcmGPe4/O7JY2HO25Z+xqTBvyfTyFbRMf8gYw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.41 h1:Q9DIKDuJix/oJnQxFpQ26L0EwVa/YNo4k2kbktrjQjE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.41/go.mod h1:x+TuqkOIG1SZS0+yN54sExGA9ZpjhPO6vPdYnpTFX1M= github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2 h1:GNU0/xtPEXMKilJZ/a8BedeuQnvu+Usi6qVm9EFfncc= github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2/go.mod h1:4jYWUecEsQtE73jPl7p3jrbYXH5ffcR4gegyCygagfg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.108.0 h1:Yp+x5PKXEmoqHsgP/pAkBy5Tyq1UlXAzM0OInh0vxWw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.108.0/go.mod h1:locV6DtXyp7Xzr2BG6jtsbeBi3YAWJ/CY4xUThYmIwQ= github.com/aws/aws-sdk-go-v2/service/signin v1.6.0 h1:agcr0j8YeFEzdXNo17Rg9MbbjLRjrimabwNtji4e+lU= github.com/aws/aws-sdk-go-v2/service/signin v1.6.0/go.mod h1:qU5PxgQ4JiUOOMotzfO3+5oUda5W+8JDVKyLQqlrJik= github.com/aws/aws-sdk-go-v2/service/sso v1.12.3/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw= @@ -1331,6 +1337,8 @@ github.com/mulgadc/bluebottle v1.18.1-0.20260903003911-3d26c3a8def8 h1:eTOKjTcXm github.com/mulgadc/bluebottle v1.18.1-0.20260903003911-3d26c3a8def8/go.mod h1:XBIMI+G2s3wuJNiEdYtlGDvCacw82OXEw50RIQhowYg= github.com/mulgadc/northstar v1.18.0 h1:ksRjIKE6qtrw/ZoYaJh8JRBh+xdnuDw8FsmFLFCiXbg= github.com/mulgadc/northstar v1.18.0/go.mod h1:z6iol77R1vsIQyIBc1JQboUPhSJ6eG4bipzfbZZbLEM= +github.com/mulgadc/northstar v1.18.1-0.20260903173023-a0e741e2831f h1:HXr5KpqRiueZrZPCAb2hBqx10QNFou9Nwu+YXVoQBsM= +github.com/mulgadc/northstar v1.18.1-0.20260903173023-a0e741e2831f/go.mod h1:/bALSgdZACw8+ANIg3lfdKcEgvwHJ1gqpY9e2TUpyn4= github.com/mulgadc/predastore v1.18.1-0.20260831064140-ff6a1dc42441 h1:5t+iuKpnRWXkKOmh5lLBoziBaWD+H71edWO/O4ceMp8= github.com/mulgadc/predastore v1.18.1-0.20260831064140-ff6a1dc42441/go.mod h1:ioGP2kHWYpW/opboAqlqQ5C2Jv+a1/z4txtenw7K7Ws= github.com/mulgadc/viperblock v1.18.0 h1:u6yu5lPfZ+RSmfzlg1OXI3whlRBnGp8t685xUzzxlns= diff --git a/spinifex/admin/admin.go b/spinifex/admin/admin.go index 514eeb2ce..c404af296 100644 --- a/spinifex/admin/admin.go +++ b/spinifex/admin/admin.go @@ -138,6 +138,13 @@ type ConfigSettings struct { // PoolDNSServers are the host-detected upstream DNS servers northstar forwards // recursive queries to, rendered into northstar.toml's [recursion] nameservers. PoolDNSServers []string + + // ResolverAllowFrom are the clients allowed to recurse, rendered into + // northstar.toml's [upstream].allow_recursion_from. Cluster node addresses: + // nodes resolve through northstar directly, and vpcd's per-tap guest DNS + // shim relays guest queries from a node address, so this covers both. + // Everyone else gets authoritative answers only. + ResolverAllowFrom []string } // PoolData is one [[network.external_pools]] block rendered into spinifex.toml.