diff --git a/.github/scripts/suite-sets.sh b/.github/scripts/suite-sets.sh index 0b9746e81..eb32d7266 100644 --- a/.github/scripts/suite-sets.sh +++ b/.github/scripts/suite-sets.sh @@ -28,6 +28,9 @@ # that runs identically on either, and because a quota left switched on would # cap every suite sharing the environment — so it must be proved to restore # itself wherever it runs, not only on one node. +# route53 belongs to both topologies because every node serves every hosted +# zone from one shared bucket: on one node it proves the zone loads at all, on +# three that each node's sync reaches the same set. # storagefault runs last in both lists and belongs to both topologies, for # different reasons. On one node it is RS(1,0), so freezing the only predastore # is a total outage and the guest-survival assertion is unambiguous. On three it @@ -37,7 +40,7 @@ # received. It is last because it SIGSTOPs a cluster-wide # service: anything sharing the environment would see the same outage and report # a failure that says nothing about itself. -E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault" +E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault" # Suites runnable against a multi-node environment. # @@ -57,7 +60,7 @@ E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota # replicated rather than sitting on one node, so it runs before the pair that # take a node away — a cluster whose buckets are already wrong should report # that, not report a recovery failing. -E2E_SUITES_MULTI="multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault" +E2E_SUITES_MULTI="multinode lb cert quota route53 kvquorum lbrecovery instancerecovery storagefault" # grep -xE alternation form, for narrowing a requested set to the eligible one. E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')" diff --git a/docs/testing/e2e-coverage.md b/docs/testing/e2e-coverage.md index bb37be01d..57d5f223b 100644 --- a/docs/testing/e2e-coverage.md +++ b/docs/testing/e2e-coverage.md @@ -6,8 +6,8 @@ Suites live in `tests/e2e//` and build to `tests/e2e/_bin/.test`. Ev | Set | Suites | | --- | --- | -| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota storagefault` | -| `E2E_SUITES_MULTI` | `multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault` | +| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault` | +| `E2E_SUITES_MULTI` | `multinode lb cert quota route53 kvquorum lbrecovery instancerecovery storagefault` | | `E2E_SUITES_NIGHTLY_SINGLE` | `single cert iam` | | `E2E_SUITES_NIGHTLY_MULTI` | `multinode cert lb` | @@ -120,6 +120,7 @@ The create/describe/connect path against a live cluster: a DB VM that actually b ### Other - **`quota`** — every enforced dimension refuses with the same `quotaExceeded` code; a launch is charged for every instance it asks for; one instance wider than the cap is refused. +- **`route53`** — hosted zones as northstar serves them, on every node. `TestNorthstarServesZonesBeyondFirstListPage` writes 1,100 zone objects (more than one `ListObjectsV2` page), requires the last one to resolve on every node and the platform base zone to survive the sync, then requires the deleted zones to stop resolving. - **`gpu`** — device visible in the guest, driver access, launch. - **`bedrock`** — `ListFoundationModels`/`GetFoundationModel` against a running cluster. Wire-format parity is unit-tested, not here. - **`ddil`** — Denied, Disrupted, Intermittent, Limited network scenarios. diff --git a/go.mod b/go.mod index 128f623ef..b3c511c6c 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.33.9 github.com/aws/aws-sdk-go-v2/credentials v1.20.9 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 github.com/cenkalti/backoff/v4 v4.3.0 github.com/charmbracelet/bubbles v1.0.0 github.com/charmbracelet/bubbletea v1.3.10 @@ -24,7 +25,7 @@ require ( github.com/miekg/dns v1.1.73 github.com/minio/highwayhash v1.0.4 github.com/mulgadc/bluebottle v1.22.0 - github.com/mulgadc/northstar v1.22.0 + github.com/mulgadc/northstar v1.22.1-0.20261023213715-059065aec064 github.com/mulgadc/predastore v1.22.0 github.com/mulgadc/viperblock v1.22.0 github.com/nats-io/nats-server/v2 v2.15.1 @@ -67,10 +68,9 @@ require ( github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect diff --git a/go.sum b/go.sum index 24cc7b17e..373a32e82 100644 --- a/go.sum +++ b/go.sum @@ -40,14 +40,14 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3 github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0= -github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 h1:NyeuyXkZL476uoAowNActm4XJO1QPKf7aassgXHpiqA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3/go.mod h1:GfoBeKMDJecgRUsGf/0RGdx0NJAh81ObTd922h8pQys= github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A= github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M= github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk= @@ -295,8 +295,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/mulgadc/bluebottle v1.22.0 h1:pGZ5/Ly3CYem62pPu2zKBRpvMpAG0LHMIMw9Yt/lkIg= github.com/mulgadc/bluebottle v1.22.0/go.mod h1:DwT91bJyUsrpg1gCJmcCOTG+tgw5J124LloHYvGAQu0= -github.com/mulgadc/northstar v1.22.0 h1:fTndlk47hK76fcNr2MqsyeKhBNVEDt8+BC6Z30ohokA= -github.com/mulgadc/northstar v1.22.0/go.mod h1:yLH8waqfHZx177Ae6dGQGTDt/02rPE9alnUKyxDNhVs= +github.com/mulgadc/northstar v1.22.1-0.20261023213715-059065aec064 h1:wnnHsRttCvLydRz5rQ5WTzk46pN7Vxxq5rSalOLkUbw= +github.com/mulgadc/northstar v1.22.1-0.20261023213715-059065aec064/go.mod h1:Halhinpgo5rzUhYemGN8uYTWu1pJsL5e/7l1eEKc7nM= github.com/mulgadc/predastore v1.22.0 h1:mUQJMTxMiRDVvemMP+34BP/+r1U0iCptc9B++u4NJ8o= github.com/mulgadc/predastore v1.22.0/go.mod h1:/cMIoKtnQG/DWyeX/0matEQrG2/abY60tNHWcBKS9Kc= github.com/mulgadc/viperblock v1.22.0 h1:8vS2AtpIDkIhGNOazQPicKXV2y/8PAy8BNrVWIOCP6I= diff --git a/tests/e2e/Makefile b/tests/e2e/Makefile index 6ea508985..701afbff2 100644 --- a/tests/e2e/Makefile +++ b/tests/e2e/Makefile @@ -18,9 +18,9 @@ SUITE_TAG ?= $(notdir $(patsubst %/...,%,$(firstword $(PKG)))) # BIN_DIR/.test so CI can ship binaries (not source + go toolchain) to # the remote VM. BIN_DIR ?= $(CURDIR)/_bin -SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink multinode storagegrowth partialblock rds quota diskperf storagefault instancerecovery lbrecovery kvquorum +SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink multinode storagegrowth partialblock rds quota diskperf storagefault instancerecovery lbrecovery kvquorum route53 -.PHONY: e2e e2e-quota e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-storagegrowth e2e-partialblock e2e-diskperf e2e-rds e2e-storagefault e2e-instancerecovery e2e-lbrecovery e2e-kvquorum e2e-all clean e2e-build +.PHONY: e2e e2e-quota e2e-route53 e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-storagegrowth e2e-partialblock e2e-diskperf e2e-rds e2e-storagefault e2e-instancerecovery e2e-lbrecovery e2e-kvquorum e2e-all clean e2e-build RUN_FLAG = $(if $(RUN),-run '$(RUN)',) @@ -65,6 +65,7 @@ e2e-diskperf: ; $(MAKE) e2e PKG=./diskperf/... TIMEOUT=180m # most of a 30m package timeout before the tests still running get to report. e2e-rds: ; $(MAKE) e2e PKG=./rds/... TIMEOUT=45m e2e-quota: ; $(MAKE) e2e PKG=./quota/... +e2e-route53: ; $(MAKE) e2e PKG=./route53/... # Freezes predastore, so its budget covers a deliberate outage plus the # recovery and re-verification that follow it, twice over for the two tests. e2e-storagefault: ; $(MAKE) e2e PKG=./storagefault/... TIMEOUT=45m diff --git a/tests/e2e/harness/northstar_zones.go b/tests/e2e/harness/northstar_zones.go new file mode 100644 index 000000000..2efd15a34 --- /dev/null +++ b/tests/e2e/harness/northstar_zones.go @@ -0,0 +1,144 @@ +//go:build e2e + +package harness + +import ( + "context" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + nsconfig "github.com/mulgadc/northstar/pkg/config" + "github.com/mulgadc/spinifex/spinifex/bootstrap/config" +) + +const northstarServerConfigPath = "/etc/spinifex/northstar/northstar.toml" + +// DNSTargets returns the nodes whose northstar the harness can query on :53: +// every cluster node in multinode mode, else the single local node. +func DNSTargets(env *Env) []Node { + if env.Mode == ModeMultinode { + if cluster, err := ClusterFromEnv(); err == nil && len(cluster.Nodes) > 0 { + return cluster.Nodes + } + } + return []Node{{Index: 1, Name: "local", Addr: gatewayDialHost(env)}} +} + +// NorthstarZoneStore returns write credentials for the bucket northstar loads +// zones from: the bucket and region from northstar.toml and the node's system +// predastore key, which is what the control-plane zone writer uses. The +// endpoint host is replaced by the node's address so it is reachable from the +// runner; certificate verification is skipped, as elsewhere in the harness. +func NorthstarZoneStore(t *testing.T, env *Env, node Node) *nsconfig.S3Config { + t.Helper() + host := node.Addr + if env.Mode != ModeMultinode { + host = "" + } + + body, err := readNodeFile(host, northstarServerConfigPath) + if err != nil { + t.Fatalf("read %s on %s: %v", northstarServerConfigPath, hostLabel(host), err) + } + path := filepath.Join(t.TempDir(), "northstar.toml") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatalf("stage northstar.toml: %v", err) + } + server, err := nsconfig.LoadServerConfig(path) + if err != nil { + t.Fatalf("parse northstar.toml from %s: %v", hostLabel(host), err) + } + if server.S3.Bucket == "" { + t.Skipf("northstar on %s serves zones from %q, not S3", hostLabel(host), server.ZoneDir) + } + + cc := nodeClusterConfig(t, host) + creds := cc.Nodes[cc.Node].Predastore + if creds.AccessKey == "" || creds.SecretKey == "" { + t.Fatalf("no system predastore credentials in spinifex.toml on %s", hostLabel(host)) + } + + endpoint := server.S3.Endpoint + if env.Mode == ModeMultinode { + endpoint = withHost(endpoint, node.Addr) + } + return &nsconfig.S3Config{ + Endpoint: endpoint, + Region: server.S3.Region, + Bucket: server.S3.Bucket, + AccessKey: creds.AccessKey, + SecretKey: creds.SecretKey, + Insecure: true, + } +} + +// WriteTestZone stores a minimal zone for domain whose apex A record is addr. +func WriteTestZone(s3cfg *nsconfig.S3Config, domain, addr string) error { + zone := nsconfig.NewZoneConfig(nsconfig.BaseZoneSeed{ + Domain: domain, + Nameservers: []nsconfig.NameserverSeed{{Host: "ns1", IP: addr}}, + }) + zone.UpsertRecord("", nsconfig.TypeA, nsconfig.ClassIN, addr, 0) + body, err := nsconfig.RenderZone(zone) + if err != nil { + return fmt.Errorf("render %s: %w", domain, err) + } + return nsconfig.WriteZoneFile(s3cfg, domain, body) +} + +// DeleteTestZone removes domain's zone object. +func DeleteTestZone(ctx context.Context, s3cfg *nsconfig.S3Config, domain string) error { + _, err := zoneStoreClient(s3cfg).DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(s3cfg.Bucket), + Key: aws.String(strings.TrimSuffix(domain, ".") + ".toml"), + }) + return err +} + +func zoneStoreClient(s3cfg *nsconfig.S3Config) *s3.Client { + endpoint := s3cfg.Endpoint + if !strings.Contains(endpoint, "://") { + endpoint = "https://" + endpoint + } + return s3.New(s3.Options{ + Region: s3cfg.Region, + BaseEndpoint: aws.String(endpoint), + UsePathStyle: true, + Credentials: credentials.NewStaticCredentialsProvider(s3cfg.AccessKey, s3cfg.SecretKey, ""), + HTTPClient: insecureHTTPClient(30 * time.Second), + }) +} + +// withHost replaces the host of an endpoint, keeping its scheme and port. +func withHost(endpoint, host string) string { + scheme := "" + rest := endpoint + if i := strings.Index(endpoint, "://"); i >= 0 { + scheme, rest = endpoint[:i+3], endpoint[i+3:] + } + if _, port, err := net.SplitHostPort(rest); err == nil { + return scheme + net.JoinHostPort(host, port) + } + return scheme + host +} + +func nodeClusterConfig(t *testing.T, host string) *config.ClusterConfig { + t.Helper() + body, err := readNodeFile(host, "/etc/spinifex/spinifex.toml") + if err != nil { + t.Fatalf("read spinifex.toml on %s: %v", hostLabel(host), err) + } + cc, err := loadClusterConfigBytes([]byte(body)) + if err != nil { + t.Fatalf("decode spinifex.toml from %s: %v", hostLabel(host), err) + } + return cc +} diff --git a/tests/e2e/route53/main_test.go b/tests/e2e/route53/main_test.go new file mode 100644 index 000000000..324c5f4a3 --- /dev/null +++ b/tests/e2e/route53/main_test.go @@ -0,0 +1,6 @@ +//go:build e2e + +// Package route53 is the hosted-zone E2E suite. It drives northstar, the +// authoritative DNS server behind Route 53, in both single and multinode +// topologies, because every node serves every zone from the same bucket. +package route53 diff --git a/tests/e2e/route53/northstar_scale_test.go b/tests/e2e/route53/northstar_scale_test.go new file mode 100644 index 000000000..948934431 --- /dev/null +++ b/tests/e2e/route53/northstar_scale_test.go @@ -0,0 +1,161 @@ +//go:build e2e + +package route53 + +import ( + "context" + "errors" + "fmt" + "net" + "sync" + "testing" + "time" + + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/require" +) + +const ( + // More zone objects than one ListObjectsV2 page holds (1,000), so a + // listing that stops at the first page leaves zones unloaded. + scaleZoneCount = 1100 + scaleZoneAddr = "192.0.2.53" + scaleWorkers = 8 + + // Two 30 s sync cycles plus reading 1,100 objects, with headroom. + scaleSyncTimeout = 4 * time.Minute +) + +// scaleZone sorts every zone ahead of the platform base zone, so a purge that +// trusts a truncated listing would take the base zone with it. +func scaleZone(i int) string { + return fmt.Sprintf("a%04d.zonescale.test", i) +} + +// Northstar loads every zone in its bucket, not just the first listing page, +// and a sync never purges a zone that is still there. The last zone written +// must resolve on every node, the platform base zone must survive the sync, +// and deleting the objects must stop them resolving. +func TestNorthstarServesZonesBeyondFirstListPage(t *testing.T) { + env := harness.LoadEnv(t) + targets := harness.DNSTargets(env) + store := harness.NorthstarZoneStore(t, env, targets[0]) + base := harness.NorthstarBaseDomain(env) + if base == "" && env.Mode == harness.ModeMultinode { + base, _ = harness.PeerNorthstarDomains(t, targets[0]) + } + require.NotEmpty(t, base, "northstar base domain is not configured") + + harness.Phase(t, "Northstar — write %d zone objects", scaleZoneCount) + t.Cleanup(func() { + if err := forEachZone(func(name string) error { + return harness.DeleteTestZone(context.Background(), store, name) + }); err != nil { + t.Logf("cleanup: %v", err) + } + }) + require.NoError(t, forEachZone(func(name string) error { + return harness.WriteTestZone(store, name, scaleZoneAddr) + })) + + first, last := scaleZone(1), scaleZone(scaleZoneCount) + harness.Phase(t, "Northstar — every node serves the first and last zone") + for _, node := range targets { + harness.EventuallyErr(t, func() error { + for _, name := range []string{first, last} { + if err := wantAddr(node, name, scaleZoneAddr); err != nil { + return err + } + } + return nil + }, scaleSyncTimeout, 5*time.Second) + } + + harness.Phase(t, "Northstar — the platform base zone survives the sync") + for _, node := range targets { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ns, err := harness.NodeResolver(node).LookupNS(ctx, base) + cancel() + require.NoError(t, err, "base zone %s on %s", base, node.Name) + require.NotEmpty(t, ns, "base zone %s on %s has no NS records", base, node.Name) + } + + harness.Phase(t, "Northstar — deleted zone objects stop resolving") + require.NoError(t, forEachZone(func(name string) error { + return harness.DeleteTestZone(context.Background(), store, name) + })) + for _, node := range targets { + harness.EventuallyErr(t, func() error { + for _, name := range []string{first, last} { + if err := wantGone(node, name); err != nil { + return err + } + } + return nil + }, scaleSyncTimeout, 5*time.Second) + } +} + +// forEachZone runs fn over every scale zone with a small worker pool and +// returns the first error. +func forEachZone(fn func(name string) error) error { + names := make(chan string) + var ( + wg sync.WaitGroup + mu sync.Mutex + firstErr error + ) + for range scaleWorkers { + wg.Add(1) + go func() { + defer wg.Done() + for name := range names { + if err := fn(name); err != nil { + mu.Lock() + if firstErr == nil { + firstErr = fmt.Errorf("%s: %w", name, err) + } + mu.Unlock() + } + } + }() + } + for i := 1; i <= scaleZoneCount; i++ { + names <- scaleZone(i) + } + close(names) + wg.Wait() + return firstErr +} + +func wantAddr(node harness.Node, name, addr string) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + addrs, err := harness.ResolveViaNode(ctx, node, name) + if err != nil { + return err + } + for _, a := range addrs { + if a == addr { + return nil + } + } + return fmt.Errorf("%s via %s = %v, want %s", name, node.Name, addrs, addr) +} + +// wantGone accepts any definite answer other than an address: NXDOMAIN when +// the name is recursed upstream, REFUSED when the listener does not recurse +// for this client. A timeout proves nothing, so it is not accepted. +func wantGone(node harness.Node, name string) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + addrs, err := harness.NodeResolver(node).LookupHost(ctx, name) + if err == nil { + return fmt.Errorf("%s via %s still resolves to %v", name, node.Name, addrs) + } + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) && dnsErr.IsTimeout { + return fmt.Errorf("%s via %s: %w", name, node.Name, err) + } + return nil +}