mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-24 05:04:03 +00:00
test: e2e proof that northstar serves zones past the first S3 page
Pins northstar to the fix that pages ListObjectsV2 before purging, and adds a route53 suite, run in both topologies, that writes 1,100 zone objects, requires every node to serve the last one while the base zone survives, and requires the deleted zones to stop resolving.
This commit is contained in:
Ben Duncan
@@ -28,6 +28,9 @@
|
||||
# that runs identically on either, and because a quota left switched on would
|
||||
# cap every suite sharing the environment — so it must be proved to restore
|
||||
# itself wherever it runs, not only on one node.
|
||||
# route53 belongs to both topologies because every node serves every hosted
|
||||
# zone from one shared bucket: on one node it proves the zone loads at all, on
|
||||
# three that each node's sync reaches the same set.
|
||||
# storagefault runs last in both lists and belongs to both topologies, for
|
||||
# different reasons. On one node it is RS(1,0), so freezing the only predastore
|
||||
# is a total outage and the guest-survival assertion is unambiguous. On three it
|
||||
@@ -37,7 +40,7 @@
|
||||
# received. It is last because it SIGSTOPs a cluster-wide
|
||||
# service: anything sharing the environment would see the same outage and report
|
||||
# a failure that says nothing about itself.
|
||||
E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"
|
||||
E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault"
|
||||
|
||||
# Suites runnable against a multi-node environment.
|
||||
#
|
||||
@@ -57,7 +60,7 @@ E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota
|
||||
# replicated rather than sitting on one node, so it runs before the pair that
|
||||
# take a node away — a cluster whose buckets are already wrong should report
|
||||
# that, not report a recovery failing.
|
||||
E2E_SUITES_MULTI="multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault"
|
||||
E2E_SUITES_MULTI="multinode lb cert quota route53 kvquorum lbrecovery instancerecovery storagefault"
|
||||
|
||||
# grep -xE alternation form, for narrowing a requested set to the eligible one.
|
||||
E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
|
||||
|
||||
@@ -6,8 +6,8 @@ Suites live in `tests/e2e/<name>/` and build to `tests/e2e/_bin/<name>.test`. Ev
|
||||
|
||||
| Set | Suites |
|
||||
| --- | --- |
|
||||
| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota storagefault` |
|
||||
| `E2E_SUITES_MULTI` | `multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault` |
|
||||
| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault` |
|
||||
| `E2E_SUITES_MULTI` | `multinode lb cert quota route53 kvquorum lbrecovery instancerecovery storagefault` |
|
||||
| `E2E_SUITES_NIGHTLY_SINGLE` | `single cert iam` |
|
||||
| `E2E_SUITES_NIGHTLY_MULTI` | `multinode cert lb` |
|
||||
|
||||
@@ -120,6 +120,7 @@ The create/describe/connect path against a live cluster: a DB VM that actually b
|
||||
### Other
|
||||
|
||||
- **`quota`** — every enforced dimension refuses with the same `quotaExceeded` code; a launch is charged for every instance it asks for; one instance wider than the cap is refused.
|
||||
- **`route53`** — hosted zones as northstar serves them, on every node. `TestNorthstarServesZonesBeyondFirstListPage` writes 1,100 zone objects (more than one `ListObjectsV2` page), requires the last one to resolve on every node and the platform base zone to survive the sync, then requires the deleted zones to stop resolving.
|
||||
- **`gpu`** — device visible in the guest, driver access, launch.
|
||||
- **`bedrock`** — `ListFoundationModels`/`GetFoundationModel` against a running cluster. Wire-format parity is unit-tested, not here.
|
||||
- **`ddil`** — Denied, Disrupted, Intermittent, Limited network scenarios.
|
||||
|
||||
@@ -9,6 +9,7 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.9
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3
|
||||
github.com/cenkalti/backoff/v4 v4.3.0
|
||||
github.com/charmbracelet/bubbles v1.0.0
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
@@ -24,7 +25,7 @@ require (
|
||||
github.com/miekg/dns v1.1.73
|
||||
github.com/minio/highwayhash v1.0.4
|
||||
github.com/mulgadc/bluebottle v1.22.0
|
||||
github.com/mulgadc/northstar v1.22.0
|
||||
github.com/mulgadc/northstar v1.22.1-0.20261023213715-059065aec064
|
||||
github.com/mulgadc/predastore v1.22.0
|
||||
github.com/mulgadc/viperblock v1.22.0
|
||||
github.com/nats-io/nats-server/v2 v2.15.1
|
||||
@@ -67,10 +68,9 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
|
||||
|
||||
@@ -40,14 +40,14 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 h1:NyeuyXkZL476uoAowNActm4XJO1QPKf7aassgXHpiqA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3/go.mod h1:GfoBeKMDJecgRUsGf/0RGdx0NJAh81ObTd922h8pQys=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
|
||||
@@ -295,8 +295,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc
|
||||
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
|
||||
github.com/mulgadc/bluebottle v1.22.0 h1:pGZ5/Ly3CYem62pPu2zKBRpvMpAG0LHMIMw9Yt/lkIg=
|
||||
github.com/mulgadc/bluebottle v1.22.0/go.mod h1:DwT91bJyUsrpg1gCJmcCOTG+tgw5J124LloHYvGAQu0=
|
||||
github.com/mulgadc/northstar v1.22.0 h1:fTndlk47hK76fcNr2MqsyeKhBNVEDt8+BC6Z30ohokA=
|
||||
github.com/mulgadc/northstar v1.22.0/go.mod h1:yLH8waqfHZx177Ae6dGQGTDt/02rPE9alnUKyxDNhVs=
|
||||
github.com/mulgadc/northstar v1.22.1-0.20261023213715-059065aec064 h1:wnnHsRttCvLydRz5rQ5WTzk46pN7Vxxq5rSalOLkUbw=
|
||||
github.com/mulgadc/northstar v1.22.1-0.20261023213715-059065aec064/go.mod h1:Halhinpgo5rzUhYemGN8uYTWu1pJsL5e/7l1eEKc7nM=
|
||||
github.com/mulgadc/predastore v1.22.0 h1:mUQJMTxMiRDVvemMP+34BP/+r1U0iCptc9B++u4NJ8o=
|
||||
github.com/mulgadc/predastore v1.22.0/go.mod h1:/cMIoKtnQG/DWyeX/0matEQrG2/abY60tNHWcBKS9Kc=
|
||||
github.com/mulgadc/viperblock v1.22.0 h1:8vS2AtpIDkIhGNOazQPicKXV2y/8PAy8BNrVWIOCP6I=
|
||||
|
||||
+3
-2
@@ -18,9 +18,9 @@ SUITE_TAG ?= $(notdir $(patsubst %/...,%,$(firstword $(PKG))))
|
||||
# BIN_DIR/<suite>.test so CI can ship binaries (not source + go toolchain) to
|
||||
# the remote VM.
|
||||
BIN_DIR ?= $(CURDIR)/_bin
|
||||
SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink multinode storagegrowth partialblock rds quota diskperf storagefault instancerecovery lbrecovery kvquorum
|
||||
SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink multinode storagegrowth partialblock rds quota diskperf storagefault instancerecovery lbrecovery kvquorum route53
|
||||
|
||||
.PHONY: e2e e2e-quota e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-storagegrowth e2e-partialblock e2e-diskperf e2e-rds e2e-storagefault e2e-instancerecovery e2e-lbrecovery e2e-kvquorum e2e-all clean e2e-build
|
||||
.PHONY: e2e e2e-quota e2e-route53 e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-storagegrowth e2e-partialblock e2e-diskperf e2e-rds e2e-storagefault e2e-instancerecovery e2e-lbrecovery e2e-kvquorum e2e-all clean e2e-build
|
||||
|
||||
RUN_FLAG = $(if $(RUN),-run '$(RUN)',)
|
||||
|
||||
@@ -65,6 +65,7 @@ e2e-diskperf: ; $(MAKE) e2e PKG=./diskperf/... TIMEOUT=180m
|
||||
# most of a 30m package timeout before the tests still running get to report.
|
||||
e2e-rds: ; $(MAKE) e2e PKG=./rds/... TIMEOUT=45m
|
||||
e2e-quota: ; $(MAKE) e2e PKG=./quota/...
|
||||
e2e-route53: ; $(MAKE) e2e PKG=./route53/...
|
||||
# Freezes predastore, so its budget covers a deliberate outage plus the
|
||||
# recovery and re-verification that follow it, twice over for the two tests.
|
||||
e2e-storagefault: ; $(MAKE) e2e PKG=./storagefault/... TIMEOUT=45m
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
//go:build e2e
|
||||
|
||||
package harness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
nsconfig "github.com/mulgadc/northstar/pkg/config"
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
|
||||
)
|
||||
|
||||
const northstarServerConfigPath = "/etc/spinifex/northstar/northstar.toml"
|
||||
|
||||
// DNSTargets returns the nodes whose northstar the harness can query on :53:
|
||||
// every cluster node in multinode mode, else the single local node.
|
||||
func DNSTargets(env *Env) []Node {
|
||||
if env.Mode == ModeMultinode {
|
||||
if cluster, err := ClusterFromEnv(); err == nil && len(cluster.Nodes) > 0 {
|
||||
return cluster.Nodes
|
||||
}
|
||||
}
|
||||
return []Node{{Index: 1, Name: "local", Addr: gatewayDialHost(env)}}
|
||||
}
|
||||
|
||||
// NorthstarZoneStore returns write credentials for the bucket northstar loads
|
||||
// zones from: the bucket and region from northstar.toml and the node's system
|
||||
// predastore key, which is what the control-plane zone writer uses. The
|
||||
// endpoint host is replaced by the node's address so it is reachable from the
|
||||
// runner; certificate verification is skipped, as elsewhere in the harness.
|
||||
func NorthstarZoneStore(t *testing.T, env *Env, node Node) *nsconfig.S3Config {
|
||||
t.Helper()
|
||||
host := node.Addr
|
||||
if env.Mode != ModeMultinode {
|
||||
host = ""
|
||||
}
|
||||
|
||||
body, err := readNodeFile(host, northstarServerConfigPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s on %s: %v", northstarServerConfigPath, hostLabel(host), err)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "northstar.toml")
|
||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||
t.Fatalf("stage northstar.toml: %v", err)
|
||||
}
|
||||
server, err := nsconfig.LoadServerConfig(path)
|
||||
if err != nil {
|
||||
t.Fatalf("parse northstar.toml from %s: %v", hostLabel(host), err)
|
||||
}
|
||||
if server.S3.Bucket == "" {
|
||||
t.Skipf("northstar on %s serves zones from %q, not S3", hostLabel(host), server.ZoneDir)
|
||||
}
|
||||
|
||||
cc := nodeClusterConfig(t, host)
|
||||
creds := cc.Nodes[cc.Node].Predastore
|
||||
if creds.AccessKey == "" || creds.SecretKey == "" {
|
||||
t.Fatalf("no system predastore credentials in spinifex.toml on %s", hostLabel(host))
|
||||
}
|
||||
|
||||
endpoint := server.S3.Endpoint
|
||||
if env.Mode == ModeMultinode {
|
||||
endpoint = withHost(endpoint, node.Addr)
|
||||
}
|
||||
return &nsconfig.S3Config{
|
||||
Endpoint: endpoint,
|
||||
Region: server.S3.Region,
|
||||
Bucket: server.S3.Bucket,
|
||||
AccessKey: creds.AccessKey,
|
||||
SecretKey: creds.SecretKey,
|
||||
Insecure: true,
|
||||
}
|
||||
}
|
||||
|
||||
// WriteTestZone stores a minimal zone for domain whose apex A record is addr.
|
||||
func WriteTestZone(s3cfg *nsconfig.S3Config, domain, addr string) error {
|
||||
zone := nsconfig.NewZoneConfig(nsconfig.BaseZoneSeed{
|
||||
Domain: domain,
|
||||
Nameservers: []nsconfig.NameserverSeed{{Host: "ns1", IP: addr}},
|
||||
})
|
||||
zone.UpsertRecord("", nsconfig.TypeA, nsconfig.ClassIN, addr, 0)
|
||||
body, err := nsconfig.RenderZone(zone)
|
||||
if err != nil {
|
||||
return fmt.Errorf("render %s: %w", domain, err)
|
||||
}
|
||||
return nsconfig.WriteZoneFile(s3cfg, domain, body)
|
||||
}
|
||||
|
||||
// DeleteTestZone removes domain's zone object.
|
||||
func DeleteTestZone(ctx context.Context, s3cfg *nsconfig.S3Config, domain string) error {
|
||||
_, err := zoneStoreClient(s3cfg).DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s3cfg.Bucket),
|
||||
Key: aws.String(strings.TrimSuffix(domain, ".") + ".toml"),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func zoneStoreClient(s3cfg *nsconfig.S3Config) *s3.Client {
|
||||
endpoint := s3cfg.Endpoint
|
||||
if !strings.Contains(endpoint, "://") {
|
||||
endpoint = "https://" + endpoint
|
||||
}
|
||||
return s3.New(s3.Options{
|
||||
Region: s3cfg.Region,
|
||||
BaseEndpoint: aws.String(endpoint),
|
||||
UsePathStyle: true,
|
||||
Credentials: credentials.NewStaticCredentialsProvider(s3cfg.AccessKey, s3cfg.SecretKey, ""),
|
||||
HTTPClient: insecureHTTPClient(30 * time.Second),
|
||||
})
|
||||
}
|
||||
|
||||
// withHost replaces the host of an endpoint, keeping its scheme and port.
|
||||
func withHost(endpoint, host string) string {
|
||||
scheme := ""
|
||||
rest := endpoint
|
||||
if i := strings.Index(endpoint, "://"); i >= 0 {
|
||||
scheme, rest = endpoint[:i+3], endpoint[i+3:]
|
||||
}
|
||||
if _, port, err := net.SplitHostPort(rest); err == nil {
|
||||
return scheme + net.JoinHostPort(host, port)
|
||||
}
|
||||
return scheme + host
|
||||
}
|
||||
|
||||
func nodeClusterConfig(t *testing.T, host string) *config.ClusterConfig {
|
||||
t.Helper()
|
||||
body, err := readNodeFile(host, "/etc/spinifex/spinifex.toml")
|
||||
if err != nil {
|
||||
t.Fatalf("read spinifex.toml on %s: %v", hostLabel(host), err)
|
||||
}
|
||||
cc, err := loadClusterConfigBytes([]byte(body))
|
||||
if err != nil {
|
||||
t.Fatalf("decode spinifex.toml from %s: %v", hostLabel(host), err)
|
||||
}
|
||||
return cc
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
//go:build e2e
|
||||
|
||||
// Package route53 is the hosted-zone E2E suite. It drives northstar, the
|
||||
// authoritative DNS server behind Route 53, in both single and multinode
|
||||
// topologies, because every node serves every zone from the same bucket.
|
||||
package route53
|
||||
@@ -0,0 +1,161 @@
|
||||
//go:build e2e
|
||||
|
||||
package route53
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
// More zone objects than one ListObjectsV2 page holds (1,000), so a
|
||||
// listing that stops at the first page leaves zones unloaded.
|
||||
scaleZoneCount = 1100
|
||||
scaleZoneAddr = "192.0.2.53"
|
||||
scaleWorkers = 8
|
||||
|
||||
// Two 30 s sync cycles plus reading 1,100 objects, with headroom.
|
||||
scaleSyncTimeout = 4 * time.Minute
|
||||
)
|
||||
|
||||
// scaleZone sorts every zone ahead of the platform base zone, so a purge that
|
||||
// trusts a truncated listing would take the base zone with it.
|
||||
func scaleZone(i int) string {
|
||||
return fmt.Sprintf("a%04d.zonescale.test", i)
|
||||
}
|
||||
|
||||
// Northstar loads every zone in its bucket, not just the first listing page,
|
||||
// and a sync never purges a zone that is still there. The last zone written
|
||||
// must resolve on every node, the platform base zone must survive the sync,
|
||||
// and deleting the objects must stop them resolving.
|
||||
func TestNorthstarServesZonesBeyondFirstListPage(t *testing.T) {
|
||||
env := harness.LoadEnv(t)
|
||||
targets := harness.DNSTargets(env)
|
||||
store := harness.NorthstarZoneStore(t, env, targets[0])
|
||||
base := harness.NorthstarBaseDomain(env)
|
||||
if base == "" && env.Mode == harness.ModeMultinode {
|
||||
base, _ = harness.PeerNorthstarDomains(t, targets[0])
|
||||
}
|
||||
require.NotEmpty(t, base, "northstar base domain is not configured")
|
||||
|
||||
harness.Phase(t, "Northstar — write %d zone objects", scaleZoneCount)
|
||||
t.Cleanup(func() {
|
||||
if err := forEachZone(func(name string) error {
|
||||
return harness.DeleteTestZone(context.Background(), store, name)
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: %v", err)
|
||||
}
|
||||
})
|
||||
require.NoError(t, forEachZone(func(name string) error {
|
||||
return harness.WriteTestZone(store, name, scaleZoneAddr)
|
||||
}))
|
||||
|
||||
first, last := scaleZone(1), scaleZone(scaleZoneCount)
|
||||
harness.Phase(t, "Northstar — every node serves the first and last zone")
|
||||
for _, node := range targets {
|
||||
harness.EventuallyErr(t, func() error {
|
||||
for _, name := range []string{first, last} {
|
||||
if err := wantAddr(node, name, scaleZoneAddr); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, scaleSyncTimeout, 5*time.Second)
|
||||
}
|
||||
|
||||
harness.Phase(t, "Northstar — the platform base zone survives the sync")
|
||||
for _, node := range targets {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
ns, err := harness.NodeResolver(node).LookupNS(ctx, base)
|
||||
cancel()
|
||||
require.NoError(t, err, "base zone %s on %s", base, node.Name)
|
||||
require.NotEmpty(t, ns, "base zone %s on %s has no NS records", base, node.Name)
|
||||
}
|
||||
|
||||
harness.Phase(t, "Northstar — deleted zone objects stop resolving")
|
||||
require.NoError(t, forEachZone(func(name string) error {
|
||||
return harness.DeleteTestZone(context.Background(), store, name)
|
||||
}))
|
||||
for _, node := range targets {
|
||||
harness.EventuallyErr(t, func() error {
|
||||
for _, name := range []string{first, last} {
|
||||
if err := wantGone(node, name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, scaleSyncTimeout, 5*time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// forEachZone runs fn over every scale zone with a small worker pool and
|
||||
// returns the first error.
|
||||
func forEachZone(fn func(name string) error) error {
|
||||
names := make(chan string)
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
firstErr error
|
||||
)
|
||||
for range scaleWorkers {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for name := range names {
|
||||
if err := fn(name); err != nil {
|
||||
mu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = fmt.Errorf("%s: %w", name, err)
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
for i := 1; i <= scaleZoneCount; i++ {
|
||||
names <- scaleZone(i)
|
||||
}
|
||||
close(names)
|
||||
wg.Wait()
|
||||
return firstErr
|
||||
}
|
||||
|
||||
func wantAddr(node harness.Node, name, addr string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
addrs, err := harness.ResolveViaNode(ctx, node, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range addrs {
|
||||
if a == addr {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("%s via %s = %v, want %s", name, node.Name, addrs, addr)
|
||||
}
|
||||
|
||||
// wantGone accepts any definite answer other than an address: NXDOMAIN when
|
||||
// the name is recursed upstream, REFUSED when the listener does not recurse
|
||||
// for this client. A timeout proves nothing, so it is not accepted.
|
||||
func wantGone(node harness.Node, name string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
addrs, err := harness.NodeResolver(node).LookupHost(ctx, name)
|
||||
if err == nil {
|
||||
return fmt.Errorf("%s via %s still resolves to %v", name, node.Name, addrs)
|
||||
}
|
||||
var dnsErr *net.DNSError
|
||||
if errors.As(err, &dnsErr) && dnsErr.IsTimeout {
|
||||
return fmt.Errorf("%s via %s: %w", name, node.Name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user