diff --git a/.github/e2e-cells.json b/.github/e2e-cells.json index b895186d2..5c2f8fa10 100644 --- a/.github/e2e-cells.json +++ b/.github/e2e-cells.json @@ -178,5 +178,17 @@ "extra": "", "suite": "lbrecovery", "timeout_minutes": 90 + }, + { + "cell": 33, + "install": "binary", + "topo": "real-multi", + "net": "static", + "host": "debian-13", + "arch": "x86_64", + "runner": "ci-multi", + "extra": "", + "suite": "kvquorum", + "timeout_minutes": 55 } ] diff --git a/.github/scripts/suite-sets.sh b/.github/scripts/suite-sets.sh index 6818406ed..0b9746e81 100644 --- a/.github/scripts/suite-sets.sh +++ b/.github/scripts/suite-sets.sh @@ -51,7 +51,13 @@ E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota # the load balancer's, so taking that node away does not take the backends with # it. It runs beside instancerecovery at the end because it also takes a node # away and owns its environment while it does. -E2E_SUITES_MULTI="multinode lb cert quota lbrecovery instancerecovery storagefault" +# +# kvquorum is multi-only and needs three, for the same reason as those two: a +# surviving majority. It is what proves the control plane's own state is +# replicated rather than sitting on one node, so it runs before the pair that +# take a node away — a cluster whose buckets are already wrong should report +# that, not report a recovery failing. +E2E_SUITES_MULTI="multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault" # grep -xE alternation form, for narrowing a requested set to the eligible one. E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')" @@ -75,6 +81,10 @@ E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')" # after a failed seal do not exist on one node, and they are the assertions # worth a nightly. E2E_SUITES_NIGHTLY_SINGLE="single cert iam" +# +# The multi-node nightly subset carries kvquorum's read-only half through +# `multinode`, which sweeps every KV bucket rather than the three the daemon +# owns. The node-loss half is too slow for a 35-minute cell and has its own. E2E_SUITES_NIGHTLY_MULTI="multinode cert lb" # How long a suite gets, for the same reason the lists above live here: the @@ -112,6 +122,10 @@ e2e_suite_timeout() { # the backends before it forwards anything. Then the node comes back and has # to be shown not to answer for an address it no longer holds. lbrecovery) echo "75m" ;; + # kvquorum audits every bucket in seconds and then spends the rest of its + # budget on a node stopping and starting: seven units down, a raft election, + # and the whole service stack back up with every stream caught up. + kvquorum) echo "40m" ;; *) echo "30m" ;; esac } diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml index bd706cc95..ff8b1606a 100644 --- a/.github/workflows/e2e-nightly.yml +++ b/.github/workflows/e2e-nightly.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: inputs: cells: - description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes" + description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes, 33 KV bucket quorum under node loss on three nodes" required: false permissions: diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index ef2614f8a..d37fbb1e8 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -30,6 +30,7 @@ import ( "github.com/mulgadc/bluebottle/pkg/masterkey" "github.com/mulgadc/bluebottle/pkg/safecast" "github.com/mulgadc/spinifex/spinifex/admin" + "github.com/mulgadc/spinifex/spinifex/clustersize" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/ebsmetadata" "github.com/mulgadc/spinifex/spinifex/ebsprovider" @@ -372,7 +373,7 @@ func init() { adminInitCmd.Flags().String("region", "ap-southeast-2", "Mulga region to create") adminInitCmd.Flags().String("az", "ap-southeast-2a", "Mulga AZ to create") adminInitCmd.Flags().String("node", "node1", "Node name, increment for additional nodes (default, node1)") - adminInitCmd.Flags().Int("nodes", 3, "Number of nodes to expect for cluster") + adminInitCmd.Flags().Int("nodes", 3, "Number of nodes to expect for cluster (1, or 3 and up)") adminInitCmd.Flags().String("host", "", "Leader node to join (if not specified, tries multicast discovery)") adminInitCmd.Flags().Int("port", 4432, "Port to bind cluster services on") adminInitCmd.Flags().String("bind", "0.0.0.0", "IP address to bind services to (e.g., 10.11.12.1 for multi-node). Default 0.0.0.0 listens on all interfaces.") @@ -1212,6 +1213,26 @@ func runimagesListCmd(cmd *cobra.Command, args []string) { pterm.Println("spx admin images import --name ") } +// validateNodeCount accepts the cluster sizes Spinifex supports: one server, or +// three and up. +// +// Two is rejected rather than allowed and warned about. On two servers OVN runs +// standalone, the storage metadata quorum has no majority to lose, and the +// control plane's own KV state has no replica count that works — one replica +// disappears with its node, and two stop accepting writes when either goes. A +// cluster that cannot survive the event it was built for is worth refusing at the +// point someone asks for it. +func validateNodeCount(nodes int) error { + if nodes < 1 { + return fmt.Errorf("a cluster needs at least one node, got %d", nodes) + } + if nodes > 1 && nodes < clustersize.MinHANodes { + return fmt.Errorf("%d nodes is not a supported cluster size: use 1 for a single server, or %d and up for a cluster that can survive losing one", + nodes, clustersize.MinHANodes) + } + return nil +} + // TODO: Move all logic to a module, use minimal application logic in viper commands. func runAdminInit(cmd *cobra.Command, args []string) { if os.Getuid() != 0 { @@ -1243,6 +1264,15 @@ func runAdminInit(cmd *cobra.Command, args []string) { clusterName, _ := cmd.Flags().GetString("cluster-name") services, _ := cmd.Flags().GetStringSlice("services") + // Rejected before anything is written, because a two-node cluster has no + // correct answer rather than a worse one: no layer of the stack has a + // majority to lose, and no replica count for its state both survives a node + // going away and still accepts writes. + if err := validateNodeCount(nodes); err != nil { + fmt.Fprintf(os.Stderr, "--nodes: %v\n", err) + os.Exit(1) + } + // Optional operator email — validated up-front so a bad address fails // before we touch any config state. Empty is allowed here; reset / repeat // inits on a box that already has an email in /etc/spinifex/spinifex.toml @@ -2824,7 +2854,7 @@ func initIAMServiceFromConfig() (*handlers_iam.IAMServiceImpl, *config.ClusterCo // Background: this runs at CLI top level, where there is no request to // inherit a deadline from and the process exits after the one command. - svc, err := handlers_iam.NewIAMServiceImpl(context.Background(), nc, masterKey, len(cfg.Nodes)) + svc, err := handlers_iam.NewIAMServiceImpl(context.Background(), nc, masterKey) if err != nil { nc.Close() return nil, nil, nil, nil, fmt.Errorf("init IAM service: %w", err) diff --git a/cmd/spinifex/cmd/admin_kv_replicas.go b/cmd/spinifex/cmd/admin_kv_replicas.go new file mode 100644 index 000000000..c715973e4 --- /dev/null +++ b/cmd/spinifex/cmd/admin_kv_replicas.go @@ -0,0 +1,201 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "text/tabwriter" + + "github.com/mulgadc/spinifex/spinifex/kvutil" + "github.com/nats-io/nats.go/jetstream" + "github.com/spf13/cobra" +) + +var kvReplicasCmd = &cobra.Command{ + Use: "replicas", + Short: "Report how many nodes each KV bucket is replicated across", + Long: `Print one line per JetStream KV bucket: how many replicas it has, how many the +cluster's size entitles it to, and which nodes currently hold it. + +A bucket with fewer replicas than the cluster has nodes lives on fewer nodes +than exist, and every node depends on those. Losing one of them makes the +bucket unreadable and unwritable from every node at once, which is a +cluster-wide outage caused by a single-node event. + +A bucket also has to be able to serve, which is a different question from how +many replicas it was configured for. ONLINE counts the replicas that are caught +up and reachable; a bucket whose ONLINE is not a majority of REPLICAS cannot +accept a write no matter what its replica count says. + +--repair raises any under-replicated bucket to the cluster's replica count, +capped at JetStream's maximum of five. It changes nothing else about a bucket +and never lowers a replica count, so it is safe to run on a healthy cluster and +safe to run twice. + +Exit status separates the two problems, so this can gate a deployment without +parsing the output: + + 0 every bucket is at the cluster's replica count and can serve + 1 at least one bucket is under-replicated — --repair is the answer + 2 at least one bucket has no quorum, or the report could not be produced — + a node needs looking at, and --repair will not help`, + Run: runKVReplicas, +} + +func init() { + kvCmd.AddCommand(kvReplicasCmd) + kvReplicasCmd.Flags().Bool("repair", false, "raise every under-replicated bucket to the cluster's node count") + kvReplicasCmd.Flags().Bool("json", false, "emit the report as JSON") +} + +func runKVReplicas(cmd *cobra.Command, _ []string) { + repair, _ := cmd.Flags().GetBool("repair") + asJSON, _ := cmd.Flags().GetBool("json") + + // Anything that stops the report being produced exits unreachable rather + // than under-replicated: an empty answer is not a cluster with no buckets, + // and --repair is not the response to it. + _, nc, err := loadConfigAndConnect() + if err != nil { + fmt.Fprintf(os.Stderr, "connect to cluster: %v\n", err) + os.Exit(exitUnreachable) + } + defer nc.Close() + + js, err := jetstream.New(nc) + if err != nil { + fmt.Fprintf(os.Stderr, "get JetStream context: %v\n", err) + os.Exit(exitUnreachable) + } + + // Background: this runs at CLI top level, where there is no request to + // inherit a deadline from and the process exits after the one command. + ctx := context.Background() + reports, err := kvutil.AuditBucketReplicas(ctx, js) + if err != nil { + fmt.Fprintf(os.Stderr, "audit KV bucket replicas: %v\n", err) + os.Exit(exitUnreachable) + } + + if repair { + repairReplicaReports(ctx, js, reports, os.Stderr) + } + + if err := writeReplicaReport(os.Stdout, reports, asJSON); err != nil { + fmt.Fprintf(os.Stderr, "write report: %v\n", err) + os.Exit(exitUnreachable) + } + + os.Exit(replicaReportExit(reports)) +} + +// Exit statuses, documented in the command's own help because a deployment gate +// reads them rather than the output. +const ( + exitHealthy = 0 + // exitUnderReplicated is the repairable problem: the buckets serve, but on + // fewer nodes than the cluster has. + exitUnderReplicated = 1 + // exitUnreachable is the problem repair cannot touch — a bucket that cannot + // accept a write, or a report that could not be produced at all. + exitUnreachable = 2 +) + +// replicaReportExit is the exit status for a set of reports. +// +// No quorum outranks under-replicated because they need different responses: one +// is fixed by raising a count, the other by finding out why a node is not +// answering. A bucket that is both should send the operator to the node. +func replicaReportExit(reports []kvutil.BucketReport) int { + status := exitHealthy + for _, r := range reports { + if !r.HasQuorum() { + return exitUnreachable + } + if r.UnderReplicated() { + status = exitUnderReplicated + } + } + return status +} + +// repairReplicaReports raises every under-replicated bucket and updates reports +// in place for the ones that were raised. +// +// One bucket that cannot be placed must not stop the others being repaired, so a +// failure is named on errOut and the sweep continues. The bucket it failed on +// stays UNDER-REPLICATED in the report, which is already what makes the exit +// status non-zero, so nothing is lost by not returning an error here. +func repairReplicaReports(ctx context.Context, js jetstream.JetStream, reports []kvutil.BucketReport, errOut io.Writer) { + for i, r := range reports { + if !r.UnderReplicated() { + continue + } + if err := kvutil.RaiseBucketReplicas(ctx, js, r.Bucket, r.Want); err != nil { + fmt.Fprintf(errOut, "repair %s: %v\n", r.Bucket, err) + continue + } + reports[i].Replicas = r.Want + } +} + +// underReplicatedCount is how many buckets are below the cluster's replica count. +func underReplicatedCount(reports []kvutil.BucketReport) int { + under := 0 + for _, r := range reports { + if r.UnderReplicated() { + under++ + } + } + return under +} + +// noQuorumCount is how many buckets cannot currently accept a write. +func noQuorumCount(reports []kvutil.BucketReport) int { + stuck := 0 + for _, r := range reports { + if !r.HasQuorum() { + stuck++ + } + } + return stuck +} + +// replicaStatus is the one word for a bucket's condition. +// +// No quorum wins over under-replicated when both are true, matching the exit +// status: a bucket that cannot serve is the more urgent of the two and is not +// what --repair is for. +func replicaStatus(r kvutil.BucketReport) string { + switch { + case !r.HasQuorum(): + return "NO-QUORUM" + case r.UnderReplicated(): + return "UNDER-REPLICATED" + default: + return "ok" + } +} + +// writeReplicaReport renders reports as a table, or as JSON when asJSON. +func writeReplicaReport(out io.Writer, reports []kvutil.BucketReport, asJSON bool) error { + if asJSON { + return json.NewEncoder(out).Encode(reports) + } + + w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "BUCKET\tREPLICAS\tWANT\tONLINE\tSTATUS\tLEADER\tHELD BY") + for _, r := range reports { + fmt.Fprintf(w, "%s\t%d\t%d\t%d\t%s\t%s\t%s\n", + r.Bucket, r.Replicas, r.Want, r.Online, replicaStatus(r), r.Leader, strings.Join(r.Peers, ",")) + } + if err := w.Flush(); err != nil { + return err + } + _, err := fmt.Fprintf(out, "\n%d buckets, %d under-replicated, %d without quorum\n", + len(reports), underReplicatedCount(reports), noQuorumCount(reports)) + return err +} diff --git a/cmd/spinifex/cmd/admin_kv_replicas_test.go b/cmd/spinifex/cmd/admin_kv_replicas_test.go new file mode 100644 index 000000000..18d2be06d --- /dev/null +++ b/cmd/spinifex/cmd/admin_kv_replicas_test.go @@ -0,0 +1,133 @@ +//test:in-package — it tests this command's unexported rendering and repair +// helpers, which exist only to be exercised without a live cluster. + +package cmd + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/mulgadc/spinifex/spinifex/clustersize" + "github.com/mulgadc/spinifex/spinifex/kvutil" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWriteReplicaReport_NamesEveryUnderReplicatedBucket(t *testing.T) { + reports := []kvutil.BucketReport{ + {Bucket: "healthy", Replicas: 3, Want: 3, Online: 3, Leader: "a", Peers: []string{"a", "b", "c"}}, + {Bucket: "short", Replicas: 1, Want: 3, Online: 1, Leader: "a", Peers: []string{"a"}}, + {Bucket: "stuck", Replicas: 3, Want: 3, Online: 1, Peers: []string{"a", "b", "c"}}, + } + + var out bytes.Buffer + require.NoError(t, writeReplicaReport(&out, reports, false)) + got := out.String() + + assert.Contains(t, got, "BUCKET") + assert.Contains(t, got, "HELD BY") + assert.Regexp(t, `healthy\s+3\s+3\s+3\s+ok\s+a\s+a,b,c`, got) + assert.Regexp(t, `short\s+1\s+3\s+1\s+UNDER-REPLICATED\s+a\s+a`, got) + // A bucket that cannot serve reads as NO-QUORUM even though its configured + // count is right, because raising a count is not the response to it. + assert.Regexp(t, `stuck\s+3\s+3\s+1\s+NO-QUORUM`, got) + assert.Contains(t, got, "3 buckets, 1 under-replicated, 1 without quorum") +} + +// TestReplicaReportExit pins the three exit classes a deployment gate reads. They +// are separate because the responses are: a raise fixes an under-replicated +// bucket, and nothing the command can do fixes one that will not answer. +func TestReplicaReportExit(t *testing.T) { + healthy := kvutil.BucketReport{Bucket: "a", Replicas: 3, Want: 3, Online: 3} + short := kvutil.BucketReport{Bucket: "b", Replicas: 1, Want: 3, Online: 1} + stuck := kvutil.BucketReport{Bucket: "c", Replicas: 3, Want: 3, Online: 1} + + assert.Equal(t, exitHealthy, replicaReportExit(nil)) + assert.Equal(t, exitHealthy, replicaReportExit([]kvutil.BucketReport{healthy})) + assert.Equal(t, exitUnderReplicated, replicaReportExit([]kvutil.BucketReport{healthy, short})) + assert.Equal(t, exitUnreachable, replicaReportExit([]kvutil.BucketReport{healthy, stuck})) + // Both problems at once sends the operator to the node rather than to --repair. + assert.Equal(t, exitUnreachable, replicaReportExit([]kvutil.BucketReport{short, stuck})) +} + +// TestWriteReplicaReport_JSONIsWhatTheE2ESuiteDecodes keeps the machine-readable +// form honest: the e2e suite and anything gating a deployment parse these field +// names, so a rename here breaks them silently. +func TestWriteReplicaReport_JSONIsWhatTheE2ESuiteDecodes(t *testing.T) { + var out bytes.Buffer + require.NoError(t, writeReplicaReport(&out, []kvutil.BucketReport{ + {Bucket: "short", Replicas: 1, Want: 3, Cluster: "spinifex", Online: 1, Leader: "a", Peers: []string{"a"}}, + }, true)) + + assert.Contains(t, out.String(), `"bucket":"short"`) + + var decoded []kvutil.BucketReport + require.NoError(t, json.Unmarshal(out.Bytes(), &decoded)) + require.Len(t, decoded, 1) + assert.Equal(t, "short", decoded[0].Bucket) + assert.Equal(t, 1, decoded[0].Replicas) + assert.Equal(t, 3, decoded[0].Want) + assert.Equal(t, 1, decoded[0].Online) + assert.Equal(t, "a", decoded[0].Leader) + assert.True(t, decoded[0].UnderReplicated()) + assert.True(t, decoded[0].HasQuorum()) +} + +func TestUnderReplicatedCount(t *testing.T) { + assert.Equal(t, 0, underReplicatedCount(nil)) + assert.Equal(t, 2, underReplicatedCount([]kvutil.BucketReport{ + {Bucket: "a", Replicas: 1, Want: 3}, + {Bucket: "b", Replicas: 3, Want: 3}, + {Bucket: "c", Replicas: 2, Want: 3}, + })) +} + +// TestRepairReplicaReports_KeepsGoingPastABucketItCannotPlace is the behaviour +// that makes --repair usable mid-growth: the buckets it can raise are raised, +// and the one it cannot is named rather than aborting the run. +func TestRepairReplicaReports_KeepsGoingPastABucketItCannotPlace(t *testing.T) { + _, nc, js := testutil.StartTestJetStream(t) + defer nc.Close() + + for _, b := range []string{"repair-alpha", "repair-beta"} { + _, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: b, History: 1}) + require.NoError(t, err) + } + + // One embedded server cannot hold two replicas, so every raise fails — and + // every bucket must still have been attempted and named. + reports := []kvutil.BucketReport{ + {Bucket: "repair-alpha", Replicas: 1, Want: 2}, + {Bucket: "repair-beta", Replicas: 1, Want: 2}, + } + var errOut bytes.Buffer + repairReplicaReports(t.Context(), js, reports, &errOut) + + assert.Contains(t, errOut.String(), "repair repair-alpha") + assert.Contains(t, errOut.String(), "repair repair-beta") + assert.Equal(t, 2, underReplicatedCount(reports), "a failed raise must not be reported as repaired") +} + +// TestRepairReplicaReports_RaisesWhatItCan covers the success side against a real +// server, so the in-place report update is proved rather than assumed. +func TestRepairReplicaReports_RaisesWhatItCan(t *testing.T) { + _, nc, js := testutil.StartTestJetStream(t) + defer nc.Close() + clustersize.DeclareForTest(t, 1) + + _, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: "repair-ok", History: 1}) + require.NoError(t, err) + + // Want 1 against a bucket already at 1: the raise is a no-op that must still + // leave the report reading as healthy rather than as repaired-and-wrong. + reports := []kvutil.BucketReport{{Bucket: "repair-ok", Replicas: 1, Want: 1}} + var errOut bytes.Buffer + repairReplicaReports(t.Context(), js, reports, &errOut) + + assert.Empty(t, strings.TrimSpace(errOut.String())) + assert.Equal(t, 0, underReplicatedCount(reports)) +} diff --git a/cmd/spinifex/cmd/admin_node_count_test.go b/cmd/spinifex/cmd/admin_node_count_test.go new file mode 100644 index 000000000..2b1ca3f71 --- /dev/null +++ b/cmd/spinifex/cmd/admin_node_count_test.go @@ -0,0 +1,39 @@ +//test:in-package — validateNodeCount is an unexported guard on this command's +// flags, and exporting it would widen the package's API for a test's convenience. + +package cmd + +import ( + "strings" + "testing" +) + +// TestValidateNodeCount pins the supported cluster sizes. Two is rejected rather +// than warned about: on two servers no layer of the stack has a majority to lose, +// and the control plane's own state has no replica count that both survives a node +// going away and still accepts writes. +func TestValidateNodeCount(t *testing.T) { + for _, nodes := range []int{1, 3, 4, 5, 9} { + if err := validateNodeCount(nodes); err != nil { + t.Errorf("validateNodeCount(%d) = %v, want nil", nodes, err) + } + } + + for _, nodes := range []int{0, -1} { + if err := validateNodeCount(nodes); err == nil { + t.Errorf("validateNodeCount(%d) returned no error", nodes) + } + } + + err := validateNodeCount(2) + if err == nil { + t.Fatal("validateNodeCount(2) returned no error") + } + // The message has to say what to do instead, because the operator typed a + // number and needs to know which one to type next. + for _, want := range []string{"2", "1", "3"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %s", err, want) + } + } +} diff --git a/cmd/spinifex/cmd/admin_ochre.go b/cmd/spinifex/cmd/admin_ochre.go index 2424dd293..131c18681 100644 --- a/cmd/spinifex/cmd/admin_ochre.go +++ b/cmd/spinifex/cmd/admin_ochre.go @@ -854,7 +854,7 @@ func runOchreWeightsStage(cmd *cobra.Command, _ []string) { ochreExit(1) return } - weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes)) + weightsStore := gateway_bedrock.NewWeightsStore(js) store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey) provider := ebsprovider.NewNATSProvider(nc, imageImportTimeout) @@ -930,7 +930,7 @@ func runOchreWeightsList(_ *cobra.Command, _ []string) { ochreExit(1) return } - weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes)) + weightsStore := gateway_bedrock.NewWeightsStore(js) store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey) checkSnapshotLive := newWeightsSnapshotChecker(store, node.Predastore.Bucket) @@ -967,7 +967,7 @@ func removeWeights(ctx context.Context, weightsStore *gateway_bedrock.WeightsSto func runOchreWeightsRemove(cmd *cobra.Command, _ []string) { modelID, _ := cmd.Flags().GetString("model-id") - appConfig, nc, err := loadConfigAndConnectFn() + _, nc, err := loadConfigAndConnectFn() if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) ochreExit(1) @@ -981,7 +981,7 @@ func runOchreWeightsRemove(cmd *cobra.Command, _ []string) { ochreExit(1) return } - weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes)) + weightsStore := gateway_bedrock.NewWeightsStore(js) entry, err := removeWeights(context.Background(), weightsStore, modelID) if err != nil { @@ -1014,7 +1014,7 @@ func resolveHFToken(ctx context.Context, cmd *cobra.Command, cfg *config.Cluster if err != nil { return "" } - credStore := gateway_bedrock.NewCredentialStore(js, masterKey, len(cfg.Nodes), nil) + credStore := gateway_bedrock.NewCredentialStore(js, masterKey, nil) token, ok, err := credStore.Resolve(ctx, utils.GlobalAccountID, vendorHuggingFace) if err != nil || !ok { return "" @@ -1088,7 +1088,7 @@ func ochreCredentialsStore() (*gateway_bedrock.CredentialStore, func(), error) { nc.Close() return nil, nil, fmt.Errorf("jetstream context: %w", err) } - return gateway_bedrock.NewCredentialStore(js, masterKey, len(cfg.Nodes), nil), func() { nc.Close() }, nil + return gateway_bedrock.NewCredentialStore(js, masterKey, nil), func() { nc.Close() }, nil } func runOchreCredentialsSet(cmd *cobra.Command, _ []string) { @@ -1186,7 +1186,7 @@ func init() { // ochreAccessStore connects to the cluster and returns the grant store along // with a cleanup that closes the connection. func ochreAccessStore() (*gateway_bedrock.ModelAccessStore, func(), error) { - cfg, nc, err := loadConfigAndConnectFn() + _, nc, err := loadConfigAndConnectFn() if err != nil { return nil, nil, fmt.Errorf("connect to cluster: %w", err) } @@ -1195,7 +1195,7 @@ func ochreAccessStore() (*gateway_bedrock.ModelAccessStore, func(), error) { nc.Close() return nil, nil, fmt.Errorf("jetstream context: %w", err) } - return gateway_bedrock.NewModelAccessStore(js, len(cfg.Nodes)), func() { nc.Close() }, nil + return gateway_bedrock.NewModelAccessStore(js), func() { nc.Close() }, nil } // ochreTargetModels resolves the --model-id / --all-models pair into the model diff --git a/cmd/spinifex/cmd/admin_ochre_pull_test.go b/cmd/spinifex/cmd/admin_ochre_pull_test.go index 947644658..d41df0632 100644 --- a/cmd/spinifex/cmd/admin_ochre_pull_test.go +++ b/cmd/spinifex/cmd/admin_ochre_pull_test.go @@ -470,7 +470,7 @@ func TestResolveHFToken_FallsBackToStoredPlatformCredential(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Join(baseDir, "config"), 0750)) require.NoError(t, handlers_iam.SaveMasterKey(filepath.Join(baseDir, "config", "master.key"), masterKey)) - credStore := gateway_bedrock.NewCredentialStore(js, masterKey, 1, nil) + credStore := gateway_bedrock.NewCredentialStore(js, masterKey, nil) require.NoError(t, credStore.PutCredential(context.Background(), utils.GlobalAccountID, vendorHuggingFace, "hf_stored_token")) cmd := newOchreWeightsPullTestCmd(t, selfHostModelID, testHFRepo, "main", "", "") @@ -621,7 +621,7 @@ func TestRunOchreCredentialsSet_RoundTripsAndDefaultsToPlatformAccount(t *testin defer verifyConn.Close() verifyJS := testutil.NewJetStream(t, verifyConn) - credStore := gateway_bedrock.NewCredentialStore(verifyJS, masterKey, 1, nil) + credStore := gateway_bedrock.NewCredentialStore(verifyJS, masterKey, nil) token, ok, err := credStore.Resolve(context.Background(), utils.GlobalAccountID, vendorHuggingFace) require.NoError(t, err) require.True(t, ok) diff --git a/cmd/spinifex/cmd/admin_ochre_test.go b/cmd/spinifex/cmd/admin_ochre_test.go index 04fb28e1f..8499fe6de 100644 --- a/cmd/spinifex/cmd/admin_ochre_test.go +++ b/cmd/spinifex/cmd/admin_ochre_test.go @@ -42,7 +42,7 @@ const ( func newWeightsStoreForTest(t *testing.T) *gateway_bedrock.WeightsStore { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - return gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, nc), 1) + return gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, nc)) } // explodingObjectStore fails the test immediately if any of its methods is @@ -775,7 +775,7 @@ func TestRunOchreWeightsList_PrintsStagedModels(t *testing.T) { } stubConnect(t, cfg, nc, nil) - seedStore := gateway_bedrock.NewWeightsStore(js, 1) + seedStore := gateway_bedrock.NewWeightsStore(js) require.NoError(t, seedStore.PutWeights(context.Background(), selfHostModelID, "s3://models/llama-3.2-1b/", "snap-0001")) out := captureStdout(t, func() { runOchreWeightsList(nil, nil) }) @@ -822,7 +822,7 @@ func TestRunOchreWeightsRemove_RoundTripPrintsSnapshotAndDropsEntry(t *testing.T ns, nc, js := testutil.StartTestJetStream(t) stubConnect(t, fakeClusterConfig(), nc, nil) - seedStore := gateway_bedrock.NewWeightsStore(js, 1) + seedStore := gateway_bedrock.NewWeightsStore(js) require.NoError(t, seedStore.PutWeights(context.Background(), selfHostModelID, "s3://models/llama-3.2-1b/", "snap-0001")) cmd := newOchreWeightsRemoveTestCmd(t, selfHostModelID) @@ -835,7 +835,7 @@ func TestRunOchreWeightsRemove_RoundTripPrintsSnapshotAndDropsEntry(t *testing.T verifyNC, err := nats.Connect(ns.ClientURL()) require.NoError(t, err) t.Cleanup(func() { verifyNC.Close() }) - verifyStore := gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, verifyNC), 1) + verifyStore := gateway_bedrock.NewWeightsStore(testutil.NewJetStream(t, verifyNC)) _, ok, err := verifyStore.GetWeights(context.Background(), selfHostModelID) require.NoError(t, err) diff --git a/cmd/spinifex/cmd/cluster.go b/cmd/spinifex/cmd/cluster.go index bdf69922c..621f734c8 100644 --- a/cmd/spinifex/cmd/cluster.go +++ b/cmd/spinifex/cmd/cluster.go @@ -167,7 +167,7 @@ func runClusterShutdown(cmd *cobra.Command, args []string) { fmt.Printf("Timeout per phase: %s\n\n", timeout) // Write cluster shutdown marker to KV - jsm, err := daemon.NewJetStreamManager(nc, 1) + jsm, err := daemon.NewJetStreamManager(nc) if err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to create JetStream manager: %v\n", err) } else { diff --git a/cmd/spinifex/cmd/nats_server_name_test.go b/cmd/spinifex/cmd/nats_server_name_test.go new file mode 100644 index 000000000..435888659 --- /dev/null +++ b/cmd/spinifex/cmd/nats_server_name_test.go @@ -0,0 +1,44 @@ +//test:in-package — the embedded nats.conf template it checks is unexported, +// and exporting it would widen the package's API for a test's convenience. + +package cmd + +import ( + "strings" + "testing" + + natssvc "github.com/mulgadc/spinifex/spinifex/services/nats" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestNATSConfServerNameMatchesTheHelper ties the rendered server_name to +// nats.ServerName. JetStream reports stream peers by server name, so anything +// mapping a replica back to the node holding it reads the prefix from that +// helper — and a template that drifted from it would break the mapping silently, +// reporting a healthy cluster as one whose buckets name no node at all. +func TestNATSConfServerNameMatchesTheHelper(t *testing.T) { + var line string + for l := range strings.SplitSeq(natsConfTemplate, "\n") { + if strings.HasPrefix(strings.TrimSpace(l), "server_name:") { + line = strings.TrimSpace(l) + break + } + } + require.NotEmpty(t, line, "nats.conf no longer sets server_name") + + got := strings.TrimSpace(strings.TrimPrefix(line, "server_name:")) + assert.Equal(t, natssvc.ServerName("{{.Node}}"), got) + + node, ok := natssvc.NodeFromServerName(natssvc.ServerName("node2")) + assert.True(t, ok) + assert.Equal(t, "node2", node) + + _, ok = natssvc.NodeFromServerName("node2") + assert.False(t, ok, "a bare node name is not a server name") + + // The bare prefix is a name we could not have generated, and mapping it to the + // empty node would put a phantom peer into anything counting nodes. + _, ok = natssvc.NodeFromServerName(natssvc.ServerNamePrefix) + assert.False(t, ok, "the prefix with no node after it is not a server name") +} diff --git a/cmd/spinifex/cmd/service.go b/cmd/spinifex/cmd/service.go index edfa0ace9..2026f1b4f 100644 --- a/cmd/spinifex/cmd/service.go +++ b/cmd/spinifex/cmd/service.go @@ -384,7 +384,6 @@ var viperblockStartCmd = &cobra.Command{ SecretKey: nodeConfig.Predastore.SecretKey, BaseDir: nodeConfig.Predastore.BaseDir, NodeName: clusterConfig.Node, - KVReplicas: len(clusterConfig.Nodes), ShardWAL: shardWAL, GCEnabled: gcEnabled, WALBaseDir: nodeConfig.Viperblock.WALBaseDir, diff --git a/docs/install/install-multi-node/README.md b/docs/install/install-multi-node/README.md index b45fb67fb..c42d5bdb6 100644 --- a/docs/install/install-multi-node/README.md +++ b/docs/install/install-multi-node/README.md @@ -30,6 +30,7 @@ resources: - [Overview](#overview) - [Cluster sizing](#cluster-sizing) +- [How the cluster's own state is replicated](#how-the-clusters-own-state-is-replicated) - [Hardware](#hardware) - [Network requirements](#network-requirements) - [Prerequisites](#prerequisites) @@ -65,11 +66,82 @@ A Spinifex cluster distributes services across multiple servers for high availab | **VPC networking** — OVN | Control-plane databases run clustered, surviving the loss of any one node. | | **Object storage** — Predastore (S3) | Objects are erasure coded `RS(2,1)`, surviving the loss of any one node's shards. | | **Block storage** — Viperblock (EBS) | Volumes are stored in Predastore, so they inherit the same durability. | +| **Cluster state** — NATS JetStream | Every key-value bucket is replicated onto all three nodes, so the control plane keeps its own state through the loss of any one. | On one or two servers none of that holds. OVN runs standalone on the first node, and the storage metadata quorum has no majority to lose. If that node goes down, running instances keep full networking — but nothing can *change*: no new VPCs, no launches, no security group edits. See [OVN control plane on multi-node clusters](/docs/vpc-networking#ovn-control-plane-on-multi-node-clusters). Servers beyond the third run the full set of services — storage, gateway and networking agents — and add their capacity to the pool. What they do not do is join the OVN database cluster, which stays at three members, so write latency there stays flat as the cluster grows. +### How the cluster's own state is replicated + +Guest data has an obvious home: volumes live in Viperblock, objects in Predastore, both erasure coded across the cluster. The cluster's own bookkeeping has a less obvious one, and it matters just as much. Every instance record, every VPC and subnet, the server roster, IAM users and access keys, quotas and DNS records are kept in **NATS JetStream key-value buckets** — Spinifex's control-plane database, replicated across the servers you installed rather than sitting on any one of them. A cluster that has lost its state has lost the ability to describe, launch, terminate or route anything, even while every guest is still running. + +**A bucket is replicated across the largest odd number of nodes the cluster has, up to five.** That number is decided once, by the `--nodes` count you give `spx admin init` in Step 4, and every service on every node reads it back out of `/etc/spinifex/spinifex.toml` before it creates anything. + +| Servers | Replicas per bucket | Quorum | Survives | +|---|---|---|---| +| 1 | 1 | 1 | nothing — a single node is a single point of failure by definition | +| 2 | — | — | rejected by `spx admin init`; see below | +| 3 | 3 | 2 | any 1 node lost | +| 4 | 3 | 2 | any 1 node lost | +| 5 | 5 | 3 | any 2 nodes lost | +| 10 | 5 | 3 | any 2 of the 5 nodes holding each bucket | + +**The count is always odd, and that is why four servers replicate three ways rather than four.** Agreement is by majority, and an even number of copies does not buy you one: four copies cost a fourth write every time and still survive exactly one loss, the same as three. On four servers NATS spreads which three nodes hold each bucket, so all four carry their share and none of them is idle. + +Five is JetStream's own ceiling on a stream's replica count, not a choice of ours. Past five servers the buckets stay at five and NATS spreads which five nodes hold each one, so the cluster keeps growing while the cost of a write does not. + +**Two servers is refused rather than allowed with a warning.** There is no replica count that works on two: one disappears with the node holding it, and two stop accepting writes when either member goes, because neither survivor is a majority on its own. OVN and the storage metadata quorum have the same problem on two servers, so `spx admin init --nodes 2` fails and tells you to use one server or three. + +#### Worked example — three servers + +Say you ran Step 4 with `--nodes 3` on `node1`, `node2` and `node3`. Every bucket the cluster creates is held by all three servers, with one of them responsible for coordinating changes to it: + +``` +KV bucket "spinifex-instance-state" KV bucket "spinifex-vpcd-reconcile" +┌──────────┬──────────┬──────────┐ ┌──────────┬──────────┬──────────┐ +│ node1 │ node2 │ node3 │ │ node1 │ node2 │ node3 │ +│ leader │ follower │ follower │ │ follower │ leader │ follower │ +└──────────┴──────────┴──────────┘ └──────────┴──────────┴──────────┘ + quorum = 2 of 3 quorum = 2 of 3 +``` + +Three things follow from that picture, and they are the whole reason three servers is the floor: + +- **A change is not confirmed until most of the cluster has it.** When you launch an instance, the API does not report success until at least two of the three servers have written the record to disk. Losing a server afterwards cannot lose that instance. +- **Losing a server costs a pause, not the data.** If `node1` goes down, `node2` and `node3` are still a majority. They take over the buckets `node1` was responsible for — seconds on a healthy cluster — and the API keeps answering throughout. Nothing becomes unreadable, because nothing lived only on `node1`. +- **The cluster keeps repairing itself.** Spinifex runs background workers that fix up VPC state, enforce quotas and publish DNS. Only one server runs each at a time, and they claim that right by writing to a bucket. If those buckets were not replicated, a server failure would stop the repair work at precisely the moment there was repair work to do. + +Contrast that with a bucket on **one** server. It lives wherever it happened to be created, which is not recorded in any config file you can inspect. Stopping, rebooting or losing that one server makes the bucket unreachable from *every* node at once — a single-server event becomes a cluster-wide outage, on a server nobody chose. + +#### What you see when a server fails + +Majority is not an implementation detail. It is what stops two halves of a damaged cluster from both believing they are in charge, and it is the reason the minimum is three servers rather than two. + +**With one server down, the cluster carries on.** `node2` and `node3` are two of three. Instances launch, VPCs change, the API answers normally, and there is nothing to do but replace the failed server. What you have lost is the *next* failure: a second server down leaves no majority and the cluster stops accepting changes until one comes back. + +**A server that is cut off stops accepting changes, on purpose.** If `node1` is not down but isolated — a failed switch, a misconfigured port — it is one server out of three, which is not a majority, so it will not record anything. An operator working on `node1` who tries to launch an instance gets a failure rather than a success. That is the behaviour you want: if `node1` accepted changes while `node2` and `node3` were also accepting them, you would end up with two different versions of the same cluster and no way to merge them. Refusing the isolated side is what makes the surviving side worth trusting. + +Everything else on `node1` keeps running. Guests continue to execute and their volumes keep serving. What stops is anything that has to be recorded: launching, terminating, attaching a volume, changing a security group. Storage writes fail alongside it, so a guest on the isolated server stalls rather than quietly diverging from the rest of the cluster. + +**Be careful reading state from an isolated server.** A server that is cut off can still answer `describe-instances` from its own copy of the data — frozen at the moment it lost contact, with nothing in the answer to say so. **Run operator commands from a server you know is with the majority**, and if one server's answers disagree with its peers', believe the peers. + +**A returning server catches up on its own.** Bring `node1` back and it synchronises from the others automatically: no command to run, no restart, no repair step. (`spx admin kv replicas --repair` exists for a different situation — a bucket configured with too few copies — not a server that is simply behind.) + +#### Growing an existing cluster + +Adding a fourth server does not, on its own, re-replicate the buckets that already exist — the nodes that hold them are perfectly healthy, so nothing prompts a change. Spinifex raises them for you at the next service start: each service checks every bucket it opens against the cluster's current node count and raises any that are short. It never lowers one. + +**A config that names a node before that node is serving is safe.** It is the ordinary middle of growing a cluster, and services start normally through it: a bucket that cannot be raised yet keeps working at the count it has, and the node logs `Could not raise KV bucket to the cluster's replica count`. The raise is retried every time a service opens the bucket, so it completes on its own once the new server is in the cluster. + +To do it immediately rather than waiting for a restart, run the audit with `--repair` from any node: + +```bash +sudo spx admin kv replicas --repair +``` + +It changes nothing about a bucket except the replica count, never lowers one, and is safe to run twice or on a healthy cluster. + ### Hardware Per server, for the three-node minimum: @@ -252,6 +324,8 @@ sudo spx admin init --force \ `--nodes 3` is the number of servers init waits for. Set it to your total node count if you are building a larger cluster. +It is also what decides how many nodes the cluster's JetStream state is replicated across, so it is worth getting right rather than raising later — see [How the cluster's own state is replicated](#how-the-clusters-own-state-is-replicated). Every joining server takes the count from server 1, so it is chosen once, here. `--nodes 2` is refused: use 1 for a single server, or 3 and up for a cluster that can survive losing one. + IPsec encrypts the Geneve overlay between servers and is on by default. Joining servers take the setting from server 1, so it is chosen once, on init. On servers that share a trusted private link, `--ipsec=false` leaves the overlay unencrypted in exchange for considerably higher throughput between instances. The init output displays the join command including the token: @@ -360,7 +434,40 @@ The top table is per-node CPU, memory and GPU usage. The bottom table is what th If capacity looks like a single server rather than the sum of your nodes, the others have not joined. -**3. The AWS API answers.** +**3. The cluster's own state is replicated across every node.** + +```bash +sudo spx admin kv replicas +``` + +``` +spinifex@node1:~$ sudo spx admin kv replicas +BUCKET REPLICAS WANT ONLINE STATUS LEADER HELD BY +spinifex-cluster-state 3 3 3 ok spinifex-nats-node1 spinifex-nats-node1,spinifex-nats-node2,spinifex-nats-node3 +spinifex-dns-reconcile 3 3 3 ok spinifex-nats-node2 spinifex-nats-node2,spinifex-nats-node1,spinifex-nats-node3 +spinifex-iam 3 3 3 ok spinifex-nats-node3 spinifex-nats-node3,spinifex-nats-node1,spinifex-nats-node2 +spinifex-instance-state 3 3 3 ok spinifex-nats-node1 spinifex-nats-node1,spinifex-nats-node2,spinifex-nats-node3 +spinifex-quota-reconcile 3 3 3 ok spinifex-nats-node2 spinifex-nats-node2,spinifex-nats-node3,spinifex-nats-node1 +spinifex-terminated-instances 3 3 3 ok spinifex-nats-node3 spinifex-nats-node3,spinifex-nats-node2,spinifex-nats-node1 +spinifex-vpcd-reconcile 3 3 3 ok spinifex-nats-node2 spinifex-nats-node2,spinifex-nats-node1,spinifex-nats-node3 + +47 buckets, 0 under-replicated, 0 without quorum +``` + +Server names are the node name behind a `spinifex-nats-` prefix, which is what NATS itself calls each server. + +What to check: + +- **`0 under-replicated, 0 without quorum`.** These are different problems. Under-replicated means a bucket lives on fewer nodes than depend on it — see [Buckets report as under-replicated](#buckets-report-as-under-replicated), and `--repair` is the fix. Without quorum means a bucket is not able to accept a write right now, whatever its replica count says, and no repair helps until you find out why a node is not answering. +- **`WANT` is the largest odd number of nodes you have**, capped at five — 3 on three or four servers, 5 on five or more. If it says 1 on a three-server cluster, the node the command ran on is not configured for three: check `--nodes` in Step 4 and the `[nodes.*]` sections of that host's `/etc/spinifex/spinifex.toml`. +- **`ONLINE` equals `REPLICAS`.** It counts the replicas that are caught up and reachable, so a bucket at `3` replicas with `ONLINE 2` is one node away from being unavailable and needs looking at before anything else is done to the cluster. +- **`LEADER` is spread across the cluster** rather than parked on one node, and `HELD BY` names as many distinct servers as `REPLICAS`. + +Exit status is what a deployment gate should read rather than the output: `0` healthy, `1` something is under-replicated, `2` something has no quorum or the report could not be produced at all. + +Run it on every node, not just one. It reads cluster-global Raft metadata, so all of them should give the same answer; one that disagrees is not fully part of the cluster. + +**4. The AWS API answers.** ```bash export AWS_PROFILE=spinifex @@ -403,6 +510,28 @@ The node has its own cluster configuration — normal for anything installed fro Safe on a freshly installed node. On one that has been in service it orphans every volume and fragment sealed under the old key, so check before forcing. +### Buckets Report as Under-Replicated + +```bash +sudo spx admin kv replicas +``` + +A bucket showing `UNDER-REPLICATED` lives on fewer nodes than the cluster has, so losing the node holding it takes it down for the whole cluster. Two things put a bucket in that state, and the fix is the same for both: + +```bash +sudo spx admin kv replicas --repair +``` + +The first cause is **a cluster that grew**: buckets created when there were three servers stay on three replicas until something raises them. Services do this themselves as they restart, and `--repair` does it now. + +The second is **a node that was down when the bucket was created**, so the Raft group formed short. Bring the node back first — `--repair` on an incomplete cluster raises the configured count, but the group cannot place the extra replica until the node is there to hold it. Confirm with `spx get nodes` before repairing. + +If `WANT` itself is wrong — 1 on a three-server cluster — the problem is not the buckets but that host's view of the cluster. Check it has all three `[nodes.*]` sections: + +```bash +sudo grep -oP '^\[nodes\.\K[^.\]]+' /etc/spinifex/spinifex.toml | sort -u +``` + ### OVN Database Cluster Not Forming ```bash diff --git a/docs/testing/e2e-coverage.md b/docs/testing/e2e-coverage.md index b0b0847d1..a23819ff3 100644 --- a/docs/testing/e2e-coverage.md +++ b/docs/testing/e2e-coverage.md @@ -7,11 +7,13 @@ Suites live in `tests/e2e//` and build to `tests/e2e/_bin/.test`. Ev | Set | Suites | | --- | --- | | `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota storagefault` | -| `E2E_SUITES_MULTI` | `multinode lb cert quota lbrecovery instancerecovery storagefault` | +| `E2E_SUITES_MULTI` | `multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault` | | `E2E_SUITES_NIGHTLY_SINGLE` | `single cert iam` | | `E2E_SUITES_NIGHTLY_MULTI` | `multinode cert lb` | -The nightly permutation sets are deliberately narrower than the full ones: those cells have a ~35 minute budget and exist to prove every install / network / host-OS combination boots and serves. `eks`, `ecs`, `rds`, `storagefault`, `instancerecovery` and `lbrecovery` each get a dedicated cell instead. +The nightly permutation sets are deliberately narrower than the full ones: those cells have a ~35 minute budget and exist to prove every install / network / host-OS combination boots and serves. `eks`, `ecs`, `rds`, `storagefault`, `instancerecovery`, `lbrecovery` and `kvquorum` each get a dedicated cell instead. + +**`multinode` carries the KV replication sweep as well**, which is why the thing `kvquorum` proves is still checked in every multi-node permutation cell even though the suite itself is not. The read-only half costs a NATS connection and a stream listing; only the node-loss half needs a cell of its own. ## The suites @@ -27,6 +29,7 @@ The largest suite, 25 top-level tests. Notable entry points: - `TestInstanceMetadata` — IMDS from inside a guest. - `TestStopStart`, `TestEarlyRebootLiveness`, `TestGuestChurnDurability`, `TestSnapshotLifecycle`, `TestSnapshotBackedLaunch`, `TestCreateImage`, `TestVolumeLifecycle`, `TestSpotInstanceLifecycle`, `TestLaunchTemplateBoot`. - `TestNegativeErrorPaths` — the `8a`–`8h` AWS error-shape cases (malformed AMI ID, invalid instance type, volume in use, detach-root forbidden, and so on). +- **`TestKVReplication`** — the single-node half of the KV replication policy, and it is not the multi-node assertion with a smaller number in it. On one node one replica per bucket is the correct answer rather than the defect, so what it proves is the opposite: that nothing declared a replica count this deployment cannot place, that `spx admin kv replicas` exits 0 and reports `WANT 1`, and that `--repair` run twice against a cluster with nothing to repair changes no bucket's replica count and leaves every bucket serving. The multi-node suites cannot cover any of that, because on three nodes the unplaceable count is the right one. It lives here rather than in its own suite so it runs in every single-node nightly cell, across all three host and network permutations. ### `natuplink` — routed NAT (`external_mode = "nat"`), one node or three @@ -42,6 +45,22 @@ Two nightly cells run it: `nat-single` (cell 19) and `nat-multi` (cell 30). Cell `VPCSetup`, `SpansMultipleNodes`, `SpreadPlacement`, `EveryRunningInstanceReported`, `BastionSSH`, and its own NAT Gateway lane: `PreNATIsolation`, `NATGatewayInternet`, `NATCleanupOrdering`. +**`TestMultinodeJetStreamReplicas` is a whole-cluster sweep, not a named list, and it lives here on purpose.** It audits every KV bucket the cluster has and fails any that is replicated across fewer nodes than the cluster has, capped at JetStream's ceiling of five. Naming buckets would only ever cover the ones somebody remembered — a replica count is set where a bucket is created, and there is a bucket-creating call in most services — so the sweep is the assertion and the three daemon-owned buckets are named only so that a run enumerating nothing still fails. Because `multinode` is in `E2E_SUITES_NIGHTLY_MULTI`, this runs in every multi-node permutation cell, and no cell can be added that quietly opts out of it. + +### `kvquorum` — the cluster's own state survives losing a node (multi-node, needs three) + +**The control plane keeps its state in JetStream KV buckets, and this is the suite that proves those buckets are actually replicated.** A bucket on one replica lives on one JetStream-chosen server recorded in no config, so losing that node makes it return `nats: no responders available for request` from *every* node at once — a cluster-wide outage caused by a single-node event. Leader leases are the worst case: acquiring one is a write, so a lease bucket without a quorum cannot be acquired by anyone, and every reconciler sharing it stops cluster-wide at exactly the moment a node has failed and there is repair work to do. + +Three entry points, in the order they run: + +- **`TestKVBucketsAreReplicatedAcrossTheCluster`** is read-only and runs first, so a cluster that is already wrong is reported as wrong rather than as a failure to survive a node loss. It sweeps every bucket, then separately requires the leader-lease buckets to be among those checked — they were the worst case of the defect, and a regression reaching only them would otherwise be one line in a long list rather than the headline. +- **`TestKVReplicasCommandReportsHealthy`** runs `spx admin kv replicas --json` on **every** node, not one. The command reads cluster-global Raft metadata, so all nodes must give the same answer; a node that disagrees is reporting on a cluster it is not fully part of, which is worth knowing before a deployment is gated on its exit status. +- **`TestKVBucketsSurviveANodeLoss`** produces the fault rather than inspecting for it. It takes away **the node leading the most buckets** — the worst one to lose, and the one a test picking arbitrarily would usually miss — then requires every survivor to still answer for every bucket, brings the node back, and requires it to carry its share again. + +**Configured, placed and working replicas are three different facts, and all three are checked.** A stream can carry the right replica count while its Raft group is short of peers, or hold the right peers while one of them is offline or still catching up, and either reads as healthy right up until the node it is really on goes away. The audit reports the leader and a count of peers NATS itself calls current and reachable, and the assertion is against that count rather than against the configured one. The last assertion is that no bucket came back with *fewer* replicas than it had: a cluster that "repaired" itself by lowering a count would pass every other assertion here and have quietly traded the durability this suite is about for a green run. + +**Its own suite and its own cell, for lbrecovery's reasons.** It takes a node away, so anything sharing the environment reports that outage as its own failure, and the question it answers — "is our internal state actually replicated" — has to read as itself rather than as one more red subtest somewhere else. Three nodes is the floor: on two, a majority is two, so losing either leaves no quorum for any stream and there is nothing to assert beyond "Raft needs a majority", which is true of NATS and not a property of ours. + ### `instancerecovery` — a guest whose node goes away (multi-node) **Asserted from both sides, and the pair is the point.** `InstanceAutoRecovery` takes a node down and requires the guest to come back on exactly one survivor, identified by the qemu process rather than by what the API says, then brings the node back and requires it to drop the local copy instead of relaunching it. `InstanceRecoveryRefusesStorageFault` freezes predastore cluster-wide first, so the guest is paused by `werror=stop` when its node goes away, and requires that **nothing moves** — a store that refuses every node cannot be fixed by moving a guest, and the relaunch would cost it the request QEMU is holding. A reconciler that cannot tell a host failure from a storage failure passes the first and fails the second. Neither turns recovery on, because there is no setting for it: both wait for every node to report the reconciler active on a cluster nothing configured, so a build that reintroduced a switch fails here. diff --git a/spinifex/accountteardown/cluster_test.go b/spinifex/accountteardown/cluster_test.go index 397ba15b7..f132cea2a 100644 --- a/spinifex/accountteardown/cluster_test.go +++ b/spinifex/accountteardown/cluster_test.go @@ -20,7 +20,7 @@ func TestNewClusterEngineCoversEveryStage(t *testing.T) { masterKey, err := handlers_iam.GenerateMasterKey() require.NoError(t, err) - svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey) require.NoError(t, err) engine, err := NewClusterEngine(t.Context(), nc, 1, svc, newFakeBuckets()) @@ -48,7 +48,7 @@ func TestClusterEnginePrechecks(t *testing.T) { masterKey, err := handlers_iam.GenerateMasterKey() require.NoError(t, err) - svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey) require.NoError(t, err) engine, err := NewClusterEngine(t.Context(), nc, 1, svc, newFakeBuckets()) diff --git a/spinifex/accountteardown/reapers_iam_test.go b/spinifex/accountteardown/reapers_iam_test.go index 71e251eb6..dd5eb84b9 100644 --- a/spinifex/accountteardown/reapers_iam_test.go +++ b/spinifex/accountteardown/reapers_iam_test.go @@ -398,7 +398,7 @@ func TestPolicyReaperDeletesAPolicyWithSeveralVersions(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) masterKey, err := handlers_iam.GenerateMasterKey() require.NoError(t, err) - svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey) require.NoError(t, err) const accountID = "000000000042" diff --git a/spinifex/clustersize/clustersize.go b/spinifex/clustersize/clustersize.go new file mode 100644 index 000000000..985b0506e --- /dev/null +++ b/spinifex/clustersize/clustersize.go @@ -0,0 +1,157 @@ +// Package clustersize holds one fact and the one number derived from it: how +// many nodes this process believes the cluster has, and therefore how many +// nodes each of its JetStream streams is replicated across. +// +// It is its own package because the fact is declared in one place, +// config.LoadConfig, and read in another, kvutil, and those sit either side of +// packages that would otherwise import each other. +// +// The number matters because a stream on one replica lives on one +// JetStream-chosen server recorded in no config. Losing that node makes the +// stream unreadable and unwritable from every node at once, so a single-node +// event becomes a cluster-wide outage. +package clustersize + +import ( + "errors" + "fmt" + "sync/atomic" + "testing" +) + +// MaxReplicas is JetStream's own ceiling on stream replicas, so a cluster +// larger than this replicates to five nodes and no further. Asking for more is +// refused by the server, which is why the count is clamped rather than passed +// through. +const MaxReplicas = 5 + +// MinHANodes is the smallest cluster in which losing a node is survivable. Below +// it no layer of the stack has a majority to keep: OVN runs standalone, the +// storage metadata quorum has nothing to lose, and a KV bucket has no replica +// count that both survives a loss and accepts writes. +const MinHANodes = 3 + +// ErrUndeclared is returned by Replicas in a process that never loaded a +// cluster config. It is an error rather than a fallback to one replica because +// one replica is the failure this package exists to prevent, and a fallback +// would reinstate it in exactly the processes nobody checked. +var ErrUndeclared = errors.New("cluster size was never declared, so the stream replica count is unknown") + +// nodes is the declared node count. Zero means undeclared. +var nodes atomic.Int64 + +// Declare records the cluster's node count for this process. It is called from +// config.LoadConfig, so every process that can reach a cluster's NATS has +// already declared it before any handler creates a bucket. +// +// The first declaration wins. Repeating the same count is a no-op, because two +// call sites legitimately read it from the same config; a different count is an +// error rather than an overwrite, since a process that disagrees with itself +// about how many nodes exist would silently pick whichever ran last. +func Declare(n int) error { + n = max(n, 0) + if nodes.CompareAndSwap(0, int64(n)) { + return nil + } + if have := Nodes(); have != n { + return fmt.Errorf("cluster size was already declared as %d nodes and cannot be redeclared as %d", have, n) + } + return nil +} + +// ResetForTest clears the declaration so a test can declare a different size. +// It panics outside a test binary: production has exactly one cluster size for +// the life of the process, and a reset available there would undo that. +func ResetForTest() { + if !testing.Testing() { + //nolint:forbidigo // Returning an error would let a caller ignore it and carry on with the count cleared. + panic("clustersize.ResetForTest called outside a test binary") + } + nodes.Store(0) +} + +// DeclareForTest declares n for the rest of the test binary, tolerating a fixture +// that already declared the same count. +// +// It is for fixtures rather than for tests. Standing up a one-node embedded server +// says something about the binary, not about one test, and several tests running +// in parallel share that one server — so restoring the previous count when any one +// of them finishes would leave its siblings undeclared mid-run. +func DeclareForTest(tb testing.TB, n int) { + tb.Helper() + if err := Declare(n); err != nil { + tb.Fatalf("declare cluster size %d: %v", n, err) + } +} + +// RedeclareForTest declares n for the duration of one test, whatever was declared +// before it, and puts the previous count back afterwards. +// +// Declaring is write-once, so a test needing a different size than its fixture set +// up cannot simply call Declare again. It must not run in parallel with anything +// else that reads the count, because there is one count per process and this +// changes it. +func RedeclareForTest(tb testing.TB, n int) { + tb.Helper() + was := Nodes() + ResetForTest() + if err := Declare(n); err != nil { + tb.Fatalf("declare cluster size %d: %v", n, err) + } + tb.Cleanup(func() { + ResetForTest() + if was > 0 { + _ = Declare(was) + } + }) +} + +// Nodes returns the declared node count, or 0 when it was never declared. +func Nodes() int { + return int(nodes.Load()) +} + +// Replicas is the replica count every stream is created with: the largest odd +// number of nodes this cluster can place a replica on, capped at JetStream's +// ceiling of five. +// +// Odd, because a majority is what a raft group needs and an even count buys +// none. Four replicas store a fourth full copy of every write and still survive +// only one loss, exactly like three; two survive nothing, because losing either +// member leaves no majority to accept a write. So four nodes replicate three +// ways and five replicate five ways, which is also the most any cluster does +// however large it grows. +// +// There is no argument and no override. A replica count is a property of the +// cluster, not of the stream or of the caller, and every stream here is read or +// written by a node other than the one that created it. +func Replicas() (int, error) { + n := Nodes() + if n < 1 { + return 0, ErrUndeclared + } + return ReplicasFor(n), nil +} + +// ReplicasFor is Replicas for a node count the caller already holds, so code that +// knows the cluster's size without having declared it — a test harness reading a +// cluster it did not build — gets its answer from the same rule rather than a +// second copy of it. +func ReplicasFor(nodes int) int { + r := min(max(nodes, 1), MaxReplicas) + if r%2 == 0 { + r-- + } + return r +} + +// Permanent reports whether err is this package's own refusal rather than a +// transient cluster condition. +// +// The retry loops that wait for JetStream quorum on a cold start must stop on +// one: no amount of waiting declares a cluster size, so a loop that cannot tell +// the two apart spends its whole budget on a misconfiguration and then reports +// a quorum problem that was never the cause. +func Permanent(err error) bool { + return errors.Is(err, ErrUndeclared) +} diff --git a/spinifex/clustersize/clustersize_test.go b/spinifex/clustersize/clustersize_test.go new file mode 100644 index 000000000..5fbe3192c --- /dev/null +++ b/spinifex/clustersize/clustersize_test.go @@ -0,0 +1,143 @@ +package clustersize_test + +import ( + "errors" + "strings" + "testing" + + "github.com/mulgadc/spinifex/spinifex/clustersize" +) + +func TestDeclareAndNodes(t *testing.T) { + t.Cleanup(clustersize.ResetForTest) + clustersize.ResetForTest() + + if got := clustersize.Nodes(); got != 0 { + t.Fatalf("undeclared cluster size = %d, want 0", got) + } + if err := clustersize.Declare(3); err != nil { + t.Fatalf("Declare(3): %v", err) + } + if got := clustersize.Nodes(); got != 3 { + t.Fatalf("Declare(3) = %d, want 3", got) + } +} + +// TestDeclareIsWriteOnce covers the three ways a second declaration can arrive. +// Repeating the same count is how two call sites reading the same config behave +// and must be accepted; a different count is a process disagreeing with itself +// about how many nodes exist, which must not be resolved by whichever ran last. +func TestDeclareIsWriteOnce(t *testing.T) { + t.Cleanup(clustersize.ResetForTest) + clustersize.ResetForTest() + + if err := clustersize.Declare(3); err != nil { + t.Fatalf("first Declare(3): %v", err) + } + if err := clustersize.Declare(3); err != nil { + t.Fatalf("repeating Declare(3) must be accepted, got %v", err) + } + err := clustersize.Declare(5) + if err == nil { + t.Fatal("Declare(5) after Declare(3) returned no error") + } + if got := clustersize.Nodes(); got != 3 { + t.Fatalf("a rejected redeclaration changed the count to %d, want 3 kept", got) + } + // The message has to name both counts, because the failure it reports is a + // disagreement and neither number alone identifies it. + for _, want := range []string{"3", "5"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not name the count %s", err, want) + } + } +} + +// TestDeclareRejectsANegativeCount: a negative node count is not a smaller +// cluster, it is a broken config, and storing it as-is would read back as a +// count rather than as undeclared. +func TestDeclareRejectsANegativeCount(t *testing.T) { + t.Cleanup(clustersize.ResetForTest) + clustersize.ResetForTest() + + if err := clustersize.Declare(-2); err != nil { + t.Fatalf("Declare(-2): %v", err) + } + if got := clustersize.Nodes(); got != 0 { + t.Fatalf("Declare(-2) = %d, want 0 (undeclared)", got) + } + if _, err := clustersize.Replicas(); !errors.Is(err, clustersize.ErrUndeclared) { + t.Fatalf("Replicas() after Declare(-2) = %v, want ErrUndeclared", err) + } +} + +// TestReplicas covers the policy and the fail-closed answer together: a bucket is +// replicated across the largest odd number of nodes the cluster can place, capped +// at five, and a process that never declared a size gets an error rather than a +// guess at one. +// +// The even counts are the point of the table. Four replicas cost a fourth full +// copy of every write and survive exactly one loss, the same as three, and two +// survive nothing at all because losing either member leaves no majority. +func TestReplicas(t *testing.T) { + t.Cleanup(clustersize.ResetForTest) + + for _, tc := range []struct{ nodes, want int }{ + {1, 1}, {2, 1}, {3, 3}, {4, 3}, {5, 5}, {6, 5}, {10, 5}, {64, 5}, + } { + clustersize.ResetForTest() + if err := clustersize.Declare(tc.nodes); err != nil { + t.Fatalf("Declare(%d): %v", tc.nodes, err) + } + got, err := clustersize.Replicas() + if err != nil { + t.Fatalf("Replicas() with %d nodes: %v", tc.nodes, err) + } + if got != tc.want { + t.Errorf("Replicas() with %d nodes = %d, want %d", tc.nodes, got, tc.want) + } + if got%2 == 0 { + t.Errorf("Replicas() with %d nodes = %d, which is even", tc.nodes, got) + } + if fromCount := clustersize.ReplicasFor(tc.nodes); fromCount != got { + t.Errorf("ReplicasFor(%d) = %d but Replicas() = %d; the two must be one rule", + tc.nodes, fromCount, got) + } + } + + clustersize.ResetForTest() + if _, err := clustersize.Replicas(); !errors.Is(err, clustersize.ErrUndeclared) { + t.Fatalf("Replicas() with an undeclared cluster size = %v, want ErrUndeclared", err) + } +} + +// TestReplicasForNeverReturnsZero: callers that hold a node count use it without +// an error return, so a count below one has to resolve to the single replica a +// lone server can hold rather than to none. +func TestReplicasForNeverReturnsZero(t *testing.T) { + for _, nodes := range []int{-1, 0} { + if got := clustersize.ReplicasFor(nodes); got != 1 { + t.Errorf("ReplicasFor(%d) = %d, want 1", nodes, got) + } + } +} + +// TestPermanent separates this package's own refusal from a cluster that has yet +// to form. The retry loops waiting for JetStream quorum stop on the first and +// keep waiting through the second, so confusing them spends a whole startup +// budget on a misconfiguration. +func TestPermanent(t *testing.T) { + if !clustersize.Permanent(clustersize.ErrUndeclared) { + t.Error("Permanent(ErrUndeclared) = false") + } + wrapped := errors.Join(errors.New("create KV bucket x"), clustersize.ErrUndeclared) + if !clustersize.Permanent(wrapped) { + t.Error("Permanent() did not see through a wrapped ErrUndeclared") + } + if clustersize.Permanent(errors.New("no suitable peers for placement")) { + t.Error("Permanent() treated a placement failure as permanent") + } + if clustersize.Permanent(nil) { + t.Error("Permanent(nil) = true") + } +} diff --git a/spinifex/config/config.go b/spinifex/config/config.go index 23c3a8180..72e2dd3c8 100644 --- a/spinifex/config/config.go +++ b/spinifex/config/config.go @@ -8,6 +8,7 @@ import ( "slices" "strings" + "github.com/mulgadc/spinifex/spinifex/clustersize" "github.com/spf13/viper" ) @@ -605,6 +606,24 @@ func LoadConfig(configPath string) (*ClusterConfig, error) { return nil, err } + // Every process that creates a KV bucket loads the cluster config first, so + // this is the one place the node count can be declared without a service + // being able to forget to. Buckets refuse to be created until it has been. + // + // A conflicting second load is reported rather than refused, because this + // function also reads config files that are not this process's own cluster — + // an installer inspecting a peer's, a tool rendering one. The first + // declaration stands either way, so nothing that already created a bucket can + // have the count changed underneath it. + if err := clustersize.Declare(len(config.Nodes)); err != nil { + slog.Warn("Ignoring a cluster size that disagrees with the one already declared; the first one stands", + "loaded_nodes", len(config.Nodes), "declared_nodes", clustersize.Nodes(), "error", err) + } + if len(config.Nodes) > 1 && len(config.Nodes) < clustersize.MinHANodes { + slog.Warn("This cluster has too few nodes to survive losing one, so its state is replicated to a single node", + "nodes", len(config.Nodes), "ha_minimum", clustersize.MinHANodes) + } + return &config, nil } diff --git a/spinifex/daemon/bedrock_deps.go b/spinifex/daemon/bedrock_deps.go index a2346acd4..011b8d900 100644 --- a/spinifex/daemon/bedrock_deps.go +++ b/spinifex/daemon/bedrock_deps.go @@ -22,11 +22,7 @@ func (d *Daemon) buildBedrockLaunchDeps() handlers_bedrock.LaunchDeps { if err != nil { slog.Warn("Bedrock: jetstream init failed; serving VMs cannot resolve staged weights", "err", err) } else { - clusterSize := 1 - if d.clusterConfig != nil { - clusterSize = len(d.clusterConfig.Nodes) - } - weights = gateway_bedrock.NewWeightsStore(js, clusterSize) + weights = gateway_bedrock.NewWeightsStore(js) } return handlers_bedrock.LaunchDeps{ diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index a4965be22..212c4e09c 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -32,6 +32,7 @@ import ( "github.com/mulgadc/bluebottle/pkg/tlsconfig" "github.com/mulgadc/spinifex/spinifex/admin" "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/clustersize" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/ebsprovider" "github.com/mulgadc/spinifex/spinifex/gpu" @@ -1667,14 +1668,18 @@ func (d *Daemon) startCluster() error { return fmt.Errorf("connect NATS: %w", err) } - if err := d.initJetStream(); err != nil { - return fmt.Errorf("initialize JetStream: %w", err) + // Declared before JetStream, not after: initJetStream creates buckets, and + // bucket creation refuses until the count is known. A daemon assembled in a + // test may never have gone through config.LoadConfig, which is where a real + // process declares it from this same source. + if d.clusterConfig != nil { + if err := clustersize.Declare(len(d.clusterConfig.Nodes)); err != nil { + return err + } } - // Set the default KV replica count before any handler creates a bucket, so - // lazily-created buckets are born at cluster-size replication instead of R1. - if d.clusterConfig != nil { - utils.SetDefaultKVReplicas(len(d.clusterConfig.Nodes)) + if err := d.initJetStream(); err != nil { + return fmt.Errorf("initialize JetStream: %w", err) } // Remove the obsolete spinifex-dhcp-leases bucket (idempotent). @@ -2397,7 +2402,7 @@ func (d *Daemon) initJetStream() error { for { attempt++ var err error - d.jsManager, err = NewJetStreamManager(d.natsConn, 1) + d.jsManager, err = NewJetStreamManager(d.natsConn) if err == nil { err = d.jsManager.InitKVBucket() } @@ -2412,10 +2417,20 @@ func (d *Daemon) initJetStream() error { if err == nil { d.jsManager.SetSyncObserver(d) - slog.Info("JetStream KV stores initialized successfully", "replicas", 1, "attempts", attempt, "elapsed_ms", otelsetup.Millis(time.Since(start))) + // Replicas cannot be undeclared here: bucket creation refuses + // before it, so reaching this line means it was declared. + replicas, _ := clustersize.Replicas() + slog.Info("JetStream KV stores initialized successfully", "replicas", replicas, "attempts", attempt, "elapsed_ms", otelsetup.Millis(time.Since(start))) break } + // A misdeclared cluster size is not something quorum arrives and fixes, + // so waiting for it spends the whole budget and then reports a quorum + // problem that was never the cause. + if clustersize.Permanent(err) { + return fmt.Errorf("initialize JetStream: %w", err) + } + elapsed := time.Since(start) if elapsed >= maxWait { return fmt.Errorf("failed to initialize JetStream after %s (%d attempts): %w", elapsed.Round(time.Second), attempt, err) @@ -2431,15 +2446,20 @@ func (d *Daemon) initJetStream() error { return nil } -// upgradeJetStreamReplicas bumps KV_* stream replication to match the cluster -// size. Runs after all buckets are created and the cluster is ready. +// upgradeJetStreamReplicas raises every KV bucket to the cluster's replica +// count. Runs after all buckets are created and the cluster is ready, which is +// when a bucket created by whichever node got there first can be repaired. func (d *Daemon) upgradeJetStreamReplicas() { - clusterSize := len(d.clusterConfig.Nodes) - if clusterSize <= 1 || d.jsManager == nil { + if d.jsManager == nil || d.jsManager.js == nil { return } - if err := d.jsManager.UpdateReplicas(clusterSize); err != nil { - slog.Warn("Failed to upgrade JetStream replicas", "targetReplicas", clusterSize, "error", err) + raised, err := kvutil.RaiseAllBucketReplicas(d.ctx, d.jsManager.js) + if err != nil { + slog.Warn("Failed to raise KV bucket replicas to the cluster's node count", "error", err) + return + } + if raised > 0 { + slog.Info("Raised KV buckets to the cluster's replica count", "buckets", raised) } } diff --git a/spinifex/daemon/daemon_handlers_test.go b/spinifex/daemon/daemon_handlers_test.go index 26765ef83..4af608046 100644 --- a/spinifex/daemon/daemon_handlers_test.go +++ b/spinifex/daemon/daemon_handlers_test.go @@ -132,7 +132,7 @@ func createFullTestDaemonWithJetStream(t *testing.T, natsURL string) *Daemon { daemon := createFullTestDaemon(t, natsURL) var err error - daemon.jsManager, err = NewJetStreamManager(daemon.natsConn, 1) + daemon.jsManager, err = NewJetStreamManager(daemon.natsConn) require.NoError(t, err) err = daemon.jsManager.InitKVBucket() require.NoError(t, err) diff --git a/spinifex/daemon/daemon_mgmt_ip_test.go b/spinifex/daemon/daemon_mgmt_ip_test.go index 3aabeb0a7..d711286e8 100644 --- a/spinifex/daemon/daemon_mgmt_ip_test.go +++ b/spinifex/daemon/daemon_mgmt_ip_test.go @@ -24,7 +24,7 @@ func newTestMgmtJSM(t *testing.T) *JetStreamManager { } t.Cleanup(nc.Close) - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) if err != nil { t.Fatalf("new JetStreamManager: %v", err) } @@ -552,7 +552,7 @@ func TestMgmtIPAllocator_Allocate_RefusedWhenKVUnhealthy(t *testing.T) { if err != nil { t.Fatal(err) } - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) if err != nil { t.Fatal(err) } diff --git a/spinifex/daemon/daemon_test.go b/spinifex/daemon/daemon_test.go index f4159e942..366f525b4 100644 --- a/spinifex/daemon/daemon_test.go +++ b/spinifex/daemon/daemon_test.go @@ -740,7 +740,7 @@ func TestDaemon_BootAllocation(t *testing.T) { defer nc.Close() daemon.natsConn = nc - daemon.jsManager, err = NewJetStreamManager(nc, 1) + daemon.jsManager, err = NewJetStreamManager(nc) require.NoError(t, err) err = daemon.jsManager.InitKVBucket() require.NoError(t, err) @@ -1692,7 +1692,7 @@ func TestTerminatedTeardownReaper_SelfHealsFailedVolumeTeardown(t *testing.T) { // A real terminated-instance KV, so Sweep can list/update/(not yet)purge // the record exactly as production does. - jsManager, err := NewJetStreamManager(daemon.natsConn, 1) + jsManager, err := NewJetStreamManager(daemon.natsConn) require.NoError(t, err) require.NoError(t, jsManager.InitTerminatedInstanceBucket()) daemon.stateStore = newStateStoreAdapter(jsManager, daemon.persistState) @@ -4545,7 +4545,7 @@ func newEBSProviderTestDaemon(t *testing.T, provider string) *Daemon { require.NoError(t, err) t.Cleanup(nc.Close) - jsManager, err := NewJetStreamManager(nc, 1) + jsManager, err := NewJetStreamManager(nc) require.NoError(t, err) ctx := context.Background() diff --git a/spinifex/daemon/dns_reconcile.go b/spinifex/daemon/dns_reconcile.go index 4dc930da6..5dbd8af45 100644 --- a/spinifex/daemon/dns_reconcile.go +++ b/spinifex/daemon/dns_reconcile.go @@ -45,9 +45,8 @@ func (d *Daemon) instanceStateWatchBuckets(context.Context) ([]*kvstore.Bucket, return nil, nil } return []*kvstore.Bucket{kvstore.NewBucket(d.jsManager.js, kvstore.Config{ - Name: InstanceStateBucket, - History: 1, - Replicas: d.jsManager.replicas, + Name: InstanceStateBucket, + History: 1, })}, nil } diff --git a/spinifex/daemon/dns_reconcile_ec2_test.go b/spinifex/daemon/dns_reconcile_ec2_test.go index 12c41055e..8318aae29 100644 --- a/spinifex/daemon/dns_reconcile_ec2_test.go +++ b/spinifex/daemon/dns_reconcile_ec2_test.go @@ -26,10 +26,10 @@ func dnsTestDaemon(t *testing.T) (*Daemon, func(id, publicIP, privateIP string, t.Helper() _, _, js := testutil.StartTestJetStream(t) store := kvstore.New[vm.InstanceRecord](js, kvstore.Config{ - Name: InstanceStateBucket, History: 1, Replicas: 1, + Name: InstanceStateBucket, History: 1, }) d := &Daemon{ - jsManager: &JetStreamManager{js: js, records: store, replicas: 1}, + jsManager: &JetStreamManager{js: js, records: store}, vmMgr: vm.NewManager(), config: &config.Config{Region: "ap-southeast-2"}, dnsBaseDomain: "spx3.net", diff --git a/spinifex/daemon/dns_reconcile_test.go b/spinifex/daemon/dns_reconcile_test.go index 207f3590f..8f09c8a78 100644 --- a/spinifex/daemon/dns_reconcile_test.go +++ b/spinifex/daemon/dns_reconcile_test.go @@ -62,7 +62,7 @@ func TestDNSWatchBuckets_ZeroValueDaemonIsNilSafe(t *testing.T) { func TestInstanceStateWatchBuckets_ReturnsSharedBucket(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - d := &Daemon{jsManager: &JetStreamManager{js: js, replicas: 1}} + d := &Daemon{jsManager: &JetStreamManager{js: js}} buckets, err := d.instanceStateWatchBuckets(t.Context()) require.NoError(t, err) @@ -111,7 +111,7 @@ func TestEksWatchBuckets_NilGuards(t *testing.T) { func TestEksWatchBuckets_DelegatesToAccountWatchBuckets(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - _, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "111111111111", 1) + _, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "111111111111") require.NoError(t, err) d := &Daemon{eksService: &handlers_eks.EKSServiceImpl{}, natsConn: nc} diff --git a/spinifex/daemon/ecs_deps.go b/spinifex/daemon/ecs_deps.go index 1542f425f..a31823285 100644 --- a/spinifex/daemon/ecs_deps.go +++ b/spinifex/daemon/ecs_deps.go @@ -43,16 +43,12 @@ func (d *Daemon) buildECSServiceDeps() handlers_ecs.Deps { return deps } - clusterSize := 1 - if d.clusterConfig != nil { - clusterSize = len(d.clusterConfig.Nodes) - } // Retried, because the constructor runs a KV migration that needs JetStream // to have responders. A daemon that started first got one attempt, failed // it, and had capacity provisioning off for the life of the process with no // symptom but a 500 from ProvisionCapacity. iamSvc, iamErr := initServiceWithRetry("ECS IAM service", func() (*handlers_iam.IAMServiceImpl, error) { - return handlers_iam.NewIAMServiceImpl(d.ctx, d.natsConn, masterKey, clusterSize) + return handlers_iam.NewIAMServiceImpl(d.ctx, d.natsConn, masterKey) }) if iamErr != nil { // Off rather than fatal: ECS CRUD, EKS and EC2 do not need this, and a diff --git a/spinifex/daemon/eks_deps.go b/spinifex/daemon/eks_deps.go index ee28153d7..ef83bd80b 100644 --- a/spinifex/daemon/eks_deps.go +++ b/spinifex/daemon/eks_deps.go @@ -36,11 +36,7 @@ func (d *Daemon) systemRoleEnsurer() handlers_iam.SystemInstanceRoleEnsurer { "err", err) return nil } - clusterSize := 1 - if d.clusterConfig != nil { - clusterSize = len(d.clusterConfig.Nodes) - } - iamSvc, iamErr := handlers_iam.NewIAMServiceImpl(d.ctx, d.natsConn, masterKey, clusterSize) + iamSvc, iamErr := handlers_iam.NewIAMServiceImpl(d.ctx, d.natsConn, masterKey) if iamErr != nil { slog.Warn("System role ensurer: IAM service init failed (retried on next launch); system VMs fall back to baked static creds", "err", iamErr) @@ -61,10 +57,8 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps { } internalSuffix := "" - clusterSize := 1 if d.clusterConfig != nil { internalSuffix = d.clusterConfig.AWS.ServicesDomain - clusterSize = len(d.clusterConfig.Nodes) } gatewayCA := "" @@ -84,7 +78,6 @@ func (d *Daemon) buildEKSServiceDeps() handlers_eks.EKSServiceDeps { GatewayBaseURL: d.resolveGatewayBaseURL(), Region: d.config.Region, HolderID: d.node, - ClusterSize: clusterSize, InternalSuffix: internalSuffix, SystemGatewayURL: d.resolveSystemGatewayBaseURL(), SystemAccessKey: d.config.Predastore.AccessKey, diff --git a/spinifex/daemon/heartbeat_test.go b/spinifex/daemon/heartbeat_test.go index a5e8d271e..10772a2cf 100644 --- a/spinifex/daemon/heartbeat_test.go +++ b/spinifex/daemon/heartbeat_test.go @@ -103,7 +103,7 @@ func TestHeartbeatKVContract(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitClusterStateBucket()) diff --git a/spinifex/daemon/instance_record_repair_test.go b/spinifex/daemon/instance_record_repair_test.go index 7c54c13b6..1a8cb4b3d 100644 --- a/spinifex/daemon/instance_record_repair_test.go +++ b/spinifex/daemon/instance_record_repair_test.go @@ -19,7 +19,7 @@ func newRecordManagerInPackage(t *testing.T) *JetStreamManager { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - m, err := NewJetStreamManager(nc, 1) + m, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitKVBucket()) require.NoError(t, m.InitTerminatedInstanceBucket()) diff --git a/spinifex/daemon/instance_records_rekey_test.go b/spinifex/daemon/instance_records_rekey_test.go index fa9a3f1f7..0a4fda3ea 100644 --- a/spinifex/daemon/instance_records_rekey_test.go +++ b/spinifex/daemon/instance_records_rekey_test.go @@ -46,7 +46,7 @@ func TestRekey_MovesInstanceStateRecordsOntoTheDot(t *testing.T) { kv := seedSlashRecord(t, nc, daemon.InstanceStateBucket, 3, &vm.VM{ID: "i-1", InstanceType: "t3.nano", LastNode: "node-1"}) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitKVBucket()) @@ -64,7 +64,7 @@ func TestRekey_MovesTerminatedRecordsOntoTheDot(t *testing.T) { kv := seedSlashRecord(t, nc, daemon.TerminatedInstanceBucket, 2, &vm.VM{ID: "i-1", InstanceType: "t3.nano", LastNode: "node-1"}) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitTerminatedInstanceBucket()) diff --git a/spinifex/daemon/instance_records_test.go b/spinifex/daemon/instance_records_test.go index 2e880a125..946f7f365 100644 --- a/spinifex/daemon/instance_records_test.go +++ b/spinifex/daemon/instance_records_test.go @@ -29,7 +29,7 @@ func newRecordManagerConn(t *testing.T) (*daemon.JetStreamManager, *nats.Conn) { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitKVBucket()) require.NoError(t, m.InitTerminatedInstanceBucket()) @@ -259,7 +259,7 @@ func TestInstanceRecord_CarriesDeletionTimestamp(t *testing.T) { // store: Tier 1 boot runs before cluster KV exists. func TestInstanceRecordAccessors_ReportAnUnopenedBucket(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) assert.Error(t, m.WriteInstanceRecord("i-1", testRecord("i-1"))) diff --git a/spinifex/daemon/instance_records_transition_test.go b/spinifex/daemon/instance_records_transition_test.go index c3e97d1ba..bacee4ea9 100644 --- a/spinifex/daemon/instance_records_transition_test.go +++ b/spinifex/daemon/instance_records_transition_test.go @@ -223,7 +223,7 @@ func TestInstanceStateMigration_CopiesStoppedInstancesForward(t *testing.T) { seedLegacyBucket(t, nc, daemon.InstanceStateBucket, daemon.StoppedInstancePrefix, &vm.VM{ID: "i-1", InstanceType: "t3.nano"}) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitKVBucket()) @@ -247,7 +247,7 @@ func TestInstanceStateMigration_RunsEveryStepFromTheOldestVersion(t *testing.T) _, err = kv.Put(context.Background(), daemon.InstanceStatePrefix+"node-1", blob) require.NoError(t, err) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitKVBucket()) @@ -277,7 +277,7 @@ func TestInstanceStateMigration_DoesNotOverwriteAFresherRecord(t *testing.T) { _, err = kv.Put(context.Background(), daemon.StoppedInstancePrefix+"i-2", untouched) require.NoError(t, err) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitKVBucket()) @@ -297,12 +297,12 @@ func TestInstanceStateMigration_IsSafeToRunTwice(t *testing.T) { kv := seedLegacyBucket(t, nc, daemon.InstanceStateBucket, daemon.StoppedInstancePrefix, &vm.VM{ID: "i-1", InstanceType: "t3.nano"}) - first, err := daemon.NewJetStreamManager(nc, 1) + first, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, first.InitKVBucket()) require.NoError(t, kvutil.WriteVersion(context.Background(), kv, 1)) - second, err := daemon.NewJetStreamManager(nc, 1) + second, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, second.InitKVBucket()) @@ -317,7 +317,7 @@ func TestTerminatedInstanceMigration_CopiesTerminatedInstancesForward(t *testing seedLegacyBucket(t, nc, daemon.TerminatedInstanceBucket, daemon.TerminatedInstancePrefix, &vm.VM{ID: "i-1", InstanceType: "t3.nano"}) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitTerminatedInstanceBucket()) diff --git a/spinifex/daemon/instance_recovery.go b/spinifex/daemon/instance_recovery.go index 21f4297bb..b1ac9ebf5 100644 --- a/spinifex/daemon/instance_recovery.go +++ b/spinifex/daemon/instance_recovery.go @@ -522,7 +522,7 @@ func (d *Daemon) newLiveness() *instancecache.Liveness { if d.jsManager == nil || d.jsManager.js == nil { return nil } - return instancecache.NewLiveness(d.jsManager.js, clusterStateConfig(d.jsManager.replicas)) + return instancecache.NewLiveness(d.jsManager.js, clusterStateConfig()) } // forgetSupersededInstances drops any local copy of an instance the records say diff --git a/spinifex/daemon/instance_running_set_test.go b/spinifex/daemon/instance_running_set_test.go index 41d1df95f..9e9a90b76 100644 --- a/spinifex/daemon/instance_running_set_test.go +++ b/spinifex/daemon/instance_running_set_test.go @@ -234,7 +234,7 @@ func TestWriteRunningSet_SeedsFromWhatIsAlreadyThere(t *testing.T) { writeRunningSet(t, first, runningVM("i-1", "t3.nano")) before := recordRevision(t, nc, "i-1") - second, err := daemon.NewJetStreamManager(nc, 1) + second, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, second.InitKVBucket()) writeRunningSet(t, second, runningVM("i-1", "t3.nano")) @@ -293,7 +293,7 @@ func TestInstanceStateMigration_SplitsTheNodeBlob(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) seedNodeBlobBucket(t, nc, localNode, runningSet(runningVM("i-1", "t3.nano"), runningVM("i-2", "t3.micro"))) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitKVBucket()) @@ -314,7 +314,7 @@ func TestInstanceStateMigration_LeavesTheNodeBlobIntact(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) kv := seedNodeBlobBucket(t, nc, localNode, runningSet(runningVM("i-1", "t3.nano"))) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitKVBucket()) writeRunningSet(t, m, runningVM("i-1", "t3.nano")) @@ -330,7 +330,7 @@ func TestInstanceStateMigration_SeedsAPresenceMarkerPerNode(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) kv := seedNodeBlobBucket(t, nc, localNode, runningSet(runningVM("i-1", "t3.nano"))) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitKVBucket()) @@ -359,7 +359,7 @@ func TestInstanceStateMigration_SplitsEveryNodesBlob(t *testing.T) { _, err = kv.Put(context.Background(), daemon.InstanceStatePrefix+"node-2", data) require.NoError(t, err) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitKVBucket()) @@ -383,7 +383,7 @@ func TestInstanceStateMigration_CarriesOwnershipOffTheBlobKey(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) seedNodeBlobBucket(t, nc, localNode, runningSet(unownedVM("i-1", "t3.nano"))) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitKVBucket()) @@ -401,7 +401,7 @@ func TestInstanceStateMigration_ANodeRecoversAnUnownedRunningSet(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) seedNodeBlobBucket(t, nc, localNode, runningSet(unownedVM("i-1", "t3.nano"), unownedVM("i-2", "t3.micro"))) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitKVBucket()) @@ -419,7 +419,7 @@ func TestInstanceStateMigration_LeavesAnOwnedRecordAlone(t *testing.T) { moved := &vm.VM{ID: "i-1", InstanceType: "t3.nano", Status: vm.StateRunning, LastNode: "node-2"} seedNodeBlobBucket(t, nc, localNode, runningSet(moved)) - m, err := daemon.NewJetStreamManager(nc, 1) + m, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, m.InitKVBucket()) @@ -433,13 +433,13 @@ func TestInstanceStateMigration_OwnershipStampIsSafeToRunTwice(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) kv := seedNodeBlobBucket(t, nc, localNode, runningSet(unownedVM("i-1", "t3.nano"))) - first, err := daemon.NewJetStreamManager(nc, 1) + first, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, first.InitKVBucket()) before := recordRevision(t, nc, "i-1") require.NoError(t, kvutil.WriteVersion(context.Background(), kv, 4)) - second, err := daemon.NewJetStreamManager(nc, 1) + second, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, second.InitKVBucket()) @@ -450,13 +450,13 @@ func TestInstanceStateMigration_BlobSplitIsSafeToRunTwice(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) kv := seedNodeBlobBucket(t, nc, localNode, runningSet(runningVM("i-1", "t3.nano"))) - first, err := daemon.NewJetStreamManager(nc, 1) + first, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, first.InitKVBucket()) before := recordRevision(t, nc, "i-1") require.NoError(t, kvutil.WriteVersion(context.Background(), kv, 2)) - second, err := daemon.NewJetStreamManager(nc, 1) + second, err := daemon.NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, second.InitKVBucket()) diff --git a/spinifex/daemon/ipsec_barrier_test.go b/spinifex/daemon/ipsec_barrier_test.go index ebd31c0f9..f90165a53 100644 --- a/spinifex/daemon/ipsec_barrier_test.go +++ b/spinifex/daemon/ipsec_barrier_test.go @@ -21,7 +21,7 @@ func newTestIPSecBarrier(t *testing.T) (*KVIPSecBarrier, *JetStreamManager) { require.NoError(t, err) t.Cleanup(nc.Close) - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitClusterStateBucket()) diff --git a/spinifex/daemon/jetstream.go b/spinifex/daemon/jetstream.go index 6f278f2d7..b64728929 100644 --- a/spinifex/daemon/jetstream.go +++ b/spinifex/daemon/jetstream.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "log/slog" - "strings" "time" "github.com/aws/aws-sdk-go/aws" @@ -83,7 +82,6 @@ type JetStreamManager struct { records *kvstore.Store[vm.InstanceRecord] // i., instance-state bucket termRecords *kvstore.Store[vm.InstanceRecord] // i., terminated bucket clusterKV jetstream.KeyValue // spinifex-cluster-state - replicas int obs KVSyncObserver running runningSetState } @@ -117,28 +115,23 @@ func (m *JetStreamManager) KVHealthy() bool { } // NewJetStreamManager creates a new JetStreamManager from a NATS connection. -// replicas specifies the number of replicas for the KV bucket (typically matches cluster node count). -func NewJetStreamManager(nc *nats.Conn, replicas int) (*JetStreamManager, error) { +func NewJetStreamManager(nc *nats.Conn) (*JetStreamManager, error) { js, err := jetstream.New(nc) if err != nil { return nil, err } - return &JetStreamManager{ - js: js, - replicas: replicas, - }, nil + return &JetStreamManager{js: js}, nil } // instanceStateConfig describes the instance-state bucket. Extracted so the // bucket can be rebuilt against a different JetStream client without the // description drifting from the one InitKVBucket creates. -func instanceStateConfig(replicas int) kvstore.Config { +func instanceStateConfig() kvstore.Config { return kvstore.Config{ Name: InstanceStateBucket, Description: "Spinifex instance state storage", History: 1, - Replicas: replicas, // The owning node republishes its record on its next write, so an // emptied bucket costs a sync rather than the records themselves. RecreateIfMissing: true, @@ -150,12 +143,11 @@ func instanceStateConfig(replicas int) kvstore.Config { } // terminatedInstanceConfig describes the terminated-instances bucket. -func terminatedInstanceConfig(replicas int) kvstore.Config { +func terminatedInstanceConfig() kvstore.Config { return kvstore.Config{ Name: TerminatedInstanceBucket, Description: "Terminated instances (auto-expire after 1 hour)", History: 1, - Replicas: replicas, TTL: 1 * time.Hour, RecreateIfMissing: true, OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error { @@ -176,7 +168,7 @@ func (m *JetStreamManager) setInstanceStateBucket(b *kvstore.Bucket) { // InitKVBucket initializes the KV bucket, creating it if it doesn't exist. func (m *JetStreamManager) InitKVBucket() error { - m.setInstanceStateBucket(kvstore.NewBucket(m.js, instanceStateConfig(m.replicas))) + m.setInstanceStateBucket(kvstore.NewBucket(m.js, instanceStateConfig())) // Opened eagerly: a bucket that cannot be created must fail startup here, // not on the first write, and Tier 1 boot must not reach cluster KV at all. @@ -185,12 +177,11 @@ func (m *JetStreamManager) InitKVBucket() error { } // clusterStateConfig describes the cluster-state bucket. -func clusterStateConfig(replicas int) kvstore.Config { +func clusterStateConfig() kvstore.Config { return kvstore.Config{ Name: ClusterStateBucket, Description: "Spinifex cluster state (heartbeats, shutdown markers, service maps)", History: 1, - Replicas: replicas, TTL: 1 * time.Hour, OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error { return migrate.DefaultRegistry.RunKV(ctx, ClusterStateBucket, kv, ClusterStateBucketVersion) @@ -204,7 +195,7 @@ func clusterStateConfig(replicas int) kvstore.Config { // The handle is kept rather than the Bucket: this one holds several unrelated // record types under one namespace, and its callers still read it as raw keys. func (m *JetStreamManager) InitClusterStateBucket() error { - kv, err := kvstore.NewBucket(m.js, clusterStateConfig(m.replicas)).KV(context.Background()) + kv, err := kvstore.NewBucket(m.js, clusterStateConfig()).KV(context.Background()) if err != nil { return err } @@ -215,7 +206,7 @@ func (m *JetStreamManager) InitClusterStateBucket() error { // InitTerminatedInstanceBucket initializes the terminated-instances KV bucket with a 1-hour TTL. // JetStream automatically purges keys after 1 hour, matching AWS behavior for terminated instances. func (m *JetStreamManager) InitTerminatedInstanceBucket() error { - m.term = kvstore.New[vm.VM](m.js, terminatedInstanceConfig(m.replicas)) + m.term = kvstore.New[vm.VM](m.js, terminatedInstanceConfig()) // A second view over the same handle, for the same reason the instance-state // bucket carries several. m.termRecords = kvstore.On[vm.InstanceRecord](m.term.Bucket) @@ -521,63 +512,6 @@ func (m *JetStreamManager) DeleteState(nodeID string) error { return nil } -// UpdateReplicas updates the replica count for ALL JetStream KV buckets. -// It iterates over every KV_* stream and bumps replicas to match the cluster size. -// This ensures service buckets (IAM, VPC, IGW, etc.) are replicated alongside daemon buckets. -// This should be called when new nodes join the cluster. -func (m *JetStreamManager) UpdateReplicas(newReplicas int) error { - if m.js == nil { - return errors.New("JetStream context not initialized") - } - - m.replicas = newReplicas - - ctx := context.Background() - // Iterate all streams and update any KV-backed stream (prefixed "KV_") - updated := 0 - lister := m.js.StreamNames(ctx) - for name := range lister.Name() { - if !strings.HasPrefix(name, "KV_") { - continue - } - - stream, err := m.js.Stream(ctx, name) - if err != nil { - slog.Warn("Failed to open stream", "stream", name, "error", err) - continue - } - info, err := stream.Info(ctx) - if err != nil { - slog.Warn("Failed to get stream info", "stream", name, "error", err) - continue - } - - if info.Config.Replicas >= newReplicas { - continue - } - - oldReplicas := info.Config.Replicas - info.Config.Replicas = newReplicas - if _, err := m.js.UpdateStream(ctx, info.Config); err != nil { - slog.Warn("Failed to update KV bucket replicas", "stream", name, "error", err) - continue - } - - bucket := strings.TrimPrefix(name, "KV_") - slog.Info("Updated KV bucket replicas", "bucket", bucket, "oldReplicas", oldReplicas, "newReplicas", newReplicas) - updated++ - } - if err := lister.Err(); err != nil { - return fmt.Errorf("list JetStream streams: %w", err) - } - - if updated > 0 { - slog.Info("KV replication update complete", "bucketsUpdated", updated, "replicas", newReplicas) - } - - return nil -} - // WriteStoppedInstance writes a stopped instance to the shared KV store, // replacing any existing record for instanceID. Uses CAS internally (bounded // retry) so a write racing a concurrent update is detected and retried diff --git a/spinifex/daemon/jetstream_test.go b/spinifex/daemon/jetstream_test.go index 93bdd2ac2..9b81f813a 100644 --- a/spinifex/daemon/jetstream_test.go +++ b/spinifex/daemon/jetstream_test.go @@ -40,7 +40,7 @@ func TestJetStreamManager_WriteAndLoadState(t *testing.T) { defer nc.Close() // Create JetStreamManager - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err, "Failed to create JetStreamManager") // Initialize the KV bucket @@ -90,7 +90,7 @@ func TestJetStreamManager_LoadState_KeyNotFound(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() @@ -112,7 +112,7 @@ func TestJetStreamManager_BucketCreation(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) // InitKVBucket should create the bucket @@ -134,7 +134,7 @@ func TestJetStreamManager_BucketReconnection(t *testing.T) { nc1, err := nats.Connect(natsURL) require.NoError(t, err) - jsm1, err := NewJetStreamManager(nc1, 1) + jsm1, err := NewJetStreamManager(nc1) require.NoError(t, err) err = jsm1.InitKVBucket() @@ -157,7 +157,7 @@ func TestJetStreamManager_BucketReconnection(t *testing.T) { require.NoError(t, err) defer nc2.Close() - jsm2, err := NewJetStreamManager(nc2, 1) + jsm2, err := NewJetStreamManager(nc2) require.NoError(t, err) err = jsm2.InitKVBucket() @@ -180,7 +180,7 @@ func TestJetStreamManager_DeleteState(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() @@ -222,7 +222,7 @@ func TestJetStreamManager_DeleteState_NonExistent(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() @@ -241,7 +241,7 @@ func TestJetStreamManager_WriteState_UpdateExisting(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() @@ -289,7 +289,7 @@ func TestJetStreamManager_MultipleNodes(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() @@ -337,7 +337,7 @@ func TestJetStreamManager_KVNotInitialized(t *testing.T) { defer nc.Close() // Create JetStreamManager but don't call InitKVBucket - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) testInstances := make(map[string]*vm.VM) @@ -351,51 +351,6 @@ func TestJetStreamManager_KVNotInitialized(t *testing.T) { assert.Error(t, err, "DeleteState should error when KV not initialized") } -// TestJetStreamManager_UpdateReplicas tests updating replica count for the KV bucket. -func TestJetStreamManager_UpdateReplicas(t *testing.T) { - natsURL := sharedJSNATSURL - - nc, err := nats.Connect(natsURL) - require.NoError(t, err) - defer nc.Close() - - // Create with 1 replica (typical for single node startup) - jsm, err := NewJetStreamManager(nc, 1) - require.NoError(t, err) - - err = jsm.InitKVBucket() - require.NoError(t, err) - - // Verify initial replica count - js, _ := jetstream.New(nc) - stream, err := js.Stream(t.Context(), "KV_"+InstanceStateBucket) - require.NoError(t, err) - streamInfo, err := stream.Info(t.Context()) - require.NoError(t, err) - assert.Equal(t, 1, streamInfo.Config.Replicas, "Should start with 1 replica") - - // Try to update to same replica count (should be a no-op) - err = jsm.UpdateReplicas(1) - assert.NoError(t, err, "Updating to same replica count should succeed") - - // Note: Increasing replicas beyond 1 requires additional NATS servers in the cluster, - // which we don't have in the test environment. In a single-node test server, - // attempting to increase replicas will fail with "insufficient resources" error. - // This test verifies the basic functionality works. -} - -// TestJetStreamManager_UpdateReplicas_NoInit tests UpdateReplicas when JS not initialized. -func TestJetStreamManager_UpdateReplicas_NoInit(t *testing.T) { - // Test with nil JetStream context - jsm := &JetStreamManager{ - js: nil, - replicas: 1, - } - - err := jsm.UpdateReplicas(3) - assert.Error(t, err, "UpdateReplicas should error when JetStream not initialized") -} - // --- Stopped instance KV tests --- // TestJetStreamManager_WriteAndLoadStoppedInstance tests round-trip write and load of a stopped instance. @@ -404,7 +359,7 @@ func TestJetStreamManager_WriteAndLoadStoppedInstance(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -439,7 +394,7 @@ func TestJetStreamManager_LoadStoppedInstance_NotFound(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -455,7 +410,7 @@ func TestJetStreamManager_DeleteStoppedInstance(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -489,7 +444,7 @@ func TestJetStreamManager_ClaimStoppedInstance_HappyPath(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -517,7 +472,7 @@ func TestJetStreamManager_ClaimStoppedInstance_NotFound(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -537,7 +492,7 @@ func TestJetStreamManager_ClaimStoppedInstance_ConcurrentClaim(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -588,7 +543,7 @@ func TestJetStreamManager_UpdateStoppedInstance_ConflictRetry(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -631,7 +586,7 @@ func TestJetStreamManager_UpdateStoppedInstance_NotFound(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -652,7 +607,7 @@ func TestJetStreamManager_UpdateStoppedInstance_NoResurrectAfterClaim(t *testing require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -696,7 +651,7 @@ func TestJetStreamManager_ListStoppedInstances(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -738,7 +693,7 @@ func TestJetStreamManager_StoppedInstances_NoInterference(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -778,7 +733,7 @@ func TestJetStreamManager_StoppedInstance_KVNotInitialized(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) // Don't call InitKVBucket @@ -803,7 +758,7 @@ func TestJetStreamManager_WriteServiceManifest(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitClusterStateBucket() require.NoError(t, err) @@ -840,7 +795,7 @@ func TestJetStreamManager_WriteServiceManifest_EmptyServices(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitClusterStateBucket() require.NoError(t, err) @@ -867,7 +822,7 @@ func TestJetStreamManager_WriteServiceManifest_ClusterKVNotInitialized(t *testin require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) // Don't call InitClusterStateBucket @@ -895,7 +850,7 @@ func TestJetStreamManager_WriteState_RecoverAfterStreamLost(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -922,7 +877,7 @@ func TestJetStreamManager_LoadState_RecoverAfterStreamLost(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -941,7 +896,7 @@ func TestJetStreamManager_DeleteState_RecoverAfterStreamLost(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -958,7 +913,7 @@ func TestJetStreamManager_WriteStoppedInstance_RecoverAfterStreamLost(t *testing require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -984,7 +939,7 @@ func TestJetStreamManager_LoadStoppedInstance_RecoverAfterStreamLost(t *testing. require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -1002,7 +957,7 @@ func TestJetStreamManager_DeleteStoppedInstance_RecoverAfterStreamLost(t *testin require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -1018,7 +973,7 @@ func TestJetStreamManager_ListStoppedInstances_RecoverAfterStreamLost(t *testing require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -1055,12 +1010,12 @@ func swapToNonJSContext(t *testing.T, jsm *JetStreamManager) { if jsm.stateB != nil { kv, err := jsm.stateB.KV(t.Context()) require.NoError(t, err) - jsm.setInstanceStateBucket(kvstore.NewOpenBucket(js, kv, instanceStateConfig(jsm.replicas))) + jsm.setInstanceStateBucket(kvstore.NewOpenBucket(js, kv, instanceStateConfig())) } if jsm.term != nil { kv, err := jsm.term.KV(t.Context()) require.NoError(t, err) - jsm.term = kvstore.Over[vm.VM](js, kv, terminatedInstanceConfig(jsm.replicas)) + jsm.term = kvstore.Over[vm.VM](js, kv, terminatedInstanceConfig()) } } @@ -1069,7 +1024,7 @@ func TestJetStreamManager_WriteState_RecoveryFailure(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -1087,7 +1042,7 @@ func TestJetStreamManager_LoadState_RecoveryFailure(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -1105,7 +1060,7 @@ func TestJetStreamManager_DeleteState_RecoveryFailure(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -1122,7 +1077,7 @@ func TestJetStreamManager_WriteStoppedInstance_RecoveryFailure(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -1140,7 +1095,7 @@ func TestJetStreamManager_LoadStoppedInstance_RecoveryFailure(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -1158,7 +1113,7 @@ func TestJetStreamManager_DeleteStoppedInstance_RecoveryFailure(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -1175,7 +1130,7 @@ func TestJetStreamManager_ListStoppedInstances_RecoveryFailure(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitKVBucket() require.NoError(t, err) @@ -1195,7 +1150,7 @@ func TestJetStreamManager_WriteAndLoadTerminatedInstance(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitTerminatedInstanceBucket()) @@ -1225,7 +1180,7 @@ func TestJetStreamManager_LoadTerminatedInstance_NotFound(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitTerminatedInstanceBucket()) @@ -1239,7 +1194,7 @@ func TestJetStreamManager_DeleteTerminatedInstance(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitTerminatedInstanceBucket()) @@ -1268,7 +1223,7 @@ func TestJetStreamManager_ListTerminatedInstances(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitTerminatedInstanceBucket()) @@ -1304,7 +1259,7 @@ func TestJetStreamManager_WriteLoadDeleteTerminatedInstance_RoundTrip(t *testing require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitTerminatedInstanceBucket()) @@ -1336,7 +1291,7 @@ func TestJetStreamManager_TerminatedInstance_KVNotInitialized(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) // Don't call InitTerminatedInstanceBucket @@ -1365,7 +1320,7 @@ func TestJetStreamManager_UpdateTerminatedInstance_ConflictRetry(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitTerminatedInstanceBucket()) @@ -1414,7 +1369,7 @@ func TestJetStreamManager_UpdateTerminatedInstance_NotFound(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitTerminatedInstanceBucket()) @@ -1432,7 +1387,7 @@ func TestJetStreamManager_WriteStoppedInstance_OverwritesConcurrentValue(t *test require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -1470,7 +1425,7 @@ func TestJetStreamManager_WriteTerminatedInstance_RecoverAfterStreamLost(t *test require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitTerminatedInstanceBucket()) @@ -1493,7 +1448,7 @@ func TestJetStreamManager_LoadTerminatedInstance_RecoverAfterStreamLost(t *testi require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitTerminatedInstanceBucket()) @@ -1509,7 +1464,7 @@ func TestJetStreamManager_DeleteTerminatedInstance_RecoverAfterStreamLost(t *tes require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitTerminatedInstanceBucket()) @@ -1524,7 +1479,7 @@ func TestJetStreamManager_ListTerminatedInstances_RecoverAfterStreamLost(t *test require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitTerminatedInstanceBucket()) @@ -1540,7 +1495,7 @@ func TestJetStreamManager_InitBuckets_WritesVersion(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -1570,7 +1525,7 @@ func TestJetStreamManager_ListStoppedInstances_SkipsVersionKey(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -1585,7 +1540,7 @@ func TestJetStreamManager_ListTerminatedInstances_SkipsVersionKey(t *testing.T) require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitTerminatedInstanceBucket()) @@ -1632,7 +1587,7 @@ func TestJetStreamManager_BestEffort_Success_NotifiesObserver(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -1651,7 +1606,7 @@ func TestJetStreamManager_BestEffort_PutError_NotifiesObserver(t *testing.T) { nc, err := nats.Connect(sharedJSNATSURL) require.NoError(t, err) - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -1675,7 +1630,7 @@ func TestJetStreamManager_BestEffort_NilObserver_NoPanic(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -1696,7 +1651,7 @@ func TestJetStreamManager_UpdateMgmtIPAM_ConflictRetry(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitClusterStateBucket()) @@ -1746,7 +1701,7 @@ func TestJetStreamManager_UpdateMgmtIPAM_NotFound(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitClusterStateBucket()) @@ -1761,7 +1716,7 @@ func TestJetStreamManager_UpdateMgmtIPAM_ClusterKVNotInitialized(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) // Don't call InitClusterStateBucket @@ -1781,7 +1736,7 @@ func TestJetStreamManager_UpdateMgmtIPAM_ClusterKVNotInitialized(t *testing.T) { func TestJetStreamManager_ListStoppedInstances_FailsOnAnUndecodableRecord(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -1806,7 +1761,7 @@ func TestJetStreamManager_UpdateStoppedInstance_AbsentKeyIsNotFound(t *testing.T require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) require.NoError(t, jsm.InitTerminatedInstanceBucket()) @@ -1842,7 +1797,7 @@ func TestJetStreamManager_LoadState_UnversionedRecordStillReads(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -1867,7 +1822,7 @@ func TestJetStreamManager_LoadState_FutureRecordIsRefused(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) @@ -1887,7 +1842,7 @@ func TestJetStreamManager_WriteState_StampsSchemaVersion(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, jsm.InitKVBucket()) diff --git a/spinifex/daemon/shutdown_test.go b/spinifex/daemon/shutdown_test.go index bd1348fc0..5247acae9 100644 --- a/spinifex/daemon/shutdown_test.go +++ b/spinifex/daemon/shutdown_test.go @@ -30,7 +30,7 @@ func TestClusterShutdownStateKVRoundTrip(t *testing.T) { require.NoError(t, err) defer nc.Close() - jsm, err := NewJetStreamManager(nc, 1) + jsm, err := NewJetStreamManager(nc) require.NoError(t, err) err = jsm.InitClusterStateBucket() require.NoError(t, err) diff --git a/spinifex/daemon/state_test.go b/spinifex/daemon/state_test.go index f4d86c623..5eb1ae974 100644 --- a/spinifex/daemon/state_test.go +++ b/spinifex/daemon/state_test.go @@ -41,7 +41,7 @@ func createDaemonWithJetStream(t *testing.T) *Daemon { daemon.config = &config.Config{BaseDir: tmpDir, DataDir: tmpDir} daemon.natsConn = nc - daemon.jsManager, err = NewJetStreamManager(nc, 1) + daemon.jsManager, err = NewJetStreamManager(nc) require.NoError(t, err) require.NoError(t, daemon.jsManager.InitKVBucket()) require.NoError(t, daemon.jsManager.InitClusterStateBucket()) diff --git a/spinifex/gateway/admin_createaccount_test.go b/spinifex/gateway/admin_createaccount_test.go index 48059b926..129e36e1c 100644 --- a/spinifex/gateway/admin_createaccount_test.go +++ b/spinifex/gateway/admin_createaccount_test.go @@ -241,7 +241,7 @@ func newCreateAccountGateway(t *testing.T) *GatewayConfig { masterKey, err := handlers_iam.GenerateMasterKey() require.NoError(t, err) - svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey) require.NoError(t, err) sub, err := nc.Subscribe(utils.SubjectEnsureDefaultVpc, func(msg *nats.Msg) { diff --git a/spinifex/gateway/admin_deleteaccount_job_test.go b/spinifex/gateway/admin_deleteaccount_job_test.go index 995827990..9aacdf48a 100644 --- a/spinifex/gateway/admin_deleteaccount_job_test.go +++ b/spinifex/gateway/admin_deleteaccount_job_test.go @@ -40,7 +40,7 @@ func newDeleteAccountGateway(t *testing.T) *GatewayConfig { masterKey, err := handlers_iam.GenerateMasterKey() require.NoError(t, err) - svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey) require.NoError(t, err) for _, subject := range teardownSubjects { diff --git a/spinifex/gateway/bedrock.go b/spinifex/gateway/bedrock.go index 09a3cde65..ddb316986 100644 --- a/spinifex/gateway/bedrock.go +++ b/spinifex/gateway/bedrock.go @@ -235,7 +235,7 @@ func (gw *GatewayConfig) bedrockLoggingConfigStore() *gateway_bedrock.LoggingCon if gw.BedrockLoggingConfig != nil { return gw.BedrockLoggingConfig } - return gateway_bedrock.NewLoggingConfigStore(nil, 1) + return gateway_bedrock.NewLoggingConfigStore(nil) } // bedrockRecorder returns gw.BedrockRecorder, or the no-op fallback when no @@ -269,7 +269,7 @@ func (gw *GatewayConfig) bedrockProvisionedStore() *gateway_bedrock.ProvisionedS if gw.BedrockProvisioned != nil { return gw.BedrockProvisioned } - return gateway_bedrock.NewProvisionedStore(nil, 1, gw.Region, nil) + return gateway_bedrock.NewProvisionedStore(nil, gw.Region, nil) } // bedrockGuardrailStore returns gw.BedrockGuardrails, or a store backed by no @@ -281,7 +281,7 @@ func (gw *GatewayConfig) bedrockGuardrailStore() *gateway_bedrock.GuardrailStore if gw.BedrockGuardrails != nil { return gw.BedrockGuardrails } - return gateway_bedrock.NewGuardrailStore(nil, 1, gw.Region) + return gateway_bedrock.NewGuardrailStore(nil, gw.Region) } // bedrockEndpointResolver returns the registry-backed resolver when one is diff --git a/spinifex/gateway/bedrock/access.go b/spinifex/gateway/bedrock/access.go index 3e4878fa3..efcc22b31 100644 --- a/spinifex/gateway/bedrock/access.go +++ b/spinifex/gateway/bedrock/access.go @@ -65,13 +65,12 @@ type ModelAccessStore struct { var _ AccessResolver = (*ModelAccessStore)(nil) // NewModelAccessStore constructs a ModelAccessStore over the cluster's -// JetStream client, replicated across replicas nodes. -func NewModelAccessStore(js jetstream.JetStream, replicas int) *ModelAccessStore { +// JetStream client. +func NewModelAccessStore(js jetstream.JetStream) *ModelAccessStore { return &ModelAccessStore{bucket: kvstore.NewBucket(js, kvstore.Config{ - Name: modelAccessBucket, - History: modelAccessHistory, - Replicas: replicas, - Missing: "bedrock: model access store has no JetStream client configured", + Name: modelAccessBucket, + History: modelAccessHistory, + Missing: "bedrock: model access store has no JetStream client configured", })} } diff --git a/spinifex/gateway/bedrock/access_test.go b/spinifex/gateway/bedrock/access_test.go index 5e3a5a312..a9afec75a 100644 --- a/spinifex/gateway/bedrock/access_test.go +++ b/spinifex/gateway/bedrock/access_test.go @@ -72,7 +72,7 @@ func TestAccessKey_EncodesModelID(t *testing.T) { // Revoke. func TestModelAccessStore_GrantResolveRevoke(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - store := NewModelAccessStore(js, 1) + store := NewModelAccessStore(js) ctx := context.Background() granted, err := store.Granted(ctx, "000000000001", selfHostTestModel) @@ -100,7 +100,7 @@ func TestModelAccessStore_GrantResolveRevoke(t *testing.T) { func TestModelAccessStore_GrantIsIdempotent(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - store := NewModelAccessStore(js, 1) + store := NewModelAccessStore(js) ctx := context.Background() require.NoError(t, store.Grant(ctx, "000000000001", selfHostTestModel)) @@ -116,7 +116,7 @@ func TestModelAccessStore_GrantIsIdempotent(t *testing.T) { // error. func TestModelAccessStore_RevokeMissingGrantSucceeds(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - store := NewModelAccessStore(js, 1) + store := NewModelAccessStore(js) assert.NoError(t, store.Revoke(context.Background(), "000000000001", selfHostTestModel)) } @@ -125,7 +125,7 @@ func TestModelAccessStore_RevokeMissingGrantSucceeds(t *testing.T) { // reversible, including the ':' that forced it. func TestModelAccessStore_List_RoundTripsModelIDs(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - store := NewModelAccessStore(js, 1) + store := NewModelAccessStore(js) ctx := context.Background() require.NoError(t, store.Grant(ctx, "000000000001", selfHostTestModel)) @@ -143,7 +143,7 @@ func TestModelAccessStore_List_RoundTripsModelIDs(t *testing.T) { func TestModelAccessStore_ListEmptyAccount(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - store := NewModelAccessStore(js, 1) + store := NewModelAccessStore(js) models, err := store.List(context.Background(), "000000000009") require.NoError(t, err) @@ -154,7 +154,7 @@ func TestModelAccessStore_ListEmptyAccount(t *testing.T) { // handlers_quota exempts the system account from every quota dimension. func TestModelAccessStore_SystemAccountBypassesGrants(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - store := NewModelAccessStore(js, 1) + store := NewModelAccessStore(js) granted, err := store.Granted(context.Background(), utils.GlobalAccountID, selfHostTestModel) require.NoError(t, err) @@ -189,7 +189,7 @@ func TestGrantedCatalogEntry_ErrorClasses(t *testing.T) { // who then revokes a grant does not have it restored by the next restart. func TestSeedAccountGrants_SeedsOnceThenRespectsRevoke(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - store := NewModelAccessStore(js, 1) + store := NewModelAccessStore(js) ctx := context.Background() const account = "000000000001" @@ -218,7 +218,7 @@ func TestSeedAccountGrants_SeedsOnceThenRespectsRevoke(t *testing.T) { // account to end up with the catalog. func TestSeedAccountGrants_LeavesOtherAccountsDenied(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - store := NewModelAccessStore(js, 1) + store := NewModelAccessStore(js) ctx := context.Background() _, err := store.SeedAccountGrants(ctx, "000000000001", CatalogModelIDs()) @@ -233,7 +233,7 @@ func TestSeedAccountGrants_LeavesOtherAccountsDenied(t *testing.T) { // read back as a grant would show up as a bogus model ID in List. func TestSeedAccountGrants_MarkerIsNotAGrant(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - store := NewModelAccessStore(js, 1) + store := NewModelAccessStore(js) ctx := context.Background() _, err := store.SeedAccountGrants(ctx, "000000000001", nil) @@ -248,7 +248,7 @@ func TestSeedAccountGrants_MarkerIsNotAGrant(t *testing.T) { // JetStream client: every accessor must report the misconfiguration rather // than panic on a nil handle. func TestModelAccessStore_RequiresJetStream(t *testing.T) { - store := NewModelAccessStore(nil, 1) + store := NewModelAccessStore(nil) ctx := context.Background() _, err := store.Granted(ctx, "000000000001", "anthropic.claude-3-5-sonnet-20240620-v1:0") diff --git a/spinifex/gateway/bedrock/admission_test.go b/spinifex/gateway/bedrock/admission_test.go index a43c98b10..5d9c4d61c 100644 --- a/spinifex/gateway/bedrock/admission_test.go +++ b/spinifex/gateway/bedrock/admission_test.go @@ -210,7 +210,7 @@ func TestAdmitSelfHost_OnDemandAdmitsThenThrottles(t *testing.T) { // resolving committed ModelUnits surfaces as that error, never masquerading as // a throttle or admitting the request. func TestAdmitSelfHost_CommittedUnitsErrorPropagates(t *testing.T) { - store := NewProvisionedStore(nil, 1, ptTestRegion, newStubEndpointProvisioner()) + store := NewProvisionedStore(nil, ptTestRegion, newStubEndpointProvisioner()) release, err := admitSelfHost(context.Background(), store, ptCallerAccount, selfHostTestModel, catalogEntry{MaxConcurrency: 1}) @@ -282,7 +282,7 @@ func TestCommittedModelUnits_EmptyAccountReturnsZero(t *testing.T) { // TestCommittedModelUnits_BucketErrorPropagates asserts an unusable store // surfaces its error rather than silently reading zero committed units. func TestCommittedModelUnits_BucketErrorPropagates(t *testing.T) { - store := NewProvisionedStore(nil, 1, ptTestRegion, newStubEndpointProvisioner()) + store := NewProvisionedStore(nil, ptTestRegion, newStubEndpointProvisioner()) _, err := committedModelUnits(context.Background(), store, ptCallerAccount, selfHostTestModel) require.Error(t, err) } diff --git a/spinifex/gateway/bedrock/credentials.go b/spinifex/gateway/bedrock/credentials.go index bc3c4277b..3283c20ab 100644 --- a/spinifex/gateway/bedrock/credentials.go +++ b/spinifex/gateway/bedrock/credentials.go @@ -38,13 +38,12 @@ var _ CredentialResolver = (*CredentialStore)(nil) // NewCredentialStore constructs a CredentialStore. platformDefaults maps // vendor to a platform-wide API key used when an account has no key of its // own; pass an empty map to disable platform defaults entirely. -func NewCredentialStore(js jetstream.JetStream, masterKey []byte, replicas int, platformDefaults map[string]string) *CredentialStore { +func NewCredentialStore(js jetstream.JetStream, masterKey []byte, platformDefaults map[string]string) *CredentialStore { return &CredentialStore{ bucket: kvstore.NewBucket(js, kvstore.Config{ - Name: bedrockCredentialsBucket, - History: bedrockCredentialsHistory, - Replicas: replicas, - Missing: "bedrock: credential store has no JetStream client configured", + Name: bedrockCredentialsBucket, + History: bedrockCredentialsHistory, + Missing: "bedrock: credential store has no JetStream client configured", }), masterKey: masterKey, platformDefaults: platformDefaults, diff --git a/spinifex/gateway/bedrock/credentials_test.go b/spinifex/gateway/bedrock/credentials_test.go index 489620339..182348ad5 100644 --- a/spinifex/gateway/bedrock/credentials_test.go +++ b/spinifex/gateway/bedrock/credentials_test.go @@ -21,7 +21,7 @@ func fixedMasterKey() []byte { // live NATS server is out of scope here). Resolve must consult // platformDefaults without dereferencing js. func TestCredentialStore_Resolve_NilJS(t *testing.T) { - store := NewCredentialStore(nil, fixedMasterKey(), 1, map[string]string{"anthropic": "sk-platform-default"}) + store := NewCredentialStore(nil, fixedMasterKey(), map[string]string{"anthropic": "sk-platform-default"}) cases := []struct { name string @@ -43,7 +43,7 @@ func TestCredentialStore_Resolve_NilJS(t *testing.T) { } func TestCredentialStore_Resolve_NoPlatformDefaults(t *testing.T) { - store := NewCredentialStore(nil, fixedMasterKey(), 1, nil) + store := NewCredentialStore(nil, fixedMasterKey(), nil) key, ok, err := store.Resolve(context.Background(), "000000000001", "anthropic") require.NoError(t, err) @@ -63,7 +63,7 @@ func TestNoopCredentialResolver_ResolvesNothing(t *testing.T) { // KV-hit branch of Resolve (Get+decrypt). func TestCredentialStore_PutAndResolve_KV(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewCredentialStore(testutil.NewJetStream(t, nc), fixedMasterKey(), 1, nil) + store := NewCredentialStore(testutil.NewJetStream(t, nc), fixedMasterKey(), nil) ctx := context.Background() require.NoError(t, store.PutCredential(ctx, "000000000001", "anthropic", "sk-test")) @@ -78,7 +78,7 @@ func TestCredentialStore_PutAndResolve_KV(t *testing.T) { // miss (jetstream.ErrKeyNotFound) that falls through to the platform default. func TestCredentialStore_Resolve_KVMissFallsBackToPlatformDefault(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewCredentialStore(testutil.NewJetStream(t, nc), fixedMasterKey(), 1, map[string]string{"anthropic": "sk-platform-default"}) + store := NewCredentialStore(testutil.NewJetStream(t, nc), fixedMasterKey(), map[string]string{"anthropic": "sk-platform-default"}) key, ok, err := store.Resolve(context.Background(), "999999999999", "anthropic") require.NoError(t, err) @@ -90,7 +90,7 @@ func TestCredentialStore_Resolve_KVMissFallsBackToPlatformDefault(t *testing.T) // unknown account with no platform defaults configured: ("", false, nil). func TestCredentialStore_Resolve_KVMissNoPlatformDefault(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewCredentialStore(testutil.NewJetStream(t, nc), fixedMasterKey(), 1, nil) + store := NewCredentialStore(testutil.NewJetStream(t, nc), fixedMasterKey(), nil) key, ok, err := store.Resolve(context.Background(), "999999999999", "anthropic") require.NoError(t, err) @@ -119,7 +119,7 @@ func TestEncryptDecryptSecret_RoundTrip(t *testing.T) { // Resolve's tolerated nil: a write has no platform-default fallback to fall // back to, so it must report the misconfiguration. func TestCredentialStore_PutCredential_RequiresJetStream(t *testing.T) { - store := NewCredentialStore(nil, fixedMasterKey(), 1, map[string]string{"anthropic": "sk-default"}) + store := NewCredentialStore(nil, fixedMasterKey(), map[string]string{"anthropic": "sk-default"}) err := store.PutCredential(context.Background(), "000000000001", "anthropic", "sk-account") require.Error(t, err) diff --git a/spinifex/gateway/bedrock/guardrail.go b/spinifex/gateway/bedrock/guardrail.go index a24aa3dd9..9567f58e9 100644 --- a/spinifex/gateway/bedrock/guardrail.go +++ b/spinifex/gateway/bedrock/guardrail.go @@ -89,15 +89,14 @@ type GuardrailStore struct { region string } -// NewGuardrailStore constructs a GuardrailStore over js, replicated across -// replicas nodes, minting ARNs for region. -func NewGuardrailStore(js jetstream.JetStream, replicas int, region string) *GuardrailStore { +// NewGuardrailStore constructs a GuardrailStore over js, minting ARNs for +// region. +func NewGuardrailStore(js jetstream.JetStream, region string) *GuardrailStore { return &GuardrailStore{ store: kvstore.New[GuardrailRecord](js, kvstore.Config{ - Name: bedrockGuardrailBucket, - History: bedrockGuardrailHistory, - Replicas: replicas, - Missing: "bedrock: guardrail store has no JetStream client configured", + Name: bedrockGuardrailBucket, + History: bedrockGuardrailHistory, + Missing: "bedrock: guardrail store has no JetStream client configured", }), region: region, } diff --git a/spinifex/gateway/bedrock/guardrail_test.go b/spinifex/gateway/bedrock/guardrail_test.go index c1ab3517d..d4638920a 100644 --- a/spinifex/gateway/bedrock/guardrail_test.go +++ b/spinifex/gateway/bedrock/guardrail_test.go @@ -20,7 +20,7 @@ const ( func newGuardrailTestStore(t *testing.T) *GuardrailStore { t.Helper() _, _, js := testutil.StartTestJetStream(t) - return NewGuardrailStore(js, 1, ptTestRegion) + return NewGuardrailStore(js, ptTestRegion) } // createGuardrailInput builds a minimal valid CreateGuardrailInput carrying diff --git a/spinifex/gateway/bedrock/logging.go b/spinifex/gateway/bedrock/logging.go index fb3f9c6d5..33da2c8f2 100644 --- a/spinifex/gateway/bedrock/logging.go +++ b/spinifex/gateway/bedrock/logging.go @@ -16,6 +16,7 @@ import ( "github.com/aws/aws-sdk-go/service/bedrock" "github.com/aws/aws-sdk-go/service/s3" "github.com/mulgadc/spinifex/spinifex/awserrors" + "github.com/mulgadc/spinifex/spinifex/clustersize" "github.com/mulgadc/spinifex/spinifex/kvstore" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/nats-io/nats.go/jetstream" @@ -51,11 +52,15 @@ const ( ) // EnsureInvocationStream idempotently creates (or updates) the invocation -// stream. Safe to call from every gateway node at boot. replicas must match -// the cluster's node count: at replicas=1, losing the one node holding this -// stream loses billing/audit records even though the control plane survives -// on the others, defeating the point of using JetStream over a lossy buffer. -func EnsureInvocationStream(ctx context.Context, js jetstream.JetStream, replicas int) (jetstream.Stream, error) { +// stream, replicated across the cluster. Safe to call from every gateway node +// at boot. On one replica, losing the one node holding this stream loses +// billing and audit records even though the control plane survives on the +// others, defeating the point of using JetStream over a lossy buffer. +func EnsureInvocationStream(ctx context.Context, js jetstream.JetStream) (jetstream.Stream, error) { + replicas, err := clustersize.Replicas() + if err != nil { + return nil, fmt.Errorf("ensure invocation stream: %w", err) + } stream, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ Name: InvocationStreamName, Subjects: []string{InvocationStreamSubject}, @@ -310,12 +315,11 @@ type LoggingConfigStore struct { var _ LoggingConfigReader = (*LoggingConfigStore)(nil) // NewLoggingConfigStore constructs a LoggingConfigStore. -func NewLoggingConfigStore(js jetstream.JetStream, replicas int) *LoggingConfigStore { +func NewLoggingConfigStore(js jetstream.JetStream) *LoggingConfigStore { return &LoggingConfigStore{store: kvstore.New[LoggingConfig](js, kvstore.Config{ - Name: bedrockLoggingConfigBucket, - History: bedrockLoggingConfigHistory, - Replicas: replicas, - Missing: "bedrock: logging config store has no JetStream client configured", + Name: bedrockLoggingConfigBucket, + History: bedrockLoggingConfigHistory, + Missing: "bedrock: logging config store has no JetStream client configured", })} } diff --git a/spinifex/gateway/bedrock/logging_test.go b/spinifex/gateway/bedrock/logging_test.go index 1d21b975f..4b81b1135 100644 --- a/spinifex/gateway/bedrock/logging_test.go +++ b/spinifex/gateway/bedrock/logging_test.go @@ -33,7 +33,7 @@ func (s stubLoggingConfigReader) Get(context.Context, string) (LoggingConfig, bo func TestLoggingConfigStore_PutGetDelete(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - store := NewLoggingConfigStore(js, 1) + store := NewLoggingConfigStore(js) ctx := context.Background() _, ok, err := store.Get(ctx, "000000000001") @@ -59,7 +59,7 @@ func TestLoggingConfigStore_PutGetDelete(t *testing.T) { func TestPutModelInvocationLoggingConfiguration_RejectsCloudWatch(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - store := NewLoggingConfigStore(js, 1) + store := NewLoggingConfigStore(js) input := &bedrock.PutModelInvocationLoggingConfigurationInput{ LoggingConfig: &bedrock.LoggingConfig{ @@ -74,7 +74,7 @@ func TestPutModelInvocationLoggingConfiguration_RejectsCloudWatch(t *testing.T) func TestPutModelInvocationLoggingConfiguration_RequiresBucketWhenDeliveryEnabled(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - store := NewLoggingConfigStore(js, 1) + store := NewLoggingConfigStore(js) input := &bedrock.PutModelInvocationLoggingConfigurationInput{ LoggingConfig: &bedrock.LoggingConfig{TextDataDeliveryEnabled: aws.Bool(true)}, @@ -89,7 +89,7 @@ func TestPutModelInvocationLoggingConfiguration_RequiresBucketWhenDeliveryEnable // undeliverable as text or image, so it must not slip past the same gate. func TestPutModelInvocationLoggingConfiguration_RequiresBucketForEmbeddingDelivery(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - store := NewLoggingConfigStore(js, 1) + store := NewLoggingConfigStore(js) input := &bedrock.PutModelInvocationLoggingConfigurationInput{ LoggingConfig: &bedrock.LoggingConfig{EmbeddingDataDeliveryEnabled: aws.Bool(true)}, @@ -105,7 +105,7 @@ func TestPutModelInvocationLoggingConfiguration_RequiresBucketForEmbeddingDelive // internal LoggingConfig. func TestModelInvocationLoggingConfiguration_RoundTripsViaSDKStructs(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) - store := NewLoggingConfigStore(js, 1) + store := NewLoggingConfigStore(js) ctx := context.Background() putInput := &bedrock.PutModelInvocationLoggingConfigurationInput{ @@ -159,7 +159,7 @@ func TestStreamRecorder_DropsAndCountsWhenStreamMissing(t *testing.T) { func TestStreamRecorder_Record_BodyOnlyWhenLoggingEnabled(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) ctx := context.Background() - _, err := EnsureInvocationStream(ctx, js, 1) + _, err := EnsureInvocationStream(ctx, js) require.NoError(t, err) consumer, err := EnsureDeliveryConsumer(ctx, js) require.NoError(t, err) @@ -198,7 +198,7 @@ func TestDeliveryConsumer_Run_WritesS3AndNeverLogsBodyText(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, err := EnsureInvocationStream(ctx, js, 1) + _, err := EnsureInvocationStream(ctx, js) require.NoError(t, err) consumer, err := EnsureDeliveryConsumer(ctx, js) require.NoError(t, err) @@ -247,7 +247,7 @@ func TestDeliveryConsumer_Run_DeliversMetadataOnlyRecordToS3(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, err := EnsureInvocationStream(ctx, js, 1) + _, err := EnsureInvocationStream(ctx, js) require.NoError(t, err) consumer, err := EnsureDeliveryConsumer(ctx, js) require.NoError(t, err) @@ -294,7 +294,7 @@ func TestDeliveryConsumer_Run_DeliversMetadataOnlyRecordToS3(t *testing.T) { // JetStream client: every accessor must report the misconfiguration rather // than panic on a nil handle. func TestLoggingConfigStore_RequiresJetStream(t *testing.T) { - store := NewLoggingConfigStore(nil, 1) + store := NewLoggingConfigStore(nil) ctx := context.Background() _, _, err := store.Get(ctx, "000000000001") diff --git a/spinifex/gateway/bedrock/prices.go b/spinifex/gateway/bedrock/prices.go index 7d74f0dd5..7f5812016 100644 --- a/spinifex/gateway/bedrock/prices.go +++ b/spinifex/gateway/bedrock/prices.go @@ -76,13 +76,12 @@ type PriceStore struct { var _ PriceResolver = (*PriceStore)(nil) // NewPriceStore constructs a PriceStore over the cluster's JetStream client, -// replicated across replicas nodes. -func NewPriceStore(js jetstream.JetStream, replicas int) *PriceStore { + +func NewPriceStore(js jetstream.JetStream) *PriceStore { return &PriceStore{store: kvstore.New[Price](js, kvstore.Config{ - Name: bedrockPricesBucket, - History: bedrockPricesHistory, - Replicas: replicas, - Missing: "bedrock: price store has no JetStream client configured", + Name: bedrockPricesBucket, + History: bedrockPricesHistory, + Missing: "bedrock: price store has no JetStream client configured", })} } diff --git a/spinifex/gateway/bedrock/prices_test.go b/spinifex/gateway/bedrock/prices_test.go index 4105583aa..cb6e5cc97 100644 --- a/spinifex/gateway/bedrock/prices_test.go +++ b/spinifex/gateway/bedrock/prices_test.go @@ -94,7 +94,7 @@ func TestResolvePrice_ResolverErrorPropagates(t *testing.T) { // TestWeightsStore_PutAndResolve_KV. func TestPriceStore_PutGetDelete(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewPriceStore(testutil.NewJetStream(t, nc), 1) + store := NewPriceStore(testutil.NewJetStream(t, nc)) ctx := context.Background() _, ok, err := store.Resolve(ctx, priceTestProviderModelID) @@ -123,7 +123,7 @@ func TestPriceStore_PutGetDelete(t *testing.T) { // JetStream client: every accessor must report the misconfiguration rather // than panic on a nil handle. func TestPriceStore_RequiresJetStream(t *testing.T) { - store := NewPriceStore(nil, 1) + store := NewPriceStore(nil) ctx := context.Background() _, _, err := store.Resolve(ctx, priceTestProviderModelID) diff --git a/spinifex/gateway/bedrock/provisioned.go b/spinifex/gateway/bedrock/provisioned.go index ed4832024..72218a8b2 100644 --- a/spinifex/gateway/bedrock/provisioned.go +++ b/spinifex/gateway/bedrock/provisioned.go @@ -81,16 +81,14 @@ type ProvisionedStore struct { endpoint EndpointProvisioner } -// NewProvisionedStore constructs a ProvisionedStore over js, replicated -// across replicas nodes, driving endpoint launches/teardowns through -// endpoint. -func NewProvisionedStore(js jetstream.JetStream, replicas int, region string, endpoint EndpointProvisioner) *ProvisionedStore { +// NewProvisionedStore constructs a ProvisionedStore over js, driving endpoint +// launches and teardowns through endpoint. +func NewProvisionedStore(js jetstream.JetStream, region string, endpoint EndpointProvisioner) *ProvisionedStore { return &ProvisionedStore{ store: kvstore.New[ProvisionedModelRecord](js, kvstore.Config{ - Name: bedrockProvisionedBucket, - History: bedrockProvisionedHistory, - Replicas: replicas, - Missing: "bedrock: provisioned store has no JetStream client configured", + Name: bedrockProvisionedBucket, + History: bedrockProvisionedHistory, + Missing: "bedrock: provisioned store has no JetStream client configured", }), region: region, endpoint: endpoint, diff --git a/spinifex/gateway/bedrock/provisioned_test.go b/spinifex/gateway/bedrock/provisioned_test.go index 236177e5b..f4c28ddd3 100644 --- a/spinifex/gateway/bedrock/provisioned_test.go +++ b/spinifex/gateway/bedrock/provisioned_test.go @@ -103,7 +103,7 @@ const ( func newProvisionedTestStore(t *testing.T, endpoint EndpointProvisioner) *ProvisionedStore { t.Helper() _, _, js := testutil.StartTestJetStream(t) - return NewProvisionedStore(js, 1, ptTestRegion, endpoint) + return NewProvisionedStore(js, ptTestRegion, endpoint) } func createInput(modelID, name string, units int64) *bedrock.CreateProvisionedModelThroughputInput { diff --git a/spinifex/gateway/bedrock/usage.go b/spinifex/gateway/bedrock/usage.go index 59ec3fc24..c06a9fbce 100644 --- a/spinifex/gateway/bedrock/usage.go +++ b/spinifex/gateway/bedrock/usage.go @@ -98,13 +98,12 @@ type UsageStore struct { var _ UsageReader = (*UsageStore)(nil) // NewUsageStore constructs a UsageStore over the cluster's JetStream client, -// replicated across replicas nodes. -func NewUsageStore(js jetstream.JetStream, replicas int) *UsageStore { + +func NewUsageStore(js jetstream.JetStream) *UsageStore { return &UsageStore{ store: kvstore.New[UsageCounters](js, kvstore.Config{ Name: bedrockUsageBucket, History: bedrockUsageHistory, - Replicas: replicas, Missing: "bedrock: usage store has no JetStream client configured", Attempts: usageCASRetries, Exhausted: usageCASExhausted, diff --git a/spinifex/gateway/bedrock/usage_test.go b/spinifex/gateway/bedrock/usage_test.go index bbddcc338..e2801f515 100644 --- a/spinifex/gateway/bedrock/usage_test.go +++ b/spinifex/gateway/bedrock/usage_test.go @@ -21,7 +21,7 @@ const ( func newTestUsageStore(t *testing.T) *UsageStore { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - return NewUsageStore(testutil.NewJetStream(t, nc), 1) + return NewUsageStore(testutil.NewJetStream(t, nc)) } // TestMicroUSDCost_Arithmetic covers D12's integer accrual, including @@ -184,7 +184,7 @@ func TestUsageConsumer_Run_RedeliveredRecordDoesNotDoubleCount(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, err := EnsureInvocationStream(ctx, js, 1) + _, err := EnsureInvocationStream(ctx, js) require.NoError(t, err) consumer, err := EnsureUsageConsumer(ctx, js) require.NoError(t, err) @@ -206,7 +206,7 @@ func TestUsageConsumer_Run_RedeliveredRecordDoesNotDoubleCount(t *testing.T) { _, err = js.Publish(ctx, InvocationStreamSubject, payload) require.NoError(t, err) - usage := NewUsageStore(js, 1) + usage := NewUsageStore(js) uc := NewUsageConsumer(usage, nil) done := make(chan struct{}) go func() { @@ -243,7 +243,7 @@ func TestUsageConsumer_Run_UnknownModelStillRecordsTokensCostUnknown(t *testing. ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, err := EnsureInvocationStream(ctx, js, 1) + _, err := EnsureInvocationStream(ctx, js) require.NoError(t, err) consumer, err := EnsureUsageConsumer(ctx, js) require.NoError(t, err) @@ -258,7 +258,7 @@ func TestUsageConsumer_Run_UnknownModelStillRecordsTokensCostUnknown(t *testing. _, err = js.Publish(ctx, InvocationStreamSubject, payload) require.NoError(t, err) - usage := NewUsageStore(js, 1) + usage := NewUsageStore(js) uc := NewUsageConsumer(usage, nil) done := make(chan struct{}) go func() { @@ -293,7 +293,7 @@ func TestBillingPeriod_MonthlyFormat(t *testing.T) { // JetStream client: both the counter bucket and the dedupe bucket must report // the misconfiguration rather than panic on a nil handle. func TestUsageStore_RequiresJetStream(t *testing.T) { - store := NewUsageStore(nil, 1) + store := NewUsageStore(nil) ctx := context.Background() _, _, err := store.Get(ctx, "000000000001", "m", "2026-08") diff --git a/spinifex/gateway/bedrock/weights.go b/spinifex/gateway/bedrock/weights.go index 9f82d9d7b..8df713fdb 100644 --- a/spinifex/gateway/bedrock/weights.go +++ b/spinifex/gateway/bedrock/weights.go @@ -57,13 +57,12 @@ type WeightsStore struct { var _ WeightsResolver = (*WeightsStore)(nil) // NewWeightsStore constructs a WeightsStore over the cluster's JetStream -// client, replicated across replicas nodes. -func NewWeightsStore(js jetstream.JetStream, replicas int) *WeightsStore { +// client. +func NewWeightsStore(js jetstream.JetStream) *WeightsStore { return &WeightsStore{store: kvstore.New[weightsRecord](js, kvstore.Config{ - Name: bedrockWeightsBucket, - History: bedrockWeightsHistory, - Replicas: replicas, - Missing: "bedrock: weights store has no JetStream client configured", + Name: bedrockWeightsBucket, + History: bedrockWeightsHistory, + Missing: "bedrock: weights store has no JetStream client configured", })} } diff --git a/spinifex/gateway/bedrock/weights_test.go b/spinifex/gateway/bedrock/weights_test.go index 3317db98d..2c821ed8b 100644 --- a/spinifex/gateway/bedrock/weights_test.go +++ b/spinifex/gateway/bedrock/weights_test.go @@ -20,7 +20,7 @@ func TestNoopWeightsResolver_ResolvesNothing(t *testing.T) { // snapshot: ("", false, nil), not an error. func TestWeightsStore_ResolveMiss(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewWeightsStore(testutil.NewJetStream(t, nc), 1) + store := NewWeightsStore(testutil.NewJetStream(t, nc)) snapshotID, ok, err := store.Resolve(context.Background(), "meta.llama3-2-1b-instruct-v1:0") require.NoError(t, err) @@ -33,7 +33,7 @@ func TestWeightsStore_ResolveMiss(t *testing.T) { // Resolve. func TestWeightsStore_PutAndResolve_KV(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewWeightsStore(testutil.NewJetStream(t, nc), 1) + store := NewWeightsStore(testutil.NewJetStream(t, nc)) ctx := context.Background() require.NoError(t, store.PutWeights(ctx, "meta.llama3-2-1b-instruct-v1:0", "s3://models/llama-3.2-1b/", "snap-0001")) @@ -49,7 +49,7 @@ func TestWeightsStore_PutAndResolve_KV(t *testing.T) { // history. func TestWeightsStore_PutOverwritesPrevious(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewWeightsStore(testutil.NewJetStream(t, nc), 1) + store := NewWeightsStore(testutil.NewJetStream(t, nc)) ctx := context.Background() require.NoError(t, store.PutWeights(ctx, "meta.llama3-2-1b-instruct-v1:0", "s3://models/llama-3.2-1b/", "snap-0001")) @@ -66,7 +66,7 @@ func TestWeightsStore_PutOverwritesPrevious(t *testing.T) { // reports ok=false, not an error. func TestWeightsStore_GetWeights_Miss(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewWeightsStore(testutil.NewJetStream(t, nc), 1) + store := NewWeightsStore(testutil.NewJetStream(t, nc)) entry, ok, err := store.GetWeights(context.Background(), "meta.llama3-2-1b-instruct-v1:0") require.NoError(t, err) @@ -80,7 +80,7 @@ func TestWeightsStore_GetWeights_Miss(t *testing.T) { // snapshot it is about to replace. func TestWeightsStore_GetWeights_ReturnsSourceURIAndSnapshot(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewWeightsStore(testutil.NewJetStream(t, nc), 1) + store := NewWeightsStore(testutil.NewJetStream(t, nc)) ctx := context.Background() require.NoError(t, store.PutWeights(ctx, "meta.llama3-2-1b-instruct-v1:0", "s3://models/llama-3.2-1b/", "snap-0001")) @@ -100,7 +100,7 @@ func TestWeightsStore_GetWeights_ReturnsSourceURIAndSnapshot(t *testing.T) { // both GetWeights and ListWeights. func TestWeightsStore_PutWeightsWithRevision_RoundTrips(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewWeightsStore(testutil.NewJetStream(t, nc), 1) + store := NewWeightsStore(testutil.NewJetStream(t, nc)) ctx := context.Background() require.NoError(t, store.PutWeightsWithRevision(ctx, "meta.llama3-2-1b-instruct-v1:0", "s3://ochre-weights/meta-llama/Llama-3.2-1B-Instruct/abc123/", "snap-0001", "abc123def456")) @@ -121,7 +121,7 @@ func TestWeightsStore_PutWeightsWithRevision_RoundTrips(t *testing.T) { // SourceRevision, not fail or fabricate one. func TestWeightsStore_PutWeights_LeavesSourceRevisionEmpty(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewWeightsStore(testutil.NewJetStream(t, nc), 1) + store := NewWeightsStore(testutil.NewJetStream(t, nc)) ctx := context.Background() require.NoError(t, store.PutWeights(ctx, "meta.llama3-2-1b-instruct-v1:0", "s3://models/llama-3.2-1b/", "snap-0001")) @@ -136,7 +136,7 @@ func TestWeightsStore_PutWeights_LeavesSourceRevisionEmpty(t *testing.T) { // bucket must report an empty list, not an error (jetstream.ErrNoKeysFound). func TestWeightsStore_ListWeights_Empty(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewWeightsStore(testutil.NewJetStream(t, nc), 1) + store := NewWeightsStore(testutil.NewJetStream(t, nc)) entries, err := store.ListWeights(context.Background()) require.NoError(t, err) @@ -148,7 +148,7 @@ func TestWeightsStore_ListWeights_Empty(t *testing.T) { // base64url KV key), its source URI and its snapshot ID, sorted by model ID. func TestWeightsStore_ListWeights_ReturnsAllStaged(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewWeightsStore(testutil.NewJetStream(t, nc), 1) + store := NewWeightsStore(testutil.NewJetStream(t, nc)) ctx := context.Background() require.NoError(t, store.PutWeights(ctx, "meta.llama3-2-1b-instruct-v1:0", "s3://models/llama-3.2-1b/", "snap-0001")) @@ -167,7 +167,7 @@ func TestWeightsStore_ListWeights_ReturnsAllStaged(t *testing.T) { // DeleteWeights drops the record and Resolve reports not-found afterwards. func TestWeightsStore_DeleteWeights_RemovesEntry(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewWeightsStore(testutil.NewJetStream(t, nc), 1) + store := NewWeightsStore(testutil.NewJetStream(t, nc)) ctx := context.Background() require.NoError(t, store.PutWeights(ctx, "meta.llama3-2-1b-instruct-v1:0", "s3://models/llama-3.2-1b/", "snap-0001")) @@ -184,7 +184,7 @@ func TestWeightsStore_DeleteWeights_RemovesEntry(t *testing.T) { // must not error on a missing key either. func TestWeightsStore_DeleteWeights_NeverStagedIsNotAnError(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) - store := NewWeightsStore(testutil.NewJetStream(t, nc), 1) + store := NewWeightsStore(testutil.NewJetStream(t, nc)) require.NoError(t, store.DeleteWeights(context.Background(), "meta.llama3-2-1b-instruct-v1:0")) } @@ -208,7 +208,7 @@ func TestSetWeightsResolver_NilRestoresNoop(t *testing.T) { // JetStream client: every accessor must report the misconfiguration rather // than panic on a nil handle. func TestWeightsStore_RequiresJetStream(t *testing.T) { - store := NewWeightsStore(nil, 1) + store := NewWeightsStore(nil) ctx := context.Background() _, _, err := store.Resolve(ctx, "meta.llama3-2-1b-instruct-v1:0") diff --git a/spinifex/gateway/ecrauth/ecrauth_test.go b/spinifex/gateway/ecrauth/ecrauth_test.go index 3504dcee5..a0cc4ade0 100644 --- a/spinifex/gateway/ecrauth/ecrauth_test.go +++ b/spinifex/gateway/ecrauth/ecrauth_test.go @@ -41,13 +41,13 @@ func TestLoadOrCreateSigningKey_CreatesThenReloads(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - key1, verify1, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + key1, verify1, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) require.NotEmpty(t, key1.Kid) require.Contains(t, verify1, key1.Kid) // Second call must reload the same persisted key, not mint a new one. - key2, verify2, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + key2, verify2, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) assert.Equal(t, key1.Kid, key2.Kid, "persisted signing key must be reused") assert.Len(t, verify2, 1) @@ -60,7 +60,7 @@ func TestLoadOrCreateSigningKey_SkipsAnUnreadableKey(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - good, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + good, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) kv, err := js.KeyValue(t.Context(), SigningBucket) @@ -68,7 +68,7 @@ func TestLoadOrCreateSigningKey_SkipsAnUnreadableKey(t *testing.T) { _, err = kv.Put(t.Context(), signingKeyName("stale-kid"), []byte("not decryptable with this master key")) require.NoError(t, err) - key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) assert.Equal(t, good.Kid, key.Kid, "the readable key must still be active") assert.Len(t, verify, 1, "an unreadable key contributes no verify key") @@ -80,14 +80,14 @@ func TestLoadOrCreateSigningKey_SkipsAKeyDeletedWhileReading(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - good, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + good, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) kv, err := js.KeyValue(t.Context(), SigningBucket) require.NoError(t, err) require.NoError(t, kv.Delete(t.Context(), signingKeyName("never-existed"))) - key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) assert.Equal(t, good.Kid, key.Kid) assert.Len(t, verify, 1) @@ -100,14 +100,14 @@ func TestLoadOrCreateSigningKey_FailsWhenNoKeyDecrypts(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - _, err := openSigningBucket(t.Context(), js, testMasterKey, 1) + _, err := openSigningBucket(t.Context(), js, testMasterKey) require.NoError(t, err) kv, err := js.KeyValue(t.Context(), SigningBucket) require.NoError(t, err) _, err = kv.Put(t.Context(), signingKeyName("stale-kid"), []byte("not decryptable with this master key")) require.NoError(t, err) - _, _, err = LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + _, _, err = LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.Error(t, err) assert.Contains(t, err.Error(), "master key does not match") @@ -119,7 +119,7 @@ func TestLoadOrCreateSigningKey_FailsWhenNoKeyDecrypts(t *testing.T) { func TestLoadOrCreateSigningKey_StoresEncrypted(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - key, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + key, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) kv, err := js.KeyValue(t.Context(), SigningBucket) @@ -133,14 +133,14 @@ func TestLoadOrCreateSigningKey_StoresEncrypted(t *testing.T) { func TestLoadOrCreateSigningKey_EmptyMasterKeyRejected(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - _, _, err := LoadOrCreateSigningKey(t.Context(), js, nil, 1) + _, _, err := LoadOrCreateSigningKey(t.Context(), js, nil) require.Error(t, err) } func TestIssuerVerifier_RoundTrip(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) iss := NewIssuer(key, testAudience) @@ -158,7 +158,7 @@ func TestIssuerVerifier_RoundTrip(t *testing.T) { func TestIssuer_MintRequiresAccountID(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - key, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + key, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) p := samplePrincipal() @@ -174,7 +174,7 @@ func TestIssuer_MintRequiresAccountID(t *testing.T) { func TestMint_RequiresCompleteIdentityPointer(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - key, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + key, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) iss := NewIssuer(key, testAudience) @@ -208,7 +208,7 @@ func TestSupportedPrincipalType(t *testing.T) { func TestVerifier_RejectsWrongAudience(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) tok, _, err := NewIssuer(key, testAudience).Mint(samplePrincipal()) @@ -221,7 +221,7 @@ func TestVerifier_RejectsWrongAudience(t *testing.T) { func TestVerifier_RejectsUnknownKid(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - key, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + key, _, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) tok, _, err := NewIssuer(key, testAudience).Mint(samplePrincipal()) @@ -235,7 +235,7 @@ func TestVerifier_RejectsUnknownKid(t *testing.T) { func TestVerifier_RejectsExpiredToken(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) // Hand-mint an already-expired token with the same signing key. @@ -263,7 +263,7 @@ func TestVerifier_RejectsExpiredToken(t *testing.T) { func TestVerifier_RejectsIncompleteIdentityPointer(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) base := func() Claims { @@ -310,7 +310,7 @@ func TestVerifier_RejectsIncompleteIdentityPointer(t *testing.T) { func TestVerifier_RejectsNonES256(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - _, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + _, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) // HS256 token must be refused regardless of signature. diff --git a/spinifex/gateway/ecrauth/rotator.go b/spinifex/gateway/ecrauth/rotator.go index 65377a45e..8b820f0c8 100644 --- a/spinifex/gateway/ecrauth/rotator.go +++ b/spinifex/gateway/ecrauth/rotator.go @@ -78,8 +78,8 @@ type Rotator struct { // NewRotator opens the signing-key bucket and builds a rotator that keeps issuer // and verifier current. replicas matches the cluster size for first-run bucket // creation. -func NewRotator(ctx context.Context, js jetstream.JetStream, masterKey []byte, replicas int, issuer *Issuer, verifier *Verifier) (*Rotator, error) { - kv, err := openSigningBucket(ctx, js, masterKey, replicas) +func NewRotator(ctx context.Context, js jetstream.JetStream, masterKey []byte, issuer *Issuer, verifier *Verifier) (*Rotator, error) { + kv, err := openSigningBucket(ctx, js, masterKey) if err != nil { return nil, err } diff --git a/spinifex/gateway/ecrauth/rotator_test.go b/spinifex/gateway/ecrauth/rotator_test.go index 2a9907d80..aa6efc905 100644 --- a/spinifex/gateway/ecrauth/rotator_test.go +++ b/spinifex/gateway/ecrauth/rotator_test.go @@ -79,12 +79,12 @@ func TestPlanRotation(t *testing.T) { func TestRotateOnce_MintsAndSwaps(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) issuer := NewIssuer(key, testAudience) verifier := NewVerifier(verify, testAudience) - kv, err := openSigningBucket(t.Context(), js, testMasterKey, 1) + kv, err := openSigningBucket(t.Context(), js, testMasterKey) require.NoError(t, err) // Token minted under the original key before rotation. @@ -120,9 +120,9 @@ func TestRotateOnce_MintsAndSwaps(t *testing.T) { func TestRotateOnce_PrunesExpiredKey(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - first, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + first, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) - kv, err := openSigningBucket(t.Context(), js, testMasterKey, 1) + kv, err := openSigningBucket(t.Context(), js, testMasterKey) require.NoError(t, err) second, err := generateSigningKey(t.Context(), kv, testMasterKey) require.NoError(t, err) @@ -151,11 +151,11 @@ func TestRotateOnce_PrunesExpiredKey(t *testing.T) { func TestRotator_RunRotatesThenStops(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + key, verify, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) issuer := NewIssuer(key, testAudience) verifier := NewVerifier(verify, testAudience) - kv, err := openSigningBucket(t.Context(), js, testMasterKey, 1) + kv, err := openSigningBucket(t.Context(), js, testMasterKey) require.NoError(t, err) r := &Rotator{ @@ -185,9 +185,9 @@ func TestRotator_RunRotatesThenStops(t *testing.T) { func TestIssuerVerifier_ConcurrentSwap(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - k1, v1, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey, 1) + k1, v1, err := LoadOrCreateSigningKey(t.Context(), js, testMasterKey) require.NoError(t, err) - kv, err := openSigningBucket(t.Context(), js, testMasterKey, 1) + kv, err := openSigningBucket(t.Context(), js, testMasterKey) require.NoError(t, err) k2, err := generateSigningKey(t.Context(), kv, testMasterKey) require.NoError(t, err) diff --git a/spinifex/gateway/ecrauth/signingkey.go b/spinifex/gateway/ecrauth/signingkey.go index e35a887b9..0fa7318d0 100644 --- a/spinifex/gateway/ecrauth/signingkey.go +++ b/spinifex/gateway/ecrauth/signingkey.go @@ -76,8 +76,8 @@ func kidFor(pub *ecdsa.PublicKey) (string, error) { // returns the active signing key. On first run it generates and persists one. // The active key is the newest by KV creation time (ties broken by kid), so all // nodes converge on the same signer and a freshly rotated key takes over. -func LoadOrCreateSigningKey(ctx context.Context, js jetstream.JetStream, masterKey []byte, replicas int) (*SigningKey, map[string]*ecdsa.PublicKey, error) { - kv, err := openSigningBucket(ctx, js, masterKey, replicas) +func LoadOrCreateSigningKey(ctx context.Context, js jetstream.JetStream, masterKey []byte) (*SigningKey, map[string]*ecdsa.PublicKey, error) { + kv, err := openSigningBucket(ctx, js, masterKey) if err != nil { return nil, nil, err } @@ -104,14 +104,14 @@ func LoadOrCreateSigningKey(ctx context.Context, js jetstream.JetStream, masterK // openSigningBucket validates inputs and returns the awsgw-keys KV handle, // creating the cluster-replicated bucket on first use. -func openSigningBucket(ctx context.Context, js jetstream.JetStream, masterKey []byte, replicas int) (jetstream.KeyValue, error) { +func openSigningBucket(ctx context.Context, js jetstream.JetStream, masterKey []byte) (jetstream.KeyValue, error) { if js == nil { return nil, errors.New("ecrauth: nil JetStream context") } if len(masterKey) == 0 { return nil, errors.New("ecrauth: empty master key") } - return kvutil.GetOrCreateBucketWithReplicas(ctx, js, SigningBucket, signingKeyHistory, replicas) + return kvutil.GetOrCreateBucket(ctx, js, SigningBucket, signingKeyHistory) } // reloadKeys reads every stored signing key, returning the verify set, each diff --git a/spinifex/gateway/ecrauth_middleware_test.go b/spinifex/gateway/ecrauth_middleware_test.go index 8a6f9d965..338b18cae 100644 --- a/spinifex/gateway/ecrauth_middleware_test.go +++ b/spinifex/gateway/ecrauth_middleware_test.go @@ -34,7 +34,7 @@ func newECRAuth(t *testing.T) (*gateway_ecrauth.Issuer, *gateway_ecrauth.Verifie t.Helper() _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - key, verify, err := gateway_ecrauth.LoadOrCreateSigningKey(t.Context(), js, ecrTestMasterKey(t), 1) + key, verify, err := gateway_ecrauth.LoadOrCreateSigningKey(t.Context(), js, ecrTestMasterKey(t)) require.NoError(t, err) return gateway_ecrauth.NewIssuer(key, ecrTestAudience), gateway_ecrauth.NewVerifier(verify, ecrTestAudience) } diff --git a/spinifex/gateway/oidc_discovery_test.go b/spinifex/gateway/oidc_discovery_test.go index b3672baf7..b0016af56 100644 --- a/spinifex/gateway/oidc_discovery_test.go +++ b/spinifex/gateway/oidc_discovery_test.go @@ -23,7 +23,7 @@ func seedDiscoveryCluster(t *testing.T) http.Handler { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, testDiscAccount, 1) + kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, testDiscAccount) if err != nil { t.Fatalf("account bucket: %v", err) } diff --git a/spinifex/gateway/spinifex_modelaccess_test.go b/spinifex/gateway/spinifex_modelaccess_test.go index a8081401f..7b42a356d 100644 --- a/spinifex/gateway/spinifex_modelaccess_test.go +++ b/spinifex/gateway/spinifex_modelaccess_test.go @@ -48,7 +48,7 @@ func accessGateway(t *testing.T) *GatewayConfig { _, _, js := testutil.StartTestJetStream(t) return &GatewayConfig{ DisableLogging: true, - BedrockAccessAdmin: gateway_bedrock.NewModelAccessStore(js, 1), + BedrockAccessAdmin: gateway_bedrock.NewModelAccessStore(js), IAMService: allowAllIAMService(), } } diff --git a/spinifex/handlers/bedrock/service.go b/spinifex/handlers/bedrock/service.go index b0ff6a0eb..26d604540 100644 --- a/spinifex/handlers/bedrock/service.go +++ b/spinifex/handlers/bedrock/service.go @@ -90,7 +90,7 @@ func NewService(nc *nats.Conn, deps ServiceDeps) *Service { return &Service{ nc: nc, deps: deps, - store: newEndpointStore(js, deps.Replicas), + store: newEndpointStore(js), leader: newLeaderBucket(js), } } diff --git a/spinifex/handlers/bedrock/store.go b/spinifex/handlers/bedrock/store.go index 18ee326dc..96f82e7d9 100644 --- a/spinifex/handlers/bedrock/store.go +++ b/spinifex/handlers/bedrock/store.go @@ -66,15 +66,13 @@ type endpointStore struct { *kvstore.Store[EndpointRecord] } -// newEndpointStore returns the endpoints store over js, replicated across -// replicas nodes. A nil js is permitted: every accessor then reports -// missingJetStream rather than panicking. -func newEndpointStore(js jetstream.JetStream, replicas int) *endpointStore { +// newEndpointStore returns the endpoints store over js. A nil js is permitted: +// every accessor then reports missingJetStream rather than panicking. +func newEndpointStore(js jetstream.JetStream) *endpointStore { return &endpointStore{Store: kvstore.New[EndpointRecord](js, kvstore.Config{ - Name: KVBucketEndpoints, - History: KVBucketEndpointsHistory, - Replicas: replicas, - Missing: missingJetStream, + Name: KVBucketEndpoints, + History: KVBucketEndpointsHistory, + Missing: missingJetStream, })} } diff --git a/spinifex/handlers/bedrock/store_test.go b/spinifex/handlers/bedrock/store_test.go index bb0849571..e6c9503ba 100644 --- a/spinifex/handlers/bedrock/store_test.go +++ b/spinifex/handlers/bedrock/store_test.go @@ -17,7 +17,7 @@ func TestEndpointKey_Base64EncodesColonBearingModelID(t *testing.T) { func newTestBucket(t *testing.T) *endpointStore { t.Helper() _, _, js := testutil.StartTestJetStream(t) - return newEndpointStore(js, 1) + return newEndpointStore(js) } func TestStore_CreateGetUpdateDelete(t *testing.T) { diff --git a/spinifex/handlers/ec2/natgw/service_impl.go b/spinifex/handlers/ec2/natgw/service_impl.go index 9af49933a..7d49ce5a1 100644 --- a/spinifex/handlers/ec2/natgw/service_impl.go +++ b/spinifex/handlers/ec2/natgw/service_impl.go @@ -84,23 +84,16 @@ func NewNatGatewayServiceImplWithNATS(ctx context.Context, natsConn *nats.Conn) }, nil } -// getOrCreateDeletedBucket creates the expiring bucket without changing an -// existing bucket's replica or placement configuration. +// getOrCreateDeletedBucket creates the expiring bucket that records a deletion +// for as long as a client may still ask about it. func getOrCreateDeletedBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) { // Terraform polls DescribeNatGateways after delete and expects state=deleted. - kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ - Bucket: KVBucketDeletedNatGateways, + return kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{ + Name: KVBucketDeletedNatGateways, Description: "Deleted NAT Gateways (auto-expire after 1 hour)", History: 1, TTL: time.Hour, }) - if err == nil { - return kv, nil - } - if !errors.Is(err, jetstream.ErrBucketExists) { - return nil, err - } - return js.KeyValue(ctx, KVBucketDeletedNatGateways) } // natGatewayEvent is published on vpc.add-nat-gateway / vpc.delete-nat-gateway topics. diff --git a/spinifex/handlers/ec2/spotinstance/service_impl.go b/spinifex/handlers/ec2/spotinstance/service_impl.go index 715924bae..240847e2c 100644 --- a/spinifex/handlers/ec2/spotinstance/service_impl.go +++ b/spinifex/handlers/ec2/spotinstance/service_impl.go @@ -71,24 +71,12 @@ func NewSpotInstanceServiceImplWithNATS(ctx context.Context, cfg *config.Config, // getOrCreateTerminalBucket opens the terminal bucket without changing its // existing replica or placement configuration, creating it only when absent. func getOrCreateTerminalBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, KVBucketSpotRequestsTerminal) - if err == nil { - return kv, nil - } - if !errors.Is(err, jetstream.ErrBucketNotFound) { - return nil, err - } - - kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ - Bucket: KVBucketSpotRequestsTerminal, + return kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{ + Name: KVBucketSpotRequestsTerminal, Description: "Terminal Spot Instance Requests (auto-expire after 1 hour)", History: 1, TTL: spotTerminalTTL, }) - if errors.Is(err, jetstream.ErrBucketExists) { - return js.KeyValue(ctx, KVBucketSpotRequestsTerminal) - } - return kv, err } // PutSpotInstanceRequests stores each request in the active bucket. diff --git a/spinifex/handlers/eks/addon_installer.go b/spinifex/handlers/eks/addon_installer.go index 0063c3067..f0a323675 100644 --- a/spinifex/handlers/eks/addon_installer.go +++ b/spinifex/handlers/eks/addon_installer.go @@ -37,16 +37,15 @@ type StagedAddonManifest struct { // The VM-side delivery slice applies it via the K3s auto-deploy dir; until then // the add-on sits CREATING. type stagingInstaller struct { - nc *nats.Conn - clusterSize int + nc *nats.Conn } var _ AddonInstaller = (*stagingInstaller)(nil) // newStagingInstaller returns a stagingInstaller bound to the daemon NATS -// connection, using clusterSize as the per-account bucket's replica count. -func newStagingInstaller(nc *nats.Conn, clusterSize int) *stagingInstaller { - return &stagingInstaller{nc: nc, clusterSize: clusterSize} +// connection. +func newStagingInstaller(nc *nats.Conn) *stagingInstaller { + return &stagingInstaller{nc: nc} } func (i *stagingInstaller) acctKV(ctx context.Context, accountID string) (jetstream.KeyValue, error) { @@ -57,7 +56,7 @@ func (i *stagingInstaller) acctKV(ctx context.Context, accountID string) (jetstr if err != nil { return nil, fmt.Errorf("jetstream: %w", err) } - return GetOrCreateAccountBucket(ctx, js, accountID, max(i.clusterSize, 1)) + return GetOrCreateAccountBucket(ctx, js, accountID) } func (i *stagingInstaller) Install(ctx context.Context, accountID, cluster string, rec *AddonRecord) error { diff --git a/spinifex/handlers/eks/addon_status_reconciler_test.go b/spinifex/handlers/eks/addon_status_reconciler_test.go index 0ca24e6c3..137b99b46 100644 --- a/spinifex/handlers/eks/addon_status_reconciler_test.go +++ b/spinifex/handlers/eks/addon_status_reconciler_test.go @@ -15,7 +15,7 @@ import ( func acctKVForTest(t *testing.T, svc *EKSServiceImpl) jetstream.KeyValue { t.Helper() js := testutil.NewJetStream(t, svc.deps.NATSConn) - kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) return kv } diff --git a/spinifex/handlers/eks/addons.go b/spinifex/handlers/eks/addons.go index 5d3c12a55..d0cc1f0b1 100644 --- a/spinifex/handlers/eks/addons.go +++ b/spinifex/handlers/eks/addons.go @@ -331,7 +331,7 @@ func (s *EKSServiceImpl) addonInstaller() AddonInstaller { if s.deps.AddonInstaller != nil { return s.deps.AddonInstaller } - return newStagingInstaller(s.deps.NATSConn, s.deps.ClusterSize) + return newStagingInstaller(s.deps.NATSConn) } // markAddonFailed best-effort flips a record to CREATE_FAILED with the error reason. diff --git a/spinifex/handlers/eks/addons_test.go b/spinifex/handlers/eks/addons_test.go index 0ad681eed..e185ef31a 100644 --- a/spinifex/handlers/eks/addons_test.go +++ b/spinifex/handlers/eks/addons_test.go @@ -243,7 +243,7 @@ func TestStagingInstaller_StagesManifest(t *testing.T) { require.NoError(t, err) js := testutil.NewJetStream(t, svc.deps.NATSConn) - kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) entry, err := kv.Get(t.Context(), AddonManifestKey("c1", albController)) require.NoError(t, err, "installer must stage a manifest for VM-side delivery") diff --git a/spinifex/handlers/eks/cluster_reconciler_state_test.go b/spinifex/handlers/eks/cluster_reconciler_state_test.go index badbe5ba0..ff64b92fd 100644 --- a/spinifex/handlers/eks/cluster_reconciler_state_test.go +++ b/spinifex/handlers/eks/cluster_reconciler_state_test.go @@ -21,9 +21,9 @@ func newStateReconcilerHarness(t *testing.T, opts ...ReconcilerOption) (*Cluster t.Helper() _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - leaderKV, err := InitLeaderBucket(t.Context(), js, 1) + leaderKV, err := InitLeaderBucket(t.Context(), js) require.NoError(t, err) - acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) require.NoError(t, PutClusterMeta(t.Context(), acctKV, sampleClusterMeta("alpha"))) diff --git a/spinifex/handlers/eks/cluster_reconciler_test.go b/spinifex/handlers/eks/cluster_reconciler_test.go index bff3a6346..448f40f91 100644 --- a/spinifex/handlers/eks/cluster_reconciler_test.go +++ b/spinifex/handlers/eks/cluster_reconciler_test.go @@ -58,9 +58,9 @@ func newReconcilerHarness(t *testing.T, healthURL string, opts ...ReconcilerOpti t.Helper() _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - leaderKV, err := InitLeaderBucket(t.Context(), js, 1) + leaderKV, err := InitLeaderBucket(t.Context(), js) require.NoError(t, err) - acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) require.NoError(t, PutClusterMeta(t.Context(), acctKV, sampleClusterMeta("alpha"))) @@ -72,9 +72,9 @@ func newReconcilerHarness(t *testing.T, healthURL string, opts ...ReconcilerOpti func TestNewClusterReconciler_EmptyInputsRejected(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - leaderKV, err := InitLeaderBucket(t.Context(), js, 1) + leaderKV, err := InitLeaderBucket(t.Context(), js) require.NoError(t, err) - acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) _, err = NewClusterReconciler(nil, acctKV, testAccountID, "alpha", "h", "") diff --git a/spinifex/handlers/eks/cluster_state_test.go b/spinifex/handlers/eks/cluster_state_test.go index c351c8ebd..bff0bb175 100644 --- a/spinifex/handlers/eks/cluster_state_test.go +++ b/spinifex/handlers/eks/cluster_state_test.go @@ -15,7 +15,7 @@ func newClusterStateTestKV(t *testing.T) jetstream.KeyValue { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) return kv } diff --git a/spinifex/handlers/eks/delete_cluster_test.go b/spinifex/handlers/eks/delete_cluster_test.go index 3bb429450..8bad32509 100644 --- a/spinifex/handlers/eks/delete_cluster_test.go +++ b/spinifex/handlers/eks/delete_cluster_test.go @@ -60,7 +60,7 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) nlb := newFakeNLBProvisioner() diff --git a/spinifex/handlers/eks/dns_reconcile_test.go b/spinifex/handlers/eks/dns_reconcile_test.go index c6a631e23..cf77a91d3 100644 --- a/spinifex/handlers/eks/dns_reconcile_test.go +++ b/spinifex/handlers/eks/dns_reconcile_test.go @@ -62,7 +62,7 @@ func TestDesiredDNSChanges_MetadataReadFailureIsNotAuthoritative(t *testing.T) { require.NoError(t, PutClusterMeta(t.Context(), fixture.kv, active)) js := testutil.NewJetStream(t, fixture.svc.deps.NATSConn) - corruptKV, err := GetOrCreateAccountBucket(t.Context(), js, "444455556666", 1) + corruptKV, err := GetOrCreateAccountBucket(t.Context(), js, "444455556666") require.NoError(t, err) _, err = corruptKV.Put(t.Context(), ClusterMetaKey("unreadable"), []byte("{not json")) require.NoError(t, err) diff --git a/spinifex/handlers/eks/eks_billable_reaper.go b/spinifex/handlers/eks/eks_billable_reaper.go index 20f432b5e..80589f159 100644 --- a/spinifex/handlers/eks/eks_billable_reaper.go +++ b/spinifex/handlers/eks/eks_billable_reaper.go @@ -69,7 +69,7 @@ func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) { continue // ENI gone or untagged: cannot confirm orphan-hood, never reap } - acctKV, err := GetOrCreateAccountBucket(ctx, js, clusterAccount, max(r.svc.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(ctx, js, clusterAccount) if err != nil { slog.WarnContext(ctx, "eks-billable: account bucket lookup failed", "account", clusterAccount, "err", err) continue diff --git a/spinifex/handlers/eks/nats_bootstrap_test.go b/spinifex/handlers/eks/nats_bootstrap_test.go index 5e891cb3b..07fc6a960 100644 --- a/spinifex/handlers/eks/nats_bootstrap_test.go +++ b/spinifex/handlers/eks/nats_bootstrap_test.go @@ -39,7 +39,7 @@ func newBootstrapHarness(t *testing.T) *natsBootstrapHarness { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) require.NoError(t, PutClusterMeta(t.Context(), kv, sampleClusterMeta(bootstrapTestCluster))) @@ -85,7 +85,7 @@ func (h *natsBootstrapHarness) waitSubsBound(base, want int) { func TestNewNATSBootstrap_RejectsBadInputs(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) _, err = NewNATSBootstrap(nil, kv, bootstrapTestMasterKey, testAccountID, "alpha") diff --git a/spinifex/handlers/eks/nodegroup.go b/spinifex/handlers/eks/nodegroup.go index cbc9cf8db..f49d079a7 100644 --- a/spinifex/handlers/eks/nodegroup.go +++ b/spinifex/handlers/eks/nodegroup.go @@ -195,7 +195,7 @@ func (s *EKSServiceImpl) nodegroupAcctKV(ctx context.Context, accountID string) if err != nil { return nil, fmt.Errorf("jetstream: %w", err) } - acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID) if err != nil { return nil, fmt.Errorf("get account bucket: %w", err) } diff --git a/spinifex/handlers/eks/nodegroup_test.go b/spinifex/handlers/eks/nodegroup_test.go index 37cd25979..1717fc7c2 100644 --- a/spinifex/handlers/eks/nodegroup_test.go +++ b/spinifex/handlers/eks/nodegroup_test.go @@ -242,7 +242,7 @@ func TestReclaimOrphanedNodegroups_TerminatesStrandedWorkers(t *testing.T) { func TestNodegroupRecord_CRUDRoundTrip(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) rec := &NodegroupRecord{ diff --git a/spinifex/handlers/eks/recovery_directive_test.go b/spinifex/handlers/eks/recovery_directive_test.go index 00236a439..b0104f4da 100644 --- a/spinifex/handlers/eks/recovery_directive_test.go +++ b/spinifex/handlers/eks/recovery_directive_test.go @@ -13,7 +13,7 @@ import ( func TestLoadRecoveryDirective_AbsentIsZeroNone(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) d, err := LoadRecoveryDirective(t.Context(), acctKV, "alpha", "i-missing") @@ -25,7 +25,7 @@ func TestLoadRecoveryDirective_AbsentIsZeroNone(t *testing.T) { func TestStoreRecoveryDirective_EpochIncrements(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) first, err := StoreRecoveryDirective(t.Context(), acctKV, "alpha", "i-0", RecoveryActionClusterReset, "", false) @@ -46,7 +46,7 @@ func TestStoreRecoveryDirective_EpochIncrements(t *testing.T) { func TestStoreRecoveryDirective_SnapshotRequiredRoundTrips(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) stored, err := StoreRecoveryDirective(t.Context(), acctKV, "alpha", "i-seed", RecoveryActionClusterReset, "etcd-frequent-20260709T010000Z.snap", true) diff --git a/spinifex/handlers/eks/service_impl.go b/spinifex/handlers/eks/service_impl.go index 938cfd1d6..e3c7ce1b1 100644 --- a/spinifex/handlers/eks/service_impl.go +++ b/spinifex/handlers/eks/service_impl.go @@ -50,11 +50,6 @@ type EKSServiceDeps struct { Region string HolderID string - // ClusterSize is the daemon's node count, used as the JetStream replica - // count for the lazily-created per-account and leader KV buckets so they - // match the cluster's other R3 streams instead of staying stuck at R1. - ClusterSize int - // InternalSuffix is the AWS-parity internal DNS suffix (e.g. spinifex.internal) // used to compose the worker's ECR registry host. InternalSuffix string @@ -305,7 +300,7 @@ func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) { // The leader bucket outlives every request, so its open is bounded by the // service's own background context rather than a caller's. ctx, cancel := context.WithCancel(context.Background()) - leaderKV, err := InitLeaderBucket(ctx, js, max(deps.ClusterSize, 1)) + leaderKV, err := InitLeaderBucket(ctx, js) if err != nil { cancel() return nil, err @@ -519,7 +514,7 @@ func (s *EKSServiceImpl) CreateCluster(ctx context.Context, input *eks.CreateClu if err != nil { return nil, logCreateErr(name, accountID, "jetstream", err) } - acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID) if err != nil { return nil, logCreateErr(name, accountID, "get account bucket", err) } @@ -1051,7 +1046,7 @@ func (s *EKSServiceImpl) DescribeCluster(ctx context.Context, input *eks.Describ if err != nil { return nil, fmt.Errorf("jetstream: %w", err) } - acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID) if err != nil { return nil, fmt.Errorf("get account bucket: %w", err) } @@ -1070,7 +1065,7 @@ func (s *EKSServiceImpl) ListClusters(ctx context.Context, input *eks.ListCluste if err != nil { return nil, eksReadUnavailableOr(err, "jetstream") } - acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID) if err != nil { return nil, eksReadUnavailableOr(err, "get account bucket") } @@ -1117,7 +1112,7 @@ func (s *EKSServiceImpl) DeleteCluster(ctx context.Context, input *eks.DeleteClu if err != nil { return nil, fmt.Errorf("jetstream: %w", err) } - acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID) if err != nil { return nil, fmt.Errorf("get account bucket: %w", err) } @@ -1536,7 +1531,7 @@ func (s *EKSServiceImpl) acctKVForCluster(ctx context.Context, accountID, cluste if err != nil { return nil, fmt.Errorf("jetstream: %w", err) } - acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(ctx, js, accountID) if err != nil { return nil, fmt.Errorf("get account bucket: %w", err) } @@ -2002,7 +1997,7 @@ func (s *EKSServiceImpl) accountBucket(ctx context.Context, accountID string) (j if err != nil { return nil, fmt.Errorf("jetstream: %w", err) } - return GetOrCreateAccountBucket(ctx, js, accountID, max(s.deps.ClusterSize, 1)) + return GetOrCreateAccountBucket(ctx, js, accountID) } // clusterNameFromARN extracts the cluster name from an EKS cluster ARN @@ -2232,7 +2227,7 @@ func (s *EKSServiceImpl) spawnReconciler(accountID, clusterName string, _ *Clust slog.Error("spawnReconciler: jetstream", "err", err) return } - acctKV, err := GetOrCreateAccountBucket(s.bgCtx, js, accountID, max(s.deps.ClusterSize, 1)) + acctKV, err := GetOrCreateAccountBucket(s.bgCtx, js, accountID) if err != nil { slog.Error("spawnReconciler: account bucket", "err", err) return diff --git a/spinifex/handlers/eks/service_impl_test.go b/spinifex/handlers/eks/service_impl_test.go index affc01c22..00bf0f58b 100644 --- a/spinifex/handlers/eks/service_impl_test.go +++ b/spinifex/handlers/eks/service_impl_test.go @@ -223,7 +223,7 @@ func TestEKSServiceImpl_NodegroupMethodsShimMode(t *testing.T) { func seedTestCluster(t *testing.T, svc *EKSServiceImpl, cluster string) { t.Helper() js := testutil.NewJetStream(t, svc.deps.NATSConn) - kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) require.NoError(t, PutClusterMeta(t.Context(), kv, &ClusterMeta{Name: cluster, Status: ClusterStatusActive})) } @@ -414,7 +414,7 @@ func TestAccessPolicy_DisassociateRemovesProjectedGroup(t *testing.T) { require.NoError(t, err) js := testutil.NewJetStream(t, svc.deps.NATSConn) - kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) rec, err := GetAccessEntryRecord(t.Context(), kv, "c1", testPrincipalARN) require.NoError(t, err) @@ -461,7 +461,7 @@ func TestEKSServiceImpl_OIDCMethodsReturnNotImplemented(t *testing.T) { func TestEKSServiceImpl_ClusterTagRoundTrip(t *testing.T) { svc := setupTestService(t) js := testutil.NewJetStream(t, svc.deps.NATSConn) - kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) const arn = "arn:aws:eks:us-east-1:111122223333:cluster/c1" @@ -503,7 +503,7 @@ func TestEKSServiceImpl_ClusterTagRoundTrip(t *testing.T) { func TestEKSServiceImpl_NodegroupTagRoundTrip(t *testing.T) { svc := setupTestService(t) js := testutil.NewJetStream(t, svc.deps.NATSConn) - kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) const arn = "arn:aws:eks:us-east-1:111122223333:nodegroup/c1/ng1/abc123" diff --git a/spinifex/handlers/eks/store.go b/spinifex/handlers/eks/store.go index 731a2769e..4575a41e1 100644 --- a/spinifex/handlers/eks/store.go +++ b/spinifex/handlers/eks/store.go @@ -232,9 +232,9 @@ func AccountWatchBuckets(ctx context.Context, nc *nats.Conn) ([]*kvstore.Bucket, // creating it on first use at the given replica count (clamped to a minimum // of 1). Idempotent: subsequent calls with the same accountID return the // existing handle. -func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string, replicas int) (jetstream.KeyValue, error) { +func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) { bucket := AccountBucketName(accountID) - kv, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, bucket, KVBucketEKSAccountHistory, replicas) + kv, err := kvutil.GetOrCreateBucket(ctx, js, bucket, KVBucketEKSAccountHistory) if err != nil { return nil, fmt.Errorf("failed to create EKS per-account KV bucket %s: %w", bucket, err) } @@ -245,13 +245,11 @@ func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accou } // InitLeaderBucket creates (or attaches to) the shared spinifex-eks-leader -// bucket used for per-cluster reconciler leader-lease CAS locks, at the given -// replica count (clamped to a minimum of 1). -func InitLeaderBucket(ctx context.Context, js jetstream.JetStream, replicas int) (jetstream.KeyValue, error) { +// bucket used for per-cluster reconciler leader-lease CAS locks. +func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) { return kvlease.OpenBucket(ctx, js, kvlease.BucketConfig{ - Name: KVBucketEKSLeader, - TTL: KVBucketEKSLeaderTTL, - Replicas: max(replicas, 1), - Version: KVBucketEKSLeaderVersion, + Name: KVBucketEKSLeader, + TTL: KVBucketEKSLeaderTTL, + Version: KVBucketEKSLeaderVersion, }) } diff --git a/spinifex/handlers/eks/store_test.go b/spinifex/handlers/eks/store_test.go index e7b57fbf2..9b719814d 100644 --- a/spinifex/handlers/eks/store_test.go +++ b/spinifex/handlers/eks/store_test.go @@ -14,11 +14,11 @@ func TestGetOrCreateAccountBucket_Idempotent(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - kv1, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + kv1, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) require.NotNil(t, kv1) - kv2, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 1) + kv2, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) require.NotNil(t, kv2) @@ -36,7 +36,7 @@ func TestGetOrCreateAccountBucket_ReplicasClamped(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID, 0) + kv, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID) require.NoError(t, err) require.NotNil(t, kv) @@ -51,12 +51,12 @@ func TestInitLeaderBucket_Idempotent(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - kv1, err := InitLeaderBucket(t.Context(), js, 1) + kv1, err := InitLeaderBucket(t.Context(), js) require.NoError(t, err) require.NotNil(t, kv1) assert.Equal(t, KVBucketEKSLeader, kv1.Bucket()) - kv2, err := InitLeaderBucket(t.Context(), js, 1) + kv2, err := InitLeaderBucket(t.Context(), js) require.NoError(t, err) require.NotNil(t, kv2) assert.Equal(t, KVBucketEKSLeader, kv2.Bucket()) @@ -68,7 +68,7 @@ func TestInitLeaderBucket_ReplicasClamped(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - kv, err := InitLeaderBucket(t.Context(), js, -1) + kv, err := InitLeaderBucket(t.Context(), js) require.NoError(t, err) require.NotNil(t, kv) diff --git a/spinifex/handlers/eks/watch_source_test.go b/spinifex/handlers/eks/watch_source_test.go index 0dbc57cfd..3110ba215 100644 --- a/spinifex/handlers/eks/watch_source_test.go +++ b/spinifex/handlers/eks/watch_source_test.go @@ -18,9 +18,9 @@ func TestAccountWatchBuckets_ReturnsOnePerAccountBucket(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - _, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "111111111111", 1) + _, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "111111111111") require.NoError(t, err) - _, err = handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "222222222222", 1) + _, err = handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "222222222222") require.NoError(t, err) _, err = kvutil.GetOrCreateBucket(t.Context(), js, "something-else", 1) require.NoError(t, err) @@ -51,7 +51,7 @@ func TestAccountWatchBuckets_NoAccountsIsNotAnError(t *testing.T) { func TestAccountWatchBuckets_BucketsAreWatchable(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - _, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "111111111111", 1) + _, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "111111111111") require.NoError(t, err) buckets, err := handlers_eks.AccountWatchBuckets(t.Context(), nc) diff --git a/spinifex/handlers/iam/account_lifecycle_test.go b/spinifex/handlers/iam/account_lifecycle_test.go index 8a78d1c04..fc3954914 100644 --- a/spinifex/handlers/iam/account_lifecycle_test.go +++ b/spinifex/handlers/iam/account_lifecycle_test.go @@ -16,7 +16,7 @@ func newLifecycleService(t *testing.T) handlers_iam.IAMService { masterKey, err := handlers_iam.GenerateMasterKey() require.NoError(t, err) - svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey) require.NoError(t, err) return svc } diff --git a/spinifex/handlers/iam/account_test.go b/spinifex/handlers/iam/account_test.go index 0971205e5..a8c1150dd 100644 --- a/spinifex/handlers/iam/account_test.go +++ b/spinifex/handlers/iam/account_test.go @@ -407,7 +407,7 @@ func TestCreateAccount_PublishesEvent(t *testing.T) { masterKey, err := GenerateMasterKey() require.NoError(t, err) - svc, err := NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + svc, err := NewIAMServiceImpl(t.Context(), nc, masterKey) require.NoError(t, err) eventCh := make(chan *nats.Msg, 1) diff --git a/spinifex/handlers/iam/managed_policies_test.go b/spinifex/handlers/iam/managed_policies_test.go index 96c2d728b..eb2871d96 100644 --- a/spinifex/handlers/iam/managed_policies_test.go +++ b/spinifex/handlers/iam/managed_policies_test.go @@ -67,7 +67,7 @@ func TestNewIAMServiceImpl_PropagatesBuiltinManagedPolicyParseError(t *testing.T builtinManagedPolicyParseErr = parseErr t.Cleanup(func() { builtinManagedPolicyParseErr = previousErr }) - _, err := NewIAMServiceImpl(t.Context(), nil, make([]byte, 32), 1) + _, err := NewIAMServiceImpl(t.Context(), nil, make([]byte, 32)) require.ErrorIs(t, err, parseErr) assert.ErrorContains(t, err, "init builtin managed policies") } diff --git a/spinifex/handlers/iam/oidc_providers.go b/spinifex/handlers/iam/oidc_providers.go index 0ff798458..8d8cb809f 100644 --- a/spinifex/handlers/iam/oidc_providers.go +++ b/spinifex/handlers/iam/oidc_providers.go @@ -50,8 +50,8 @@ func OIDCProviderARN(accountID, issuerHostPath string) string { } // GetOrCreateIAMAccountBucket opens the per-account IAM bucket, creating it on first use. -func GetOrCreateIAMAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string, replicas int) (jetstream.KeyValue, error) { - return kvutil.GetOrCreateBucketWithReplicas(ctx, js, IAMAccountBucketName(accountID), 1, max(replicas, 1)) +func GetOrCreateIAMAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) { + return kvutil.GetOrCreateBucket(ctx, js, IAMAccountBucketName(accountID), 1) } // OIDCProviderRecord is the stored shape of a registered OIDC identity provider. @@ -141,7 +141,7 @@ func (s *IAMServiceImpl) CreateOpenIDConnectProvider(accountID string, input *ia return nil, fmt.Errorf("marshal OIDC provider: %w", err) } - kv, err := GetOrCreateIAMAccountBucket(ctx, s.js, accountID, s.replicas) + kv, err := GetOrCreateIAMAccountBucket(ctx, s.js, accountID) if err != nil { return nil, fmt.Errorf("open IAM account bucket: %w", err) } diff --git a/spinifex/handlers/iam/provision_test.go b/spinifex/handlers/iam/provision_test.go index 13a48e22d..54536ef13 100644 --- a/spinifex/handlers/iam/provision_test.go +++ b/spinifex/handlers/iam/provision_test.go @@ -20,7 +20,7 @@ func newIAMService(t *testing.T) *handlers_iam.IAMServiceImpl { masterKey, err := handlers_iam.GenerateMasterKey() require.NoError(t, err) - svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey) require.NoError(t, err) return svc } diff --git a/spinifex/handlers/iam/retry.go b/spinifex/handlers/iam/retry.go index b76b6b143..73d820725 100644 --- a/spinifex/handlers/iam/retry.go +++ b/spinifex/handlers/iam/retry.go @@ -3,10 +3,12 @@ package handlers_iam import ( "context" "fmt" - "github.com/mulgadc/spinifex/spinifex/otelsetup" "log/slog" "time" + "github.com/mulgadc/spinifex/spinifex/clustersize" + "github.com/mulgadc/spinifex/spinifex/otelsetup" + "github.com/nats-io/nats.go" ) @@ -16,7 +18,7 @@ import ( // leave the service nil for the process lifetime. Blocks up to maxWait, then // returns the last error. Callers that legitimately run without a master key // must guard the call themselves. -func NewIAMServiceWithRetry(ctx context.Context, natsConn *nats.Conn, masterKey []byte, clusterSize int) (*IAMServiceImpl, error) { +func NewIAMServiceWithRetry(ctx context.Context, natsConn *nats.Conn, masterKey []byte) (*IAMServiceImpl, error) { const maxWait = 5 * time.Minute retryDelay := 500 * time.Millisecond start := time.Now() @@ -24,7 +26,7 @@ func NewIAMServiceWithRetry(ctx context.Context, natsConn *nats.Conn, masterKey for { attempt++ - svc, err := NewIAMServiceImpl(ctx, natsConn, masterKey, clusterSize) + svc, err := NewIAMServiceImpl(ctx, natsConn, masterKey) if err == nil { if attempt > 1 { slog.Info("IAM service initialized after retry", "attempts", attempt, "elapsed_ms", otelsetup.Millis(time.Since(start))) @@ -32,6 +34,13 @@ func NewIAMServiceWithRetry(ctx context.Context, natsConn *nats.Conn, masterKey return svc, nil } + // An undeclared cluster size is a configuration fault, not a cluster that + // has yet to form, so retrying it turns a clear error into five minutes + // of waiting followed by a misleading one. + if clustersize.Permanent(err) { + return nil, err + } + elapsed := time.Since(start) if elapsed >= maxWait { return nil, fmt.Errorf("IAM service unavailable after %s (%d attempts): %w", elapsed.Round(time.Second), attempt, err) diff --git a/spinifex/handlers/iam/retry_test.go b/spinifex/handlers/iam/retry_test.go new file mode 100644 index 000000000..0491b819a --- /dev/null +++ b/spinifex/handlers/iam/retry_test.go @@ -0,0 +1,38 @@ +package handlers_iam_test + +import ( + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/clustersize" + handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/stretchr/testify/require" +) + +// TestNewIAMServiceWithRetry_StopsOnAConfigFault covers the difference between a +// cluster that has yet to form and one that was misconfigured. +// +// The retry loop exists for the first: on a cold multi-node boot the KV store +// needs a quorum that does not exist yet, so waiting is right. An undeclared +// cluster size is the second, and waiting five minutes for it produces a timeout +// that names quorum as the cause when the cause was a config the process never +// loaded. This test bounds that: it must come back fast, and it must come back +// saying what is actually wrong. +func TestNewIAMServiceWithRetry_StopsOnAConfigFault(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + clustersize.RedeclareForTest(t, 0) + + masterKey, err := handlers_iam.GenerateMasterKey() + require.NoError(t, err) + + start := time.Now() + _, err = handlers_iam.NewIAMServiceWithRetry(t.Context(), nc, masterKey) + elapsed := time.Since(start) + + require.Error(t, err, "an undeclared cluster size must not look like a working IAM service") + require.ErrorIs(t, err, clustersize.ErrUndeclared, + "the error has to name the config fault, not the quorum the loop was waiting for") + require.Lessf(t, elapsed, 10*time.Second, + "returned after %s; a permanent fault must not consume the retry budget", elapsed) +} diff --git a/spinifex/handlers/iam/service_impl.go b/spinifex/handlers/iam/service_impl.go index 106b25649..3e1210b0b 100644 --- a/spinifex/handlers/iam/service_impl.go +++ b/spinifex/handlers/iam/service_impl.go @@ -106,16 +106,13 @@ type IAMServiceImpl struct { instanceProfilesBucket jetstream.KeyValue groupsBucket jetstream.KeyValue key *masterkey.Key - // replicas is the JetStream replication factor for lazily-created per-account buckets. - replicas int } var _ IAMService = (*IAMServiceImpl)(nil) // NewIAMServiceImpl creates a new IAM service backed by NATS JetStream KV. -// clusterSize sets the replication factor; pass 1 for single-node or test setups. // The context bounds bucket creation and the schema migrations only. -func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byte, clusterSize int) (*IAMServiceImpl, error) { +func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byte) (*IAMServiceImpl, error) { if builtinManagedPolicyParseErr != nil { return nil, fmt.Errorf("init builtin managed policies: %w", builtinManagedPolicyParseErr) } @@ -123,14 +120,12 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt return nil, fmt.Errorf("master key must be 32 bytes, got %d", len(masterKey)) } - replicas := max(clusterSize, 1) - js, err := jetstream.New(natsConn) if err != nil { return nil, fmt.Errorf("get JetStream context: %w", err) } - usersBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketUsers, 10, replicas) + usersBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketUsers, 10) if err != nil { return nil, fmt.Errorf("init users bucket: %w", err) } @@ -138,7 +133,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt return nil, fmt.Errorf("migrate %s: %w", KVBucketUsers, err) } - accessKeysBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketAccessKeys, 5, replicas) + accessKeysBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketAccessKeys, 5) if err != nil { return nil, fmt.Errorf("init access keys bucket: %w", err) } @@ -146,7 +141,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt return nil, fmt.Errorf("migrate %s: %w", KVBucketAccessKeys, err) } - policiesBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketPolicies, 10, replicas) + policiesBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketPolicies, 10) if err != nil { return nil, fmt.Errorf("init policies bucket: %w", err) } @@ -154,7 +149,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt return nil, fmt.Errorf("migrate %s: %w", KVBucketPolicies, err) } - accountsBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketAccounts, 5, replicas) + accountsBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketAccounts, 5) if err != nil { return nil, fmt.Errorf("init accounts bucket: %w", err) } @@ -162,7 +157,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt return nil, fmt.Errorf("migrate %s: %w", KVBucketAccounts, err) } - accountCounterBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketAccountCounter, 5, replicas) + accountCounterBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketAccountCounter, 5) if err != nil { return nil, fmt.Errorf("init account counter bucket: %w", err) } @@ -170,7 +165,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt return nil, fmt.Errorf("migrate %s: %w", KVBucketAccountCounter, err) } - rolesBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketRoles, 10, replicas) + rolesBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketRoles, 10) if err != nil { return nil, fmt.Errorf("init roles bucket: %w", err) } @@ -178,7 +173,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt return nil, fmt.Errorf("migrate %s: %w", KVBucketRoles, err) } - instanceProfilesBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketInstanceProfiles, 10, replicas) + instanceProfilesBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketInstanceProfiles, 10) if err != nil { return nil, fmt.Errorf("init instance profiles bucket: %w", err) } @@ -186,7 +181,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt return nil, fmt.Errorf("migrate %s: %w", KVBucketInstanceProfiles, err) } - groupsBucket, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketGroups, 10, replicas) + groupsBucket, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketGroups, 10) if err != nil { return nil, fmt.Errorf("init groups bucket: %w", err) } @@ -206,8 +201,7 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt "accounts_bucket", KVBucketAccounts, "roles_bucket", KVBucketRoles, "instance_profiles_bucket", KVBucketInstanceProfiles, - "groups_bucket", KVBucketGroups, - "replicas", replicas) + "groups_bucket", KVBucketGroups) return &IAMServiceImpl{ js: js, @@ -221,7 +215,6 @@ func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byt instanceProfilesBucket: instanceProfilesBucket, groupsBucket: groupsBucket, key: key, - replicas: replicas, }, nil } diff --git a/spinifex/handlers/iam/service_impl_test.go b/spinifex/handlers/iam/service_impl_test.go index 19c395787..287b7e1df 100644 --- a/spinifex/handlers/iam/service_impl_test.go +++ b/spinifex/handlers/iam/service_impl_test.go @@ -29,7 +29,7 @@ func setupTestIAMService(t *testing.T) *IAMServiceImpl { masterKey, err := GenerateMasterKey() require.NoError(t, err) - svc, err := NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + svc, err := NewIAMServiceImpl(t.Context(), nc, masterKey) require.NoError(t, err) return svc } @@ -2381,7 +2381,7 @@ func TestSensitiveDataNotLogged_MasterKey(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) masterKey, err := GenerateMasterKey() require.NoError(t, err) - _, err = NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + _, err = NewIAMServiceImpl(t.Context(), nc, masterKey) require.NoError(t, err) logOutput := buf.String() diff --git a/spinifex/handlers/quota/bedrock.go b/spinifex/handlers/quota/bedrock.go index 85ce7a7ed..3333bbc7e 100644 --- a/spinifex/handlers/quota/bedrock.go +++ b/spinifex/handlers/quota/bedrock.go @@ -167,11 +167,11 @@ type bedrockRPMSnapshot struct { // RunBedrockRPMSync periodically snapshots every account this gateway has // rate-limited into kv, until ctx is cancelled. A no-op when the RPM // dimension is disabled, so a default-off deployment starts no ticker. -func (s *Service) RunBedrockRPMSync(ctx context.Context, js jetstream.JetStream, replicas int) { +func (s *Service) RunBedrockRPMSync(ctx context.Context, js jetstream.JetStream) { if s == nil || !s.limits.RequestsPerMinuteEnabled { return } - kv, err := kvutil.GetOrCreateBucketWithReplicas(ctx, js, KVBucketBedrockRPMUsage, 1, replicas) + kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketBedrockRPMUsage, 1) if err != nil { slog.Warn("bedrock rpm sync: open usage bucket failed, sync disabled", "err", err) return diff --git a/spinifex/handlers/quota/bedrock_test.go b/spinifex/handlers/quota/bedrock_test.go index 746934c64..313d52000 100644 --- a/spinifex/handlers/quota/bedrock_test.go +++ b/spinifex/handlers/quota/bedrock_test.go @@ -195,7 +195,7 @@ func TestRunBedrockRPMSync_DisabledNoop(t *testing.T) { s := New(Limits{}, nil) done := make(chan struct{}) go func() { - s.RunBedrockRPMSync(context.Background(), nil, 1) + s.RunBedrockRPMSync(context.Background(), nil) close(done) }() select { diff --git a/spinifex/handlers/quota/records_test.go b/spinifex/handlers/quota/records_test.go index c0ba852c4..34e4c7119 100644 --- a/spinifex/handlers/quota/records_test.go +++ b/spinifex/handlers/quota/records_test.go @@ -21,9 +21,8 @@ func recordStore(t *testing.T) *kvstore.Store[vm.InstanceRecord] { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) return kvstore.New[vm.InstanceRecord](testutil.NewJetStream(t, nc), kvstore.Config{ - Name: "spinifex-instance-state", - History: 1, - Replicas: 1, + Name: "spinifex-instance-state", + History: 1, }) } diff --git a/spinifex/handlers/sts/assume_role_with_web_identity_test.go b/spinifex/handlers/sts/assume_role_with_web_identity_test.go index 3dff7ccd9..160faabc3 100644 --- a/spinifex/handlers/sts/assume_role_with_web_identity_test.go +++ b/spinifex/handlers/sts/assume_role_with_web_identity_test.go @@ -74,7 +74,7 @@ func newWebIdentityFixture(t *testing.T, svc *STSServiceImpl, accountID string) federatedARN := handlers_iam.OIDCProviderARN(accountID, issuerHostPath) // Publish JWKS to the per-cluster EKS bucket. - kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), svc.js, accountID, 1) + kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), svc.js, accountID) require.NoError(t, err) raw, err := json.Marshal(jwks) require.NoError(t, err) @@ -364,7 +364,7 @@ func TestAssumeRoleWithWebIdentity_ProviderNotRegistered(t *testing.T) { federatedARN := handlers_iam.OIDCProviderARN(testCallerAccountID, issuerHostPath) // Publish JWKS — but skip the IAM provider registration. - kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), svc.js, testCallerAccountID, 1) + kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), svc.js, testCallerAccountID) require.NoError(t, err) jwk := signingJWK(t, &priv.PublicKey) jwk.Alg, jwk.Use, jwk.Kid = "ES256", "sig", kid diff --git a/spinifex/handlers/sts/oidc_jwks_test.go b/spinifex/handlers/sts/oidc_jwks_test.go index 1a8f1d937..018d7d403 100644 --- a/spinifex/handlers/sts/oidc_jwks_test.go +++ b/spinifex/handlers/sts/oidc_jwks_test.go @@ -80,7 +80,7 @@ func TestFetchClusterJWKS_RoundTrip(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "123456789012", 1) + kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "123456789012") require.NoError(t, err) want := &JWKS{Keys: []JWK{ @@ -104,7 +104,7 @@ func TestFetchClusterJWKS_EmptyKeysIsError(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "123456789012", 1) + kv, err := handlers_eks.GetOrCreateAccountBucket(t.Context(), js, "123456789012") require.NoError(t, err) _, err = kv.Put(t.Context(), handlers_eks.OIDCJWKSKey("empty-cluster"), []byte(`{"keys":[]}`)) diff --git a/spinifex/handlers/sts/service_impl.go b/spinifex/handlers/sts/service_impl.go index fe25b12d0..e7f1b334e 100644 --- a/spinifex/handlers/sts/service_impl.go +++ b/spinifex/handlers/sts/service_impl.go @@ -37,10 +37,9 @@ type STSServiceImpl struct { var _ STSService = (*STSServiceImpl)(nil) // NewSTSServiceImpl constructs an STSServiceImpl. masterKey must be the 32-byte -// key shared with IAMServiceImpl. clusterSize sets the JetStream replication factor; -// pass 1 for single-node or test setups. The context bounds bucket creation and -// the schema migration only. -func NewSTSServiceImpl(ctx context.Context, natsConn *nats.Conn, iamSvc handlers_iam.IAMService, masterKey []byte, clusterSize int) (*STSServiceImpl, error) { +// key shared with IAMServiceImpl. The context bounds bucket creation and the +// schema migration only. +func NewSTSServiceImpl(ctx context.Context, natsConn *nats.Conn, iamSvc handlers_iam.IAMService, masterKey []byte) (*STSServiceImpl, error) { if natsConn == nil { return nil, errors.New("nil NATS connection") } @@ -51,21 +50,18 @@ func NewSTSServiceImpl(ctx context.Context, natsConn *nats.Conn, iamSvc handlers return nil, fmt.Errorf("master key must be %d bytes, got %d", masterKeySize, len(masterKey)) } - replicas := max(clusterSize, 1) - js, err := jetstream.New(natsConn) if err != nil { return nil, fmt.Errorf("get JetStream context: %w", err) } - sessions, err := initSessionCredentialsStore(ctx, js, replicas) + sessions, err := initSessionCredentialsStore(ctx, js) if err != nil { return nil, fmt.Errorf("init session credentials bucket: %w", err) } slog.Info("STS service initialized", - "sessions_bucket", KVBucketSessionCredentials, - "replicas", replicas) + "sessions_bucket", KVBucketSessionCredentials) return &STSServiceImpl{ natsConn: natsConn, diff --git a/spinifex/handlers/sts/service_impl_test.go b/spinifex/handlers/sts/service_impl_test.go index 1b8ba311d..d730eb5e9 100644 --- a/spinifex/handlers/sts/service_impl_test.go +++ b/spinifex/handlers/sts/service_impl_test.go @@ -24,17 +24,17 @@ func newTestSetup(t *testing.T) (*STSServiceImpl, *nats.Conn) { masterKey, err := handlers_iam.GenerateMasterKey() require.NoError(t, err) - iamSvc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + iamSvc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey) require.NoError(t, err) - stsSvc, err := NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey, 1) + stsSvc, err := NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey) require.NoError(t, err) return stsSvc, nc } func TestNewSTSServiceImpl_RejectsNilNATSConn(t *testing.T) { masterKey := bytes.Repeat([]byte{0x01}, masterKeySize) - _, err := NewSTSServiceImpl(t.Context(), nil, nopIAMService{}, masterKey, 1) + _, err := NewSTSServiceImpl(t.Context(), nil, nopIAMService{}, masterKey) require.Error(t, err) assert.Contains(t, err.Error(), "NATS") } @@ -42,7 +42,7 @@ func TestNewSTSServiceImpl_RejectsNilNATSConn(t *testing.T) { func TestNewSTSServiceImpl_RejectsNilIAMService(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) masterKey := bytes.Repeat([]byte{0x01}, masterKeySize) - _, err := NewSTSServiceImpl(t.Context(), nc, nil, masterKey, 1) + _, err := NewSTSServiceImpl(t.Context(), nc, nil, masterKey) require.Error(t, err) assert.Contains(t, err.Error(), "IAM") } @@ -51,7 +51,7 @@ func TestNewSTSServiceImpl_RejectsWrongMasterKeySize(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) for _, size := range []int{0, 1, 16, 31, 33, 64} { t.Run("size", func(t *testing.T) { - _, err := NewSTSServiceImpl(t.Context(), nc, nopIAMService{}, bytes.Repeat([]byte{0xaa}, size), 1) + _, err := NewSTSServiceImpl(t.Context(), nc, nopIAMService{}, bytes.Repeat([]byte{0xaa}, size)) require.Error(t, err) assert.Contains(t, err.Error(), "master key") }) @@ -68,10 +68,10 @@ func TestNewSTSServiceImpl_NormalisesNegativeClusterSize(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) masterKey, err := handlers_iam.GenerateMasterKey() require.NoError(t, err) - iamSvc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey, 1) + iamSvc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey) require.NoError(t, err) - svc, err := NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey, 0) + svc, err := NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey) require.NoError(t, err) require.NotNil(t, svc) } diff --git a/spinifex/handlers/sts/session_credentials.go b/spinifex/handlers/sts/session_credentials.go index e3bcd26c9..9deb6d7e0 100644 --- a/spinifex/handlers/sts/session_credentials.go +++ b/spinifex/handlers/sts/session_credentials.go @@ -66,12 +66,11 @@ type SessionCredential struct { // sessionCredentialsConfig describes the session-credentials bucket. History is // fixed at 1: credentials are write-once at mint and delete-once at expiry. -func sessionCredentialsConfig(replicas int) kvstore.Config { +func sessionCredentialsConfig() kvstore.Config { return kvstore.Config{ - Name: KVBucketSessionCredentials, - History: 1, - Replicas: replicas, - Missing: "session credentials KV bucket not initialized", + Name: KVBucketSessionCredentials, + History: 1, + Missing: "session credentials KV bucket not initialized", OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error { return migrate.DefaultRegistry.RunKV(ctx, KVBucketSessionCredentials, kv, KVBucketSessionCredentialsVersion) }, @@ -79,9 +78,8 @@ func sessionCredentialsConfig(replicas int) kvstore.Config { } // initSessionCredentialsStore opens (or creates) the session-credentials bucket. -func initSessionCredentialsStore(ctx context.Context, js jetstream.JetStream, replicas int) (*kvstore.Store[SessionCredential], error) { - // kvstore clamps replicas to a minimum of 1, so a zero clusterSize still creates. - store := kvstore.New[SessionCredential](js, sessionCredentialsConfig(replicas)) +func initSessionCredentialsStore(ctx context.Context, js jetstream.JetStream) (*kvstore.Store[SessionCredential], error) { + store := kvstore.New[SessionCredential](js, sessionCredentialsConfig()) // Opened eagerly: a bucket that cannot be created must fail service // construction, not the first AssumeRole that needs to mint into it. if _, err := store.KV(ctx); err != nil { diff --git a/spinifex/handlers/sts/session_credentials_test.go b/spinifex/handlers/sts/session_credentials_test.go index 9df45beca..0711e74c9 100644 --- a/spinifex/handlers/sts/session_credentials_test.go +++ b/spinifex/handlers/sts/session_credentials_test.go @@ -18,7 +18,7 @@ import ( func setupBucket(t *testing.T) *kvstore.Store[SessionCredential] { t.Helper() _, nc, _ := testutil.StartTestJetStream(t) - store, err := initSessionCredentialsStore(t.Context(), testutil.NewJetStream(t, nc), 1) + store, err := initSessionCredentialsStore(t.Context(), testutil.NewJetStream(t, nc)) require.NoError(t, err) return store } @@ -53,7 +53,7 @@ func TestInitSessionCredentialsBucket_StampsVersion(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - store, err := initSessionCredentialsStore(t.Context(), js, 1) + store, err := initSessionCredentialsStore(t.Context(), js) require.NoError(t, err) kv, err := store.KV(t.Context()) require.NoError(t, err) @@ -67,11 +67,11 @@ func TestInitSessionCredentialsBucket_Idempotent(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - store1, err := initSessionCredentialsStore(t.Context(), js, 1) + store1, err := initSessionCredentialsStore(t.Context(), js) require.NoError(t, err) // Reopen — must return the same bucket without error. - store2, err := initSessionCredentialsStore(t.Context(), js, 1) + store2, err := initSessionCredentialsStore(t.Context(), js) require.NoError(t, err) assert.Equal(t, store1.Name(), store2.Name()) } diff --git a/spinifex/idempotency/store.go b/spinifex/idempotency/store.go index 2a86d8a65..533ca4f8c 100644 --- a/spinifex/idempotency/store.go +++ b/spinifex/idempotency/store.go @@ -14,6 +14,8 @@ import ( "time" "github.com/nats-io/nats.go/jetstream" + + "github.com/mulgadc/spinifex/spinifex/kvutil" ) const ( @@ -59,14 +61,7 @@ type Store[T any] struct { // from NewStore so several typed stores can share one bucket without rebinding // it per request. func OpenBucket(ctx context.Context, js jetstream.JetStream, bucket string, ttl time.Duration) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, bucket) - if errors.Is(err, jetstream.ErrBucketNotFound) { - kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ - Bucket: bucket, - History: 1, - TTL: ttl, - }) - } + kv, err := kvutil.GetOrCreateBucketWithTTL(ctx, js, bucket, 1, ttl) if err != nil { return nil, fmt.Errorf("open idempotency bucket %s: %w", bucket, err) } diff --git a/spinifex/instancecache/cache_test.go b/spinifex/instancecache/cache_test.go index afc4b1aab..c51d18a42 100644 --- a/spinifex/instancecache/cache_test.go +++ b/spinifex/instancecache/cache_test.go @@ -44,7 +44,7 @@ func newTestCache(t *testing.T, mutate ...func(*Config)) (*Cache, jetstream.KeyV require.NoError(t, err) cfg := Config{ - Bucket: kvstore.Config{Name: bucket, History: 1, Replicas: 1}, + Bucket: kvstore.Config{Name: bucket, History: 1}, Prefix: testPrefix, RetryInterval: 20 * time.Millisecond, } @@ -243,7 +243,7 @@ func TestPeriodicResync_TTLExpiredRecordRemovedWithNoWatchEvent(t *testing.T) { require.NoError(t, err) c := New(js, Config{ - Bucket: kvstore.Config{Name: bucket, History: 1, Replicas: 1}, + Bucket: kvstore.Config{Name: bucket, History: 1}, Prefix: testPrefix, RetryInterval: 20 * time.Millisecond, }) diff --git a/spinifex/instancecache/liveness_test.go b/spinifex/instancecache/liveness_test.go index 105a3b460..ee67248ee 100644 --- a/spinifex/instancecache/liveness_test.go +++ b/spinifex/instancecache/liveness_test.go @@ -23,7 +23,7 @@ func newTestLiveness(t *testing.T) (*instancecache.Liveness, jetstream.KeyValue) kv, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: bucket, History: 1}) require.NoError(t, err) - l := instancecache.NewLiveness(js, kvstore.Config{Name: bucket, History: 1, Replicas: 1}) + l := instancecache.NewLiveness(js, kvstore.Config{Name: bucket, History: 1}) return l, kv } diff --git a/spinifex/instancecache/resume_from_revision_test.go b/spinifex/instancecache/resume_from_revision_test.go index e3f1e9864..8709eeeca 100644 --- a/spinifex/instancecache/resume_from_revision_test.go +++ b/spinifex/instancecache/resume_from_revision_test.go @@ -25,7 +25,7 @@ func TestResumeFromRevision_ReplaysWhatTheSnapshotMissed(t *testing.T) { _, _, js := testutil.StartTestJetStream(t) ctx := context.Background() - store := kvstore.New[vm.InstanceRecord](js, kvstore.Config{Name: "resume-bucket", History: 1, Replicas: 1}) + store := kvstore.New[vm.InstanceRecord](js, kvstore.Config{Name: "resume-bucket", History: 1}) rec := func(id string) *vm.InstanceRecord { r := &vm.InstanceRecord{} diff --git a/spinifex/kvlease/bucket.go b/spinifex/kvlease/bucket.go index a3bbd9454..1675d2593 100644 --- a/spinifex/kvlease/bucket.go +++ b/spinifex/kvlease/bucket.go @@ -7,6 +7,8 @@ import ( "log/slog" "time" + "github.com/mulgadc/spinifex/spinifex/clustersize" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/migrate" "github.com/mulgadc/spinifex/spinifex/otelsetup" "github.com/nats-io/nats.go" @@ -28,10 +30,6 @@ type BucketConfig struct { // does not park the key forever. It must match the lease's own Config.TTL. TTL time.Duration - // Replicas is the replica count applied on a genuine first creation only. - // Zero leaves it to the server, which means one. - Replicas int - // Version runs the registered schema migrations for the bucket after it is // opened. Zero skips them, for a bucket holding nothing but lease keys. Version int @@ -42,18 +40,32 @@ type BucketConfig struct { // // Attach before create, because callers re-open this on every reconcile tick: // CreateKeyValue against a bucket that already exists is a STREAM.CREATE the -// meta leader answers with an error, so creating first bills one per tick. It is -// also what makes Replicas mean "on first creation" rather than a config change -// against a live bucket. +// meta leader answers with an error, so creating first bills one per tick. +// +// A leader lease is the worst thing here to lose to a single node. Nobody can +// acquire a lease whose bucket has no quorum, so every reconciler that shares +// it stops at once, cluster-wide, at exactly the moment a node has failed. func OpenBucket(ctx context.Context, js jetstream.JetStream, cfg BucketConfig) (jetstream.KeyValue, error) { + replicas, err := clustersize.Replicas() + if err != nil { + return nil, fmt.Errorf("kvlease: open or create lease bucket %s: %w", cfg.Name, err) + } + kv, err := js.KeyValue(ctx, cfg.Name) - if errors.Is(err, jetstream.ErrBucketNotFound) { + switch { + case errors.Is(err, jetstream.ErrBucketNotFound): kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ Bucket: cfg.Name, History: 1, TTL: cfg.TTL, - Replicas: cfg.Replicas, + Replicas: replicas, }) + case err == nil: + // A failed raise must not fail the open. This runs on every reconcile + // tick, so refusing here would stop every reconciler sharing the bucket + // — the outage this replica count exists to prevent — over a bucket that + // is present and quorate at the count it already has. + kvutil.TryRaiseBucketReplicas(ctx, js, cfg.Name, replicas) } if err != nil { return nil, fmt.Errorf("kvlease: open or create lease bucket %s: %w", cfg.Name, err) @@ -82,6 +94,11 @@ func NATSBucket(nc *nats.Conn, bucket string, ttl time.Duration) BucketFunc { if err == nil { return kv, nil } + // An undeclared cluster size does not become declared by waiting, so + // it surfaces now rather than after the whole retry window. + if clustersize.Permanent(err) { + return nil, err + } if time.Now().After(deadline) { return nil, fmt.Errorf("kvlease: KV bucket %q unreachable after %s: %w", bucket, bucketRetryFor, err) } diff --git a/spinifex/kvlease/bucket_test.go b/spinifex/kvlease/bucket_test.go new file mode 100644 index 000000000..02ba270ea --- /dev/null +++ b/spinifex/kvlease/bucket_test.go @@ -0,0 +1,76 @@ +package kvlease_test + +import ( + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/clustersize" + "github.com/mulgadc/spinifex/spinifex/kvlease" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestOpenBucket_OpensALeaseBucketTheClusterCannotYetRaise is the assertion that +// keeps this fix from becoming the outage it prevents. +// +// OpenBucket runs on every reconcile tick. If a raise it cannot place failed the +// open, no reconciler could acquire its lease, and every one of them would stop +// cluster-wide — over a lease bucket that is present and quorate at the replica +// count it already has. The config naming a node that is not serving yet is the +// documented way a cluster grows, so this is a state to tolerate, not to refuse. +func TestOpenBucket_OpensALeaseBucketTheClusterCannotYetRaise(t *testing.T) { + _, nc, js := testutil.StartTestJetStream(t) + defer nc.Close() + + cfg := kvlease.BucketConfig{Name: "lease-growing-cluster", TTL: time.Minute} + kv, err := kvlease.OpenBucket(t.Context(), js, cfg) + require.NoError(t, err) + _, err = kv.Put(t.Context(), "leader", []byte("node1")) + require.NoError(t, err) + + // One embedded server cannot hold three replicas, so the raise must fail. + clustersize.RedeclareForTest(t, 3) + + reopened, err := kvlease.OpenBucket(t.Context(), js, cfg) + require.NoError(t, err, "a reconciler must still be able to reach its lease bucket") + entry, err := reopened.Get(t.Context(), "leader") + require.NoError(t, err) + assert.Equal(t, "node1", string(entry.Value())) +} + +// TestOpenBucket_RefusesWhenClusterSizeIsUndeclared pins the fail-closed half: +// creating a lease bucket without knowing the cluster size must not guess at one +// replica, which is the state that makes a lease unacquirable when a node fails. +func TestOpenBucket_RefusesWhenClusterSizeIsUndeclared(t *testing.T) { + _, nc, js := testutil.StartTestJetStream(t) + defer nc.Close() + + clustersize.RedeclareForTest(t, 0) + + _, err := kvlease.OpenBucket(t.Context(), js, kvlease.BucketConfig{Name: "lease-undeclared", TTL: time.Minute}) + require.ErrorIs(t, err, clustersize.ErrUndeclared) +} + +// TestNATSBucket_StopsOnAConfigFault covers the retry loop rather than the open. +// +// The loop exists because a cold multi-node start has no JetStream quorum yet, and +// waiting for one is right. An undeclared cluster size never becomes declared by +// waiting, so a loop that cannot tell them apart holds a reconciler for its whole +// retry window and then reports the bucket as unreachable — which was never the +// problem. +func TestNATSBucket_StopsOnAConfigFault(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + defer nc.Close() + + clustersize.RedeclareForTest(t, 0) + + start := time.Now() + _, err := kvlease.NATSBucket(nc, "lease-config-fault", time.Minute)(t.Context()) + elapsed := time.Since(start) + + require.ErrorIs(t, err, clustersize.ErrUndeclared, + "the error has to name the config fault, not the bucket it never reached") + require.Lessf(t, elapsed, 10*time.Second, + "returned after %s; a permanent fault must not consume the retry window", elapsed) +} diff --git a/spinifex/kvstore/bucket.go b/spinifex/kvstore/bucket.go index a911e1270..2c21f88d7 100644 --- a/spinifex/kvstore/bucket.go +++ b/spinifex/kvstore/bucket.go @@ -23,7 +23,6 @@ type Config struct { // Description is set on the bucket at creation, including a recovery // recreate, so a recreated bucket is not left anonymous next to its peers. Description string - Replicas int TTL time.Duration Missing string Attempts int @@ -393,15 +392,13 @@ func (b *Bucket) open(ctx context.Context) (jetstream.KeyValue, error) { Name: b.cfg.Name, Description: b.cfg.Description, History: b.cfg.History, - Replicas: b.cfg.Replicas, TTL: b.cfg.TTL, }) - case b.cfg.Replicas > 0 || b.cfg.Description != "": + case b.cfg.Description != "": return kvutil.GetOrCreateBucketWithOptions(ctx, b.js, kvutil.BucketOptions{ Name: b.cfg.Name, Description: b.cfg.Description, History: b.cfg.History, - Replicas: b.cfg.Replicas, }) default: return kvutil.GetOrCreateBucket(ctx, b.js, b.cfg.Name, b.cfg.History) diff --git a/spinifex/kvutil/bucket_creation_invariants_test.go b/spinifex/kvutil/bucket_creation_invariants_test.go new file mode 100644 index 000000000..ad347ef84 --- /dev/null +++ b/spinifex/kvutil/bucket_creation_invariants_test.go @@ -0,0 +1,116 @@ +package kvutil_test + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +// createFuncs are the JetStream calls that bring a KV bucket into existence. +// Each one takes a KeyValueConfig whose zero Replicas means one, so each one is +// a way to create a bucket on a single node without saying so. +var createFuncs = []string{"CreateKeyValue(", "CreateOrUpdateKeyValue("} + +// exempt are the only packages allowed to call them: the chokepoint itself, the +// lease-bucket opener it shares its rule with, and the test helpers that build +// buckets for a single embedded server. +var exempt = []string{ + filepath.Join("spinifex", "kvutil"), + filepath.Join("spinifex", "kvlease"), + filepath.Join("spinifex", "testutil"), +} + +// TestKVBuckets_AreOnlyCreatedThroughTheChokepoint keeps the replica-count rule +// enforceable by keeping the number of places that can break it at one. +// +// A KV bucket created with the default single replica puts a service every node +// depends on onto one node, so losing that node is a cluster-wide outage from a +// single-node event. The rule is that the cluster's node count decides the +// replica count, and a rule that lives in one function stays true only while +// that function is the only way through. This is what has failed before: there +// were five independent ways to create a bucket and the replica count was +// opt-in in all of them. +func TestKVBuckets_AreOnlyCreatedThroughTheChokepoint(t *testing.T) { + root := repoRoot(t) + + type hit struct { + file string + line int + call string + } + var hits []hit + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + switch d.Name() { + case ".git", "vendor", "node_modules", "tests": + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + return relErr + } + for _, dir := range exempt { + if strings.HasPrefix(rel, dir+string(filepath.Separator)) { + return nil + } + } + + data, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + for i, line := range strings.Split(string(data), "\n") { + for _, fn := range createFuncs { + if strings.Contains(line, fn) { + hits = append(hits, hit{file: rel, line: i + 1, call: strings.TrimSpace(line)}) + } + } + } + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + + if len(hits) > 0 { + var b strings.Builder + for _, h := range hits { + fmt.Fprintf(&b, "\n %s:%d: %s", h.file, h.line, h.call) + } + t.Fatalf("a KV bucket is created outside kvutil, so its replica count is "+ + "whatever the literal happened to say — and an omitted Replicas is one, "+ + "on a cluster where every node depends on the bucket.\n"+ + "Use kvutil.GetOrCreateBucket, GetOrCreateBucketWithTTL or "+ + "GetOrCreateBucketWithOptions, which take the count from the cluster:%s", b.String()) + } +} + +func repoRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatalf("no go.mod above %s", dir) + } + dir = parent + } +} diff --git a/spinifex/kvutil/export_test.go b/spinifex/kvutil/export_test.go new file mode 100644 index 000000000..6957757ff --- /dev/null +++ b/spinifex/kvutil/export_test.go @@ -0,0 +1,28 @@ +package kvutil + +import ( + "time" + + "github.com/nats-io/nats.go/jetstream" +) + +// NewRaiseThrottleForTest exposes the throttle the open path uses, so its +// behaviour can be driven by an injected clock rather than by sleeping. +func NewRaiseThrottleForTest(every time.Duration) *RaiseThrottleForTest { + return &RaiseThrottleForTest{t: newRaiseThrottle(every)} +} + +// RaiseThrottleForTest wraps the unexported throttle for the external test +// package. +type RaiseThrottleForTest struct{ t *raiseThrottle } + +// Allow reports whether bucket may be attempted at now. +func (r *RaiseThrottleForTest) Allow(bucket string, now time.Time) bool { + return r.t.allow(bucket, now) +} + +// NewBucketReportForTest exposes how one stream's info becomes a report, which is +// where configured replicas and working replicas are told apart. +func NewBucketReportForTest(bucket string, want int, info *jetstream.StreamInfo) BucketReport { + return newBucketReport(bucket, want, info) +} diff --git a/spinifex/kvutil/kvutil.go b/spinifex/kvutil/kvutil.go index c4d93107d..8e22d4481 100644 --- a/spinifex/kvutil/kvutil.go +++ b/spinifex/kvutil/kvutil.go @@ -17,84 +17,84 @@ import ( "time" "github.com/mulgadc/bluebottle/pkg/safecast" + "github.com/mulgadc/spinifex/spinifex/clustersize" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" ) -// GetOrCreateBucket creates or opens a KV bucket at the cluster's default -// replica count (see utils.SetDefaultKVReplicas), so buckets created lazily -// after boot are quorate on multi-node rather than stuck at R1. +// GetOrCreateBucket creates or opens a KV bucket replicated across the cluster. func GetOrCreateBucket(ctx context.Context, js jetstream.KeyValueManager, bucket string, history int) (jetstream.KeyValue, error) { - return GetOrCreateBucketWithReplicas(ctx, js, bucket, history, utils.DefaultKVReplicas()) + return getOrCreateBucket(ctx, js, jetstream.KeyValueConfig{ + Bucket: bucket, + History: safecast.IntToUint8(history), + }) } // GetOrCreateBucketWithTTL is GetOrCreateBucket for buckets whose entries // should age out on their own — request-dedupe records and other short-lived // state that would otherwise accumulate without a sweeper. TTL applies at -// creation only, like Replicas. +// creation only; the replica count does not. func GetOrCreateBucketWithTTL(ctx context.Context, js jetstream.KeyValueManager, bucket string, history int, ttl time.Duration) (jetstream.KeyValue, error) { return getOrCreateBucket(ctx, js, jetstream.KeyValueConfig{ - Bucket: bucket, - History: safecast.IntToUint8(history), - Replicas: max(utils.DefaultKVReplicas(), 1), - TTL: ttl, - }) -} - -// GetOrCreateBucketWithReplicas creates or opens a KV bucket at the given -// replica count (clamped to a minimum of 1). The replica count applies to -// creation only: an existing bucket is opened with the config it already has, -// so one created before the cluster grew is upgraded on rebalance, not here. -func GetOrCreateBucketWithReplicas(ctx context.Context, js jetstream.KeyValueManager, bucket string, history, replicas int) (jetstream.KeyValue, error) { - return getOrCreateBucket(ctx, js, jetstream.KeyValueConfig{ - Bucket: bucket, - History: safecast.IntToUint8(history), - Replicas: max(replicas, 1), + Bucket: bucket, + History: safecast.IntToUint8(history), + TTL: ttl, }) } // BucketOptions is GetOrCreateBucket's full argument set, for callers needing a -// combination the three named helpers do not cover. A zero Replicas means the -// cluster default, matching those helpers. +// combination the named helpers do not cover. There is deliberately no Replicas +// field: see Replicas. type BucketOptions struct { Name string Description string History int - Replicas int TTL time.Duration } -// GetOrCreateBucketWithOptions creates or opens a KV bucket from opts. Like the -// named helpers, every field but Name applies at creation only: an existing -// bucket is opened with the config it already has. +// GetOrCreateBucketWithOptions creates or opens a KV bucket from opts. Every +// field but Name applies at creation only: an existing bucket is opened with +// the config it already has, apart from its replica count. func GetOrCreateBucketWithOptions(ctx context.Context, js jetstream.KeyValueManager, opts BucketOptions) (jetstream.KeyValue, error) { - replicas := opts.Replicas - if replicas == 0 { - replicas = DefaultReplicas() - } return getOrCreateBucket(ctx, js, jetstream.KeyValueConfig{ Bucket: opts.Name, Description: opts.Description, History: safecast.IntToUint8(opts.History), - Replicas: max(replicas, 1), TTL: opts.TTL, }) } -// DefaultReplicas is the cluster's default KV replica count, exposed so callers -// building a BucketOptions can tell "unset" from a deliberate 1. -func DefaultReplicas() int { return utils.DefaultKVReplicas() } - func getOrCreateBucket(ctx context.Context, js jetstream.KeyValueManager, cfg jetstream.KeyValueConfig) (jetstream.KeyValue, error) { bucket := cfg.Bucket - kv, err := js.CreateKeyValue(ctx, cfg) + replicas, err := clustersize.Replicas() + if err != nil { + return nil, fmt.Errorf("create KV bucket %s: %w", bucket, err) + } + cfg.Replicas = replicas + + // Attach before create, so whether the bucket exists is this function's own + // answer rather than an inference from which error the server reported + // first. A create asking for more replicas than the cluster can place is + // refused before the name is looked at, which would otherwise read as + // "create failed" for a bucket that is present and serving. + kv, err := js.KeyValue(ctx, bucket) + switch { + case err == nil: + return openAndRaise(ctx, js, kv, bucket, replicas), nil + case !errors.Is(err, jetstream.ErrBucketNotFound): + return nil, fmt.Errorf("open KV bucket %s: %w", bucket, err) + } + + kv, err = js.CreateKeyValue(ctx, cfg) if err == nil { return kv, nil } - // A previous boot or a concurrent daemon already created it, so opening it - // is the expected outcome. Every other create failure is real and surfaces. + // A concurrent daemon created it in the gap, so opening it is the expected + // outcome. Every other create failure is real and surfaces — a create at + // fewer replicas than the cluster wants is the defect, so it must not be + // reached by falling back. if !errors.Is(err, jetstream.ErrBucketExists) { return nil, fmt.Errorf("create KV bucket %s: %w", bucket, err) } @@ -102,7 +102,19 @@ func getOrCreateBucket(ctx context.Context, js jetstream.KeyValueManager, cfg je if err != nil { return nil, fmt.Errorf("open KV bucket %s: %w", bucket, err) } - return kv, nil + return openAndRaise(ctx, js, kv, bucket, replicas), nil +} + +// openAndRaise returns an already-open bucket, raising it to the cluster's +// replica count on a best-effort basis. +// +// An existing bucket may predate this rule, or predate the cluster growing, and +// raising it on open is what makes a formed cluster self-heal rather than wait +// for someone to run a repair. The daemon's sweep and +// `spx admin kv replicas --repair` finish the job. +func openAndRaise(ctx context.Context, js jetstream.KeyValueManager, kv jetstream.KeyValue, bucket string, replicas int) jetstream.KeyValue { + TryRaiseBucketReplicas(ctx, js, bucket, replicas) + return kv } // IsStreamUnavailable reports whether err means the KV bucket's underlying diff --git a/spinifex/kvutil/kvutil_test.go b/spinifex/kvutil/kvutil_test.go index a15f9ccab..a78677798 100644 --- a/spinifex/kvutil/kvutil_test.go +++ b/spinifex/kvutil/kvutil_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/mulgadc/spinifex/spinifex/clustersize" "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go/jetstream" @@ -31,13 +32,13 @@ func streamReplicas(t *testing.T, js jetstream.JetStream, bucket string) int { return info.Config.Replicas } -func TestGetOrCreateBucket_CreatesAtDefaultReplicas(t *testing.T) { +func TestGetOrCreateBucket_CreatesAtTheClusterReplicaCount(t *testing.T) { js := startJetStream(t) kv, err := GetOrCreateBucket(t.Context(), js, "regression-bucket", 5) require.NoError(t, err) require.NotNil(t, kv) - assert.Equal(t, 1, streamReplicas(t, js, "regression-bucket")) + assert.Equal(t, 1, streamReplicas(t, js, "regression-bucket"), "one embedded server is a one-node cluster") } // TestGetOrCreateBucket_OpensExisting covers the second-boot path: a bucket that @@ -58,33 +59,63 @@ func TestGetOrCreateBucket_OpensExisting(t *testing.T) { assert.Equal(t, "value", string(entry.Value())) } -func TestGetOrCreateBucketWithReplicas_ClampsBelowOne(t *testing.T) { +// TestGetOrCreateBucket_RefusesWhenClusterSizeIsUndeclared pins the fail-closed +// choice: an undeclared size must not fall back to one replica, because one +// replica on a multi-node cluster is the outage this package exists to prevent. +func TestGetOrCreateBucket_RefusesWhenClusterSizeIsUndeclared(t *testing.T) { js := startJetStream(t) + clustersize.RedeclareForTest(t, 0) - kv, err := GetOrCreateBucketWithReplicas(t.Context(), js, "clamped-zero", 1, 0) - require.NoError(t, err) - require.NotNil(t, kv) - assert.Equal(t, 1, streamReplicas(t, js, "clamped-zero")) + _, err := GetOrCreateBucket(t.Context(), js, "undeclared", 1) + require.ErrorIs(t, err, clustersize.ErrUndeclared) - kv, err = GetOrCreateBucketWithReplicas(t.Context(), js, "clamped-negative", 1, -3) - require.NoError(t, err) - require.NotNil(t, kv) - assert.Equal(t, 1, streamReplicas(t, js, "clamped-negative")) + _, err = js.KeyValue(t.Context(), "undeclared") + require.ErrorIs(t, err, jetstream.ErrBucketNotFound, "a refused create must not leave a bucket behind") } -// TestGetOrCreateBucketWithReplicas_SurfacesCreateFailure pins the reason the -// open is scoped to "bucket exists": a create that fails for any other reason -// must report that reason, not the "bucket not found" a blind reopen produces. -func TestGetOrCreateBucketWithReplicas_SurfacesCreateFailure(t *testing.T) { +// TestGetOrCreateBucket_SurfacesCreateFailure pins the reason the open is scoped +// to "bucket exists": a create that fails for any other reason must report that +// reason, not the "bucket not found" a blind reopen produces. +func TestGetOrCreateBucket_SurfacesCreateFailure(t *testing.T) { js := startJetStream(t) - // The embedded single-node server rejects Replicas > 1. - _, err := GetOrCreateBucketWithReplicas(t.Context(), js, "over-replicated", 1, 3) + // The embedded single-node server rejects a three-node replica count. + clustersize.RedeclareForTest(t, 3) + + _, err := GetOrCreateBucket(t.Context(), js, "over-replicated", 1) require.Error(t, err) assert.NotErrorIs(t, err, jetstream.ErrBucketNotFound) assert.Contains(t, err.Error(), "create KV bucket over-replicated") } +// TestGetOrCreateBucket_OpensAnExistingBucketTheClusterCannotYetRaise pins the +// other half of the line, and it is the half that matters in production: a +// bucket that exists and works must be handed back even when it cannot be +// raised to the count the config now asks for. +// +// That happens on the documented growth path — the config names a new node +// before that node is serving — and refusing would stop every service on the +// host over a bucket that is perfectly usable. The sweep and +// `kv replicas --repair` are what finish the raise. +func TestGetOrCreateBucket_OpensAnExistingBucketTheClusterCannotYetRaise(t *testing.T) { + js := startJetStream(t) + + kv, err := GetOrCreateBucket(t.Context(), js, "growing-cluster", 1) + require.NoError(t, err) + _, err = kv.Put(t.Context(), "key", []byte("value")) + require.NoError(t, err) + + // One embedded server cannot hold three replicas, so the raise must fail. + clustersize.RedeclareForTest(t, 3) + + reopened, err := GetOrCreateBucket(t.Context(), js, "growing-cluster", 1) + require.NoError(t, err, "an existing, working bucket must open even when it cannot be raised") + entry, err := reopened.Get(t.Context(), "key") + require.NoError(t, err) + assert.Equal(t, "value", string(entry.Value())) + assert.Equal(t, 1, streamReplicas(t, js, "growing-cluster"), "the raise must not have been faked") +} + func TestDeleteBucketIfExists(t *testing.T) { js := startJetStream(t) diff --git a/spinifex/kvutil/replicas.go b/spinifex/kvutil/replicas.go new file mode 100644 index 000000000..d16b96da8 --- /dev/null +++ b/spinifex/kvutil/replicas.go @@ -0,0 +1,249 @@ +package kvutil + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sort" + "sync" + "time" + + "github.com/mulgadc/spinifex/spinifex/clustersize" + "github.com/nats-io/nats.go/jetstream" +) + +// streamPrefix is how JetStream names the stream behind a KV bucket. Raising a +// bucket's replica count is a stream update, so the name has to be built here. +const streamPrefix = "KV_" + +// BucketReport is one bucket's replication, as the cluster currently holds it. +// +// Configured replicas and working replicas are separate fields because they are +// separate facts. A stream can report the count it was asked for while a peer is +// offline or still catching up, and that reads as healthy right up until the +// node actually serving it goes away. +type BucketReport struct { + Bucket string `json:"bucket"` + Replicas int `json:"replicas"` + Want int `json:"want"` + Cluster string `json:"cluster,omitempty"` + + // Leader is the server currently accepting writes, empty when there is none. + Leader string `json:"leader,omitempty"` + + // Online counts the peers that are caught up and reachable, the leader + // included. This is the number a majority is measured against. + Online int `json:"online"` + + // Peers names every server assigned to the bucket, leader first. + Peers []string `json:"peers,omitempty"` +} + +// UnderReplicated reports whether this bucket has fewer replicas than the +// cluster can give it. A bucket at the cluster's replica count is correct; one +// below it is a cluster-wide service living on fewer nodes than exist. +func (r BucketReport) UnderReplicated() bool { return r.Replicas < r.Want } + +// HasQuorum reports whether enough of the bucket's replicas are working for it +// to accept a write. +// +// Measured against the configured count rather than the wanted one: a bucket +// that has not been raised yet is still perfectly able to serve at the size it +// has, and calling that a quorum failure would confuse a durability shortfall +// with an outage. +func (r BucketReport) HasQuorum() bool { return r.Online > r.Replicas/2 } + +// Healthy reports whether the bucket is both at the cluster's replica count and +// able to serve. +func (r BucketReport) Healthy() bool { return !r.UnderReplicated() && r.HasQuorum() } + +// AuditBucketReplicas reports every KV bucket's replica count against what the +// cluster's size entitles it to. It reads and changes nothing, so it is safe to +// run against a live cluster and is what both the admin command and the e2e +// assertion are built on. +func AuditBucketReplicas(ctx context.Context, js jetstream.JetStream) ([]BucketReport, error) { + want, err := clustersize.Replicas() + if err != nil { + return nil, err + } + + names, err := BucketNames(ctx, js) + if err != nil { + return nil, err + } + sort.Strings(names) + + reports := make([]BucketReport, 0, len(names)) + for _, name := range names { + info, err := js.Stream(ctx, streamPrefix+name) + if err != nil { + // A bucket deleted between the listing and the read is not a finding. + if errors.Is(err, jetstream.ErrStreamNotFound) { + continue + } + return nil, fmt.Errorf("read KV bucket %s: %w", name, err) + } + cached, err := info.Info(ctx) + if err != nil { + return nil, fmt.Errorf("read KV bucket %s: %w", name, err) + } + reports = append(reports, newBucketReport(name, want, cached)) + } + return reports, nil +} + +// newBucketReport reads one stream's replication and placement out of its info. +// +// A non-clustered server reports no cluster block at all. Its single replica is +// nonetheless online, because the read that produced this info was answered, and +// reporting zero there would make every single-node bucket look like it had lost +// quorum. +func newBucketReport(bucket string, want int, info *jetstream.StreamInfo) BucketReport { + report := BucketReport{Bucket: bucket, Replicas: info.Config.Replicas, Want: want} + if info.Cluster == nil { + report.Online = 1 + return report + } + + report.Cluster = info.Cluster.Name + report.Leader = info.Cluster.Leader + if report.Leader != "" { + report.Peers = append(report.Peers, report.Leader) + report.Online++ + } + for _, peer := range info.Cluster.Replicas { + report.Peers = append(report.Peers, peer.Name) + // Current and not offline is JetStream's own statement that this peer is + // caught up and reachable, which is what a majority has to be counted + // from. A peer that is merely assigned cannot acknowledge a write. + if peer.Current && !peer.Offline { + report.Online++ + } + } + return report +} + +// RaiseAllBucketReplicas raises every KV bucket to the cluster's replica count +// and reports how many it had to change. +// +// It is the cluster-wide counterpart to the raise a bucket gets when a service +// opens it: a node joining changes the answer for every bucket at once, and +// most of them will not be opened again until something restarts. +// +// Every bucket is attempted even after one fails, and the failures come back +// joined. Stopping at the first would leave the rest of a cluster's buckets +// short because one of them could not be placed, which is the opposite of what +// a sweep is for. +func RaiseAllBucketReplicas(ctx context.Context, js jetstream.JetStream) (int, error) { + reports, err := AuditBucketReplicas(ctx, js) + if err != nil { + return 0, err + } + raised := 0 + var failures []error + for _, r := range reports { + if !r.UnderReplicated() { + continue + } + if err := RaiseBucketReplicas(ctx, js, r.Bucket, r.Want); err != nil { + failures = append(failures, err) + continue + } + raised++ + } + return raised, errors.Join(failures...) +} + +// raiseRetryEvery is how often the open path will re-attempt a raise for the +// same bucket. It is a floor on retries, not a deadline: the next open after the +// window tries again, so a cluster that gains a node still converges without an +// operator. +const raiseRetryEvery = 1 * time.Minute + +// openRaises throttles the raise attempts made on the open path. +var openRaises = newRaiseThrottle(raiseRetryEvery) + +// TryRaiseBucketReplicas raises a bucket toward want on the path a service takes +// when it opens one, warning rather than failing when it cannot. +// +// A failed raise is not a failed open. The config can legitimately name nodes +// that are not serving yet — that is what growing a cluster looks like — and +// refusing would stop every service on the host over a bucket that is present +// and quorate at the count it has. +// +// Throttled per bucket, because callers reopen buckets on every reconcile tick. +// Without it a cluster that cannot place the replica bills a stream update to the +// meta leader and writes a warning line every few seconds, for every bucket, on +// every node — load and noise arriving exactly when a cluster is already unwell. +func TryRaiseBucketReplicas(ctx context.Context, js jetstream.KeyValueManager, bucket string, want int) { + if !openRaises.allow(bucket, time.Now()) { + return + } + if err := RaiseBucketReplicas(ctx, js, bucket, want); err != nil { + slog.WarnContext(ctx, "Could not raise KV bucket to the cluster's replica count; it stays where it is until the cluster can hold more", + "bucket", bucket, "want", want, "error", err) + } +} + +// raiseThrottle records when each bucket was last attempted, so a caller on a +// reconcile loop retries on a schedule of its own rather than the loop's. +type raiseThrottle struct { + mu sync.Mutex + every time.Duration + last map[string]time.Time +} + +func newRaiseThrottle(every time.Duration) *raiseThrottle { + return &raiseThrottle{every: every, last: make(map[string]time.Time)} +} + +// allow reports whether bucket may be attempted at now, recording the attempt +// when it may. The first attempt for a bucket is always allowed, so a raise is +// never delayed by the throttle existing. +func (t *raiseThrottle) allow(bucket string, now time.Time) bool { + t.mu.Lock() + defer t.mu.Unlock() + if last, seen := t.last[bucket]; seen && now.Sub(last) < t.every { + return false + } + t.last[bucket] = now + return true +} + +// RaiseBucketReplicas raises a bucket to want replicas, leaving every other +// setting as it is. +// +// It updates the stream rather than the bucket deliberately. A KV update takes +// a KeyValueConfig, which cannot express a stream's full configuration, so +// applying one would quietly reset whatever the caller's literal omitted — +// history and TTL among them. Reading the live config and changing one field +// cannot do that. +// +// Lowering is never done: a bucket with more replicas than the cluster is +// entitled to is not a fault, and a cluster that has lost nodes must not have +// its durability reduced to match. +func RaiseBucketReplicas(ctx context.Context, js jetstream.KeyValueManager, bucket string, want int) error { + streams, ok := js.(jetstream.StreamManager) + if !ok { + return fmt.Errorf("raise KV bucket %s to %d replicas: no stream manager available", bucket, want) + } + + stream, err := streams.Stream(ctx, streamPrefix+bucket) + if err != nil { + return fmt.Errorf("read KV bucket %s: %w", bucket, err) + } + cfg := stream.CachedInfo().Config + if cfg.Replicas >= want { + return nil + } + + was := cfg.Replicas + cfg.Replicas = want + if _, err := streams.UpdateStream(ctx, cfg); err != nil { + return fmt.Errorf("raise KV bucket %s from %d to %d replicas: %w", bucket, was, want, err) + } + slog.InfoContext(ctx, "raised a KV bucket to the cluster's replica count", + "bucket", bucket, "was", was, "now", want) + return nil +} diff --git a/spinifex/kvutil/replicas_health_test.go b/spinifex/kvutil/replicas_health_test.go new file mode 100644 index 000000000..3c44e1ea4 --- /dev/null +++ b/spinifex/kvutil/replicas_health_test.go @@ -0,0 +1,130 @@ +package kvutil_test + +import ( + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/kvutil" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestNewBucketReport_TellsConfiguredFromWorking is the distinction the whole +// health report exists for. A stream can carry the replica count it was asked for +// while a peer is offline or still catching up, and that reads as healthy right up +// until the node actually serving it goes away. +func TestNewBucketReport_TellsConfiguredFromWorking(t *testing.T) { + info := &jetstream.StreamInfo{ + Config: jetstream.StreamConfig{Replicas: 3}, + Cluster: &jetstream.ClusterInfo{ + Name: "spinifex", + Leader: "spinifex-nats-node1", + Replicas: []*jetstream.PeerInfo{ + {Name: "spinifex-nats-node2", Current: true}, + {Name: "spinifex-nats-node3", Current: false}, + }, + }, + } + + got := kvutil.NewBucketReportForTest("spinifex-instance-state", 3, info) + + assert.Equal(t, 3, got.Replicas, "the configured count is what the stream asked for") + assert.Equal(t, 2, got.Online, "a peer that is not current cannot acknowledge a write") + assert.Equal(t, "spinifex-nats-node1", got.Leader) + assert.Equal(t, []string{"spinifex-nats-node1", "spinifex-nats-node2", "spinifex-nats-node3"}, got.Peers, + "every assigned peer is named, leader first") + assert.False(t, got.UnderReplicated()) + assert.True(t, got.HasQuorum(), "two of three is a majority") + assert.True(t, got.Healthy()) +} + +// TestNewBucketReport_OfflinePeerIsNotOnline: JetStream reports current and offline +// separately, and a peer can be marked current from a stale view while being +// unreachable. Both have to disqualify it. +func TestNewBucketReport_OfflinePeerIsNotOnline(t *testing.T) { + info := &jetstream.StreamInfo{ + Config: jetstream.StreamConfig{Replicas: 3}, + Cluster: &jetstream.ClusterInfo{ + Leader: "spinifex-nats-node1", + Replicas: []*jetstream.PeerInfo{ + {Name: "spinifex-nats-node2", Current: true, Offline: true}, + {Name: "spinifex-nats-node3", Current: true, Offline: true}, + }, + }, + } + + got := kvutil.NewBucketReportForTest("b", 3, info) + + assert.Equal(t, 1, got.Online) + assert.False(t, got.HasQuorum(), "one of three cannot accept a write") + assert.False(t, got.Healthy()) + assert.False(t, got.UnderReplicated(), "no quorum is not the same finding as too few replicas") +} + +// TestNewBucketReport_NoLeaderIsNoQuorum: a raft group mid-election has no writer, +// and reporting it as serving would hide exactly the window this matters in. +func TestNewBucketReport_NoLeaderIsNoQuorum(t *testing.T) { + info := &jetstream.StreamInfo{ + Config: jetstream.StreamConfig{Replicas: 3}, + Cluster: &jetstream.ClusterInfo{ + Replicas: []*jetstream.PeerInfo{{Name: "spinifex-nats-node2", Current: true}}, + }, + } + + got := kvutil.NewBucketReportForTest("b", 3, info) + + assert.Empty(t, got.Leader) + assert.Equal(t, 1, got.Online) + assert.False(t, got.HasQuorum()) +} + +// TestNewBucketReport_SingleNodeServerReportsNoCluster is the single-node case, and +// it must not read as an outage. A non-clustered server sends no cluster block at +// all, and its one replica is serving — the read that produced this info was +// answered by it. +func TestNewBucketReport_SingleNodeServerReportsNoCluster(t *testing.T) { + got := kvutil.NewBucketReportForTest("b", 1, &jetstream.StreamInfo{ + Config: jetstream.StreamConfig{Replicas: 1}, + }) + + assert.Equal(t, 1, got.Replicas) + assert.Equal(t, 1, got.Online) + assert.True(t, got.HasQuorum()) + assert.True(t, got.Healthy()) +} + +// TestHasQuorumIsMeasuredAgainstConfiguredReplicas: a bucket not yet raised is +// still able to serve at the size it has, and calling that a quorum failure would +// confuse a durability shortfall with an outage. +func TestHasQuorumIsMeasuredAgainstConfiguredReplicas(t *testing.T) { + r := kvutil.BucketReport{Bucket: "b", Replicas: 1, Want: 3, Online: 1} + assert.True(t, r.HasQuorum(), "one of one is a majority even when three are wanted") + assert.True(t, r.UnderReplicated()) + assert.False(t, r.Healthy()) +} + +// TestRaiseThrottle_LetsTheFirstAttemptThroughThenWaits is what keeps a reconcile +// loop from billing a stream update and a warning line every few seconds while a +// cluster cannot place a replica. The first attempt is never delayed, because the +// throttle must not slow down a raise that would have succeeded. +func TestRaiseThrottle_LetsTheFirstAttemptThroughThenWaits(t *testing.T) { + th := kvutil.NewRaiseThrottleForTest(time.Minute) + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + require.True(t, th.Allow("alpha", start), "the first attempt must never be throttled") + assert.False(t, th.Allow("alpha", start.Add(time.Second)), "a retry one second later is the noise this prevents") + assert.False(t, th.Allow("alpha", start.Add(59*time.Second))) + assert.True(t, th.Allow("alpha", start.Add(time.Minute)), "past the window it tries again, so a cluster that grows still converges") +} + +// TestRaiseThrottle_IsPerBucket: one bucket that cannot be placed must not silence +// the raise for every other bucket on the node. +func TestRaiseThrottle_IsPerBucket(t *testing.T) { + th := kvutil.NewRaiseThrottleForTest(time.Minute) + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + require.True(t, th.Allow("alpha", now)) + assert.True(t, th.Allow("beta", now), "beta has never been attempted") + assert.False(t, th.Allow("alpha", now)) +} diff --git a/spinifex/kvutil/replicas_test.go b/spinifex/kvutil/replicas_test.go new file mode 100644 index 000000000..74e75c093 --- /dev/null +++ b/spinifex/kvutil/replicas_test.go @@ -0,0 +1,162 @@ +package kvutil_test + +import ( + "testing" + + "github.com/mulgadc/spinifex/spinifex/clustersize" + "github.com/mulgadc/spinifex/spinifex/kvutil" + "github.com/mulgadc/spinifex/spinifex/testutil" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAuditBucketReplicas_ReportsEveryBucketAgainstTheClusterSize(t *testing.T) { + js := newJetStream(t) + + for _, bucket := range []string{"audit-alpha", "audit-beta"} { + _, err := kvutil.GetOrCreateBucket(t.Context(), js, bucket, 1) + require.NoError(t, err) + } + + reports, err := kvutil.AuditBucketReplicas(t.Context(), js) + require.NoError(t, err) + require.Len(t, reports, 2) + assert.Equal(t, "audit-alpha", reports[0].Bucket, "reports are sorted by bucket name") + + for _, r := range reports { + assert.Equal(t, 1, r.Replicas) + assert.Equal(t, 1, r.Want) + assert.False(t, r.UnderReplicated()) + } + + // The cluster grew. Every existing bucket is now behind it, which is exactly + // what the audit has to say before anything can repair it. + clustersize.RedeclareForTest(t, 3) + + reports, err = kvutil.AuditBucketReplicas(t.Context(), js) + require.NoError(t, err) + require.Len(t, reports, 2) + for _, r := range reports { + assert.Equal(t, 1, r.Replicas) + assert.Equal(t, 3, r.Want) + assert.True(t, r.UnderReplicated(), "%s", r.Bucket) + } +} + +func TestAuditBucketReplicas_RefusesWhenClusterSizeIsUndeclared(t *testing.T) { + js := newJetStream(t) + clustersize.RedeclareForTest(t, 0) + + _, err := kvutil.AuditBucketReplicas(t.Context(), js) + require.ErrorIs(t, err, clustersize.ErrUndeclared) +} + +// TestRaiseBucketReplicas_NeverLowers is the property that makes the raise safe +// to run on every service start: a bucket already at or above the count is left +// alone, so a cluster that has lost a node does not have its durability cut to +// match. +func TestRaiseBucketReplicas_NeverLowers(t *testing.T) { + js := newJetStream(t) + + _, err := kvutil.GetOrCreateBucket(t.Context(), js, "no-downgrade", 1) + require.NoError(t, err) + + require.NoError(t, kvutil.RaiseBucketReplicas(t.Context(), js, "no-downgrade", 1)) + assert.Equal(t, 1, streamConfig(t, js, "no-downgrade").Replicas) + + // Zero and negative are below the current count, so they are no-ops too + // rather than an update that would ask the server for an invalid config. + require.NoError(t, kvutil.RaiseBucketReplicas(t.Context(), js, "no-downgrade", 0)) + assert.Equal(t, 1, streamConfig(t, js, "no-downgrade").Replicas) +} + +// TestRaiseBucketReplicas_PreservesTheRestOfTheConfig is why this updates the +// stream rather than the KV bucket: a KeyValueConfig cannot express a stream's +// full configuration, so applying one would reset whatever it omits. +func TestRaiseBucketReplicas_PreservesTheRestOfTheConfig(t *testing.T) { + js := newJetStream(t) + + _, err := kvutil.GetOrCreateBucketWithOptions(t.Context(), js, kvutil.BucketOptions{ + Name: "config-preserved", + Description: "a bucket with settings worth not losing", + History: 7, + }) + require.NoError(t, err) + + before := streamConfig(t, js, "config-preserved") + require.NoError(t, kvutil.RaiseBucketReplicas(t.Context(), js, "config-preserved", 1)) + after := streamConfig(t, js, "config-preserved") + + assert.Equal(t, before.Description, after.Description) + assert.Equal(t, before.MaxMsgsPerSubject, after.MaxMsgsPerSubject, "history is MaxMsgsPerSubject on the stream") + assert.Equal(t, before.MaxAge, after.MaxAge) +} + +// TestRaiseBucketReplicas_SurfacesAMissingBucket keeps a raise against a bucket +// that is not there from reading as success. +func TestRaiseBucketReplicas_SurfacesAMissingBucket(t *testing.T) { + js := newJetStream(t) + + err := kvutil.RaiseBucketReplicas(t.Context(), js, "never-created", 1) + require.Error(t, err) + assert.ErrorIs(t, err, jetstream.ErrStreamNotFound) +} + +func streamConfig(t *testing.T, js jetstream.JetStream, bucket string) jetstream.StreamConfig { + t.Helper() + stream, err := js.Stream(t.Context(), "KV_"+bucket) + require.NoError(t, err) + info, err := stream.Info(t.Context()) + require.NoError(t, err) + return info.Config +} + +// newJetStream starts an embedded one-node JetStream for a test in this package. +func newJetStream(t *testing.T) jetstream.JetStream { + t.Helper() + _, nc, _ := testutil.StartTestJetStream(t) + return testutil.NewJetStream(t, nc) +} + +// TestRaiseAllBucketReplicas_RaisesWhatItCanAndReportsWhatItCannot covers the +// sweep's two halves at once. It is the cluster-wide counterpart to the raise a +// bucket gets when a service opens it: a node joining changes the answer for +// every bucket, and most will not be reopened until something restarts. +func TestRaiseAllBucketReplicas_RaisesWhatItCanAndReportsWhatItCannot(t *testing.T) { + js := newJetStream(t) + + for _, b := range []string{"sweep-alpha", "sweep-beta"} { + _, err := kvutil.GetOrCreateBucket(t.Context(), js, b, 1) + require.NoError(t, err) + } + + // Nothing is short at one node, so the sweep must change nothing and say so + // rather than rewriting every bucket it looked at. + raised, err := kvutil.RaiseAllBucketReplicas(t.Context(), js) + require.NoError(t, err) + assert.Zero(t, raised, "a healthy cluster needs no repair") + + // The cluster grew beyond what one embedded server can place, so every + // bucket is short and every raise fails. The sweep must attempt all of them + // and join the failures rather than stopping at the first. + clustersize.RedeclareForTest(t, 3) + + raised, err = kvutil.RaiseAllBucketReplicas(t.Context(), js) + require.Error(t, err) + assert.Zero(t, raised) + assert.Contains(t, err.Error(), "sweep-alpha") + assert.Contains(t, err.Error(), "sweep-beta", "the sweep stopped at the first failure") + assert.Equal(t, 1, streamConfig(t, js, "sweep-alpha").Replicas, "a failed raise must not have changed anything") +} + +// TestRaiseAllBucketReplicas_RefusesWhenClusterSizeIsUndeclared keeps the sweep +// from reading an undeclared size as "one replica is what everything wants", +// which would report a whole cluster of single-replica buckets as healthy. +func TestRaiseAllBucketReplicas_RefusesWhenClusterSizeIsUndeclared(t *testing.T) { + js := newJetStream(t) + clustersize.RedeclareForTest(t, 0) + + _, err := kvutil.RaiseAllBucketReplicas(t.Context(), js) + require.ErrorIs(t, err, clustersize.ErrUndeclared) +} diff --git a/spinifex/services/awsgw/awsgw.go b/spinifex/services/awsgw/awsgw.go index a46a520be..79c830a44 100644 --- a/spinifex/services/awsgw/awsgw.go +++ b/spinifex/services/awsgw/awsgw.go @@ -4,7 +4,6 @@ import ( "context" "crypto/tls" "crypto/x509" - "errors" "fmt" "log/slog" "net" @@ -33,6 +32,7 @@ import ( handlers_sts "github.com/mulgadc/spinifex/spinifex/handlers/sts" "github.com/mulgadc/spinifex/spinifex/instancecache" "github.com/mulgadc/spinifex/spinifex/kvstore" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/network/reconcile" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/reconciler" @@ -150,53 +150,15 @@ func loadAWSGWConfig(path string) (awsgwTOML, error) { // openAccountUsageBucket opens (or idempotently creates) the gateway-owned // per-account vCPU usage bucket. History is 1: each account key holds a single // CAS-updated integer counter, so no revision beyond the latest is worth keeping. -func openAccountUsageBucket(ctx context.Context, js jetstream.KeyValueManager, replicas int) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, handlers_quota.KVBucketAccountUsage) - if err == nil { - return kv, nil - } - if !errors.Is(err, jetstream.ErrBucketNotFound) { - return nil, fmt.Errorf("open account usage bucket: %w", err) - } - - kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ - Bucket: handlers_quota.KVBucketAccountUsage, - History: 1, - Replicas: max(replicas, 1), - }) - if errors.Is(err, jetstream.ErrBucketExists) { - return js.KeyValue(ctx, handlers_quota.KVBucketAccountUsage) - } - if err != nil { - return nil, fmt.Errorf("create account usage bucket: %w", err) - } - return kv, nil +func openAccountUsageBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) { + return kvutil.GetOrCreateBucket(ctx, js, handlers_quota.KVBucketAccountUsage, 1) } // openAccountQuotaBucket opens (or idempotently creates) the per-account quota // override bucket. History is 1: each key holds the current override set, and // no earlier revision of a limit is worth keeping. -func openAccountQuotaBucket(ctx context.Context, js jetstream.KeyValueManager, replicas int) (jetstream.KeyValue, error) { - kv, err := js.KeyValue(ctx, handlers_quota.KVBucketAccountQuota) - if err == nil { - return kv, nil - } - if !errors.Is(err, jetstream.ErrBucketNotFound) { - return nil, fmt.Errorf("open account quota bucket: %w", err) - } - - kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ - Bucket: handlers_quota.KVBucketAccountQuota, - History: 1, - Replicas: max(replicas, 1), - }) - if errors.Is(err, jetstream.ErrBucketExists) { - return js.KeyValue(ctx, handlers_quota.KVBucketAccountQuota) - } - if err != nil { - return nil, fmt.Errorf("create account quota bucket: %w", err) - } - return kv, nil +func openAccountQuotaBucket(ctx context.Context, js jetstream.KeyValueManager) (jetstream.KeyValue, error) { + return kvutil.GetOrCreateBucket(ctx, js, handlers_quota.KVBucketAccountQuota, 1) } func launchService(config *config.ClusterConfig) error { @@ -239,14 +201,14 @@ func launchService(config *config.ClusterConfig) error { // Initialize IAM service with NATS KV backend (required for auth). // On multi-node clusters, JetStream KV requires cluster quorum which may // not be available yet if nodes start concurrently. Retry with backoff. - iamService, err := handlers_iam.NewIAMServiceWithRetry(janitorCtx, natsConn, masterKey, len(config.Nodes)) + iamService, err := handlers_iam.NewIAMServiceWithRetry(janitorCtx, natsConn, masterKey) if err != nil { return fmt.Errorf("initialize IAM service: %w", err) } // STS service shares the IAM master key (single envelope for at-rest // secrets + session-token HMACs) and resolves roles via IAMService. - stsService, err := handlers_sts.NewSTSServiceImpl(janitorCtx, natsConn, iamService, masterKey, len(config.Nodes)) + stsService, err := handlers_sts.NewSTSServiceImpl(janitorCtx, natsConn, iamService, masterKey) if err != nil { return fmt.Errorf("initialize STS service: %w", err) } @@ -338,7 +300,7 @@ func launchService(config *config.ClusterConfig) error { if err != nil { return fmt.Errorf("jetstream client: %w", err) } - signingKey, verifyKeys, err := gateway_ecrauth.LoadOrCreateSigningKey(janitorCtx, js, masterKey, len(config.Nodes)) + signingKey, verifyKeys, err := gateway_ecrauth.LoadOrCreateSigningKey(janitorCtx, js, masterKey) if err != nil { return fmt.Errorf("ECR auth bridge: load signing key: %w", err) } @@ -368,19 +330,19 @@ func launchService(config *config.ClusterConfig) error { if key := os.Getenv("OCHRE_ANTHROPIC_API_KEY"); key != "" { bedrockPlatformDefaults["anthropic"] = key } - bedrockCredentials := gateway_bedrock.NewCredentialStore(js, masterKey, len(config.Nodes), bedrockPlatformDefaults) + bedrockCredentials := gateway_bedrock.NewCredentialStore(js, masterKey, bedrockPlatformDefaults) // Bedrock self-host weights: a model's serving spec (VRAM, instance type, // vLLM args) ships in-tree, but which staged snapshot serves it is // deployment-local state. tieredCatalog/GetFoundationModel read this // through the package-level resolver rather than a parameter, since they // are called from gateway/bedrock.go's fixed-arity route table. - gateway_bedrock.SetWeightsResolver(gateway_bedrock.NewWeightsStore(js, len(config.Nodes))) + gateway_bedrock.SetWeightsResolver(gateway_bedrock.NewWeightsStore(js)) // Bedrock model access: grants live in the bedrock-model-access KV bucket // and are deny-by-default, so a fresh deployment serves no models until an // operator grants them (spx admin ochre access grant). - bedrockAccess := gateway_bedrock.NewModelAccessStore(js, len(config.Nodes)) + bedrockAccess := gateway_bedrock.NewModelAccessStore(js) // Deny-by-default would otherwise leave a fresh install with a catalog // nobody can see, so seed the platform admin account — the operator's own @@ -432,13 +394,13 @@ func launchService(config *config.ClusterConfig) error { // gateway_bedrock's narrow EndpointProvisioner (see provisioned_adapter.go // for why the adapter, not handlers_bedrock.EndpointService, is what // gateway_bedrock depends on). - bedrockProvisioned := gateway_bedrock.NewProvisionedStore(js, len(config.Nodes), nodeConfig.Region, + bedrockProvisioned := gateway_bedrock.NewProvisionedStore(js, nodeConfig.Region, handlers_bedrock.NewProvisionedEndpointAdapter(bedrockEndpointSvc)) // Bedrock guardrails: control-plane CRUD only at this stage — the record // stores the full policy config so a later stage's filter engine and // inference enforcement can read it back, but nothing enforces it yet. - bedrockGuardrails := gateway_bedrock.NewGuardrailStore(js, len(config.Nodes), nodeConfig.Region) + bedrockGuardrails := gateway_bedrock.NewGuardrailStore(js, nodeConfig.Region) // bedrock-agent knowledge-base + data-source resource metadata: gateway- // owned (D-arch), opened directly against JetStream the same way the @@ -454,8 +416,8 @@ func launchService(config *config.ClusterConfig) error { // deliveryConsumer to any account with a configured destination bucket // and a metadata-only log line. bedrockLoggingConfig separately gates // whether the record written to that bucket includes body text. - bedrockLoggingConfig := gateway_bedrock.NewLoggingConfigStore(js, len(config.Nodes)) - if _, err := gateway_bedrock.EnsureInvocationStream(janitorCtx, js, len(config.Nodes)); err != nil { + bedrockLoggingConfig := gateway_bedrock.NewLoggingConfigStore(js) + if _, err := gateway_bedrock.EnsureInvocationStream(janitorCtx, js); err != nil { return fmt.Errorf("bedrock: ensure invocation stream: %w", err) } deliveryConsumer, err := gateway_bedrock.EnsureDeliveryConsumer(janitorCtx, js) @@ -470,8 +432,8 @@ func launchService(config *config.ClusterConfig) error { // updates per-account/model/period counters, deduping on RequestID so // at-least-once redelivery never double-counts. bedrockPrices resolves // KV price overrides over the catalog's in-tree defaults. - bedrockUsage := gateway_bedrock.NewUsageStore(js, len(config.Nodes)) - bedrockPrices := gateway_bedrock.NewPriceStore(js, len(config.Nodes)) + bedrockUsage := gateway_bedrock.NewUsageStore(js) + bedrockPrices := gateway_bedrock.NewPriceStore(js) usageConsumer, err := gateway_bedrock.EnsureUsageConsumer(janitorCtx, js) if err != nil { return fmt.Errorf("bedrock: ensure usage metering consumer: %w", err) @@ -526,7 +488,7 @@ func launchService(config *config.ClusterConfig) error { // Rotate the ECR signing key on a 30-day cadence, retaining the previous keys // until their tokens expire. The rotator keeps the issuer/verifier current as // keys roll. Bound to the same lifetime context as the STS janitor. - keyRotator, err := gateway_ecrauth.NewRotator(janitorCtx, js, masterKey, len(config.Nodes), gw.ECRTokenIssuer, gw.ECRTokenVerifier) + keyRotator, err := gateway_ecrauth.NewRotator(janitorCtx, js, masterKey, gw.ECRTokenIssuer, gw.ECRTokenVerifier) if err != nil { return fmt.Errorf("ECR auth bridge: signing-key rotator: %w", err) } @@ -542,11 +504,11 @@ func launchService(config *config.ClusterConfig) error { // config builds a no-op Service whose Exempt short-circuits every check. var usageBucket, quotaOverrides jetstream.KeyValue if quotaCfg.Enabled { - usageBucket, err = openAccountUsageBucket(janitorCtx, js, len(config.Nodes)) + usageBucket, err = openAccountUsageBucket(janitorCtx, js) if err != nil { return fmt.Errorf("init account usage bucket: %w", err) } - quotaOverrides, err = openAccountQuotaBucket(janitorCtx, js, len(config.Nodes)) + quotaOverrides, err = openAccountQuotaBucket(janitorCtx, js) if err != nil { return fmt.Errorf("init account quota bucket: %w", err) } @@ -570,13 +532,13 @@ func launchService(config *config.ClusterConfig) error { // quotaCfg.Enabled); this loop only pushes a periodic observability // snapshot to KV and is itself a no-op when the RPM dimension is // disabled. - go gw.Quota.RunBedrockRPMSync(janitorCtx, js, len(config.Nodes)) + go gw.Quota.RunBedrockRPMSync(janitorCtx, js) // Instance cache: a read-only informer over the live instance record space, // started now so it is warm well before anything reads it. The describe // path still serves from the fan-out and KV; only status synthesis reads it. instanceCache := instancecache.New(js, instancecache.Config{ - Bucket: kvstore.Config{Name: daemon.InstanceStateBucket, History: 1, Replicas: len(config.Nodes)}, + Bucket: kvstore.Config{Name: daemon.InstanceStateBucket, History: 1}, Prefix: daemon.InstanceRecordPrefix, }) go instanceCache.Run(janitorCtx) @@ -586,7 +548,7 @@ func launchService(config *config.ClusterConfig) error { gw.InstanceStatus = gateway_ec2_instance.StatusSynthesis{ Records: instanceCache, Liveness: instancecache.NewLiveness(js, kvstore.Config{ - Name: daemon.ClusterStateBucket, History: 1, Replicas: len(config.Nodes), + Name: daemon.ClusterStateBucket, History: 1, }), } @@ -632,7 +594,7 @@ func launchService(config *config.ClusterConfig) error { func runQuotaReconcile(ctx context.Context, quota *handlers_quota.Service, natsConn *nats.Conn, js jetstream.JetStream, accounts handlers_quota.AccountLister, replicas int) { holder, _ := os.Hostname() - cfg := kvstore.Config{Name: daemon.InstanceStateBucket, History: 1, Replicas: replicas} + cfg := kvstore.Config{Name: daemon.InstanceStateBucket, History: 1} list := handlers_quota.RecordVCPULister( kvstore.New[vm.InstanceRecord](js, cfg), daemon.InstanceRecordPrefix) diff --git a/spinifex/services/awsgw/quota_reconcile_test.go b/spinifex/services/awsgw/quota_reconcile_test.go index c9f770c1b..e15a0b895 100644 --- a/spinifex/services/awsgw/quota_reconcile_test.go +++ b/spinifex/services/awsgw/quota_reconcile_test.go @@ -24,7 +24,7 @@ func TestOpenAccountUsageBucketPreservesExistingConfiguration(t *testing.T) { }) require.NoError(t, err) - bucket, err := openAccountUsageBucket(t.Context(), js, 1) + bucket, err := openAccountUsageBucket(t.Context(), js) require.NoError(t, err) status, err := bucket.Status(t.Context()) require.NoError(t, err) @@ -63,7 +63,7 @@ func instanceRecordBucket(t *testing.T, js jetstream.JetStream) jetstream.KeyVal func TestRunQuotaReconcileCountsFromTheRecordSpace(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - bucket, err := openAccountUsageBucket(t.Context(), js, 1) + bucket, err := openAccountUsageBucket(t.Context(), js) require.NoError(t, err) quota := handlers_quota.New(handlers_quota.Limits{Enabled: true, VCPUs: 100}, bucket) @@ -99,7 +99,7 @@ func TestRunQuotaReconcileCountsFromTheRecordSpace(t *testing.T) { func TestRunQuotaReconcileFollowsARecordChange(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - bucket, err := openAccountUsageBucket(t.Context(), js, 1) + bucket, err := openAccountUsageBucket(t.Context(), js) require.NoError(t, err) quota := handlers_quota.New(handlers_quota.Limits{Enabled: true, VCPUs: 100}, bucket) @@ -135,7 +135,7 @@ func TestRunQuotaReconcileFollowsARecordChange(t *testing.T) { func TestRunQuotaReconcileRecomputesOnlyTheChangedAccount(t *testing.T) { _, nc, _ := testutil.StartTestJetStream(t) js := testutil.NewJetStream(t, nc) - bucket, err := openAccountUsageBucket(t.Context(), js, 1) + bucket, err := openAccountUsageBucket(t.Context(), js) require.NoError(t, err) quota := handlers_quota.New(handlers_quota.Limits{Enabled: true, VCPUs: 100}, bucket) diff --git a/spinifex/services/nats/servername.go b/spinifex/services/nats/servername.go new file mode 100644 index 000000000..647443cbb --- /dev/null +++ b/spinifex/services/nats/servername.go @@ -0,0 +1,27 @@ +package nats + +import "strings" + +// ServerNamePrefix is what nats.conf prefixes a node name with to build +// server_name. JetStream reports stream peers by server name, so anything +// mapping a replica back to the node holding it has to go through here rather +// than assume the two are the same string — they are not. +const ServerNamePrefix = "spinifex-nats-" + +// ServerName is the NATS server_name for a node in this cluster. +func ServerName(node string) string { return ServerNamePrefix + node } + +// NodeFromServerName is ServerName's inverse. The second result is false for a +// name this cluster did not generate, so a caller can tell "not our node" from +// "a node called the empty string". +// +// The bare prefix is rejected too. It is a name we could not have generated, and +// mapping it to the empty node would silently create a phantom peer in anything +// counting nodes. +func NodeFromServerName(server string) (string, bool) { + node, ok := strings.CutPrefix(server, ServerNamePrefix) + if !ok || node == "" { + return "", false + } + return node, true +} diff --git a/spinifex/services/viperblockd/provider_handlers.go b/spinifex/services/viperblockd/provider_handlers.go index 5522804ba..5cd9e872c 100644 --- a/spinifex/services/viperblockd/provider_handlers.go +++ b/spinifex/services/viperblockd/provider_handlers.go @@ -68,7 +68,7 @@ func registerProviderSubjects(cfg *Config, nc *nats.Conn) error { // that reaches an engine open without one refuses, and refusing every // publish is a worse failure than not starting. if cfg.leases == nil { - leases, err := newVolumeLeases(context.Background(), nc, cfg.leaseOwner(), cfg.KVReplicas) + leases, err := newVolumeLeases(context.Background(), nc, cfg.leaseOwner()) if err != nil { return fmt.Errorf("volume leases: %w", err) } @@ -79,7 +79,7 @@ func registerProviderSubjects(cfg *Config, nc *nats.Conn) error { // Same reasoning as the lease store: a mount that cannot consult the dirty // marker cannot tell a stale cross-node start from a routine one. if cfg.dirty == nil { - dirty, err := newVolumeDirty(context.Background(), nc, cfg.leaseOwner(), cfg.KVReplicas) + dirty, err := newVolumeDirty(context.Background(), nc, cfg.leaseOwner()) if err != nil { return fmt.Errorf("volume dirty markers: %w", err) } diff --git a/spinifex/services/viperblockd/viperblockd.go b/spinifex/services/viperblockd/viperblockd.go index 1c347ef14..a91a670d5 100644 --- a/spinifex/services/viperblockd/viperblockd.go +++ b/spinifex/services/viperblockd/viperblockd.go @@ -142,11 +142,6 @@ type Config struct { // If empty, falls back to generic ebs.mount / ebs.unmount with queue group (single-node compat). NodeName string - // KVReplicas is the replica count for the volume lease and dirty buckets, - // which is the cluster's node count. 0 means one, for a single node and for - // tests. - KVReplicas int - // NBDTransport controls the transport type: "socket" (default) or "tcp" // Socket is faster for local connections, TCP required for remote/DPU scenarios NBDTransport types.NBDTransport @@ -720,7 +715,7 @@ func launchService(cfg *Config) (err error) { // Bound before recovery, which opens engines: without the store every // engine open refuses, and the daemon would come up unable to adopt the // exports that outlived it. - leases, err := newVolumeLeases(context.Background(), nc, cfg.leaseOwner(), cfg.KVReplicas) + leases, err := newVolumeLeases(context.Background(), nc, cfg.leaseOwner()) if err != nil { return fmt.Errorf("volume leases: %w", err) } diff --git a/spinifex/services/viperblockd/viperblockd_integration_test.go b/spinifex/services/viperblockd/viperblockd_integration_test.go index ccd0dc951..016add161 100644 --- a/spinifex/services/viperblockd/viperblockd_integration_test.go +++ b/spinifex/services/viperblockd/viperblockd_integration_test.go @@ -17,6 +17,7 @@ import ( "testing" "time" + "github.com/mulgadc/spinifex/spinifex/clustersize" "github.com/mulgadc/spinifex/spinifex/types" "github.com/nats-io/nats-server/v2/server" natstest "github.com/nats-io/nats-server/v2/test" @@ -42,6 +43,9 @@ func setupEmbeddedNATS(t *testing.T) (*server.Server, string) { } t.Cleanup(ns.Shutdown) + // One embedded server is a one-node cluster, and bucket creation refuses to + // guess a replica count it was never told. + clustersize.DeclareForTest(t, 1) return ns, ns.ClientURL() } @@ -54,7 +58,7 @@ func installTestVolumeLeases(t *testing.T, cfg *Config, natsURL string) { require.NoError(t, err) t.Cleanup(nc.Close) - leases, err := newVolumeLeases(t.Context(), nc, cfg.leaseOwner(), 1) + leases, err := newVolumeLeases(t.Context(), nc, cfg.leaseOwner()) require.NoError(t, err) cfg.leases = leases } diff --git a/spinifex/services/viperblockd/volume_bucket_replicas_test.go b/spinifex/services/viperblockd/volume_bucket_replicas_test.go index ec207fff7..5acbb5de7 100644 --- a/spinifex/services/viperblockd/volume_bucket_replicas_test.go +++ b/spinifex/services/viperblockd/volume_bucket_replicas_test.go @@ -1,11 +1,12 @@ package viperblockd //test:in-package — both bucket constructors are unexported, and the replica -//count they are given has no exported surface to read it back from. +//count they are created with has no exported surface to read it back from. import ( "testing" + "github.com/mulgadc/spinifex/spinifex/clustersize" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/stretchr/testify/assert" @@ -15,28 +16,28 @@ import ( // The lease bucket decides which node may write a volume, and the dirty bucket // is read on every mount. Created on one replica, both live on one node, and a // cluster that loses that node can neither fence nor mount anywhere — which is -// the failure the whole recovery path is built to survive. So the replica count -// has to reach NATS, and these assert that it does in both directions: a count -// the server cannot satisfy fails loudly rather than quietly creating one -// replica, and the count asked for is the count the bucket is created with. -func TestVolumeBuckets_AreCreatedWithTheReplicaCountTheyAreGiven(t *testing.T) { +// the failure the whole recovery path is built to survive. So the cluster's node +// count has to reach NATS, and these assert that it does in both directions: a +// count the server cannot satisfy fails loudly rather than quietly creating one +// replica, and the cluster's count is the count the bucket is created with. +func TestVolumeBuckets_AreCreatedAtTheClusterReplicaCount(t *testing.T) { _, natsURL := setupEmbeddedNATS(t) nc, err := nats.Connect(natsURL) require.NoError(t, err) t.Cleanup(nc.Close) - leases, err := newVolumeLeases(t.Context(), nc, "node-a", 1) + leases, err := newVolumeLeases(t.Context(), nc, "node-a") require.NoError(t, err) assert.Equal(t, 1, bucketReplicas(t, leases.kv)) - dirty, err := newVolumeDirty(t.Context(), nc, "node-a", 1) + dirty, err := newVolumeDirty(t.Context(), nc, "node-a") require.NoError(t, err) assert.Equal(t, 1, bucketReplicas(t, dirty.kv)) } -// A single embedded server cannot hold three replicas, so asking for them is -// how a test proves the number is not being dropped on the way. +// A single embedded server cannot hold three replicas, so declaring a +// three-node cluster is how a test proves the number is not dropped on the way. func TestVolumeBuckets_RefuseAReplicaCountTheServerCannotHold(t *testing.T) { _, natsURL := setupEmbeddedNATS(t) @@ -44,13 +45,34 @@ func TestVolumeBuckets_RefuseAReplicaCountTheServerCannotHold(t *testing.T) { require.NoError(t, err) t.Cleanup(nc.Close) - _, err = newVolumeLeases(t.Context(), nc, "node-a", 3) + clustersize.RedeclareForTest(t, 3) + + _, err = newVolumeLeases(t.Context(), nc, "node-a") require.Error(t, err, "a lease bucket that cannot be made quorate must fail the service, not serve unreplicated") - _, err = newVolumeDirty(t.Context(), nc, "node-a", 3) + _, err = newVolumeDirty(t.Context(), nc, "node-a") require.Error(t, err, "a dirty bucket that cannot be made quorate must fail the service, not serve unreplicated") } +// TestVolumeBuckets_RefuseAnUndeclaredClusterSize pins the fail-closed choice at +// the call site that matters most: a process that never loaded a cluster config +// must not create these buckets on one node by default. +func TestVolumeBuckets_RefuseAnUndeclaredClusterSize(t *testing.T) { + _, natsURL := setupEmbeddedNATS(t) + + nc, err := nats.Connect(natsURL) + require.NoError(t, err) + t.Cleanup(nc.Close) + + clustersize.RedeclareForTest(t, 0) + + _, err = newVolumeLeases(t.Context(), nc, "node-a") + require.Error(t, err) + + _, err = newVolumeDirty(t.Context(), nc, "node-a") + require.Error(t, err) +} + func bucketReplicas(t *testing.T, kv jetstream.KeyValue) int { t.Helper() diff --git a/spinifex/services/viperblockd/volume_dirty.go b/spinifex/services/viperblockd/volume_dirty.go index 11745c2b4..c4dd43bdc 100644 --- a/spinifex/services/viperblockd/volume_dirty.go +++ b/spinifex/services/viperblockd/volume_dirty.go @@ -8,6 +8,7 @@ import ( "log/slog" "time" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/otelsetup" "github.com/mulgadc/spinifex/spinifex/services/viperblockd/vbwire" "github.com/nats-io/nats.go" @@ -26,22 +27,18 @@ type volumeDirty struct { } // newVolumeDirty binds the dirty bucket, creating it if this is the first node -// up. owner identifies this node in the entries it writes, and replicas is the -// cluster size: this is read on the mount path, so a single replica would make -// every mount depend on one node being up. -func newVolumeDirty(ctx context.Context, nc *nats.Conn, owner string, replicas int) (*volumeDirty, error) { +// up. owner identifies this node in the entries it writes. This is read on the +// mount path, so a single replica would make every mount in the cluster depend +// on one node being up. +func newVolumeDirty(ctx context.Context, nc *nats.Conn, owner string) (*volumeDirty, error) { js, err := jetstream.New(nc) if err != nil { return nil, fmt.Errorf("jetstream: %w", err) } - // Create-or-update rather than create: a bucket left behind by a build that - // made it single-replica is raised here, and a cluster cannot be asked to - // lose its markers to be repaired. - kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ - Bucket: vbwire.DirtyBucket, + kv, err := kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{ + Name: vbwire.DirtyBucket, Description: "volumes whose last seal failed, keyed by volume, naming the node holding the current copy", History: 1, - Replicas: max(replicas, 1), }) if err != nil { return nil, fmt.Errorf("volume dirty bucket: %w", err) diff --git a/spinifex/services/viperblockd/volume_dirty_test.go b/spinifex/services/viperblockd/volume_dirty_test.go index 12fee61d7..546697e02 100644 --- a/spinifex/services/viperblockd/volume_dirty_test.go +++ b/spinifex/services/viperblockd/volume_dirty_test.go @@ -20,7 +20,7 @@ func newTestDirty(t *testing.T, natsURL, owner string) *volumeDirty { require.NoError(t, err) t.Cleanup(nc.Close) - dirty, err := newVolumeDirty(t.Context(), nc, owner, 1) + dirty, err := newVolumeDirty(t.Context(), nc, owner) require.NoError(t, err) return dirty } diff --git a/spinifex/services/viperblockd/volume_lease.go b/spinifex/services/viperblockd/volume_lease.go index 175bbeed9..8ed803402 100644 --- a/spinifex/services/viperblockd/volume_lease.go +++ b/spinifex/services/viperblockd/volume_lease.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" ) @@ -127,24 +128,20 @@ type volumeLeases struct { held map[string]*volumeLease } -// newVolumeLeases binds the lease bucket, creating it if this is the first -// node up. owner identifies this node in the entries it writes, and replicas is -// the cluster size: this bucket decides who may write a volume, so a single -// replica would put every volume in the cluster behind one node staying up. -func newVolumeLeases(ctx context.Context, nc *nats.Conn, owner string, replicas int) (*volumeLeases, error) { +// newVolumeLeases binds the lease bucket, creating it if this is the first node +// up. owner identifies this node in the entries it writes. This bucket decides +// who may write a volume, so a single replica would put every volume in the +// cluster behind one node staying up. +func newVolumeLeases(ctx context.Context, nc *nats.Conn, owner string) (*volumeLeases, error) { js, err := jetstream.New(nc) if err != nil { return nil, fmt.Errorf("jetstream: %w", err) } - // Create-or-update rather than create: a bucket left behind by a build that - // made it single-replica is raised here, and a cluster cannot be asked to - // lose its leases to be repaired. - kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ - Bucket: volumeLeaseBucket, + kv, err := kvutil.GetOrCreateBucketWithOptions(ctx, js, kvutil.BucketOptions{ + Name: volumeLeaseBucket, Description: "one entry per volume with a viperblock engine open on it", TTL: volumeLeaseTTL, History: 1, - Replicas: max(replicas, 1), }) if err != nil { return nil, fmt.Errorf("volume lease bucket: %w", err) diff --git a/spinifex/services/viperblockd/volume_lease_test.go b/spinifex/services/viperblockd/volume_lease_test.go index 4aee0012b..62dc02b8c 100644 --- a/spinifex/services/viperblockd/volume_lease_test.go +++ b/spinifex/services/viperblockd/volume_lease_test.go @@ -24,7 +24,7 @@ func newTestLeases(t *testing.T, natsURL, owner string) *volumeLeases { require.NoError(t, err) t.Cleanup(nc.Close) - leases, err := newVolumeLeases(t.Context(), nc, owner, 1) + leases, err := newVolumeLeases(t.Context(), nc, owner) require.NoError(t, err) return leases } @@ -561,7 +561,7 @@ func TestVolumeLease_UnreachableStoreIsRetryable(t *testing.T) { require.NoError(t, err) t.Cleanup(nc.Close) - leases, err := newVolumeLeases(t.Context(), nc, "node-a", 1) + leases, err := newVolumeLeases(t.Context(), nc, "node-a") require.NoError(t, err) // Close the connection under the bound store, which is the shape of a diff --git a/spinifex/testutil/nats.go b/spinifex/testutil/nats.go index 42ad5aa86..8a2acd73e 100644 --- a/spinifex/testutil/nats.go +++ b/spinifex/testutil/nats.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/mulgadc/spinifex/spinifex/clustersize" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -73,6 +74,11 @@ func StartTestJetStream(t *testing.T) (*server.Server, *nats.Conn, jetstream.Jet js, err := jetstream.New(nc) require.NoError(t, err) + + // One embedded server is a one-node cluster, and bucket creation refuses to + // guess. A test that wants a different count calls RedeclareForTest, because + // declaring is write-once and this has already done it. + clustersize.DeclareForTest(t, 1) return ns, nc, js } diff --git a/spinifex/utils/account.go b/spinifex/utils/account.go index 823bfe4b2..a4f70e2c8 100644 --- a/spinifex/utils/account.go +++ b/spinifex/utils/account.go @@ -1,27 +1,5 @@ package utils -import ( - "sync/atomic" -) - -// defaultKVReplicas is the replica count applied to buckets created via -// GetOrCreateKVBucket. The daemon sets it once at boot to the cluster size so -// lazily-created buckets are born quorate on multi-node; 0 means unset and is -// treated as 1 (single-node / tests). -var defaultKVReplicas atomic.Int64 - -// SetDefaultKVReplicas sets the replica count for buckets subsequently created -// via GetOrCreateKVBucket. Clamped to a minimum of 1. Called once at daemon -// boot with the cluster size, before any handler creates a bucket. -func SetDefaultKVReplicas(n int) { - defaultKVReplicas.Store(int64(max(n, 1))) -} - -// DefaultKVReplicas returns the current default replica count (minimum 1). -func DefaultKVReplicas() int { - return max(int(defaultKVReplicas.Load()), 1) -} - // VersionKey is the well-known KV key used to store a bucket's schema version. // The schema-version helpers that read and write it live in package kvutil, // alongside the other context-aware KV helpers. diff --git a/spinifex/utils/account_replicas_test.go b/spinifex/utils/account_replicas_test.go deleted file mode 100644 index 7ba5ae27e..000000000 --- a/spinifex/utils/account_replicas_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package utils - -import ( - "testing" -) - -func TestDefaultKVReplicas(t *testing.T) { - t.Cleanup(func() { defaultKVReplicas.Store(0) }) - - if got := DefaultKVReplicas(); got != 1 { - t.Fatalf("unset default = %d, want 1", got) - } - SetDefaultKVReplicas(0) - if got := DefaultKVReplicas(); got != 1 { - t.Fatalf("SetDefaultKVReplicas(0) = %d, want 1 (clamped)", got) - } - SetDefaultKVReplicas(3) - if got := DefaultKVReplicas(); got != 3 { - t.Fatalf("SetDefaultKVReplicas(3) = %d, want 3", got) - } -} diff --git a/spinifex/utils/images.go b/spinifex/utils/images.go index 7d629b792..a6e6464f8 100644 --- a/spinifex/utils/images.go +++ b/spinifex/utils/images.go @@ -591,28 +591,6 @@ var AvailableImages = map[string]Images{ Tags: map[string]string{"gpu-vendor": "nvidia"}, }, - // Bedrock self-host serving AMI. Resolved by spinifex:managed-by=bedrock + - // spinifex:bedrock-role=vllm-serving tags — LaunchServingVM's - // resolveServingAMI filters on both, never by name. - "ubuntu-26.04-vllm-serving-x86_64": { - Name: "ubuntu-26.04-vllm-serving-x86_64", - Description: "Ubuntu 26.04 Bedrock co-served bundle image — NVIDIA GPU base + vLLM and TEI baked side by side (uv-managed venv, TEI staged from HuggingFace's CUDA image), one bedrock-serve@ instance per bundle member against its own read-only weights mount", - Distro: "ubuntu", - Version: "26.04", - Arch: "x86_64", - Platform: "Linux/UNIX", - CreatedAt: time.Date(2026, 8, 3, 0, 0, 0, 0, time.UTC), - URL: "https://iso.mulgadc.com/system-ami/ubuntu-26.04-vllm-serving-x86_64.qcow2", - Checksum: "https://iso.mulgadc.com/system-ami/ubuntu-26.04-vllm-serving-x86_64.qcow2.sha256", - ChecksumType: "sha256", - BootMode: "uefi", - Tags: map[string]string{ - "spinifex:managed-by": "bedrock", - "spinifex:bedrock-role": "vllm-serving", - "gpu-vendor": "nvidia", - }, - }, - "ubuntu-26.04-amd-gpu-x86_64": { Name: "ubuntu-26.04-amd-gpu-x86_64", Description: "Ubuntu 26.04 AMD GPU base image — linux-firmware, ROCm CLI, Python toolchain, Docker", @@ -742,6 +720,35 @@ var AvailableImages = map[string]Images{ }, } +// WithheldImages are catalog entries whose artifact is not published, so an +// import of one would fail for every operator. They are kept here rather than +// deleted because the entry is the definition: moving one back into +// AvailableImages is the whole of publishing it. +var WithheldImages = map[string]Images{ + + // Bedrock self-host serving AMI. Resolved by spinifex:managed-by=bedrock + + // spinifex:bedrock-role=vllm-serving tags — LaunchServingVM's + // resolveServingAMI filters on both, never by name. + "ubuntu-26.04-vllm-serving-x86_64": { + Name: "ubuntu-26.04-vllm-serving-x86_64", + Description: "Ubuntu 26.04 Bedrock co-served bundle image — NVIDIA GPU base + vLLM and TEI baked side by side (uv-managed venv, TEI staged from HuggingFace's CUDA image), one bedrock-serve@ instance per bundle member against its own read-only weights mount", + Distro: "ubuntu", + Version: "26.04", + Arch: "x86_64", + Platform: "Linux/UNIX", + CreatedAt: time.Date(2026, 8, 3, 0, 0, 0, 0, time.UTC), + URL: "https://iso.mulgadc.com/system-ami/ubuntu-26.04-vllm-serving-x86_64.qcow2", + Checksum: "https://iso.mulgadc.com/system-ami/ubuntu-26.04-vllm-serving-x86_64.qcow2.sha256", + ChecksumType: "sha256", + BootMode: "uefi", + Tags: map[string]string{ + "spinifex:managed-by": "bedrock", + "spinifex:bedrock-role": "vllm-serving", + "gpu-vendor": "nvidia", + }, + }, +} + // ExtractDiskImageFromFile turns a downloaded artifact at imagepath into a bootable disk // image and returns its path. tar-based archives are expanded into tmpdir and their listing // is scanned for the member that carries the payload; single-file compressed artifacts diff --git a/spinifex/utils/images_test.go b/spinifex/utils/images_test.go index eb3807b01..027a29c34 100644 --- a/spinifex/utils/images_test.go +++ b/spinifex/utils/images_test.go @@ -21,12 +21,19 @@ import ( // resolveServingAMI filters purely on these two tags and never on the image // name, so a typo in either hides the serving image from Ochre and every // endpoint launch dies at "bedrock: no vllm-serving AMI found". -func TestAvailableImages_VLLMServingCarriesBedrockTags(t *testing.T) { +// +// The entry is withheld while its artifact is unpublished, so the contract is +// checked where the definition lives rather than dropped along with it. +func TestWithheldImages_VLLMServingCarriesBedrockTags(t *testing.T) { const key = "ubuntu-26.04-vllm-serving-x86_64" - img, ok := AvailableImages[key] + if _, offered := AvailableImages[key]; offered { + t.Fatalf("%s is offered for import while its artifact is unpublished", key) + } + + img, ok := WithheldImages[key] if !ok { - t.Fatalf("%s missing from AvailableImages", key) + t.Fatalf("%s missing from WithheldImages", key) } for tag, want := range map[string]string{ diff --git a/spinifex/vpcd/imds_instance_state.go b/spinifex/vpcd/imds_instance_state.go index 202006e44..bb2ca73c1 100644 --- a/spinifex/vpcd/imds_instance_state.go +++ b/spinifex/vpcd/imds_instance_state.go @@ -86,7 +86,7 @@ type recordLoader interface { // this node does not hold locally resolves to nothing, and IMDS serves that as // "no such instance" — a guest that boots with no key and no error anywhere. func newInstanceRecordLoader(ctx context.Context, nc *nats.Conn) (recordLoader, error) { - jsm, err := daemon.NewJetStreamManager(nc, 1) + jsm, err := daemon.NewJetStreamManager(nc) if err != nil { return nil, fmt.Errorf("jetstream manager: %w", err) } diff --git a/tests/e2e/Makefile b/tests/e2e/Makefile index 413ba5e22..6ea508985 100644 --- a/tests/e2e/Makefile +++ b/tests/e2e/Makefile @@ -18,9 +18,9 @@ SUITE_TAG ?= $(notdir $(patsubst %/...,%,$(firstword $(PKG)))) # BIN_DIR/.test so CI can ship binaries (not source + go toolchain) to # the remote VM. BIN_DIR ?= $(CURDIR)/_bin -SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink multinode storagegrowth partialblock rds quota diskperf storagefault instancerecovery lbrecovery +SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink multinode storagegrowth partialblock rds quota diskperf storagefault instancerecovery lbrecovery kvquorum -.PHONY: e2e e2e-quota e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-storagegrowth e2e-partialblock e2e-diskperf e2e-rds e2e-storagefault e2e-instancerecovery e2e-lbrecovery e2e-all clean e2e-build +.PHONY: e2e e2e-quota e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-storagegrowth e2e-partialblock e2e-diskperf e2e-rds e2e-storagefault e2e-instancerecovery e2e-lbrecovery e2e-kvquorum e2e-all clean e2e-build RUN_FLAG = $(if $(RUN),-run '$(RUN)',) @@ -74,6 +74,9 @@ e2e-instancerecovery: ; $(MAKE) e2e PKG=./instancerecovery/... TIMEOUT=60m # Stands a load balancer and its backends up before it breaks anything, then # takes the load balancer's node away and brings it back. e2e-lbrecovery: ; $(MAKE) e2e PKG=./lbrecovery/... TIMEOUT=75m +# Audits every KV bucket, then takes a node away and brings it back. Most of its +# budget is the node stopping and starting, not the assertions. +e2e-kvquorum: ; $(MAKE) e2e PKG=./kvquorum/... MODE=multinode TIMEOUT=40m e2e-all: ; $(MAKE) e2e e2e-build: diff --git a/tests/e2e/harness/jetstream.go b/tests/e2e/harness/jetstream.go index f04f3587e..da6a7fd7f 100644 --- a/tests/e2e/harness/jetstream.go +++ b/tests/e2e/harness/jetstream.go @@ -3,9 +3,15 @@ package harness import ( + "fmt" + "sort" + "strings" "testing" + "github.com/mulgadc/spinifex/spinifex/clustersize" + "github.com/mulgadc/spinifex/spinifex/kvutil" "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" ) @@ -42,3 +48,149 @@ func KVReplicaFactors(t *testing.T, env *Env, buckets ...string) map[string]int } return out } + +// WantKVReplicas is how many nodes a KV bucket must be replicated across on a +// cluster of this size. +// +// It defers to the production rule rather than restating it. A test that carried +// its own copy would keep passing after the policy changed, which is the one +// thing this assertion must not do. +func WantKVReplicas(nodes int) int { + return clustersize.ReplicasFor(nodes) +} + +// DialClusterNATS opens a NATS connection to the cluster, using the caller's +// env for the address and credentials. The connection is closed on cleanup. +func DialClusterNATS(t *testing.T, env *Env) *nats.Conn { + t.Helper() + host, token, ca := natsConn(t, env) + nc, err := utils.ConnectNATS(host, token, ca) + if err != nil { + t.Fatalf("connect NATS %s: %v", host, err) + } + t.Cleanup(nc.Close) + return nc +} + +// KVReplication reports every KV bucket in the cluster: its replica count, what +// the cluster's size entitles it to, and which nodes hold it. +// +// The node count is declared here because this process never loaded a cluster +// config, and the audit refuses to guess one rather than assuming a single +// replica. Declaring is write-once, so the reset makes a second call with the +// same count safe and a third with a different one a visible failure. +func KVReplication(t *testing.T, env *Env, nodes int) []kvutil.BucketReport { + t.Helper() + clustersize.ResetForTest() + if err := clustersize.Declare(nodes); err != nil { + t.Fatalf("declare cluster size %d: %v", nodes, err) + } + js, err := jetstream.New(DialClusterNATS(t, env)) + if err != nil { + t.Fatalf("jetstream context: %v", err) + } + reports, err := kvutil.AuditBucketReplicas(t.Context(), js) + if err != nil { + t.Fatalf("audit KV bucket replicas: %v", err) + } + return reports +} + +// RequireKVQuorum fails unless every KV bucket in the cluster is replicated +// across every node, and unless each one is actually held by that many distinct +// nodes rather than merely configured for it. +// +// A bucket below the node count lives on fewer nodes than depend on it, so +// losing one of them makes it unreadable and unwritable from every node at +// once — a cluster-wide outage caused by a single-node event. This runs in +// every multi-node suite rather than one of them, because a bucket created by +// any service on any cell is the one that can be wrong. +func RequireKVQuorum(t *testing.T, env *Env, nodes int) { + t.Helper() + want := WantKVReplicas(nodes) + + reports := KVReplication(t, env, nodes) + if len(reports) == 0 { + t.Fatal("the cluster reports no KV buckets at all, so nothing was checked") + } + + var problems []string + for _, r := range reports { + if r.Replicas < want { + problems = append(problems, fmt.Sprintf( + "%s is on %d replicas, want %d: losing the node holding it takes it down cluster-wide", + r.Bucket, r.Replicas, want)) + continue + } + // Configured, placed and serving are three facts. A stream can carry the + // right count while its group is short of peers or those peers are not + // caught up, and either reads as healthy right up until the node it is + // really on goes away. + if held := distinctPeers(r.Peers); held < want { + problems = append(problems, fmt.Sprintf( + "%s is configured for %d replicas but held by %d node(s) (%s)", + r.Bucket, r.Replicas, held, strings.Join(r.Peers, ","))) + continue + } + if r.Online < want { + problems = append(problems, fmt.Sprintf( + "%s is held by %d node(s) but only %d are caught up and reachable (leader %s)", + r.Bucket, len(r.Peers), r.Online, r.Leader)) + } + } + if len(problems) > 0 { + sort.Strings(problems) + t.Fatalf("%d of %d KV buckets are not replicated across the cluster's %d nodes:\n %s", + len(problems), len(reports), nodes, strings.Join(problems, "\n ")) + } + Detail(t, "kv_buckets", len(reports), "replicas", want, "nodes", nodes) +} + +// RequireKVSingleNode fails unless a one-node cluster's buckets are exactly what +// one node can give them: one replica each, all of them serving. +// +// It is the counterpart assertion to RequireKVQuorum, and it is not the same +// check with a smaller number. On one node R1 is correct rather than a defect, so +// what has to be proved is the opposite: that nothing declared a count this +// deployment cannot place, and that a bucket which cannot be raised has not been +// left unreadable by the attempt. +func RequireKVSingleNode(t *testing.T, env *Env) { + t.Helper() + + reports := KVReplication(t, env, 1) + if len(reports) == 0 { + t.Fatal("the cluster reports no KV buckets at all, so nothing was checked") + } + + var problems []string + for _, r := range reports { + if r.Replicas != 1 { + problems = append(problems, fmt.Sprintf( + "%s is configured for %d replicas on a single node, which JetStream cannot place", + r.Bucket, r.Replicas)) + continue + } + if !r.HasQuorum() { + problems = append(problems, fmt.Sprintf("%s is not serving: %d of %d replicas online", + r.Bucket, r.Online, r.Replicas)) + } + } + if len(problems) > 0 { + sort.Strings(problems) + t.Fatalf("%d of %d KV buckets are wrong for a single-node cluster:\n %s", + len(problems), len(reports), strings.Join(problems, "\n ")) + } + Detail(t, "kv_buckets", len(reports), "replicas", 1, "nodes", 1) +} + +// distinctPeers counts the unique node names holding a stream, since the leader +// is reported separately from the followers and could repeat. +func distinctPeers(peers []string) int { + seen := make(map[string]struct{}, len(peers)) + for _, p := range peers { + if p != "" { + seen[p] = struct{}{} + } + } + return len(seen) +} diff --git a/tests/e2e/harness/multinode_cluster.go b/tests/e2e/harness/multinode_cluster.go index 4870b6f67..3cd2e93e8 100644 --- a/tests/e2e/harness/multinode_cluster.go +++ b/tests/e2e/harness/multinode_cluster.go @@ -30,6 +30,9 @@ type natsRoutezResponse struct { // WaitNATSPeers polls every node's NATS /routez until each reports at least want // distinct peers (timeout 60s, interval 2s). NATS monitor binds 127.0.0.1:8222 // only, so queries run via PeerSSH + curl rather than dialling node.Addr directly. +// +// want is a count of *other* nodes, because /routez reports routes to peers and +// not a node's own membership. A fully formed cluster is len(c.Nodes)-1. // Nodes passed to WithSkipNodes are excluded — use it when a node has been // stopped so its dead monitor port doesn't fail the poll. func (c *Cluster) WaitNATSPeers(t *testing.T, want int, opts ...PollOpt) { diff --git a/tests/e2e/kvquorum/main_test.go b/tests/e2e/kvquorum/main_test.go new file mode 100644 index 000000000..5a534864d --- /dev/null +++ b/tests/e2e/kvquorum/main_test.go @@ -0,0 +1,123 @@ +//go:build e2e + +// Package kvquorum proves that the cluster's own state survives losing a node. +// +// Every control-plane service keeps its state in a JetStream KV bucket, and a +// bucket is replicated across however many nodes it was created with. At one +// replica it lives on one JetStream-chosen server recorded in no config, so +// losing that node makes the bucket unreadable and unwritable from every node +// at once. Leader leases are the worst case: nobody can acquire one whose +// bucket has no quorum, so every reconciler sharing it stops cluster-wide at +// exactly the moment a node has failed. +// +// It is its own suite because it takes a node away, so anything sharing the +// environment would report that outage as its own failure, and because the +// question it answers — "is our internal state actually replicated" — has to +// read as itself rather than as one more red subtest somewhere else. +package kvquorum + +import ( + "fmt" + "os" + "sync" + "testing" + "time" + + "github.com/mulgadc/spinifex/tests/e2e/harness" +) + +const ( + // How long a survivor is given to answer for every bucket after a node + // goes away. A raft group only has to elect a new leader for the streams + // the departed node led, which is seconds on a healthy cluster; the budget + // is generous because a failure here is the whole point of the suite and a + // flake would be read as one. + quorumRecoveryBudget = 2 * time.Minute + + // How long the returning node is given to rejoin and carry its share again. + rejoinBudget = 5 * time.Minute + + // How often a survivor is polled while quorum is expected to return. + pollInterval = 5 * time.Second +) + +var ( + pkgFixOnce sync.Once + pkgFix *Fixture + pkgFixErr error +) + +func TestMain(m *testing.M) { + code := m.Run() + if pkgFix != nil && pkgFix.Harness != nil { + if err := pkgFix.Harness.Close(); err != nil { + fmt.Fprintf(os.Stderr, "e2e teardown: %v\n", err) + code = 1 + } + } + os.Exit(code) +} + +// Fixture carries per-process state shared across this package's tests. +type Fixture struct { + Env *harness.Env + AWS *harness.AWSClient + Harness *harness.Fixture + Cluster *harness.Cluster +} + +// requireFixture returns the package-scoped Fixture, building it on first call. +// +// Three nodes is the floor. On two, a majority is two, so losing either one +// leaves no quorum for any stream and there is nothing to assert beyond "raft +// needs a majority" — which is true of NATS and not a property of ours. +func requireFixture(t *testing.T) *Fixture { + t.Helper() + pkgFixOnce.Do(func() { + if os.Getenv("SPINIFEX_E2E") == "" { + return + } + env := harness.LoadEnv(t) + cluster, err := harness.ClusterFromEnv() + if err != nil { + pkgFixErr = fmt.Errorf("this suite needs node SSH to take a node away: %w", err) + return + } + awsCli := harness.NewAWSClient(t, env) + h, err := harness.NewProcessFixture(awsCli) + if err != nil { + pkgFixErr = err + return + } + pkgFix = &Fixture{Env: env, AWS: awsCli, Harness: h, Cluster: cluster} + }) + if pkgFixErr != nil { + t.Fatalf("kvquorum fixture init: %v", pkgFixErr) + } + if pkgFix == nil { + t.Skip("SPINIFEX_E2E unset") + } + if n := len(pkgFix.Cluster.Nodes); n < 3 { + t.Skipf("quorum under node loss needs a surviving majority, have %d node(s)", n) + } + return pkgFix +} + +// TestKVBucketsAreReplicatedAcrossTheCluster is read-only and runs first, so a +// cluster that is already wrong is reported as wrong rather than as a failure +// to survive a node loss. +func TestKVBucketsAreReplicatedAcrossTheCluster(t *testing.T) { + runBucketsAreReplicated(t, requireFixture(t)) +} + +// TestKVReplicasCommandReportsHealthy proves the operator surface agrees with +// the cluster, on every node, and that its exit status can gate a deployment. +func TestKVReplicasCommandReportsHealthy(t *testing.T) { + runReplicasCommand(t, requireFixture(t)) +} + +// TestKVBucketsSurviveANodeLoss is the assertion the suite exists for, and the +// only one that produces the fault rather than inspecting for it. +func TestKVBucketsSurviveANodeLoss(t *testing.T) { + runBucketsSurviveANodeLoss(t, requireFixture(t)) +} diff --git a/tests/e2e/kvquorum/node_loss_test.go b/tests/e2e/kvquorum/node_loss_test.go new file mode 100644 index 000000000..d2beafaef --- /dev/null +++ b/tests/e2e/kvquorum/node_loss_test.go @@ -0,0 +1,189 @@ +//go:build e2e + +package kvquorum + +import ( + "fmt" + "sort" + "strings" + "testing" + + "github.com/mulgadc/spinifex/spinifex/kvutil" + natssvc "github.com/mulgadc/spinifex/spinifex/services/nats" + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/require" +) + +// runBucketsSurviveANodeLoss produces the fault rather than inspecting for it. +// +// The node taken away is the one leading the most buckets, which is the worst +// one to lose and the one a test that picked arbitrarily would usually miss. +// On a correctly replicated cluster the survivors elect new leaders and keep +// answering; on a cluster of single-replica buckets, every bucket that lived +// on that node answers "no responders" from every node at once, which is the +// outage this whole change exists to remove. +func runBucketsSurviveANodeLoss(t *testing.T, fix *Fixture) { + harness.Phase(t, "KV quorum — the cluster's state survives losing a node") + + nodes := len(fix.Cluster.Nodes) + want := harness.WantKVReplicas(nodes) + + harness.Step(t, "audit the cluster before anything is broken") + before := harness.KVReplication(t, fix.Env, nodes) + require.NotEmpty(t, before, "no KV buckets to lose") + harness.RequireKVQuorum(t, fix.Env, nodes) + + victim, led := busiestLeader(fix.Cluster.Nodes, before) + require.NotZerof(t, victim.Index, + "no bucket named a leader this test can map to a node; peers were %s", peerNames(before)) + harness.Detail(t, "victim", victim.Name, "buckets_led", led, "buckets_total", len(before)) + + survivors := fix.Cluster.Peers(victim) + + harness.Step(t, "take %s away", victim.Name) + // Registered before the stop, so a panic or a cancelled run still brings + // the node back rather than leaving the cluster a node short. + t.Cleanup(func() { + harness.StartNode(t, victim) + harness.WaitNodeServiceReady(t, victim, harness.WithTimeout(rejoinBudget)) + }) + harness.StopNode(t, victim) + + harness.Step(t, "every survivor must still answer for every bucket") + for _, node := range survivors { + requireNodeSeesEveryBucket(t, node, before, want) + } + + harness.Step(t, "bring %s back", victim.Name) + harness.StartNode(t, victim) + harness.WaitNodeServiceReady(t, victim, harness.WithTimeout(rejoinBudget)) + fix.Cluster.WaitNATSPeers(t, nodes-1, harness.WithTimeout(rejoinBudget)) + + harness.Step(t, "the returning node must carry its share again") + for _, node := range fix.Cluster.Nodes { + requireNodeSeesEveryBucket(t, node, before, want) + } + + // Configured replicas must be unchanged. A cluster that repaired itself by + // lowering a count would pass every assertion above and have quietly traded + // the durability this suite is about for a green run. + harness.Step(t, "no bucket lost replicas on the way through") + after := harness.KVReplication(t, fix.Env, nodes) + requireNoBucketLostReplicas(t, before, after) + harness.RequireKVQuorum(t, fix.Env, nodes) +} + +// requireNodeSeesEveryBucket polls a node until its own audit names every +// bucket the cluster had before the outage, with none under-replicated. +// +// Polling rather than asserting once: a raft group whose leader has gone has to +// elect a new one, and that is the interval this test is required to tolerate. +// What it is not required to tolerate is a bucket that never comes back. +func requireNodeSeesEveryBucket(t *testing.T, node harness.Node, before []kvutil.BucketReport, want int) { + t.Helper() + wantBuckets := bucketNames(before) + + harness.EventuallyErr(t, func() error { + reports, err := nodeReplicaReport(t, node) + if err != nil { + return fmt.Errorf("%s: %w", node.Name, err) + } + if missing := missingFrom(wantBuckets, bucketNames(reports)); len(missing) > 0 { + return fmt.Errorf("%s cannot read %d bucket(s): %s", + node.Name, len(missing), strings.Join(missing, ", ")) + } + if under := underReplicated(reports, want); len(under) > 0 { + return fmt.Errorf("%s reports %d under-replicated bucket(s): %s", + node.Name, len(under), strings.Join(under, ", ")) + } + return nil + }, quorumRecoveryBudget, pollInterval) + + harness.Detail(t, "node", node.Name, "buckets_readable", len(wantBuckets)) +} + +// requireNoBucketLostReplicas fails if any bucket came back with fewer replicas +// than it had, which is a silent downgrade rather than a repair. +func requireNoBucketLostReplicas(t *testing.T, before, after []kvutil.BucketReport) { + t.Helper() + was := make(map[string]int, len(before)) + for _, r := range before { + was[r.Bucket] = r.Replicas + } + for _, r := range after { + if prev, ok := was[r.Bucket]; ok && r.Replicas < prev { + t.Fatalf("KV bucket %s went from %d replicas to %d across the outage", r.Bucket, prev, r.Replicas) + } + } +} + +// busiestLeader returns the node leading the most buckets, and how many. +// Leadership is where the loss hurts most: a follower going away costs a raft +// group a peer, a leader going away costs it an election. +// +// The leader is read from the field that holds it rather than guessed at the head +// of the peer list. It is a NATS server name, which is the node name behind a +// prefix rather than the node name itself, so it goes through the helper that +// owns that prefix. +func busiestLeader(nodes []harness.Node, reports []kvutil.BucketReport) (harness.Node, int) { + led := make(map[string]int, len(nodes)) + for _, r := range reports { + if node, ok := natssvc.NodeFromServerName(r.Leader); ok { + led[node]++ + } + } + + var best harness.Node + bestN := -1 + for _, n := range nodes { + if c := led[n.Name]; c > bestN { + best, bestN = n, c + } + } + if bestN <= 0 { + return harness.Node{}, 0 + } + return best, bestN +} + +// bucketNames returns the sorted bucket names in a report set. +func bucketNames(reports []kvutil.BucketReport) []string { + out := make([]string, 0, len(reports)) + for _, r := range reports { + out = append(out, r.Bucket) + } + sort.Strings(out) + return out +} + +// missingFrom returns every name in want that is absent from got. +func missingFrom(want, got []string) []string { + have := make(map[string]struct{}, len(got)) + for _, g := range got { + have[g] = struct{}{} + } + var out []string + for _, w := range want { + if _, ok := have[w]; !ok { + out = append(out, w) + } + } + return out +} + +// peerNames is diagnostic only: it says what the audit did report when no +// leader could be mapped to a node, which is otherwise an opaque failure. +func peerNames(reports []kvutil.BucketReport) string { + seen := map[string]struct{}{} + for _, r := range reports { + for _, p := range r.Peers { + seen[p] = struct{}{} + } + } + out := make([]string, 0, len(seen)) + for p := range seen { + out = append(out, p) + } + sort.Strings(out) + return strings.Join(out, ",") +} diff --git a/tests/e2e/kvquorum/replication_test.go b/tests/e2e/kvquorum/replication_test.go new file mode 100644 index 000000000..248f623be --- /dev/null +++ b/tests/e2e/kvquorum/replication_test.go @@ -0,0 +1,148 @@ +//go:build e2e + +package kvquorum + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/kvutil" + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/require" +) + +// replicasCmd is the operator surface under test. No --config: on a node the +// command finds /etc/spinifex/spinifex.toml itself, which is how anyone +// gating a deployment on it would actually run it. +const replicasCmd = "sudo spx admin kv replicas --json" + +// runBucketsAreReplicated sweeps every KV bucket in the cluster rather than a +// named list. A replica count is set where a bucket is created, and there is a +// bucket-creating call in most services, so a named list only ever covers the +// ones somebody remembered. +func runBucketsAreReplicated(t *testing.T, fix *Fixture) { + harness.Phase(t, "KV quorum — every bucket is replicated across the cluster") + + nodes := len(fix.Cluster.Nodes) + want := harness.WantKVReplicas(nodes) + harness.Detail(t, "cluster_nodes", nodes, "want_replicas", want) + + harness.Step(t, "audit every KV bucket") + reports := harness.KVReplication(t, fix.Env, nodes) + require.NotEmpty(t, reports, "the cluster reports no KV buckets at all, so nothing was checked") + + for _, r := range reports { + harness.Detail(t, "bucket", r.Bucket, "replicas", r.Replicas, "held_by", strings.Join(r.Peers, ",")) + } + harness.RequireKVQuorum(t, fix.Env, nodes) + + // The leader-lease buckets are named as well as swept. They were the worst + // case of the defect this suite guards — created with no replica count at + // all — and a regression that reached only them would otherwise be one more + // line in a long list rather than the headline. + harness.Step(t, "confirm the leader-lease buckets are among those checked") + leases := leaseBuckets(reports) + require.NotEmpty(t, leases, + "no leader-lease bucket was found; either the cluster has never run a reconciler or the audit missed them") + harness.Detail(t, "lease_buckets", strings.Join(leases, ",")) +} + +// leaseBuckets returns the names of the reconciler leader-lease buckets, which +// are the ones a reconciler must acquire before it can do anything at all. +func leaseBuckets(reports []kvutil.BucketReport) []string { + var out []string + for _, r := range reports { + if strings.HasSuffix(r.Bucket, "-leader") || strings.HasSuffix(r.Bucket, "-reconcile") { + out = append(out, r.Bucket) + } + } + sort.Strings(out) + return out +} + +// runReplicasCommand proves the operator surface on every node, not one. The +// command reads cluster-global raft metadata, so every node must give the same +// answer — a node that disagrees is reporting on a cluster it is not fully part +// of, which is worth knowing before anything is gated on its exit status. +func runReplicasCommand(t *testing.T, fix *Fixture) { + harness.Phase(t, "KV quorum — spx admin kv replicas agrees on every node") + + want := harness.WantKVReplicas(len(fix.Cluster.Nodes)) + + counts := make(map[string]int, len(fix.Cluster.Nodes)) + for _, node := range fix.Cluster.Nodes { + harness.Step(t, "run the replica audit on %s", node.Name) + reports, err := nodeReplicaReport(t, node) + require.NoErrorf(t, err, + "%s could not audit its own KV buckets, so nothing can be gated on its exit status", node.Name) + + under := underReplicated(reports, want) + harness.Detail(t, "node", node.Name, "buckets", len(reports), "under_replicated", len(under)) + require.Emptyf(t, under, + "%s reports %d under-replicated bucket(s): %s", node.Name, len(under), strings.Join(under, ", ")) + counts[node.Name] = len(reports) + } + + harness.Step(t, "confirm every node sees the same set of buckets") + var first string + for _, node := range fix.Cluster.Nodes { + if first == "" { + first = node.Name + continue + } + require.Equalf(t, counts[first], counts[node.Name], + "%s sees %d KV buckets and %s sees %d; the two are not looking at the same cluster", + first, counts[first], node.Name, counts[node.Name]) + } +} + +// nodeReplicaReport runs the replica audit on one node and decodes it. A +// non-zero exit is returned as an error rather than fatalled, because callers +// polling a recovering cluster need to retry it. +func nodeReplicaReport(t *testing.T, node harness.Node) ([]kvutil.BucketReport, error) { + t.Helper() + ssh := harness.NewPeerSSH() + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + out, err := ssh.Run(ctx, node.Addr, replicasCmd) + if err != nil { + // The command exits 1 when a bucket is under-replicated, and it still + // prints its report, so the output is decoded either way — a report + // naming the bucket is a better failure than "exit status 1". + if len(out) == 0 { + return nil, fmt.Errorf("%s: %w", node.Name, err) + } + } + var reports []kvutil.BucketReport + if jsonErr := json.Unmarshal(trimToJSON(out), &reports); jsonErr != nil { + return nil, fmt.Errorf("%s: decode replica report (%q): %w", node.Name, string(out), jsonErr) + } + return reports, nil +} + +// trimToJSON drops anything before the JSON document, since sudo and the login +// shell can both write to the same stream. +func trimToJSON(out []byte) []byte { + if i := strings.IndexAny(string(out), "[{"); i > 0 { + return out[i:] + } + return out +} + +// underReplicated names every bucket below want replicas. +func underReplicated(reports []kvutil.BucketReport, want int) []string { + var out []string + for _, r := range reports { + if r.Replicas < want { + out = append(out, fmt.Sprintf("%s(%d/%d)", r.Bucket, r.Replicas, want)) + } + } + sort.Strings(out) + return out +} diff --git a/tests/e2e/multinode/jetstream_replicas_test.go b/tests/e2e/multinode/jetstream_replicas_test.go index 48573dece..e91977fe5 100644 --- a/tests/e2e/multinode/jetstream_replicas_test.go +++ b/tests/e2e/multinode/jetstream_replicas_test.go @@ -11,34 +11,49 @@ import ( "github.com/stretchr/testify/require" ) -// jetStreamReplicaBuckets are the daemon-owned KV buckets created at replicas=1 -// during initJetStream and bumped to the cluster size by upgradeJetStreamReplicas. -// A regression there leaves them stuck at 1, silently losing EC2/cluster state -// on a single node loss — exactly what this guards (the upgrade path is 0% unit). +// jetStreamReplicaBuckets are the daemon-owned KV buckets. They are named +// explicitly as well as swept, so a run that somehow enumerates nothing still +// fails rather than passing on an empty set. var jetStreamReplicaBuckets = []string{ daemon.InstanceStateBucket, daemon.ClusterStateBucket, daemon.TerminatedInstanceBucket, } -// runJetStreamReplicas asserts every daemon KV bucket's configured replica factor -// matches the cluster node count, proving upgradeJetStreamReplicas ran end-to-end -// (the assembled NATS path no unit test reaches). Read-only over NATS. +// runJetStreamReplicas asserts that every KV bucket in the cluster is +// replicated across every node, not just the three the daemon owns. +// +// The sweep is the assertion that matters. A replica count is set where a +// bucket is created, and there is a bucket-creating call in most services, so +// naming buckets here would only ever cover the ones somebody remembered. A +// bucket below the node count lives on fewer nodes than depend on it: losing +// one of them makes it unreadable from every node at once, which is how a +// single-node event becomes a cluster-wide outage. func runJetStreamReplicas(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — JetStream KV Replica Factor") - want := len(fix.Cluster.Nodes) - if want < 2 { - t.Skipf("cluster has %d node(s); replica upgrade is a no-op below 2", want) + nodes := len(fix.Cluster.Nodes) + if nodes < 2 { + t.Skipf("cluster has %d node(s); replication across nodes is a no-op below 2", nodes) } - harness.Detail(t, "cluster_nodes", want, "buckets", strings.Join(jetStreamReplicaBuckets, ",")) + want := harness.WantKVReplicas(nodes) + harness.Detail(t, "cluster_nodes", nodes, "want_replicas", want) - harness.Step(t, "introspect KV stream replica factors") + harness.Step(t, "introspect the daemon's own KV stream replica factors") got := harness.KVReplicaFactors(t, fix.Env, jetStreamReplicaBuckets...) for _, b := range jetStreamReplicaBuckets { harness.Detail(t, "bucket", b, "replicas", got[b]) require.Equalf(t, want, got[b], - "KV bucket %s replicas=%d want=%d — upgradeJetStreamReplicas did not match cluster size; a single node loss would lose this bucket", + "KV bucket %s replicas=%d want=%d — a single node loss would take this bucket down cluster-wide", b, got[b], want) } + + harness.Step(t, "sweep every KV bucket in the cluster") + reports := harness.KVReplication(t, fix.Env, nodes) + names := make([]string, 0, len(reports)) + for _, r := range reports { + names = append(names, r.Bucket) + } + harness.Detail(t, "buckets", len(reports), "names", strings.Join(names, ",")) + harness.RequireKVQuorum(t, fix.Env, nodes) } diff --git a/tests/e2e/single/imagecatalog_test.go b/tests/e2e/single/imagecatalog_test.go index 2bb2f36c3..2ee348a16 100644 --- a/tests/e2e/single/imagecatalog_test.go +++ b/tests/e2e/single/imagecatalog_test.go @@ -34,6 +34,10 @@ const catalogHeadTimeout = 30 * time.Second // purpose. It is the check most likely to catch a problem that is not ours, // and the sooner it is seen the smaller the window in which a released binary // points at a dead URL. +// +// utils.WithheldImages is deliberately not walked: those are the entries whose +// artifact is known to be unpublished, so HEADing them would assert a failure +// we already know about and hide the ones we do not. func runImageCatalogReachable(t *testing.T, _ *Fixture) { harness.Phase(t, "Single — Image catalog URLs still answer") diff --git a/tests/e2e/single/kvreplication_test.go b/tests/e2e/single/kvreplication_test.go new file mode 100644 index 000000000..3f47dddce --- /dev/null +++ b/tests/e2e/single/kvreplication_test.go @@ -0,0 +1,118 @@ +//go:build e2e + +package single + +import ( + "context" + "encoding/json" + "fmt" + "os/exec" + "strings" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/kvutil" + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/require" +) + +// runKVReplication proves what a single-node cluster's own state should look +// like, which is not the multi-node assertion with a smaller number in it. +// +// On one node R1 is the correct answer rather than the defect, so the thing worth +// proving is the opposite: that nothing asked for a replica count this deployment +// cannot place, that the operator surface says so plainly, and that repairing a +// cluster with nothing to repair changes nothing. The multi-node suites cannot +// cover any of that, because on three nodes the unplaceable count is the right +// one. +func runKVReplication(t *testing.T, fix *Fixture) { + harness.Phase(t, "Single — the cluster's own state on one node") + + harness.Step(t, "every bucket is on the one replica a single node can hold") + harness.RequireKVSingleNode(t, fix.Env) + + harness.Step(t, "the operator surface reports a healthy single node") + before, err := localReplicaReport(t) + require.NoError(t, err, + "spx admin kv replicas failed on a healthy single node, so nothing can be gated on it") + require.NotEmpty(t, before, "the command reported no buckets at all, so nothing was checked") + harness.Detail(t, "buckets", len(before), "want_replicas", before[0].Want) + + // Exit 0 is the contract a deployment gate reads. On one node every bucket is + // at the count it should be and serving, so anything else is a bug in the + // audit rather than in the cluster. + require.Equal(t, 1, before[0].Want, + "a single-node cluster wants one replica per bucket, not %d", before[0].Want) + for _, r := range before { + require.Truef(t, r.HasQuorum(), + "%s reports %d of %d replicas online on a single node", r.Bucket, r.Online, r.Replicas) + } + + // Twice, because --repair is documented as safe to run on a healthy cluster + // and safe to run twice. A repair that rewrote a stream would show up as a + // changed replica count or a bucket that stopped serving. + harness.Step(t, "--repair is idempotent and changes nothing it should not") + for i := 1; i <= 2; i++ { + out, repairErr := runLocal(t, "sudo -n spx admin kv replicas --repair --json") + require.NoErrorf(t, repairErr, "repair pass %d failed: %s", i, string(out)) + } + + after, err := localReplicaReport(t) + require.NoError(t, err, "the audit stopped working after two repair passes") + requireSameReplication(t, before, after) + harness.Detail(t, "repair_passes", 2, "buckets", len(after)) +} + +// requireSameReplication fails if any bucket's replication changed across the +// repair passes, naming the bucket rather than reporting that two lists differ. +func requireSameReplication(t *testing.T, before, after []kvutil.BucketReport) { + t.Helper() + was := make(map[string]kvutil.BucketReport, len(before)) + for _, r := range before { + was[r.Bucket] = r + } + require.Len(t, after, len(before), "the number of KV buckets changed across the repair passes") + for _, now := range after { + prev, ok := was[now.Bucket] + require.Truef(t, ok, "%s appeared during a repair that should create nothing", now.Bucket) + require.Equalf(t, prev.Replicas, now.Replicas, + "%s went from %d replicas to %d across a repair that had nothing to repair", + now.Bucket, prev.Replicas, now.Replicas) + require.Truef(t, now.HasQuorum(), + "%s stopped serving after a repair that had nothing to repair", now.Bucket) + } +} + +// localReplicaReport runs the replica audit on this node and decodes it. The +// suite runs on the node it is testing, so there is no ssh hop. +func localReplicaReport(t *testing.T) ([]kvutil.BucketReport, error) { + t.Helper() + out, err := runLocal(t, "sudo -n spx admin kv replicas --json") + if err != nil && len(out) == 0 { + return nil, err + } + var reports []kvutil.BucketReport + if jsonErr := json.Unmarshal(trimToJSON(out), &reports); jsonErr != nil { + return nil, fmt.Errorf("decode replica report (%q): %w", string(out), jsonErr) + } + return reports, err +} + +// runLocal runs a shell command on this node, returning its combined output +// whether or not it succeeded — the audit prints its report even when it exits +// non-zero, and a report naming a bucket is a better failure than an exit status. +func runLocal(t *testing.T, cmd string) ([]byte, error) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + return exec.CommandContext(ctx, "sh", "-c", cmd).CombinedOutput() +} + +// trimToJSON drops anything before the JSON document, since sudo and the login +// shell can both write to the same stream. +func trimToJSON(out []byte) []byte { + if i := strings.IndexAny(string(out), "[{"); i > 0 { + return out[i:] + } + return out +} diff --git a/tests/e2e/single/tests_test.go b/tests/e2e/single/tests_test.go index a1d4b5b8f..8d11a67c3 100644 --- a/tests/e2e/single/tests_test.go +++ b/tests/e2e/single/tests_test.go @@ -30,6 +30,13 @@ func TestAccountScoping(t *testing.T) { runAccountScoping(t, requireSingleNodeFixture(t)) } +// TestKVReplication reads the cluster's own state and runs a repair that has +// nothing to repair, so it mutates nothing another test can see. +func TestKVReplication(t *testing.T) { + t.Parallel() + runKVReplication(t, requireSingleNodeFixture(t)) +} + // Reaches the public internet rather than the cluster: it asks whether the // upstreams our image catalog names are still publishing what it claims. func TestImageCatalogReachable(t *testing.T) { diff --git a/tests/integration/harness.go b/tests/integration/harness.go index bd7f4f8e5..dfcd87c72 100644 --- a/tests/integration/harness.go +++ b/tests/integration/harness.go @@ -137,10 +137,10 @@ func startGateway(t *testing.T, collector *conformanceCollector, opts ...Option) masterKey, err := handlers_iam.GenerateMasterKey() require.NoError(t, err) - iamSvc, err := handlers_iam.NewIAMServiceWithRetry(t.Context(), nc, masterKey, 1) + iamSvc, err := handlers_iam.NewIAMServiceWithRetry(t.Context(), nc, masterKey) require.NoError(t, err) - stsSvc, err := handlers_sts.NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey, 1) + stsSvc, err := handlers_sts.NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey) require.NoError(t, err) encryptedSecret, err := handlers_iam.EncryptSecret(testSecretAccessKey, masterKey) @@ -155,10 +155,10 @@ func startGateway(t *testing.T, collector *conformanceCollector, opts ...Option) // ECR auth bridge signing key: reuses the IAM master key to encrypt the // signing key at rest in the same embedded JetStream KV, matching // production's awsgw-keys wiring (services/awsgw/awsgw.go). - signingKey, verifyKeys, err := gateway_ecrauth.LoadOrCreateSigningKey(t.Context(), js, masterKey, 1) + signingKey, verifyKeys, err := gateway_ecrauth.LoadOrCreateSigningKey(t.Context(), js, masterKey) require.NoError(t, err) - bedrockAccess := gateway_bedrock.NewModelAccessStore(js, 1) + bedrockAccess := gateway_bedrock.NewModelAccessStore(js) cfg := &gateway.GatewayConfig{ DisableLogging: true, diff --git a/tests/integration/service_daemon_lite.go b/tests/integration/service_daemon_lite.go index 7315f0d61..857b7ee6a 100644 --- a/tests/integration/service_daemon_lite.go +++ b/tests/integration/service_daemon_lite.go @@ -35,7 +35,7 @@ func StartServiceDaemonLite(t *testing.T, gw *Gateway) { subscribeServiceMethods(t, nc, "elbv2", elbv2) eks, err := handlers_eks.NewEKSServiceImpl(handlers_eks.EKSServiceDeps{ - Config: cfg, NATSConn: nc, MasterKey: masterKey, Region: testRegion, HolderID: "integration-test-node", ClusterSize: 1, + Config: cfg, NATSConn: nc, MasterKey: masterKey, Region: testRegion, HolderID: "integration-test-node", }) require.NoError(t, err, "construct EKS service") t.Cleanup(eks.Shutdown) diff --git a/tests/integration/shared_nats.go b/tests/integration/shared_nats.go index ac285f2f8..0a3345609 100644 --- a/tests/integration/shared_nats.go +++ b/tests/integration/shared_nats.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/mulgadc/spinifex/spinifex/clustersize" "github.com/mulgadc/spinifex/tests/fixtures/scratch" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" @@ -43,6 +44,14 @@ const natsStorePrefix = "spinifex-integration-nats-" // startSharedNATS boots the one embedded, JetStream-enabled NATS server the // whole package's tests connect into. func startSharedNATS() (*sharedNATS, error) { + // One embedded, non-clustered server, so every bucket this binary creates is + // on one replica. Bucket creation refuses until it is told, and this package + // builds its gateway by hand rather than through config.LoadConfig, which is + // where a real process declares it. + if err := clustersize.Declare(1); err != nil { + return nil, err + } + // The store cannot be a t.TempDir(): the server outlives every individual // test, so the first test to finish would delete it out from under it. scratch.SweepAbandoned(os.TempDir(), natsStorePrefix, scratch.DefaultMaxAge)