style(daemon): align JetStreamManager fields

A mismatch now exits 1 so the unit shows failed. A recorded value that
does not decode or names no supported provider is a distinct permanent
refusal quoting the raw value. Permanence is a sentinel matched with
errors.Is, and the cluster-config bucket is declared beside the others.

spinifex/bootstrap/config/config.go

@@ -5,6 +5,7 @@ package config
 import (
 	"fmt"
 	"log/slog"
+	"maps"
 	"net/netip"
 	"os"
 	"path/filepath"
@@ -355,6 +356,11 @@ type EBSConfig struct {
 	CacheSizeMB *int `json:"CacheSizeMB" mapstructure:"cache_size_mb"`
 }
 
+// SupportedEBSProvider reports whether name is a provider a node may run.
+func SupportedEBSProvider(name string) bool {
+	return name == EBSProviderViperblockd || name == EBSProviderQEMUNBD
+}
+
 // ResolvedProvider normalizes an empty Provider to EBSProviderViperblockd,
 // the default provider.
 func (c EBSConfig) ResolvedProvider() string {
@@ -801,6 +807,27 @@ func validateVultrPool(cc *ClusterConfig, p ExternalPool) error {
 	return nil
 }
 
+// validateSingleEBSProvider refuses nodes whose explicit [nodes.<name>.ebs]
+// providers differ. A peer stanza without one is unknown, not the default: the
+// generated config omits it, and the cluster's KV lock is the authoritative check.
+func validateSingleEBSProvider(cc *ClusterConfig) error {
+	var first, want string
+	for _, name := range slices.Sorted(maps.Keys(cc.Nodes)) {
+		if !viper.IsSet("nodes." + name + ".ebs.provider") {
+			continue
+		}
+		got := cc.Nodes[name].EBS.ResolvedProvider()
+		if first == "" {
+			first, want = name, got
+			continue
+		}
+		if got != want {
+			return fmt.Errorf("config: [nodes.%s.ebs] provider resolves to %q but [nodes.%s.ebs] resolves to %q; every node in a cluster must use the same EBS provider", first, want, name, got)
+		}
+	}
+	return nil
+}
+
 func validateClusterConfig(cc *ClusterConfig) error {
 	if viper.IsSet("network.external_dhcp") {
 		return fmt.Errorf("config: [network] external_dhcp is no longer supported; remove the key (static WAN-pool allocation only)")
@@ -809,9 +836,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		if viper.IsSet("nodes." + nodeName + ".vpcd.dhcp_bind_bridge") {
 			return fmt.Errorf("config: [nodes.%s.vpcd] dhcp_bind_bridge is no longer supported; remove the key (vpcd no longer runs a DHCP client)", nodeName)
 		}
-		switch nodeCfg.EBS.Provider {
-		case "", EBSProviderViperblockd, EBSProviderQEMUNBD:
-		case EBSProviderEmbedded:
+		switch {
+		case nodeCfg.EBS.Provider == "", SupportedEBSProvider(nodeCfg.EBS.Provider):
+		case nodeCfg.EBS.Provider == EBSProviderEmbedded:
 			// Refuse rather than silently upgrade: the embedded engine is gone,
 			// and starting anyway would migrate this node's volumes to
 			// ebsmetadata one-way without the operator ever asking for it.
@@ -829,6 +856,9 @@ func validateClusterConfig(cc *ClusterConfig) error {
 			return fmt.Errorf("config: [nodes.%s.ebs] cache_size_mb=%d must not be negative; use 0 to disable the cache, or remove the key for the default of %d", nodeName, *c, DefaultCacheSizeMB)
 		}
 	}
+	if err := validateSingleEBSProvider(cc); err != nil {
+		return err
+	}
 
 	if err := validateNorthstarCluster(cc); err != nil {
 		return err

spinifex/bootstrap/config/config_test.go

@@ -1108,6 +1108,45 @@ provider = "bogus"
 	assert.Contains(t, err.Error(), `provider="bogus"`)
 }
 
+func TestLoadConfig_EBSProvider_SingleProviderPerCluster(t *testing.T) {
+	stanza := func(node, provider string) string {
+		return fmt.Sprintf("[nodes.%s.ebs]\nprovider = %q\n", node, provider)
+	}
+	cases := []struct {
+		name    string
+		ebs     []string
+		wantErr string
+	}{
+		{"all unset", []string{"", "", ""}, ""},
+		{"explicit viperblockd beside unset peers", []string{"", stanza("n2", EBSProviderViperblockd), ""}, ""},
+		{"explicit qemunbd beside unset peers", []string{stanza("n1", EBSProviderQEMUNBD), "", ""}, ""},
+		{"all explicit qemunbd", []string{stanza("n1", EBSProviderQEMUNBD), stanza("n2", EBSProviderQEMUNBD), stanza("n3", EBSProviderQEMUNBD)}, ""},
+		{"two explicit providers differ", []string{stanza("n1", EBSProviderQEMUNBD), stanza("n2", EBSProviderViperblockd), ""},
+			`[nodes.n1.ebs] provider resolves to "qemunbd" but [nodes.n2.ebs] resolves to "viperblockd"`},
+		{"only the third node differs", []string{stanza("n1", EBSProviderViperblockd), stanza("n2", EBSProviderViperblockd), stanza("n3", EBSProviderQEMUNBD)},
+			`[nodes.n1.ebs] provider resolves to "viperblockd" but [nodes.n3.ebs] resolves to "qemunbd"`},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			body := "node = \"n1\"\n"
+			for i, ebs := range tc.ebs {
+				body += fmt.Sprintf("\n[nodes.n%d]\nregion = \"us-east-1\"\n%s", i+1, ebs)
+			}
+			resetViper(t)
+			path := filepath.Join(t.TempDir(), "spinifex.toml")
+			require.NoError(t, os.WriteFile(path, []byte(body), 0600))
+
+			_, err := LoadConfig(path)
+			if tc.wantErr == "" {
+				require.NoError(t, err)
+				return
+			}
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.wantErr)
+		})
+	}
+}
+
 func TestEBSConfig_ResolvedTunablesDefault(t *testing.T) {
 	var c EBSConfig
 	assert.Equal(t, DefaultNBDKitThreads, c.ResolvedThreads())

spinifex/operator/cli/admin_test.go

@@ -1748,3 +1748,31 @@ func TestApplyNetworkConfig_PropagatesPoolDNSServersWithoutSelf(t *testing.T) {
 	require.Len(t, settings.Pools, 1)
 	assert.Equal(t, []string{"1.1.1.1", "10.0.0.6", "8.8.8.8"}, settings.Pools[0].DNSServers)
 }
+
+// A generated config writes no [nodes.<peer>.ebs] stanza, so a node switched to
+// a non-default provider must still load: an absent peer provider is unknown.
+func TestSpinifexTomlTemplate_NonDefaultProviderLoadsBesidePeers(t *testing.T) {
+	allNodes := map[string]formation.NodeInfo{
+		"node1": {Name: "node1", BindIP: "10.0.0.1", AdvertiseIP: "192.168.1.31"},
+		"node2": {Name: "node2", BindIP: "10.0.0.2", AdvertiseIP: "192.168.1.32"},
+		"node3": {Name: "node3", BindIP: "10.0.0.3", AdvertiseIP: "192.168.1.33"},
+	}
+	settings := multinodeNorthstarSettings("/etc/spinifex/northstar/northstar.toml")
+	settings.RemoteNodes = buildRemoteNodes(allNodes, "node1", settings.NorthstarConfigPath)
+	require.Len(t, settings.RemoteNodes, 2)
+
+	path := filepath.Join(t.TempDir(), "spinifex.toml")
+	require.NoError(t, admin.GenerateConfigFile(path, spinifexTomlTemplate, settings))
+	rendered, err := os.ReadFile(path)
+	require.NoError(t, err)
+	const defaultLine = "provider = \"viperblockd\"\n"
+	require.Equal(t, 1, strings.Count(string(rendered), defaultLine), "the template must set the provider for its own node only")
+	switched := strings.Replace(string(rendered), defaultLine, "provider = \"qemunbd\"\n", 1)
+	require.NoError(t, os.WriteFile(path, []byte(switched), 0o600))
+
+	cfg, err := config.LoadConfig(path)
+	require.NoError(t, err)
+	assert.Equal(t, config.EBSProviderQEMUNBD, cfg.Nodes["node1"].EBS.ResolvedProvider())
+	assert.Contains(t, cfg.Nodes, "node2")
+	assert.Contains(t, cfg.Nodes, "node3")
+}

spinifex/operator/cli/templates/spinifex.toml

@@ -136,6 +136,9 @@ encryption_key_file = "{{.EncryptionKeyFile}}"
 {{- end}}
 
 [nodes.{{.Node}}.ebs]
+# provider: every node in a cluster must name the same one, and the daemon
+# refuses to start on a provider other than the one the cluster recorded.
+# Changing it needs the cluster wiped and reformed.
 provider = "viperblockd"
 # Per-volume export tunables. Both are applied per nbdkit process and there is
 # one of those per volume, so the host cost of each is the value times the

spinifex/operator/upgrade/VERSIONS.md

@@ -21,6 +21,7 @@ Single source of truth for the current schema version of every config file Spini
 |---|---|---|
 | `spinifex-instance-state` | `5` | `daemon.InstanceStateBucketVersion` |
 | `spinifex-cluster-state` | `1` | `daemon.ClusterStateBucketVersion` |
+| `spinifex-cluster-config` | `1` | `daemon.ClusterConfigBucketVersion` |
 | `spinifex-terminated-instances` | `3` | `daemon.TerminatedInstanceBucketVersion` |
 
 ## Registered migrations

spinifex/runtime/daemon/daemon.go

@@ -1943,6 +1943,17 @@ func (d *Daemon) startCluster() error {
 		return fmt.Errorf("initialize JetStream: %w", err)
 	}
 
+	// Before any service is built: a node on a different provider from the
+	// cluster's would address existing volumes through the wrong implementation.
+	// A refusal exits so the unit shows failed rather than a healthy idle daemon.
+	if err := d.claimEBSProvider(); err != nil {
+		if errors.Is(err, errStartupPermanent) {
+			slog.Error("Refusing to start: EBS provider lock", "err", err)
+			d.exitFunc(1)
+		}
+		return fmt.Errorf("EBS provider lock: %w", err)
+	}
+
 	// Remove the obsolete spinifex-dhcp-leases bucket (idempotent).
 	if js, jsErr := jetstream.New(d.natsConn); jsErr == nil {
 		if err := kvutil.DeleteBucketIfExists(d.ctx, js, "spinifex-dhcp-leases"); err != nil {
@@ -2709,6 +2720,10 @@ func (d *Daemon) initJetStream() error {
 			err = d.jsManager.InitTerminatedInstanceBucket()
 		}
 
+		if err == nil {
+			err = d.jsManager.InitClusterConfigBucket()
+		}
+
 		if err == nil {
 			d.jsManager.SetSyncObserver(d)
 			// Replicas cannot be undeclared here: bucket creation refuses
@@ -2757,6 +2772,29 @@ func (d *Daemon) upgradeJetStreamReplicas() {
 	}
 }
 
+// claimEBSProvider records this node's EBS provider as the cluster's, or
+// refuses when the cluster already records a different one.
+func (d *Daemon) claimEBSProvider() error {
+	provider := d.config.EBS.ResolvedProvider()
+	rev, err := initServiceWithRetry("EBS provider lock", func() (uint64, error) {
+		ctx, cancel := context.WithTimeout(context.Background(), ebsProviderClaimTimeout)
+		defer cancel()
+		return d.jsManager.ClaimEBSProvider(ctx, d.node, provider)
+	})
+	if err != nil {
+		return err
+	}
+	slog.Info("EBS provider matches the cluster's", "provider", provider, "revision", rev)
+	return nil
+}
+
+// ebsProviderClaimTimeout bounds one attempt at the provider lock's KV calls.
+const ebsProviderClaimTimeout = 10 * time.Second
+
+// errStartupPermanent marks an error that a startup retry loop cannot clear,
+// so the loop returns it at once rather than waiting out its budget.
+var errStartupPermanent = errors.New("permanent startup refusal")
+
 // initRetrySleep is the sleep seam used by initServiceWithRetry; tests override it.
 var initRetrySleep = time.Sleep
 
@@ -2777,6 +2815,10 @@ func initServiceWithRetry[T any](name string, initFn func() (T, error)) (T, erro
 			}
 			return result, nil
 		}
+		if errors.Is(err, errStartupPermanent) {
+			var zero T
+			return zero, err
+		}
 
 		elapsed := time.Since(start)
 		if elapsed >= maxWait {

spinifex/runtime/daemon/ebs_provider_lock.go

@@ -0,0 +1,103 @@
+package daemon
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// EBSProviderKey is the ClusterConfigBucket key recording the one EBS
+// provider the cluster runs.
+const EBSProviderKey = "ebs-provider"
+
+// EBSProviderRecord is the value at EBSProviderKey.
+type EBSProviderRecord struct {
+	Provider string `json:"provider"`
+	// RecordedBy names the node whose Create won, for the refusal message.
+	RecordedBy string `json:"recorded_by"`
+}
+
+// EBSProviderMismatchError is the refusal a daemon returns when its configured
+// provider is not the one the cluster recorded. Retrying cannot clear it.
+type EBSProviderMismatchError struct {
+	Node       string
+	Configured string
+	Recorded   EBSProviderRecord
+}
+
+func (e *EBSProviderMismatchError) Error() string {
+	return fmt.Sprintf("EBS provider mismatch: node %s is configured with provider %q, but this cluster runs %q (recorded by node %s in KV bucket %s, key %s); "+
+		"set [nodes.%s.ebs] provider = %q to join it, or change the cluster's provider by wiping and reforming the cluster, since volumes record no provider to migrate by",
+		e.Node, e.Configured, e.Recorded.Provider, e.Recorded.RecordedBy, ClusterConfigBucket, EBSProviderKey, e.Node, e.Recorded.Provider)
+}
+
+func (e *EBSProviderMismatchError) Unwrap() error { return errStartupPermanent }
+
+// EBSProviderRecordInvalidError reports a recorded value that does not decode
+// or names no supported provider, so there is nothing to compare against.
+type EBSProviderRecordInvalidError struct {
+	Raw    []byte
+	Reason string
+}
+
+func (e *EBSProviderRecordInvalidError) Error() string {
+	return fmt.Sprintf("EBS provider lock unreadable: KV bucket %s, key %s holds %q (%s); "+
+		"correct the value to {\"provider\":\"<provider>\"} naming the provider the cluster's volumes were created on, or delete the key so the next daemon records its own",
+		ClusterConfigBucket, EBSProviderKey, e.Raw, e.Reason)
+}
+
+func (e *EBSProviderRecordInvalidError) Unwrap() error { return errStartupPermanent }
+
+// ClaimEBSProvider records provider as the cluster's EBS provider if none is
+// recorded, and otherwise checks it against the recorded one. Create decides
+// a race, so one provider is ever recorded; it returns that record's revision.
+func (m *JetStreamManager) ClaimEBSProvider(ctx context.Context, node, provider string) (uint64, error) {
+	if m.clusterConfig == nil {
+		return 0, errors.New("cluster config KV not initialized")
+	}
+	claim := &EBSProviderRecord{Provider: provider, RecordedBy: node}
+	rev, err := kvstore.On[EBSProviderRecord](m.clusterConfig).Create(ctx, EBSProviderKey, claim)
+	if err == nil {
+		return rev, nil
+	}
+	if !errors.Is(err, kvstore.ErrExists) {
+		return 0, fmt.Errorf("record EBS provider: %w", err)
+	}
+	recorded, rev, err := m.readEBSProvider(ctx)
+	if err != nil {
+		return 0, err
+	}
+	if recorded.Provider != provider {
+		return 0, &EBSProviderMismatchError{Node: node, Configured: provider, Recorded: *recorded}
+	}
+	return rev, nil
+}
+
+// readEBSProvider reads the raw value rather than through the store's codec,
+// so a malformed one can be quoted in the refusal.
+func (m *JetStreamManager) readEBSProvider(ctx context.Context) (*EBSProviderRecord, uint64, error) {
+	kv, err := m.clusterConfig.KV(ctx)
+	if err != nil {
+		return nil, 0, err
+	}
+	entry, err := kv.Get(ctx, EBSProviderKey)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			return nil, 0, fmt.Errorf("recorded EBS provider vanished after Create lost: %w", err)
+		}
+		return nil, 0, fmt.Errorf("read recorded EBS provider: %w", err)
+	}
+	var rec EBSProviderRecord
+	if err := json.Unmarshal(entry.Value(), &rec); err != nil {
+		return nil, 0, &EBSProviderRecordInvalidError{Raw: entry.Value(), Reason: "not a JSON record: " + err.Error()}
+	}
+	if !config.SupportedEBSProvider(rec.Provider) {
+		return nil, 0, &EBSProviderRecordInvalidError{Raw: entry.Value(), Reason: fmt.Sprintf("provider %q is not a supported provider", rec.Provider)}
+	}
+	return &rec, entry.Revision(), nil
+}

spinifex/runtime/daemon/ebs_provider_lock_test.go

@@ -0,0 +1,272 @@
+package daemon
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"slices"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// newLockManager opens the cluster-config bucket over its own connection, as
+// a separate daemon would.
+func newLockManager(t *testing.T, url string) *JetStreamManager {
+	t.Helper()
+	nc, err := nats.Connect(url)
+	require.NoError(t, err)
+	t.Cleanup(nc.Close)
+	m, err := NewJetStreamManager(nc)
+	require.NoError(t, err)
+	require.NoError(t, m.InitClusterConfigBucket())
+	return m
+}
+
+func recordedEBSProvider(t *testing.T, js jetstream.JetStream) (EBSProviderRecord, uint64) {
+	t.Helper()
+	kv, err := js.KeyValue(t.Context(), ClusterConfigBucket)
+	require.NoError(t, err)
+	entry, err := kv.Get(t.Context(), EBSProviderKey)
+	require.NoError(t, err)
+	var rec EBSProviderRecord
+	require.NoError(t, json.Unmarshal(entry.Value(), &rec))
+	return rec, entry.Revision()
+}
+
+func TestClusterConfigBucket_NoTTLAndSchemaVersion(t *testing.T) {
+	ns, _, js := testutil.StartTestJetStream(t)
+	newLockManager(t, ns.ClientURL())
+
+	kv, err := js.KeyValue(t.Context(), ClusterConfigBucket)
+	require.NoError(t, err)
+	status, err := kv.Status(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, int64(1), status.History())
+	assert.Zero(t, status.TTL(), "a TTL would expire the lock and let a second provider in")
+
+	v, err := kvutil.ReadVersion(t.Context(), kv)
+	require.NoError(t, err)
+	assert.Equal(t, 1, v)
+}
+
+func TestClaimEBSProvider_FirstClaimRecordsAndMatchingNodesJoin(t *testing.T) {
+	ns, _, js := testutil.StartTestJetStream(t)
+	first := newLockManager(t, ns.ClientURL())
+	second := newLockManager(t, ns.ClientURL())
+
+	rev, err := first.ClaimEBSProvider(t.Context(), "node1", config.EBSProviderQEMUNBD)
+	require.NoError(t, err)
+
+	joined, err := second.ClaimEBSProvider(t.Context(), "node2", config.EBSProviderQEMUNBD)
+	require.NoError(t, err)
+	assert.Equal(t, rev, joined)
+	restarted, err := first.ClaimEBSProvider(t.Context(), "node1", config.EBSProviderQEMUNBD)
+	require.NoError(t, err, "a restart on the same provider must start")
+	assert.Equal(t, rev, restarted)
+
+	rec, revAfter := recordedEBSProvider(t, js)
+	assert.Equal(t, EBSProviderRecord{Provider: config.EBSProviderQEMUNBD, RecordedBy: "node1"}, rec)
+	assert.Equal(t, rev, revAfter, "a matching claim must not rewrite the key")
+}
+
+func TestClaimEBSProvider_DifferentProviderRefused(t *testing.T) {
+	ns, _, js := testutil.StartTestJetStream(t)
+	first := newLockManager(t, ns.ClientURL())
+	second := newLockManager(t, ns.ClientURL())
+
+	_, err := first.ClaimEBSProvider(t.Context(), "node1", config.EBSProviderViperblockd)
+	require.NoError(t, err)
+	_, err = second.ClaimEBSProvider(t.Context(), "node2", config.EBSProviderQEMUNBD)
+
+	var mismatch *EBSProviderMismatchError
+	require.ErrorAs(t, err, &mismatch)
+	assert.ErrorIs(t, err, errStartupPermanent)
+	assert.Equal(t, "node2", mismatch.Node)
+	assert.Equal(t, config.EBSProviderQEMUNBD, mismatch.Configured)
+	assert.Equal(t, EBSProviderRecord{Provider: config.EBSProviderViperblockd, RecordedBy: "node1"}, mismatch.Recorded)
+	for _, want := range []string{`"qemunbd"`, `"viperblockd"`, "node1", ClusterConfigBucket, EBSProviderKey, `[nodes.node2.ebs] provider = "viperblockd"`, "wiping and reforming"} {
+		assert.Contains(t, err.Error(), want)
+	}
+
+	rec, _ := recordedEBSProvider(t, js)
+	assert.Equal(t, config.EBSProviderViperblockd, rec.Provider, "a refused claim must leave the recorded provider in place")
+}
+
+// A value that does not decode, or names no supported provider, is refused as
+// unreadable rather than as a mismatch, and quotes what the key holds.
+func TestClaimEBSProvider_InvalidRecordRefused(t *testing.T) {
+	cases := []struct {
+		name   string
+		raw    string
+		reason string
+	}{
+		{"not JSON", "qemunbd", "not a JSON record"},
+		{"empty provider", `{"provider":"","recorded_by":"node1"}`, `provider "" is not a supported provider`},
+		{"unknown provider", `{"provider":"cephd","recorded_by":"node1"}`, `provider "cephd" is not a supported provider`},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			ns, _, js := testutil.StartTestJetStream(t)
+			m := newLockManager(t, ns.ClientURL())
+			kv, err := js.KeyValue(t.Context(), ClusterConfigBucket)
+			require.NoError(t, err)
+			_, err = kv.Put(t.Context(), EBSProviderKey, []byte(tc.raw))
+			require.NoError(t, err)
+
+			_, err = m.ClaimEBSProvider(t.Context(), "node2", config.EBSProviderViperblockd)
+			var invalid *EBSProviderRecordInvalidError
+			require.ErrorAs(t, err, &invalid)
+			assert.ErrorIs(t, err, errStartupPermanent)
+			assert.NotErrorAs(t, err, new(*EBSProviderMismatchError))
+			for _, want := range []string{ClusterConfigBucket, EBSProviderKey, strconv.Quote(tc.raw), tc.reason} {
+				assert.Contains(t, err.Error(), want)
+			}
+		})
+	}
+}
+
+// Two daemons on different providers starting together: whichever Create wins
+// is recorded, and the other is refused.
+func TestClaimEBSProvider_ConcurrentDifferentProvidersLeaveOneKeyAndOneRefusal(t *testing.T) {
+	for range 20 {
+		ns, _, js := testutil.StartTestJetStream(t)
+		managers := []*JetStreamManager{newLockManager(t, ns.ClientURL()), newLockManager(t, ns.ClientURL())}
+		providers := []string{config.EBSProviderViperblockd, config.EBSProviderQEMUNBD}
+		nodes := []string{"node1", "node2"}
+
+		errs := make([]error, 2)
+		revs := make([]uint64, 2)
+		start := make(chan struct{})
+		var wg sync.WaitGroup
+		for i := range managers {
+			wg.Go(func() {
+				<-start
+				revs[i], errs[i] = managers[i].ClaimEBSProvider(context.Background(), nodes[i], providers[i])
+			})
+		}
+		close(start)
+		wg.Wait()
+
+		rec, rev := recordedEBSProvider(t, js)
+		winner := slices.Index(providers, rec.Provider)
+		require.GreaterOrEqual(t, winner, 0, "recorded provider %q is neither claimant's", rec.Provider)
+		assert.Equal(t, nodes[winner], rec.RecordedBy)
+		require.NoError(t, errs[winner])
+		assert.Equal(t, revs[winner], rev, "the key must still be the winner's Create")
+
+		var mismatch *EBSProviderMismatchError
+		require.ErrorAs(t, errs[1-winner], &mismatch, "loser must be refused")
+		assert.Equal(t, rec.Provider, mismatch.Recorded.Provider)
+	}
+}
+
+// A permanent refusal returns on the first attempt rather than being retried
+// for the startup budget.
+func TestInitServiceWithRetry_PermanentErrorNotRetried(t *testing.T) {
+	stubInitRetrySleep(t)
+	calls := 0
+	_, err := initServiceWithRetry("svc", func() (int, error) {
+		calls++
+		if calls > 1 {
+			t.Fatal("a permanent error was retried")
+		}
+		return 0, fmt.Errorf("refused: %w", errStartupPermanent)
+	})
+	require.ErrorIs(t, err, errStartupPermanent)
+	assert.Equal(t, 1, calls)
+}
+
+// An unset provider resolves to viperblockd for the check, and the daemon
+// refuses on the first mismatch: a retry would need a sleep first.
+func TestDaemonClaimEBSProvider_MismatchRefusesWithoutRetry(t *testing.T) {
+	ns, _, _ := testutil.StartTestJetStream(t)
+	prev := initRetrySleep
+	initRetrySleep = func(time.Duration) { t.Fatal("a mismatch was retried") }
+	t.Cleanup(func() { initRetrySleep = prev })
+
+	first := &Daemon{node: "node1", config: &config.Config{EBS: config.EBSConfig{Provider: config.EBSProviderQEMUNBD}}, jsManager: newLockManager(t, ns.ClientURL())}
+	require.NoError(t, first.claimEBSProvider())
+
+	second := &Daemon{node: "node2", config: &config.Config{}, jsManager: newLockManager(t, ns.ClientURL())}
+	err := second.claimEBSProvider()
+	var mismatch *EBSProviderMismatchError
+	require.ErrorAs(t, err, &mismatch)
+	assert.Equal(t, config.EBSProviderViperblockd, mismatch.Configured)
+}
+
+// A transient KV failure is retried until the bucket answers.
+func TestDaemonClaimEBSProvider_RetriesUntilBucketOpens(t *testing.T) {
+	ns, _, js := testutil.StartTestJetStream(t)
+	m := newLockManager(t, ns.ClientURL())
+	bucket := m.clusterConfig
+	m.clusterConfig = nil
+
+	sleeps := 0
+	prev := initRetrySleep
+	initRetrySleep = func(time.Duration) {
+		sleeps++
+		m.clusterConfig = bucket
+	}
+	t.Cleanup(func() { initRetrySleep = prev })
+
+	d := &Daemon{node: "node1", config: &config.Config{}, jsManager: m}
+	require.NoError(t, d.claimEBSProvider())
+	assert.Equal(t, 1, sleeps)
+	rec, _ := recordedEBSProvider(t, js)
+	assert.Equal(t, config.EBSProviderViperblockd, rec.Provider)
+}
+
+// A daemon whose provider differs from the recorded one exits 1 before any
+// cluster service is built or any instance restored.
+func TestStartCluster_EBSProviderMismatchExits(t *testing.T) {
+	t.Setenv("SPINIFEX_REQUIRE_NATS", "")
+	ns, _, js := testutil.StartTestJetStream(t)
+	seed := newLockManager(t, ns.ClientURL())
+	_, err := seed.ClaimEBSProvider(t.Context(), "node-2", config.EBSProviderQEMUNBD)
+	require.NoError(t, err)
+
+	tmpDir := t.TempDir()
+	cfg := &config.ClusterConfig{
+		Node: "node-1",
+		Nodes: map[string]config.Config{
+			"node-1": {BaseDir: tmpDir, DataDir: tmpDir, NATS: config.NATSConfig{Host: ns.ClientURL()}},
+		},
+	}
+	d, err := NewDaemon(cfg)
+	require.NoError(t, err)
+	t.Cleanup(d.cancel)
+	exitCh := make(chan int, 1)
+	d.exitFunc = func(code int) { exitCh <- code }
+
+	err = d.startCluster()
+	var mismatch *EBSProviderMismatchError
+	require.ErrorAs(t, err, &mismatch)
+	assert.Equal(t, config.EBSProviderViperblockd, mismatch.Configured)
+	select {
+	case code := <-exitCh:
+		assert.Equal(t, 1, code)
+	default:
+		t.Fatal("a provider mismatch must request exit(1)")
+	}
+
+	assert.Nil(t, d.instanceService, "no cluster service may be built after the refusal")
+	assert.Nil(t, d.imageService)
+	assert.Nil(t, d.snapshotService)
+	assert.Nil(t, d.volumeService)
+	assert.Nil(t, d.vpcService)
+	assert.Nil(t, d.eksService)
+	assert.Zero(t, d.vmMgr.Count(), "no instance may be restored after the refusal")
+
+	rec, _ := recordedEBSProvider(t, js)
+	assert.Equal(t, config.EBSProviderQEMUNBD, rec.Provider)
+}

spinifex/runtime/daemon/jetstream.go

@@ -25,6 +25,9 @@ const (
 	InstanceStateBucket = "spinifex-instance-state"
 	// ClusterStateBucket is the name of the KV bucket for cluster state (heartbeats, shutdown markers, service maps).
 	ClusterStateBucket = clusterv1.ClusterStateBucket
+	// ClusterConfigBucket holds settings fixed when a cluster forms. Unlike
+	// ClusterStateBucket it has no TTL, so a key written once stays written.
+	ClusterConfigBucket = "spinifex-cluster-config"
 	// InstanceStatePrefix is the key prefix for the per-node running-set blobs
 	// the record space replaces. Nothing writes it after the cutover; the
 	// migration reads it and a rolled-back node still finds what it left.
@@ -53,6 +56,7 @@ const (
 	// on a bucket stamped 5 rather than reading the keys it no longer owns.
 	InstanceStateBucketVersion      = 5
 	ClusterStateBucketVersion       = 1
+	ClusterConfigBucketVersion      = 1
 	TerminatedInstanceBucketVersion = 3
 )
 
@@ -80,11 +84,12 @@ type JetStreamManager struct {
 	// The per-resource key space the three views above are moving onto. Each
 	// is a third view over a bucket one of them already holds, so the two
 	// spaces share a handle and a recovery driven through either repairs both.
-	records     *kvstore.Store[vm.InstanceRecord] // i.<id>, instance-state bucket
-	termRecords *kvstore.Store[vm.InstanceRecord] // i.<id>, terminated bucket
-	clusterKV   jetstream.KeyValue                // spinifex-cluster-state
-	obs         KVSyncObserver
-	running     runningSetState
+	records       *kvstore.Store[vm.InstanceRecord] // i.<id>, instance-state bucket
+	termRecords   *kvstore.Store[vm.InstanceRecord] // i.<id>, terminated bucket
+	clusterKV     jetstream.KeyValue                // spinifex-cluster-state
+	clusterConfig *kvstore.Bucket                   // spinifex-cluster-config
+	obs           KVSyncObserver
+	running       runningSetState
 }
 
 // checkNodeStateVersion reports whether a node record read from KV is one this
@@ -204,6 +209,29 @@ func (m *JetStreamManager) InitClusterStateBucket() error {
 	return nil
 }
 
+// clusterConfigConfig describes the cluster-config bucket. RecreateIfMissing
+// stays false: a lost lock is not reconstructible by recovery, only by the
+// next daemon start, which claims it again.
+func clusterConfigConfig() kvstore.Config {
+	return kvstore.Config{
+		Name:        ClusterConfigBucket,
+		Description: "Spinifex cluster configuration fixed at formation (EBS provider)",
+		History:     1,
+		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
+			return migrate.DefaultRegistry.RunKV(ctx, ClusterConfigBucket, kv, ClusterConfigBucketVersion)
+		},
+		Missing: "cluster config KV bucket not initialized",
+	}
+}
+
+// InitClusterConfigBucket initializes the cluster-config KV bucket, creating it
+// if it doesn't exist.
+func (m *JetStreamManager) InitClusterConfigBucket() error {
+	m.clusterConfig = kvstore.NewBucket(m.js, clusterConfigConfig())
+	_, err := m.clusterConfig.KV(context.Background())
+	return err
+}
+
 // InitTerminatedInstanceBucket initializes the terminated-instances KV bucket with a 1-hour TTL.
 // JetStream automatically purges keys after 1 hour, matching AWS behavior for terminated instances.
 func (m *JetStreamManager) InitTerminatedInstanceBucket() error {