From 975a23e5f649e052b9fe23a2e39ffc3f6eba6080 Mon Sep 17 00:00:00 2001 From: Paul Lorenz Date: Wed, 6 May 2026 18:41:32 -0400 Subject: [PATCH] Add a recover mechanism for when a controller cluster can't form a quorum. Fixes #3849 - adds 'ziti ops cluster recover ', an offline CLI that opens a stopped controller's data directory, forces the raft configuration to a single local node via raft.RecoverCluster, and aligns the FSM-tracked member list in ctrl-ha.db so stale peers don't leak through IsPeerMember or CtrlAddresses on restart - removes the previous in-process recovery path: the cluster.recover config flag and the corresponding RaftConfig.Recover field are gone, along with the os.Exit branch in Controller.Init that consumed them - adds BoltDbFsm.OverwriteServers and GetCachedServers so offline tooling can update and inspect the FSM-side server list without a live raft instance; OverwriteServers runs before raft.RecoverCluster so the snapshot it produces captures the corrected configuration - updates Broker.AcceptClusterEvent to call DeleteRemovedPeers on every ClusterLeadershipGained, making the controllers entity table self-healing for any membership change a non-leader missed (offline recovery, or a 'cluster remove' applied while another node was leader) - wires the new subcommand into both V1 and V2 CLI roots so it's reachable regardless of ZITI_CLI_LAYOUT - switches filesystem path joins in the raft package and the recover command from path.Join to filepath.Join - tests bootstrap a two-node configuration, run recoverDataDir, then verify the post-recovery snapshot, the FSM-cached server list, and Fsm.GetCurrentState (after starting a real raft instance) all report the survivor only --- CHANGELOG.md | 21 ++++ controller/config/config_raft.go | 1 - controller/env/broker.go | 7 ++ controller/raft/fsm.go | 31 ++++- controller/raft/raft.go | 18 +-- version | 2 +- ziti/cmd/cmd.go | 9 +- ziti/cmd/ops/cluster/recover.go | 166 ++++++++++++++++++++++++++ ziti/cmd/ops/cluster/recover_test.go | 169 +++++++++++++++++++++++++++ 9 files changed, 402 insertions(+), 22 deletions(-) create mode 100644 ziti/cmd/ops/cluster/recover.go create mode 100644 ziti/cmd/ops/cluster/recover_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 64299b750..0d8875751 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,5 +2,26 @@ ## What's New +* [Cluster Quorum Recover](#cluster_quorum_recovery) - A mechanism for recovering clusters that have irrevocably lost the ability to form a quorum + +## Cluster Quorum Recovery + +A new offline CLI command, `ziti ops cluster recover `, lets +operators rebuild a stuck HA controller cluster after losing quorum. Use it when +enough controllers are permanently gone that `ziti ops cluster add` and +`ziti ops cluster remove` fail with "no leader" — for example, a 2-node cluster +where one node is unrecoverable, or a 3-node cluster that lost two nodes at once. + +The command must be run while the surviving controller process is stopped. It +reads the same controller config the controller would, opens the raft data +directory, calls `raft.RecoverCluster` to force the configuration down to a +single local node, and aligns the FSM-tracked member list and snapshot data so +no stale peers leak through on restart. After it succeeds, restart the +controller and add new peers normally with `ziti ops cluster add`. + ## Component Updates and Bug Fixes +* github.com/openziti/ziti/v2: [v2.0.0 -> v2.1.0](https://github.com/openziti/ziti/compare/v2.0.0...v2.1.0) + * [Issue #3849](https://github.com/openziti/ziti/issues/3849) - Add a recover mechanism for when a controller cluster can't form a quorum + + diff --git a/controller/config/config_raft.go b/controller/config/config_raft.go index b45788d7a..26ea7f965 100644 --- a/controller/config/config_raft.go +++ b/controller/config/config_raft.go @@ -9,7 +9,6 @@ import ( ) type RaftConfig struct { - Recover bool DataDir string RestartSelf bool AdvertiseAddress transport.Address diff --git a/controller/env/broker.go b/controller/env/broker.go index f022e68fa..03a725a2d 100644 --- a/controller/env/broker.go +++ b/controller/env/broker.go @@ -101,6 +101,13 @@ func (broker *Broker) GetRouterSyncStrategy() RouterSyncStrategy { func (broker *Broker) AcceptClusterEvent(clusterEvent *event.ClusterEvent) { if clusterEvent.EventType == event.ClusterLeadershipGained { broker.ae.Managers.Controller.UpdateControllerState(clusterEvent.Peers, false) + // Reconcile the controllers entity table against the current raft + // configuration on every leadership transition so rows for peers no + // longer in the cluster get pruned. This makes the entity table + // self-healing for any membership change a non-leader missed (e.g. a + // 'cluster remove' applied while another node was leader, or an + // offline recovery via 'ziti ops cluster recover'). + broker.ae.Managers.Controller.DeleteRemovedPeers(clusterEvent.Peers) } else if clusterEvent.EventType == event.ClusterHasLeader && !broker.ae.HostController.IsRaftLeader() { //gained a leader and it isn't us broker.ae.Managers.Controller.UpdateSelfOnNewLeader() diff --git a/controller/raft/fsm.go b/controller/raft/fsm.go index 4d14257cb..6a43f1147 100644 --- a/controller/raft/fsm.go +++ b/controller/raft/fsm.go @@ -24,7 +24,7 @@ import ( "io" "os" "os/exec" - "path" + "path/filepath" "sync/atomic" "time" @@ -49,7 +49,7 @@ const ( func NewFsm(dataDir string, restartSelf bool, decoders command.Decoders, indexTracker IndexTracker, eventDispatcher event.Dispatcher) *BoltDbFsm { return &BoltDbFsm{ decoders: decoders, - dbPath: path.Join(dataDir, "ctrl-ha.db"), + dbPath: filepath.Join(dataDir, "ctrl-ha.db"), indexTracker: indexTracker, eventDispatcher: eventDispatcher, restartSelf: restartSelf, @@ -122,6 +122,33 @@ func (self *BoltDbFsm) Close() error { return self.db.Close() } +// GetCachedServers returns the FSM-tracked cluster member list cached from +// ctrl-ha.db. Unlike GetCurrentState it does not consult a live raft instance, +// so it is safe to call from offline tooling (e.g. the recover CLI) and +// returns nil if Init has not yet populated the cache. +func (self *BoltDbFsm) GetCachedServers() *ServersWithIndex { + return self.currentState.Load() +} + +// OverwriteServers replaces the FSM-tracked cluster member list in the +// underlying bolt database with the supplied servers. Intended for offline +// recovery (see ziti ops cluster recover): raft.RecoverCluster updates raft's +// own configuration but does not touch this FSM-side cache, so without an +// explicit overwrite the controller would keep advertising stale peers via +// CtrlAddresses and accept their reconnects via IsPeerMember. +func (self *BoltDbFsm) OverwriteServers(servers []raft.Server) error { + if err := self.db.Update(nil, func(ctx boltz.MutateContext) error { + return self.storeServers(ctx.Tx(), servers) + }); err != nil { + return err + } + self.currentState.Store(&ServersWithIndex{ + Servers: servers, + Index: self.index, + }) + return nil +} + func (self *BoltDbFsm) GetDb() boltz.Db { self.dbReferenced.Store(true) return self.db diff --git a/controller/raft/raft.go b/controller/raft/raft.go index 0f7df0618..76f8ae461 100644 --- a/controller/raft/raft.go +++ b/controller/raft/raft.go @@ -22,7 +22,7 @@ import ( "errors" "fmt" "os" - "path" + "path/filepath" "reflect" "strconv" "sync" @@ -608,7 +608,7 @@ func (self *Controller) Init() error { self.Configure(raftConfig, conf) // Create the log store and stable store. - raftBoltFile := path.Join(raftConfig.DataDir, "raft.db") + raftBoltFile := filepath.Join(raftConfig.DataDir, "raft.db") var err error self.raftStore, err = raftboltdb.NewBoltStore(raftBoltFile) if err != nil { @@ -642,20 +642,6 @@ func (self *Controller) Init() error { raftTransport := raft.NewNetworkTransportWithLogger(self.Mesh, 3, 10*time.Second, raftConfig.Logger) - if raftConfig.Recover { - err := raft.RecoverCluster(conf, self.Fsm, self.raftStore, self.raftStore, snapshotStore, raftTransport, raft.Configuration{ - Servers: []raft.Server{ - {ID: conf.LocalID, Address: localAddr}, - }, - }) - if err != nil { - return fmt.Errorf("failed to recover cluster (%w)", err) - } - - logrus.Info("raft configuration reset to only include local node. exiting.") - os.Exit(0) - } - r, err := raft.NewRaft(conf, self.Fsm, self.raftStore, self.raftStore, snapshotStore, raftTransport) if err != nil { return fmt.Errorf("failed to initialise raft (%w)", err) diff --git a/version b/version index cd5ac039d..879b416e6 100644 --- a/version +++ b/version @@ -1 +1 @@ -2.0 +2.1 diff --git a/ziti/cmd/cmd.go b/ziti/cmd/cmd.go index f352e69dc..855de488b 100644 --- a/ziti/cmd/cmd.go +++ b/ziti/cmd/cmd.go @@ -29,6 +29,7 @@ import ( edgeSubCmd "github.com/openziti/ziti/v2/controller/subcmd" "github.com/openziti/ziti/v2/ziti/cmd/ascode/importer" "github.com/openziti/ziti/v2/ziti/cmd/ops" + "github.com/openziti/ziti/v2/ziti/cmd/ops/cluster" "github.com/openziti/ziti/v2/ziti/cmd/ops/database" "github.com/openziti/ziti/v2/ziti/cmd/ops/verify" ext_jwt_signer "github.com/openziti/ziti/v2/ziti/cmd/ops/verify/ext-jwt-signer" @@ -236,7 +237,9 @@ func NewV1CmdRoot(in io.Reader, out, err io.Writer, cmd *cobra.Command) *cobra.C } opsCommands.AddCommand(database.NewCmdDb(out, err)) - opsCommands.AddCommand(fabric.NewClusterCmd(p)) + clusterCmd := fabric.NewClusterCmd(p) + clusterCmd.AddCommand(cluster.NewCmdRecover(out, err)) + opsCommands.AddCommand(clusterCmd) opsCommands.AddCommand(ops.NewCmdLogFormat(out, err)) opsCommands.AddCommand(ops.NewUnwrapIdentityFileCommand(out, err)) opsCommands.AddCommand(verify.NewVerifyCommand(out, err, context.Background())) @@ -410,7 +413,9 @@ func NewV2CmdRoot(in io.Reader, out, err io.Writer, cmd *cobra.Command) *cobra.C dbCmd.AddCommand(fabric.NewDbCheckIntegrityCmd(p)) dbCmd.AddCommand(fabric.NewDbCheckIntegrityStatusCmd(p)) opsCommands.AddCommand(dbCmd) - opsCommands.AddCommand(fabric.NewClusterCmd(p)) + clusterCmd := fabric.NewClusterCmd(p) + clusterCmd.AddCommand(cluster.NewCmdRecover(out, err)) + opsCommands.AddCommand(clusterCmd) // Group utility tools under ops tools toolsCmd := &cobra.Command{ diff --git a/ziti/cmd/ops/cluster/recover.go b/ziti/cmd/ops/cluster/recover.go new file mode 100644 index 000000000..4d571452c --- /dev/null +++ b/ziti/cmd/ops/cluster/recover.go @@ -0,0 +1,166 @@ +/* + Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package cluster + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/hashicorp/raft" + raftboltdb "github.com/hashicorp/raft-boltdb/v2" + "github.com/openziti/ziti/v2/controller/command" + "github.com/openziti/ziti/v2/controller/config" + "github.com/openziti/ziti/v2/controller/event" + zitiraft "github.com/openziti/ziti/v2/controller/raft" + "github.com/spf13/cobra" +) + +const recoverLong = `Force the local raft state at the controller's data directory to a single-node +configuration so the surviving controller can come back up after losing quorum. + +Use this when a cluster has lost quorum (e.g. one of two nodes is permanently +gone) and 'ziti agent cluster add'/'cluster remove' fail with "no leader". After +recovery, restart the controller and add new peers with 'ziti agent cluster add'. + +The controller process MUST be stopped before running this command. The +operation is destructive: the existing peer membership recorded in the raft log +is discarded.` + +// NewCmdRecover builds the 'ziti ops cluster recover' command, which calls +// raft.RecoverCluster on a stopped controller's data directory to force the +// configuration to a single local node. +func NewCmdRecover(out io.Writer, errOut io.Writer) *cobra.Command { + var skipConfirm bool + + cmd := &cobra.Command{ + Use: "recover ", + Short: "Force the local raft state to a single-node configuration to recover from quorum loss", + Long: recoverLong, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runRecover(out, os.Stdin, args[0], skipConfirm) + }, + } + + cmd.Flags().BoolVar(&skipConfirm, "yes", false, "skip the confirmation prompt") + + return cmd +} + +func runRecover(out io.Writer, in io.Reader, configPath string, skipConfirm bool) error { + cfg, err := config.LoadConfig(configPath) + if err != nil { + return fmt.Errorf("failed to load controller config %q: %w", configPath, err) + } + if cfg.Raft == nil { + return fmt.Errorf("controller config %q has no 'cluster' section; this controller is not configured for HA", configPath) + } + if cfg.Id == nil { + return fmt.Errorf("controller config %q has no identity loaded; cannot determine local raft ID", configPath) + } + if cfg.Raft.AdvertiseAddress == nil { + return fmt.Errorf("controller config %q has no cluster advertise address", configPath) + } + + localID := raft.ServerID(cfg.Id.Token) + localAddr := raft.ServerAddress(cfg.Raft.AdvertiseAddress.String()) + dataDir := cfg.Raft.DataDir + + _, _ = fmt.Fprintf(out, "About to force the raft configuration in %s to a single node:\n", dataDir) + _, _ = fmt.Fprintf(out, " ID: %s\n", localID) + _, _ = fmt.Fprintf(out, " Address: %s\n", localAddr) + _, _ = fmt.Fprintln(out, "Existing peer membership recorded in the raft log will be discarded.") + _, _ = fmt.Fprintln(out, "The controller process must be stopped before continuing.") + + if !skipConfirm { + _, _ = fmt.Fprint(out, "Continue? [y/N]: ") + reader := bufio.NewReader(in) + line, err := reader.ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("failed to read confirmation: %w", err) + } + answer := strings.ToLower(strings.TrimSpace(line)) + if answer != "y" && answer != "yes" { + return fmt.Errorf("recovery aborted by user") + } + } + + if err := recoverDataDir(dataDir, localID, localAddr); err != nil { + return err + } + + _, _ = fmt.Fprintln(out) + _, _ = fmt.Fprintln(out, "Cluster configuration recovered.") + _, _ = fmt.Fprintln(out, "Restart the controller; it will start as a single-node cluster.") + _, _ = fmt.Fprintln(out, "Add new peers with 'ziti agent cluster add'.") + return nil +} + +// recoverDataDir opens the raft stores under dataDir and calls +// raft.RecoverCluster to force the configuration to a single server matching +// localID and localAddr. The data directory must belong to a stopped +// controller — BoltDB takes an exclusive file lock. +func recoverDataDir(dataDir string, localID raft.ServerID, localAddr raft.ServerAddress) error { + boltPath := filepath.Join(dataDir, "raft.db") + boltStore, err := raftboltdb.NewBoltStore(boltPath) + if err != nil { + return fmt.Errorf("failed to open raft bolt store %q: %w", boltPath, err) + } + defer func() { _ = boltStore.Close() }() + + logger := zitiraft.NewHcLogrusLogger() + snapshotStore, err := raft.NewFileSnapshotStoreWithLogger(dataDir, 5, logger) + if err != nil { + return fmt.Errorf("failed to open snapshot store under %q: %w", dataDir, err) + } + + fsm := zitiraft.NewFsm(dataDir, false, command.GetDefaultDecoders(), zitiraft.NewIndexTracker(), event.DispatcherMock{}) + if err = fsm.Init(); err != nil { + return fmt.Errorf("failed to initialize fsm at %q: %w", dataDir, err) + } + defer func() { _ = fsm.Close() }() + + raftConf := raft.DefaultConfig() + raftConf.LocalID = localID + raftConf.Logger = logger + raftConf.NoSnapshotRestoreOnStart = true + + _, transport := raft.NewInmemTransport(localAddr) + + configuration := raft.Configuration{ + Servers: []raft.Server{{ID: localID, Address: localAddr, Suffrage: raft.Voter}}, + } + + // Update ctrl-ha.db's servers bucket BEFORE RecoverCluster runs, so the + // snapshot file fsm.Snapshot() writes inside RecoverCluster captures the + // corrected configuration. Otherwise any node that later joins via + // InstallSnapshot would see the stale member list until a subsequent + // LogConfiguration entry overwrites it. + if err = fsm.OverwriteServers(configuration.Servers); err != nil { + return fmt.Errorf("failed to update FSM-tracked servers list in ctrl-ha.db: %w", err) + } + + if err = raft.RecoverCluster(raftConf, fsm, boltStore, boltStore, snapshotStore, transport, configuration); err != nil { + return fmt.Errorf("raft.RecoverCluster failed: %w", err) + } + return nil +} diff --git a/ziti/cmd/ops/cluster/recover_test.go b/ziti/cmd/ops/cluster/recover_test.go new file mode 100644 index 000000000..d7938a8dc --- /dev/null +++ b/ziti/cmd/ops/cluster/recover_test.go @@ -0,0 +1,169 @@ +/* + Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package cluster + +import ( + "path/filepath" + "testing" + + "github.com/hashicorp/raft" + raftboltdb "github.com/hashicorp/raft-boltdb/v2" + "github.com/openziti/ziti/v2/controller/command" + "github.com/openziti/ziti/v2/controller/event" + zitiraft "github.com/openziti/ziti/v2/controller/raft" + "github.com/stretchr/testify/require" +) + +func TestRecoverDataDir_ReducesMultiNodeConfigToSingleNode(t *testing.T) { + r := require.New(t) + dataDir := t.TempDir() + + const survivorID = raft.ServerID("survivor") + const survivorAddr = raft.ServerAddress("127.0.0.1:6262") + const deadID = raft.ServerID("dead") + const deadAddr = raft.ServerAddress("127.0.0.1:6263") + + bootstrap := func() { + boltStore, err := raftboltdb.NewBoltStore(filepath.Join(dataDir, "raft.db")) + r.NoError(err) + defer func() { _ = boltStore.Close() }() + + snapStore, err := raft.NewFileSnapshotStoreWithLogger(dataDir, 5, raft.DefaultConfig().Logger) + r.NoError(err) + + conf := raft.DefaultConfig() + conf.LocalID = survivorID + _, transport := raft.NewInmemTransport(survivorAddr) + + err = raft.BootstrapCluster(conf, boltStore, boltStore, snapStore, transport, raft.Configuration{ + Servers: []raft.Server{ + {ID: survivorID, Address: survivorAddr}, + {ID: deadID, Address: deadAddr}, + }, + }) + r.NoError(err) + } + bootstrap() + + r.NoError(recoverDataDir(dataDir, survivorID, survivorAddr)) + + snapStore, err := raft.NewFileSnapshotStoreWithLogger(dataDir, 5, raft.DefaultConfig().Logger) + r.NoError(err) + + snaps, err := snapStore.List() + r.NoError(err) + r.NotEmpty(snaps, "expected RecoverCluster to produce at least one snapshot") + + latest := snaps[0] + r.Equal(1, len(latest.Configuration.Servers), "post-recovery configuration should have a single server") + r.Equal(survivorID, latest.Configuration.Servers[0].ID) + r.Equal(survivorAddr, latest.Configuration.Servers[0].Address) + + // The FSM-tracked servers list in ctrl-ha.db must also reflect the recovered + // configuration so the controller does not keep accepting reconnects from + // the removed peer via IsPeerMember on next startup. + fsm := zitiraft.NewFsm(dataDir, false, command.GetDefaultDecoders(), zitiraft.NewIndexTracker(), event.DispatcherMock{}) + r.NoError(fsm.Init()) + defer func() { _ = fsm.Close() }() + + state := fsm.GetCachedServers() + r.NotNil(state, "FSM should have loaded the servers list from ctrl-ha.db") + r.Equal(1, len(state.Servers), "FSM-tracked servers should reflect the recovered single-node config") + r.Equal(survivorID, state.Servers[0].ID) + r.Equal(survivorAddr, state.Servers[0].Address) +} + +// TestRecoverDataDir_GetCurrentStateAfterRaftStart simulates the production +// startup path: post-recovery, the controller boots a real raft instance +// against the recovered stores, then 'getClusterPeersForEvent' calls +// Fsm.GetCurrentState(raft) to build clusterEvent.Peers. That peers list +// drives DeleteRemovedPeers in broker.AcceptClusterEvent. This test ensures +// the chain returns exactly the recovered single-server configuration so +// DeleteRemovedPeers cannot accidentally prune (or fail to prune) the wrong +// rows. +func TestRecoverDataDir_GetCurrentStateAfterRaftStart(t *testing.T) { + r := require.New(t) + dataDir := t.TempDir() + + const survivorID = raft.ServerID("survivor") + const survivorAddr = raft.ServerAddress("127.0.0.1:7262") + const deadID = raft.ServerID("dead") + const deadAddr = raft.ServerAddress("127.0.0.1:7263") + + bootstrap := func() { + boltStore, err := raftboltdb.NewBoltStore(filepath.Join(dataDir, "raft.db")) + r.NoError(err) + defer func() { _ = boltStore.Close() }() + + snapStore, err := raft.NewFileSnapshotStoreWithLogger(dataDir, 5, raft.DefaultConfig().Logger) + r.NoError(err) + + conf := raft.DefaultConfig() + conf.LocalID = survivorID + _, transport := raft.NewInmemTransport(survivorAddr) + + err = raft.BootstrapCluster(conf, boltStore, boltStore, snapStore, transport, raft.Configuration{ + Servers: []raft.Server{ + {ID: survivorID, Address: survivorAddr}, + {ID: deadID, Address: deadAddr}, + }, + }) + r.NoError(err) + } + bootstrap() + + r.NoError(recoverDataDir(dataDir, survivorID, survivorAddr)) + + // Simulate the controller restarting against the recovered stores. + boltStore, err := raftboltdb.NewBoltStore(filepath.Join(dataDir, "raft.db")) + r.NoError(err) + defer func() { _ = boltStore.Close() }() + + snapStore, err := raft.NewFileSnapshotStoreWithLogger(dataDir, 5, raft.DefaultConfig().Logger) + r.NoError(err) + + fsm := zitiraft.NewFsm(dataDir, false, command.GetDefaultDecoders(), zitiraft.NewIndexTracker(), event.DispatcherMock{}) + r.NoError(fsm.Init()) + defer func() { _ = fsm.Close() }() + + conf := raft.DefaultConfig() + conf.LocalID = survivorID + conf.NoSnapshotRestoreOnStart = true + _, transport := raft.NewInmemTransport(survivorAddr) + + rNode, err := raft.NewRaft(conf, fsm, boltStore, boltStore, snapStore, transport) + r.NoError(err) + defer func() { _ = rNode.Shutdown().Error() }() + + // raft.GetConfiguration() should match the recovered single-server config + // regardless of leadership state — it reads from the snapshot meta. + cfgFuture := rNode.GetConfiguration() + r.NoError(cfgFuture.Error()) + cfg := cfgFuture.Configuration() + r.Equal(1, len(cfg.Servers), "raft.GetConfiguration() should report the recovered single-server config") + r.Equal(survivorID, cfg.Servers[0].ID) + r.Equal(survivorAddr, cfg.Servers[0].Address) + + // This is the exact call getClusterPeersForEvent makes when building + // clusterEvent.Peers for ClusterLeadershipGained. It must return the + // survivor only — no dead-peer leakage from any cached state. + current := fsm.GetCurrentState(rNode) + r.NotNil(current) + r.Equal(1, len(current.Servers), "Fsm.GetCurrentState should report the recovered single-server config") + r.Equal(survivorID, current.Servers[0].ID) + r.Equal(survivorAddr, current.Servers[0].Address) +}