Add a recover mechanism for when a controller cluster can't form a quorum. Fixes #3849

- adds 'ziti ops cluster recover <controller-config>', an offline CLI
  that opens a stopped controller's data directory, forces the raft
  configuration to a single local node via raft.RecoverCluster, and
  aligns the FSM-tracked member list in ctrl-ha.db so stale peers don't
  leak through IsPeerMember or CtrlAddresses on restart
- removes the previous in-process recovery path: the cluster.recover
  config flag and the corresponding RaftConfig.Recover field are gone,
  along with the os.Exit branch in Controller.Init that consumed them
- adds BoltDbFsm.OverwriteServers and GetCachedServers so offline
  tooling can update and inspect the FSM-side server list without a
  live raft instance; OverwriteServers runs before raft.RecoverCluster
  so the snapshot it produces captures the corrected configuration
- updates Broker.AcceptClusterEvent to call DeleteRemovedPeers on every
  ClusterLeadershipGained, making the controllers entity table
  self-healing for any membership change a non-leader missed (offline
  recovery, or a 'cluster remove' applied while another node was leader)
- wires the new subcommand into both V1 and V2 CLI roots so it's
  reachable regardless of ZITI_CLI_LAYOUT
- switches filesystem path joins in the raft package and the recover
  command from path.Join to filepath.Join
- tests bootstrap a two-node configuration, run recoverDataDir, then
  verify the post-recovery snapshot, the FSM-cached server list, and
  Fsm.GetCurrentState (after starting a real raft instance) all report
  the survivor only

CHANGELOG.md

@@ -2,5 +2,26 @@
 
 ## What's New
 
+* [Cluster Quorum Recover](#cluster_quorum_recovery) - A mechanism for recovering clusters that have irrevocably lost the ability to form a quorum
+
+## Cluster Quorum Recovery
+
+A new offline CLI command, `ziti ops cluster recover <controller-config>`, lets
+operators rebuild a stuck HA controller cluster after losing quorum. Use it when
+enough controllers are permanently gone that `ziti ops cluster add` and
+`ziti ops cluster remove` fail with "no leader" — for example, a 2-node cluster
+where one node is unrecoverable, or a 3-node cluster that lost two nodes at once.
+
+The command must be run while the surviving controller process is stopped. It
+reads the same controller config the controller would, opens the raft data
+directory, calls `raft.RecoverCluster` to force the configuration down to a
+single local node, and aligns the FSM-tracked member list and snapshot data so
+no stale peers leak through on restart. After it succeeds, restart the
+controller and add new peers normally with `ziti ops cluster add`.
+
 ## Component Updates and Bug Fixes
 
+* github.com/openziti/ziti/v2: [v2.0.0 -> v2.1.0](https://github.com/openziti/ziti/compare/v2.0.0...v2.1.0)
+    * [Issue #3849](https://github.com/openziti/ziti/issues/3849) - Add a recover mechanism for when a controller cluster can't form a quorum
+
+

controller/config/config_raft.go

@@ -9,7 +9,6 @@ import (
 )
 
 type RaftConfig struct {
-	Recover               bool
 	DataDir               string
 	RestartSelf           bool
 	AdvertiseAddress      transport.Address

controller/env/broker.go

@@ -101,6 +101,13 @@ func (broker *Broker) GetRouterSyncStrategy() RouterSyncStrategy {
 func (broker *Broker) AcceptClusterEvent(clusterEvent *event.ClusterEvent) {
 	if clusterEvent.EventType == event.ClusterLeadershipGained {
 		broker.ae.Managers.Controller.UpdateControllerState(clusterEvent.Peers, false)
+		// Reconcile the controllers entity table against the current raft
+		// configuration on every leadership transition so rows for peers no
+		// longer in the cluster get pruned. This makes the entity table
+		// self-healing for any membership change a non-leader missed (e.g. a
+		// 'cluster remove' applied while another node was leader, or an
+		// offline recovery via 'ziti ops cluster recover').
+		broker.ae.Managers.Controller.DeleteRemovedPeers(clusterEvent.Peers)
 	} else if clusterEvent.EventType == event.ClusterHasLeader && !broker.ae.HostController.IsRaftLeader() {
 		//gained a leader and it isn't us
 		broker.ae.Managers.Controller.UpdateSelfOnNewLeader()

controller/raft/fsm.go

@@ -24,7 +24,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
-	"path"
+	"path/filepath"
 	"sync/atomic"
 	"time"
 
@@ -49,7 +49,7 @@ const (
 func NewFsm(dataDir string, restartSelf bool, decoders command.Decoders, indexTracker IndexTracker, eventDispatcher event.Dispatcher) *BoltDbFsm {
 	return &BoltDbFsm{
 		decoders:        decoders,
-		dbPath:          path.Join(dataDir, "ctrl-ha.db"),
+		dbPath:          filepath.Join(dataDir, "ctrl-ha.db"),
 		indexTracker:    indexTracker,
 		eventDispatcher: eventDispatcher,
 		restartSelf:     restartSelf,
@@ -122,6 +122,33 @@ func (self *BoltDbFsm) Close() error {
 	return self.db.Close()
 }
 
+// GetCachedServers returns the FSM-tracked cluster member list cached from
+// ctrl-ha.db. Unlike GetCurrentState it does not consult a live raft instance,
+// so it is safe to call from offline tooling (e.g. the recover CLI) and
+// returns nil if Init has not yet populated the cache.
+func (self *BoltDbFsm) GetCachedServers() *ServersWithIndex {
+	return self.currentState.Load()
+}
+
+// OverwriteServers replaces the FSM-tracked cluster member list in the
+// underlying bolt database with the supplied servers. Intended for offline
+// recovery (see ziti ops cluster recover): raft.RecoverCluster updates raft's
+// own configuration but does not touch this FSM-side cache, so without an
+// explicit overwrite the controller would keep advertising stale peers via
+// CtrlAddresses and accept their reconnects via IsPeerMember.
+func (self *BoltDbFsm) OverwriteServers(servers []raft.Server) error {
+	if err := self.db.Update(nil, func(ctx boltz.MutateContext) error {
+		return self.storeServers(ctx.Tx(), servers)
+	}); err != nil {
+		return err
+	}
+	self.currentState.Store(&ServersWithIndex{
+		Servers: servers,
+		Index:   self.index,
+	})
+	return nil
+}
+
 func (self *BoltDbFsm) GetDb() boltz.Db {
 	self.dbReferenced.Store(true)
 	return self.db

controller/raft/raft.go

@@ -22,7 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path"
+	"path/filepath"
 	"reflect"
 	"strconv"
 	"sync"
@@ -608,7 +608,7 @@ func (self *Controller) Init() error {
 	self.Configure(raftConfig, conf)
 
 	// Create the log store and stable store.
-	raftBoltFile := path.Join(raftConfig.DataDir, "raft.db")
+	raftBoltFile := filepath.Join(raftConfig.DataDir, "raft.db")
 	var err error
 	self.raftStore, err = raftboltdb.NewBoltStore(raftBoltFile)
 	if err != nil {
@@ -642,20 +642,6 @@ func (self *Controller) Init() error {
 
 	raftTransport := raft.NewNetworkTransportWithLogger(self.Mesh, 3, 10*time.Second, raftConfig.Logger)
 
-	if raftConfig.Recover {
-		err := raft.RecoverCluster(conf, self.Fsm, self.raftStore, self.raftStore, snapshotStore, raftTransport, raft.Configuration{
-			Servers: []raft.Server{
-				{ID: conf.LocalID, Address: localAddr},
-			},
-		})
-		if err != nil {
-			return fmt.Errorf("failed to recover cluster (%w)", err)
-		}
-
-		logrus.Info("raft configuration reset to only include local node. exiting.")
-		os.Exit(0)
-	}
-
 	r, err := raft.NewRaft(conf, self.Fsm, self.raftStore, self.raftStore, snapshotStore, raftTransport)
 	if err != nil {
 		return fmt.Errorf("failed to initialise raft (%w)", err)

version

@@ -1 +1 @@
-2.0
+2.1

ziti/cmd/cmd.go

@@ -29,6 +29,7 @@ import (
 	edgeSubCmd "github.com/openziti/ziti/v2/controller/subcmd"
 	"github.com/openziti/ziti/v2/ziti/cmd/ascode/importer"
 	"github.com/openziti/ziti/v2/ziti/cmd/ops"
+	"github.com/openziti/ziti/v2/ziti/cmd/ops/cluster"
 	"github.com/openziti/ziti/v2/ziti/cmd/ops/database"
 	"github.com/openziti/ziti/v2/ziti/cmd/ops/verify"
 	ext_jwt_signer "github.com/openziti/ziti/v2/ziti/cmd/ops/verify/ext-jwt-signer"
@@ -236,7 +237,9 @@ func NewV1CmdRoot(in io.Reader, out, err io.Writer, cmd *cobra.Command) *cobra.C
 	}
 
 	opsCommands.AddCommand(database.NewCmdDb(out, err))
-	opsCommands.AddCommand(fabric.NewClusterCmd(p))
+	clusterCmd := fabric.NewClusterCmd(p)
+	clusterCmd.AddCommand(cluster.NewCmdRecover(out, err))
+	opsCommands.AddCommand(clusterCmd)
 	opsCommands.AddCommand(ops.NewCmdLogFormat(out, err))
 	opsCommands.AddCommand(ops.NewUnwrapIdentityFileCommand(out, err))
 	opsCommands.AddCommand(verify.NewVerifyCommand(out, err, context.Background()))
@@ -410,7 +413,9 @@ func NewV2CmdRoot(in io.Reader, out, err io.Writer, cmd *cobra.Command) *cobra.C
 	dbCmd.AddCommand(fabric.NewDbCheckIntegrityCmd(p))
 	dbCmd.AddCommand(fabric.NewDbCheckIntegrityStatusCmd(p))
 	opsCommands.AddCommand(dbCmd)
-	opsCommands.AddCommand(fabric.NewClusterCmd(p))
+	clusterCmd := fabric.NewClusterCmd(p)
+	clusterCmd.AddCommand(cluster.NewCmdRecover(out, err))
+	opsCommands.AddCommand(clusterCmd)
 
 	// Group utility tools under ops tools
 	toolsCmd := &cobra.Command{

ziti/cmd/ops/cluster/recover.go

@@ -0,0 +1,166 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package cluster
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/hashicorp/raft"
+	raftboltdb "github.com/hashicorp/raft-boltdb/v2"
+	"github.com/openziti/ziti/v2/controller/command"
+	"github.com/openziti/ziti/v2/controller/config"
+	"github.com/openziti/ziti/v2/controller/event"
+	zitiraft "github.com/openziti/ziti/v2/controller/raft"
+	"github.com/spf13/cobra"
+)
+
+const recoverLong = `Force the local raft state at the controller's data directory to a single-node
+configuration so the surviving controller can come back up after losing quorum.
+
+Use this when a cluster has lost quorum (e.g. one of two nodes is permanently
+gone) and 'ziti agent cluster add'/'cluster remove' fail with "no leader". After
+recovery, restart the controller and add new peers with 'ziti agent cluster add'.
+
+The controller process MUST be stopped before running this command. The
+operation is destructive: the existing peer membership recorded in the raft log
+is discarded.`
+
+// NewCmdRecover builds the 'ziti ops cluster recover' command, which calls
+// raft.RecoverCluster on a stopped controller's data directory to force the
+// configuration to a single local node.
+func NewCmdRecover(out io.Writer, errOut io.Writer) *cobra.Command {
+	var skipConfirm bool
+
+	cmd := &cobra.Command{
+		Use:   "recover <controller-config>",
+		Short: "Force the local raft state to a single-node configuration to recover from quorum loss",
+		Long:  recoverLong,
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runRecover(out, os.Stdin, args[0], skipConfirm)
+		},
+	}
+
+	cmd.Flags().BoolVar(&skipConfirm, "yes", false, "skip the confirmation prompt")
+
+	return cmd
+}
+
+func runRecover(out io.Writer, in io.Reader, configPath string, skipConfirm bool) error {
+	cfg, err := config.LoadConfig(configPath)
+	if err != nil {
+		return fmt.Errorf("failed to load controller config %q: %w", configPath, err)
+	}
+	if cfg.Raft == nil {
+		return fmt.Errorf("controller config %q has no 'cluster' section; this controller is not configured for HA", configPath)
+	}
+	if cfg.Id == nil {
+		return fmt.Errorf("controller config %q has no identity loaded; cannot determine local raft ID", configPath)
+	}
+	if cfg.Raft.AdvertiseAddress == nil {
+		return fmt.Errorf("controller config %q has no cluster advertise address", configPath)
+	}
+
+	localID := raft.ServerID(cfg.Id.Token)
+	localAddr := raft.ServerAddress(cfg.Raft.AdvertiseAddress.String())
+	dataDir := cfg.Raft.DataDir
+
+	_, _ = fmt.Fprintf(out, "About to force the raft configuration in %s to a single node:\n", dataDir)
+	_, _ = fmt.Fprintf(out, "  ID:      %s\n", localID)
+	_, _ = fmt.Fprintf(out, "  Address: %s\n", localAddr)
+	_, _ = fmt.Fprintln(out, "Existing peer membership recorded in the raft log will be discarded.")
+	_, _ = fmt.Fprintln(out, "The controller process must be stopped before continuing.")
+
+	if !skipConfirm {
+		_, _ = fmt.Fprint(out, "Continue? [y/N]: ")
+		reader := bufio.NewReader(in)
+		line, err := reader.ReadString('\n')
+		if err != nil && !errors.Is(err, io.EOF) {
+			return fmt.Errorf("failed to read confirmation: %w", err)
+		}
+		answer := strings.ToLower(strings.TrimSpace(line))
+		if answer != "y" && answer != "yes" {
+			return fmt.Errorf("recovery aborted by user")
+		}
+	}
+
+	if err := recoverDataDir(dataDir, localID, localAddr); err != nil {
+		return err
+	}
+
+	_, _ = fmt.Fprintln(out)
+	_, _ = fmt.Fprintln(out, "Cluster configuration recovered.")
+	_, _ = fmt.Fprintln(out, "Restart the controller; it will start as a single-node cluster.")
+	_, _ = fmt.Fprintln(out, "Add new peers with 'ziti agent cluster add'.")
+	return nil
+}
+
+// recoverDataDir opens the raft stores under dataDir and calls
+// raft.RecoverCluster to force the configuration to a single server matching
+// localID and localAddr. The data directory must belong to a stopped
+// controller — BoltDB takes an exclusive file lock.
+func recoverDataDir(dataDir string, localID raft.ServerID, localAddr raft.ServerAddress) error {
+	boltPath := filepath.Join(dataDir, "raft.db")
+	boltStore, err := raftboltdb.NewBoltStore(boltPath)
+	if err != nil {
+		return fmt.Errorf("failed to open raft bolt store %q: %w", boltPath, err)
+	}
+	defer func() { _ = boltStore.Close() }()
+
+	logger := zitiraft.NewHcLogrusLogger()
+	snapshotStore, err := raft.NewFileSnapshotStoreWithLogger(dataDir, 5, logger)
+	if err != nil {
+		return fmt.Errorf("failed to open snapshot store under %q: %w", dataDir, err)
+	}
+
+	fsm := zitiraft.NewFsm(dataDir, false, command.GetDefaultDecoders(), zitiraft.NewIndexTracker(), event.DispatcherMock{})
+	if err = fsm.Init(); err != nil {
+		return fmt.Errorf("failed to initialize fsm at %q: %w", dataDir, err)
+	}
+	defer func() { _ = fsm.Close() }()
+
+	raftConf := raft.DefaultConfig()
+	raftConf.LocalID = localID
+	raftConf.Logger = logger
+	raftConf.NoSnapshotRestoreOnStart = true
+
+	_, transport := raft.NewInmemTransport(localAddr)
+
+	configuration := raft.Configuration{
+		Servers: []raft.Server{{ID: localID, Address: localAddr, Suffrage: raft.Voter}},
+	}
+
+	// Update ctrl-ha.db's servers bucket BEFORE RecoverCluster runs, so the
+	// snapshot file fsm.Snapshot() writes inside RecoverCluster captures the
+	// corrected configuration. Otherwise any node that later joins via
+	// InstallSnapshot would see the stale member list until a subsequent
+	// LogConfiguration entry overwrites it.
+	if err = fsm.OverwriteServers(configuration.Servers); err != nil {
+		return fmt.Errorf("failed to update FSM-tracked servers list in ctrl-ha.db: %w", err)
+	}
+
+	if err = raft.RecoverCluster(raftConf, fsm, boltStore, boltStore, snapshotStore, transport, configuration); err != nil {
+		return fmt.Errorf("raft.RecoverCluster failed: %w", err)
+	}
+	return nil
+}

ziti/cmd/ops/cluster/recover_test.go

@@ -0,0 +1,169 @@
+/*
+	Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package cluster
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/hashicorp/raft"
+	raftboltdb "github.com/hashicorp/raft-boltdb/v2"
+	"github.com/openziti/ziti/v2/controller/command"
+	"github.com/openziti/ziti/v2/controller/event"
+	zitiraft "github.com/openziti/ziti/v2/controller/raft"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRecoverDataDir_ReducesMultiNodeConfigToSingleNode(t *testing.T) {
+	r := require.New(t)
+	dataDir := t.TempDir()
+
+	const survivorID = raft.ServerID("survivor")
+	const survivorAddr = raft.ServerAddress("127.0.0.1:6262")
+	const deadID = raft.ServerID("dead")
+	const deadAddr = raft.ServerAddress("127.0.0.1:6263")
+
+	bootstrap := func() {
+		boltStore, err := raftboltdb.NewBoltStore(filepath.Join(dataDir, "raft.db"))
+		r.NoError(err)
+		defer func() { _ = boltStore.Close() }()
+
+		snapStore, err := raft.NewFileSnapshotStoreWithLogger(dataDir, 5, raft.DefaultConfig().Logger)
+		r.NoError(err)
+
+		conf := raft.DefaultConfig()
+		conf.LocalID = survivorID
+		_, transport := raft.NewInmemTransport(survivorAddr)
+
+		err = raft.BootstrapCluster(conf, boltStore, boltStore, snapStore, transport, raft.Configuration{
+			Servers: []raft.Server{
+				{ID: survivorID, Address: survivorAddr},
+				{ID: deadID, Address: deadAddr},
+			},
+		})
+		r.NoError(err)
+	}
+	bootstrap()
+
+	r.NoError(recoverDataDir(dataDir, survivorID, survivorAddr))
+
+	snapStore, err := raft.NewFileSnapshotStoreWithLogger(dataDir, 5, raft.DefaultConfig().Logger)
+	r.NoError(err)
+
+	snaps, err := snapStore.List()
+	r.NoError(err)
+	r.NotEmpty(snaps, "expected RecoverCluster to produce at least one snapshot")
+
+	latest := snaps[0]
+	r.Equal(1, len(latest.Configuration.Servers), "post-recovery configuration should have a single server")
+	r.Equal(survivorID, latest.Configuration.Servers[0].ID)
+	r.Equal(survivorAddr, latest.Configuration.Servers[0].Address)
+
+	// The FSM-tracked servers list in ctrl-ha.db must also reflect the recovered
+	// configuration so the controller does not keep accepting reconnects from
+	// the removed peer via IsPeerMember on next startup.
+	fsm := zitiraft.NewFsm(dataDir, false, command.GetDefaultDecoders(), zitiraft.NewIndexTracker(), event.DispatcherMock{})
+	r.NoError(fsm.Init())
+	defer func() { _ = fsm.Close() }()
+
+	state := fsm.GetCachedServers()
+	r.NotNil(state, "FSM should have loaded the servers list from ctrl-ha.db")
+	r.Equal(1, len(state.Servers), "FSM-tracked servers should reflect the recovered single-node config")
+	r.Equal(survivorID, state.Servers[0].ID)
+	r.Equal(survivorAddr, state.Servers[0].Address)
+}
+
+// TestRecoverDataDir_GetCurrentStateAfterRaftStart simulates the production
+// startup path: post-recovery, the controller boots a real raft instance
+// against the recovered stores, then 'getClusterPeersForEvent' calls
+// Fsm.GetCurrentState(raft) to build clusterEvent.Peers. That peers list
+// drives DeleteRemovedPeers in broker.AcceptClusterEvent. This test ensures
+// the chain returns exactly the recovered single-server configuration so
+// DeleteRemovedPeers cannot accidentally prune (or fail to prune) the wrong
+// rows.
+func TestRecoverDataDir_GetCurrentStateAfterRaftStart(t *testing.T) {
+	r := require.New(t)
+	dataDir := t.TempDir()
+
+	const survivorID = raft.ServerID("survivor")
+	const survivorAddr = raft.ServerAddress("127.0.0.1:7262")
+	const deadID = raft.ServerID("dead")
+	const deadAddr = raft.ServerAddress("127.0.0.1:7263")
+
+	bootstrap := func() {
+		boltStore, err := raftboltdb.NewBoltStore(filepath.Join(dataDir, "raft.db"))
+		r.NoError(err)
+		defer func() { _ = boltStore.Close() }()
+
+		snapStore, err := raft.NewFileSnapshotStoreWithLogger(dataDir, 5, raft.DefaultConfig().Logger)
+		r.NoError(err)
+
+		conf := raft.DefaultConfig()
+		conf.LocalID = survivorID
+		_, transport := raft.NewInmemTransport(survivorAddr)
+
+		err = raft.BootstrapCluster(conf, boltStore, boltStore, snapStore, transport, raft.Configuration{
+			Servers: []raft.Server{
+				{ID: survivorID, Address: survivorAddr},
+				{ID: deadID, Address: deadAddr},
+			},
+		})
+		r.NoError(err)
+	}
+	bootstrap()
+
+	r.NoError(recoverDataDir(dataDir, survivorID, survivorAddr))
+
+	// Simulate the controller restarting against the recovered stores.
+	boltStore, err := raftboltdb.NewBoltStore(filepath.Join(dataDir, "raft.db"))
+	r.NoError(err)
+	defer func() { _ = boltStore.Close() }()
+
+	snapStore, err := raft.NewFileSnapshotStoreWithLogger(dataDir, 5, raft.DefaultConfig().Logger)
+	r.NoError(err)
+
+	fsm := zitiraft.NewFsm(dataDir, false, command.GetDefaultDecoders(), zitiraft.NewIndexTracker(), event.DispatcherMock{})
+	r.NoError(fsm.Init())
+	defer func() { _ = fsm.Close() }()
+
+	conf := raft.DefaultConfig()
+	conf.LocalID = survivorID
+	conf.NoSnapshotRestoreOnStart = true
+	_, transport := raft.NewInmemTransport(survivorAddr)
+
+	rNode, err := raft.NewRaft(conf, fsm, boltStore, boltStore, snapStore, transport)
+	r.NoError(err)
+	defer func() { _ = rNode.Shutdown().Error() }()
+
+	// raft.GetConfiguration() should match the recovered single-server config
+	// regardless of leadership state — it reads from the snapshot meta.
+	cfgFuture := rNode.GetConfiguration()
+	r.NoError(cfgFuture.Error())
+	cfg := cfgFuture.Configuration()
+	r.Equal(1, len(cfg.Servers), "raft.GetConfiguration() should report the recovered single-server config")
+	r.Equal(survivorID, cfg.Servers[0].ID)
+	r.Equal(survivorAddr, cfg.Servers[0].Address)
+
+	// This is the exact call getClusterPeersForEvent makes when building
+	// clusterEvent.Peers for ClusterLeadershipGained. It must return the
+	// survivor only — no dead-peer leakage from any cached state.
+	current := fsm.GetCurrentState(rNode)
+	r.NotNil(current)
+	r.Equal(1, len(current.Servers), "Fsm.GetCurrentState should report the recovered single-server config")
+	r.Equal(survivorID, current.Servers[0].ID)
+	r.Equal(survivorAddr, current.Servers[0].Address)
+}