Retry bootstrap members until bootstrapped. Fixes #1523

controller/raft/raft.go

@@ -683,6 +683,7 @@ func (self *Controller) addConfiguredBootstrapMembers() error {
 
 		if id, addr, err := self.Mesh.GetPeerInfo(bootstrapMember, time.Second*5); err != nil {
 			pfxlog.Logger().WithError(err).Errorf("unable to get id for bootstrap member [%v]", bootstrapMember)
+			go self.retryBootstrapMember(bootstrapMember)
 		} else {
 			self.addBootstrapServer(raft.Server{
 				Suffrage: raft.Voter,
@@ -694,6 +695,34 @@ func (self *Controller) addConfiguredBootstrapMembers() error {
 	return nil
 }
 
+func (self *Controller) retryBootstrapMember(bootstrapMember string) {
+	ticker := time.NewTicker(6 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		<-ticker.C
+
+		// If we've bootstrapped, exit out
+		if self.bootstrapped.Load() {
+			return
+		}
+
+		if id, addr, err := self.Mesh.GetPeerInfo(bootstrapMember, time.Second*5); err != nil {
+			pfxlog.Logger().WithError(err).Errorf("unable to get id for bootstrap member [%v]", bootstrapMember)
+		} else {
+			req := &cmd_pb.AddPeerRequest{
+				Addr:    string(addr),
+				Id:      string(id),
+				IsVoter: true,
+			}
+
+			if err = self.Join(req); err == nil {
+				return
+			}
+		}
+	}
+}
+
 // Join adds the given node to the raft cluster
 func (self *Controller) Join(req *cmd_pb.AddPeerRequest) error {
 	self.clusterLock.Lock()