feat: overhaul hp_agent lifecycle handling

* Added backoff and liveness probes for better management
* Switched IPC to a simple text based system
* Lookups don't directly touch the agent now
* Use the database as a source of truth

internal/hpagent/handler.go

@@ -2,26 +2,15 @@ package hpagent
 
 import (
 	"bufio"
+	"context"
+	"fmt"
 
-	"encoding/json"
 	"os"
-	"sync"
 
 	"github.com/tale/headplane/internal/tsnet"
 	"github.com/tale/headplane/internal/util"
-	"tailscale.com/tailcfg"
 )
 
-// Represents messages from the Headplane master
-type RecvMessage struct {
-	NodeIDs []string
-}
-
-type SendMessage struct {
-	Type string
-	Data any
-}
-
 // Starts listening for messages from stdin
 func FollowMaster(agent *tsnet.TSAgent) {
 	log := util.GetLogger()
@@ -30,55 +19,34 @@ func FollowMaster(agent *tsnet.TSAgent) {
 
 	for scanner.Scan() {
 		line := scanner.Bytes()
-		var msg RecvMessage
-		err := json.Unmarshal(line, &msg)
-		if err != nil {
-			log.Error("Unable to decode message from master: %s", err)
+		directive := string(line)
+
+		log.Debug("Received directive from master: %s", directive)
+		switch directive {
+		case "SHUTDOWN":
+			log.Debug("Received SHUTDOWN directive from master, shutting down agent")
+			agent.Shutdown()
+			return
+
+		case "START":
+			log.Debug("Received START directive from master, starting agent")
+			// TODO: Start the agent here instead of in main
+			fmt.Println("READY " + agent.ID)
 			continue
-		}
 
-		log.Debug("Recieved message from master: %v", line)
-
-		if len(msg.NodeIDs) == 0 {
-			log.Debug("Message recieved had no node IDs")
-			log.Debug("Full message: %s", line)
+		case "PING":
+			log.Debug("Received PING directive from master, responding with PONG")
+			fmt.Println("PONG " + agent.ID)
 			continue
+
+		case "REFRESH":
+			log.Debug("Received REFRESH directive from master, refreshing status for all nodes")
+			err := agent.DispatchHostInfo(context.Background())
+			if err != nil {
+				log.Error("Error refreshing host info: %s", err)
+				fmt.Println("ERR " + err.Error())
+			}
 		}
-
-		// Accumulate the results since we invoke via gofunc
-		results := make(map[string]*tailcfg.HostinfoView)
-		mu := sync.Mutex{}
-		wg := sync.WaitGroup{}
-
-		for _, nodeID := range msg.NodeIDs {
-			wg.Add(1)
-			go func(nodeID string) {
-				defer wg.Done()
-				result, err := agent.GetStatusForPeer(nodeID)
-				if err != nil {
-					log.Error("Unable to get status for node %s: %s", nodeID, err)
-					return
-				}
-
-				if result == nil {
-					log.Debug("No status for node %s", nodeID)
-					return
-				}
-
-				mu.Lock()
-				results[nodeID] = result
-				mu.Unlock()
-			}(nodeID)
-		}
-
-		wg.Wait()
-
-		// Send the results back to the Headplane master
-		log.Debug("Sending status back to master: %v", results)
-		log.Msg(&SendMessage{
-			Type: "status",
-			Data: results,
-		})
 	}
 
 	if err := scanner.Err(); err != nil {

internal/tsnet/peers.go

@@ -3,10 +3,14 @@ package tsnet
 import (
 	"context"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/tale/headplane/internal/util"
+	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 
@@ -64,3 +68,81 @@ func (s *TSAgent) GetStatusForPeer(id string) (*tailcfg.HostinfoView, error) {
 	log.Debug("Got whois for peer %s: %v", id, whois)
 	return &whois.Node.Hostinfo, nil
 }
+
+// Dispatches ALL the HostInfo entries in our Tailnet to the master
+func (s *TSAgent) DispatchHostInfo(ctx context.Context) error {
+	log := util.GetLogger()
+
+	stat, err := s.Lc.Status(ctx)
+	if err != nil {
+		log.Debug("Failed to get status: %s", err)
+		return fmt.Errorf("failed to get status: %w", err)
+	}
+
+	// Do lookups for all peers with a hint of parallelism for speed!
+	const maxParallel = 8
+	sema := make(chan struct{}, maxParallel)
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+
+	nodeMap := make(map[key.NodePublic]*ipnstate.PeerStatus)
+	nodeMap[stat.Self.PublicKey] = stat.Self
+	for nodeKey, peer := range stat.Peer {
+		if peer == nil {
+			log.Debug("Skipping nil peer for node key: %s", nodeKey)
+			continue
+		}
+
+		nodeMap[nodeKey] = peer
+	}
+
+	for nodeKey, peer := range nodeMap {
+		idBytes, err := nodeKey.MarshalText()
+		if err != nil {
+			log.Debug("Failed to marshal node key: %s", err)
+			continue
+		}
+
+		nodeID := string(idBytes)
+		wg.Add(1)
+		sema <- struct{}{}
+
+		go func() {
+			defer wg.Done()
+			defer func() { <-sema }()
+
+			wctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+			defer cancel()
+
+			ip := peer.TailscaleIPs[0].String()
+			if len(ip) == 0 {
+				log.Debug("Peer %s has no Tailscale IPs", nodeID)
+				return
+			}
+
+			whois, err := s.Lc.WhoIs(wctx, ip)
+			if err != nil {
+				log.Debug("WhoIs failed for %s (%s): %s", nodeID, ip, err)
+				return
+			}
+
+			if whois == nil || whois.Node == nil {
+				log.Debug("WhoIs returned nil node for %s (%s)", nodeID, ip)
+				return
+			}
+
+			data, err := json.Marshal(whois.Node.Hostinfo)
+			if err != nil {
+				log.Debug("Failed to marshal hostinfo for %s (%s): %s", nodeID, ip, err)
+				return
+			}
+
+			mu.Lock()
+			fmt.Println("HOSTINFO " + nodeID + " " + string(data))
+			mu.Unlock()
+		}()
+	}
+
+	wg.Wait()
+	return nil
+}