mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-27 05:57:27 +00:00
7d02f9f04e
An operator's split of a Proxmox node and its pulse-agent (report-merge or unlink) split the registry's rows in #2741, but the monitor's own link (Node.LinkedAgentID, Host.LinkedNodeID) stayed. Everything that reads that link kept treating two machines as one: the agent's SMART inventory as the node's disk fallback, its --disk-exclude patterns on the node's disks, guest discovery through the agent, deployment reporting the node as already running an agent, service discovery, the agent's container filesystem readings, the node's temperatures and the node/agent alert correlation. The resource store's exclusion is now the one record of the split. MonitorAdapter decides it with the registry's own rule for a node's link and confirms a split against the store's latest decisions, and models.State holds both sides of a split link back on every link write, before a link can steer where node views fold. The agents API's node link is the operator's newer decision, so it removes the pair's split from the store (TakeExclusion) before recording manual intent, and puts it back, unless the operator decided again meanwhile, if the intent fails to persist (RestoreExclusion). A split recorded after a manual link ends that intent on the agent's next report that finds the split confirmed, so the node is no longer reserved for that agent. A rebuild that cannot read the store's decisions still loads none, in the registry as in monitoring; that needs a change in registry.go and monitor_adapter.go and is left for a follow-up.
2268 lines
68 KiB
Go
2268 lines
68 KiB
Go
package models
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestUpsertHost(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Test insert new host
|
|
host1 := Host{
|
|
ID: "host-1",
|
|
Hostname: "server-1",
|
|
Status: "online",
|
|
}
|
|
state.UpsertHost(host1)
|
|
|
|
hosts := state.GetHosts()
|
|
if len(hosts) != 1 {
|
|
t.Fatalf("Expected 1 host, got %d", len(hosts))
|
|
}
|
|
if hosts[0].ID != "host-1" {
|
|
t.Errorf("Expected host ID 'host-1', got %q", hosts[0].ID)
|
|
}
|
|
|
|
// Test update existing host
|
|
host1Updated := Host{
|
|
ID: "host-1",
|
|
Hostname: "server-1",
|
|
Status: "offline",
|
|
}
|
|
state.UpsertHost(host1Updated)
|
|
|
|
hosts = state.GetHosts()
|
|
if len(hosts) != 1 {
|
|
t.Fatalf("Expected 1 host after update, got %d", len(hosts))
|
|
}
|
|
if hosts[0].Status != "offline" {
|
|
t.Errorf("Expected status 'offline', got %q", hosts[0].Status)
|
|
}
|
|
|
|
// Test multiple hosts and sorting
|
|
state.UpsertHost(Host{ID: "host-2", Hostname: "alpha-server"})
|
|
state.UpsertHost(Host{ID: "host-3", Hostname: "zulu-server"})
|
|
|
|
hosts = state.GetHosts()
|
|
if len(hosts) != 3 {
|
|
t.Fatalf("Expected 3 hosts, got %d", len(hosts))
|
|
}
|
|
|
|
// Hosts should be sorted by hostname
|
|
if hosts[0].Hostname != "alpha-server" {
|
|
t.Errorf("First host should be 'alpha-server', got %q", hosts[0].Hostname)
|
|
}
|
|
}
|
|
|
|
func TestUpsertHostPreservesReportedCommandAuthority(t *testing.T) {
|
|
state := NewState()
|
|
state.UpsertHost(Host{
|
|
ID: "host-authority",
|
|
AgentPrivilege: &AgentPrivilegeStatus{
|
|
RunningAsRoot: true,
|
|
CommandAuthority: "monitoring-only",
|
|
},
|
|
})
|
|
|
|
hosts := state.GetHosts()
|
|
if len(hosts) != 1 || hosts[0].AgentPrivilege == nil {
|
|
t.Fatalf("reported privilege missing from state: %+v", hosts)
|
|
}
|
|
if got := hosts[0].AgentPrivilege.CommandAuthority; got != "monitoring-only" {
|
|
t.Fatalf("command authority = %q, want monitoring-only", got)
|
|
}
|
|
}
|
|
|
|
func TestGetHosts_Copy(t *testing.T) {
|
|
state := NewState()
|
|
|
|
state.UpsertHost(Host{ID: "host-1", Hostname: "server-1"})
|
|
|
|
hosts1 := state.GetHosts()
|
|
hosts2 := state.GetHosts()
|
|
|
|
// Modify one slice
|
|
if len(hosts1) > 0 {
|
|
hosts1[0].Hostname = "modified"
|
|
}
|
|
|
|
// Other slice should be unchanged
|
|
if len(hosts2) > 0 && hosts2[0].Hostname == "modified" {
|
|
t.Error("GetHosts should return a copy, not the same slice")
|
|
}
|
|
|
|
// Original state should be unchanged
|
|
hosts3 := state.GetHosts()
|
|
if hosts3[0].Hostname == "modified" {
|
|
t.Error("State should be unchanged by modifications to returned slice")
|
|
}
|
|
}
|
|
|
|
func TestRemoveHost(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Insert hosts
|
|
state.UpsertHost(Host{ID: "host-1", Hostname: "server1"})
|
|
state.UpsertHost(Host{ID: "host-2", Hostname: "server2"})
|
|
|
|
// Remove existing host
|
|
removed, ok := state.RemoveHost("host-1")
|
|
if !ok {
|
|
t.Error("Expected RemoveHost to return true for existing host")
|
|
}
|
|
if removed.ID != "host-1" {
|
|
t.Errorf("Expected removed host ID 'host-1', got %q", removed.ID)
|
|
}
|
|
|
|
hosts := state.GetHosts()
|
|
if len(hosts) != 1 {
|
|
t.Fatalf("Expected 1 host after removal, got %d", len(hosts))
|
|
}
|
|
if hosts[0].ID != "host-2" {
|
|
t.Errorf("Remaining host should be 'host-2', got %q", hosts[0].ID)
|
|
}
|
|
|
|
// Remove non-existing host
|
|
removed, ok = state.RemoveHost("non-existent")
|
|
if ok {
|
|
t.Error("Expected RemoveHost to return false for non-existent host")
|
|
}
|
|
if removed.ID != "" {
|
|
t.Errorf("Expected empty Host for non-existent removal, got ID %q", removed.ID)
|
|
}
|
|
}
|
|
|
|
func TestSetHostStatus(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Test with no hosts
|
|
changed := state.SetHostStatus("host-1", "online")
|
|
if changed {
|
|
t.Error("SetHostStatus should return false when host doesn't exist")
|
|
}
|
|
|
|
// Add host and set status
|
|
state.UpsertHost(Host{ID: "host-1", Hostname: "server1", Status: "offline"})
|
|
|
|
changed = state.SetHostStatus("host-1", "online")
|
|
if !changed {
|
|
t.Error("SetHostStatus should return true when host exists")
|
|
}
|
|
|
|
hosts := state.GetHosts()
|
|
if hosts[0].Status != "online" {
|
|
t.Errorf("Expected status 'online', got %q", hosts[0].Status)
|
|
}
|
|
|
|
// Set same status (no change)
|
|
changed = state.SetHostStatus("host-1", "online")
|
|
if !changed {
|
|
t.Error("SetHostStatus should return true even when status unchanged")
|
|
}
|
|
}
|
|
|
|
func TestTouchHost(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
|
|
// Test with non-existent host
|
|
ok := state.TouchHost("host-1", now)
|
|
if ok {
|
|
t.Error("TouchHost should return false for non-existent host")
|
|
}
|
|
|
|
// Add host and touch it
|
|
state.UpsertHost(Host{ID: "host-1", Hostname: "server1"})
|
|
|
|
later := now.Add(time.Hour)
|
|
ok = state.TouchHost("host-1", later)
|
|
if !ok {
|
|
t.Error("TouchHost should return true for existing host")
|
|
}
|
|
|
|
hosts := state.GetHosts()
|
|
if !hosts[0].LastSeen.Equal(later) {
|
|
t.Errorf("LastSeen should be updated to %v, got %v", later, hosts[0].LastSeen)
|
|
}
|
|
}
|
|
|
|
func TestSetConnectionHealth(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Set connection healthy
|
|
state.SetConnectionHealth("pve-cluster-1", true)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if healthy, ok := snapshot.ConnectionHealth["pve-cluster-1"]; !ok || !healthy {
|
|
t.Error("Expected connection to be healthy")
|
|
}
|
|
|
|
// Set connection unhealthy
|
|
state.SetConnectionHealth("pve-cluster-1", false)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if healthy, ok := snapshot.ConnectionHealth["pve-cluster-1"]; !ok || healthy {
|
|
t.Error("Expected connection to be unhealthy")
|
|
}
|
|
|
|
// Multiple connections
|
|
state.SetConnectionHealth("pve-cluster-2", true)
|
|
state.SetConnectionHealth("pbs-instance-1", true)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.ConnectionHealth) != 3 {
|
|
t.Errorf("Expected 3 connection health entries, got %d", len(snapshot.ConnectionHealth))
|
|
}
|
|
}
|
|
|
|
func TestPVETagStyleSnapshotRebuildsAggregateByInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
state.MergePVETagStyle("pve-a", PVETagStyle{
|
|
Colors: map[string]string{
|
|
"Production": "#00ff00",
|
|
},
|
|
CaseSensitive: true,
|
|
})
|
|
state.MergePVETagStyle("pve-b", PVETagStyle{
|
|
Colors: map[string]string{
|
|
"Backup": "#112233",
|
|
},
|
|
})
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if got := snapshot.PVETagStyles["pve-a"].Colors["Production"]; got != "#00ff00" {
|
|
t.Fatalf("expected pve-a tag style to preserve case-sensitive color, got %q", got)
|
|
}
|
|
if !snapshot.PVETagStyles["pve-a"].CaseSensitive {
|
|
t.Fatal("expected pve-a tag style to preserve case-sensitive flag")
|
|
}
|
|
if got := snapshot.PVETagStyles["pve-b"].Colors["backup"]; got != "#112233" {
|
|
t.Fatalf("expected pve-b tag style to normalize non-case-sensitive tag, got %q", got)
|
|
}
|
|
|
|
state.MergePVETagStyle("pve-a", PVETagStyle{})
|
|
snapshot = state.GetSnapshot()
|
|
if _, ok := snapshot.PVETagColors["Production"]; ok {
|
|
t.Fatalf("expected stale pve-a aggregate color to be removed, got %#v", snapshot.PVETagColors)
|
|
}
|
|
if got := snapshot.PVETagColors["backup"]; got != "#112233" {
|
|
t.Fatalf("expected pve-b aggregate color to remain, got %q", got)
|
|
}
|
|
}
|
|
|
|
func TestRemoveConnectionHealth(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Set up connection health entries
|
|
state.SetConnectionHealth("pve-cluster-1", true)
|
|
state.SetConnectionHealth("pve-cluster-2", false)
|
|
|
|
// Remove one
|
|
state.RemoveConnectionHealth("pve-cluster-1")
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if _, ok := snapshot.ConnectionHealth["pve-cluster-1"]; ok {
|
|
t.Error("Expected pve-cluster-1 to be removed")
|
|
}
|
|
if _, ok := snapshot.ConnectionHealth["pve-cluster-2"]; !ok {
|
|
t.Error("Expected pve-cluster-2 to still exist")
|
|
}
|
|
|
|
// Remove non-existent (should not panic)
|
|
state.RemoveConnectionHealth("non-existent")
|
|
|
|
// Remove remaining
|
|
state.RemoveConnectionHealth("pve-cluster-2")
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.ConnectionHealth) != 0 {
|
|
t.Errorf("Expected empty connection health map, got %d entries", len(snapshot.ConnectionHealth))
|
|
}
|
|
}
|
|
|
|
func TestUpdatePBSBackups(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
|
|
// Add backups from first instance
|
|
backups1 := []PBSBackup{
|
|
{ID: "backup-1", Instance: "pbs-1", BackupTime: now},
|
|
{ID: "backup-2", Instance: "pbs-1", BackupTime: now.Add(-time.Hour)},
|
|
}
|
|
state.UpdatePBSBackups("pbs-1", backups1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PBSBackups) != 2 {
|
|
t.Fatalf("Expected 2 backups, got %d", len(snapshot.PBSBackups))
|
|
}
|
|
|
|
// Add backups from second instance
|
|
backups2 := []PBSBackup{
|
|
{ID: "backup-3", Instance: "pbs-2", BackupTime: now.Add(-30 * time.Minute)},
|
|
}
|
|
state.UpdatePBSBackups("pbs-2", backups2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PBSBackups) != 3 {
|
|
t.Fatalf("Expected 3 backups, got %d", len(snapshot.PBSBackups))
|
|
}
|
|
|
|
// Update first instance (should replace its backups)
|
|
backups1Updated := []PBSBackup{
|
|
{ID: "backup-4", Instance: "pbs-1", BackupTime: now.Add(time.Hour)},
|
|
}
|
|
state.UpdatePBSBackups("pbs-1", backups1Updated)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PBSBackups) != 2 {
|
|
t.Fatalf("Expected 2 backups after update, got %d", len(snapshot.PBSBackups))
|
|
}
|
|
|
|
// Verify pbs-1 backups were replaced
|
|
hasOldBackup := false
|
|
for _, b := range snapshot.PBSBackups {
|
|
if b.ID == "backup-1" || b.ID == "backup-2" {
|
|
hasOldBackup = true
|
|
break
|
|
}
|
|
}
|
|
if hasOldBackup {
|
|
t.Error("Old pbs-1 backups should have been replaced")
|
|
}
|
|
}
|
|
|
|
func TestUpdatePMGBackups(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
|
|
// Add backups from first instance
|
|
backups1 := []PMGBackup{
|
|
{Filename: "backup-1.conf", Instance: "pmg-1", BackupTime: now},
|
|
{Filename: "backup-2.conf", Instance: "pmg-1", BackupTime: now.Add(-time.Hour)},
|
|
}
|
|
state.UpdatePMGBackups("pmg-1", backups1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PMGBackups) != 2 {
|
|
t.Fatalf("Expected 2 backups, got %d", len(snapshot.PMGBackups))
|
|
}
|
|
|
|
// Backups should be sorted by time (newest first)
|
|
if snapshot.PMGBackups[0].BackupTime.Before(snapshot.PMGBackups[1].BackupTime) {
|
|
t.Error("Backups should be sorted by time descending")
|
|
}
|
|
|
|
// Add backups from second instance
|
|
backups2 := []PMGBackup{
|
|
{Filename: "backup-3.conf", Instance: "pmg-2", BackupTime: now.Add(-30 * time.Minute)},
|
|
}
|
|
state.UpdatePMGBackups("pmg-2", backups2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PMGBackups) != 3 {
|
|
t.Fatalf("Expected 3 backups, got %d", len(snapshot.PMGBackups))
|
|
}
|
|
|
|
// Update first instance with empty (should remove its backups)
|
|
state.UpdatePMGBackups("pmg-1", []PMGBackup{})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PMGBackups) != 1 {
|
|
t.Fatalf("Expected 1 backup after clearing pmg-1, got %d", len(snapshot.PMGBackups))
|
|
}
|
|
if snapshot.PMGBackups[0].Instance != "pmg-2" {
|
|
t.Error("Remaining backup should be from pmg-2")
|
|
}
|
|
}
|
|
|
|
func TestUpdatePhysicalDisks(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Add disks from first instance
|
|
disks1 := []PhysicalDisk{
|
|
{ID: "disk-1", Instance: "pve-1", Node: "node1", DevPath: "/dev/sda"},
|
|
{ID: "disk-2", Instance: "pve-1", Node: "node1", DevPath: "/dev/sdb"},
|
|
}
|
|
state.UpdatePhysicalDisks("pve-1", disks1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PhysicalDisks) != 2 {
|
|
t.Fatalf("Expected 2 disks, got %d", len(snapshot.PhysicalDisks))
|
|
}
|
|
|
|
// Add disks from second instance
|
|
disks2 := []PhysicalDisk{
|
|
{ID: "disk-3", Instance: "pve-2", Node: "node2", DevPath: "/dev/sda"},
|
|
}
|
|
state.UpdatePhysicalDisks("pve-2", disks2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PhysicalDisks) != 3 {
|
|
t.Fatalf("Expected 3 disks, got %d", len(snapshot.PhysicalDisks))
|
|
}
|
|
|
|
// Update first instance (should replace its disks)
|
|
disks1Updated := []PhysicalDisk{
|
|
{ID: "disk-4", Instance: "pve-1", Node: "node1", DevPath: "/dev/nvme0n1"},
|
|
}
|
|
state.UpdatePhysicalDisks("pve-1", disks1Updated)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PhysicalDisks) != 2 {
|
|
t.Fatalf("Expected 2 disks after update, got %d", len(snapshot.PhysicalDisks))
|
|
}
|
|
|
|
// Verify sorting by node then devpath
|
|
if snapshot.PhysicalDisks[0].Node > snapshot.PhysicalDisks[1].Node {
|
|
t.Error("Disks should be sorted by node")
|
|
}
|
|
}
|
|
|
|
func TestUpdateStorageForInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Add storage from first instance
|
|
storage1 := []Storage{
|
|
{ID: "storage-1", Instance: "pve-1", Name: "local"},
|
|
{ID: "storage-2", Instance: "pve-1", Name: "ceph-pool"},
|
|
}
|
|
state.UpdateStorageForInstance("pve-1", storage1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.Storage) != 2 {
|
|
t.Fatalf("Expected 2 storage entries, got %d", len(snapshot.Storage))
|
|
}
|
|
|
|
// Add storage from second instance
|
|
storage2 := []Storage{
|
|
{ID: "storage-3", Instance: "pve-2", Name: "local"},
|
|
}
|
|
state.UpdateStorageForInstance("pve-2", storage2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.Storage) != 3 {
|
|
t.Fatalf("Expected 3 storage entries, got %d", len(snapshot.Storage))
|
|
}
|
|
|
|
// Update first instance with empty (should remove its storage)
|
|
state.UpdateStorageForInstance("pve-1", []Storage{})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.Storage) != 1 {
|
|
t.Fatalf("Expected 1 storage after clearing pve-1, got %d", len(snapshot.Storage))
|
|
}
|
|
}
|
|
|
|
func TestUpdatePBSInstances(t *testing.T) {
|
|
state := NewState()
|
|
|
|
instances := []PBSInstance{
|
|
{ID: "pbs-1", Name: "Backup Server 1"},
|
|
{ID: "pbs-2", Name: "Backup Server 2"},
|
|
}
|
|
state.UpdatePBSInstances(instances)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PBSInstances) != 2 {
|
|
t.Fatalf("Expected 2 PBS instances, got %d", len(snapshot.PBSInstances))
|
|
}
|
|
|
|
// Replace with different instances
|
|
newInstances := []PBSInstance{
|
|
{ID: "pbs-3", Name: "New Backup Server"},
|
|
}
|
|
state.UpdatePBSInstances(newInstances)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PBSInstances) != 1 {
|
|
t.Fatalf("Expected 1 PBS instance after replacement, got %d", len(snapshot.PBSInstances))
|
|
}
|
|
if snapshot.PBSInstances[0].ID != "pbs-3" {
|
|
t.Error("Expected pbs-3 instance")
|
|
}
|
|
}
|
|
|
|
func TestUpdatePBSInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Add first instance
|
|
state.UpdatePBSInstance(PBSInstance{ID: "pbs-1", Name: "Server 1"})
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PBSInstances) != 1 {
|
|
t.Fatalf("Expected 1 PBS instance, got %d", len(snapshot.PBSInstances))
|
|
}
|
|
|
|
// Add second instance
|
|
state.UpdatePBSInstance(PBSInstance{ID: "pbs-2", Name: "Server 2"})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PBSInstances) != 2 {
|
|
t.Fatalf("Expected 2 PBS instances, got %d", len(snapshot.PBSInstances))
|
|
}
|
|
|
|
// Update existing instance
|
|
state.UpdatePBSInstance(PBSInstance{ID: "pbs-1", Name: "Server 1 Updated"})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PBSInstances) != 2 {
|
|
t.Fatalf("Expected 2 PBS instances after update, got %d", len(snapshot.PBSInstances))
|
|
}
|
|
|
|
// Find updated instance
|
|
var found bool
|
|
for _, inst := range snapshot.PBSInstances {
|
|
if inst.ID == "pbs-1" && inst.Name == "Server 1 Updated" {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
t.Error("Expected pbs-1 to be updated with new name")
|
|
}
|
|
}
|
|
|
|
func TestPBSInstanceJobHealthEvidenceNormalizeAndClone(t *testing.T) {
|
|
state := NewState()
|
|
observedAt := time.Unix(1700000000, 0).UTC()
|
|
state.UpdatePBSInstance(PBSInstance{
|
|
ID: "pbs-1",
|
|
Name: "pbs-main",
|
|
JobHealthEvidence: []PBSJobHealthEvidence{{
|
|
ID: "sync-1",
|
|
Family: "sync",
|
|
Store: "fast",
|
|
LastRunState: "OK",
|
|
LastRunUPID: "UPID:sync:1",
|
|
LastRunEndtime: 1699999900,
|
|
NextRun: 1700003600,
|
|
Confidence: "direct-task-match",
|
|
EvidenceSource: "pbs-job-config",
|
|
EvidenceScope: "configured-job",
|
|
Freshness: PBSJobHealthFreshness{
|
|
ObservedAt: observedAt,
|
|
State: "observed",
|
|
},
|
|
Posture: "healthy",
|
|
}},
|
|
})
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PBSInstances) != 1 || len(snapshot.PBSInstances[0].JobHealthEvidence) != 1 {
|
|
t.Fatalf("expected cloned PBS job health evidence, got %+v", snapshot.PBSInstances)
|
|
}
|
|
snapshot.PBSInstances[0].JobHealthEvidence[0].LastRunState = "changed"
|
|
|
|
fresh := state.GetSnapshot()
|
|
if got := fresh.PBSInstances[0].JobHealthEvidence[0].LastRunState; got != "OK" {
|
|
t.Fatalf("state clone leaked mutation, got last-run-state %q", got)
|
|
}
|
|
if got := fresh.PBSInstances[0].JobHealthEvidence[0]; got.EvidenceSource != "pbs-job-config" || got.EvidenceScope != "configured-job" {
|
|
t.Fatalf("state clone did not preserve evidence source/scope: %+v", got)
|
|
}
|
|
|
|
normalized := (PBSInstance{}).NormalizeCollections()
|
|
if normalized.JobHealthEvidence == nil {
|
|
t.Fatal("JobHealthEvidence should normalize to an initialized empty slice")
|
|
}
|
|
}
|
|
|
|
func TestUpdatePMGInstances(t *testing.T) {
|
|
state := NewState()
|
|
|
|
instances := []PMGInstance{
|
|
{ID: "pmg-1", Name: "Mail Gateway 1"},
|
|
{ID: "pmg-2", Name: "Mail Gateway 2"},
|
|
}
|
|
state.UpdatePMGInstances(instances)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PMGInstances) != 2 {
|
|
t.Fatalf("Expected 2 PMG instances, got %d", len(snapshot.PMGInstances))
|
|
}
|
|
|
|
// Replace with empty
|
|
state.UpdatePMGInstances([]PMGInstance{})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PMGInstances) != 0 {
|
|
t.Fatalf("Expected 0 PMG instances after clearing, got %d", len(snapshot.PMGInstances))
|
|
}
|
|
}
|
|
|
|
func TestUpdatePMGInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Add first instance
|
|
state.UpdatePMGInstance(PMGInstance{ID: "pmg-1", Name: "Gateway 1"})
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PMGInstances) != 1 {
|
|
t.Fatalf("Expected 1 PMG instance, got %d", len(snapshot.PMGInstances))
|
|
}
|
|
|
|
// Add second instance
|
|
state.UpdatePMGInstance(PMGInstance{ID: "pmg-2", Name: "Gateway 2"})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PMGInstances) != 2 {
|
|
t.Fatalf("Expected 2 PMG instances, got %d", len(snapshot.PMGInstances))
|
|
}
|
|
|
|
// Update existing instance
|
|
state.UpdatePMGInstance(PMGInstance{ID: "pmg-1", Name: "Gateway 1 Updated"})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PMGInstances) != 2 {
|
|
t.Fatalf("Expected 2 PMG instances after update, got %d", len(snapshot.PMGInstances))
|
|
}
|
|
|
|
// Verify update
|
|
var found bool
|
|
for _, inst := range snapshot.PMGInstances {
|
|
if inst.ID == "pmg-1" && inst.Name == "Gateway 1 Updated" {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
t.Error("Expected pmg-1 to be updated with new name")
|
|
}
|
|
}
|
|
|
|
func TestUpdateCephClustersForInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Add clusters from first instance
|
|
clusters1 := []CephCluster{
|
|
{ID: "ceph-1", Instance: "pve-1", Name: "ceph-pool-1"},
|
|
{ID: "ceph-2", Instance: "pve-1", Name: "ceph-pool-2"},
|
|
}
|
|
state.UpdateCephClustersForInstance("pve-1", clusters1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.CephClusters) != 2 {
|
|
t.Fatalf("Expected 2 clusters, got %d", len(snapshot.CephClusters))
|
|
}
|
|
|
|
// Add clusters from second instance
|
|
clusters2 := []CephCluster{
|
|
{ID: "ceph-3", Instance: "pve-2", Name: "ceph-pool-1"},
|
|
}
|
|
state.UpdateCephClustersForInstance("pve-2", clusters2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.CephClusters) != 3 {
|
|
t.Fatalf("Expected 3 clusters, got %d", len(snapshot.CephClusters))
|
|
}
|
|
|
|
// Update first instance with empty (should remove its clusters)
|
|
state.UpdateCephClustersForInstance("pve-1", []CephCluster{})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.CephClusters) != 1 {
|
|
t.Fatalf("Expected 1 cluster after clearing pve-1, got %d", len(snapshot.CephClusters))
|
|
}
|
|
if snapshot.CephClusters[0].Instance != "pve-2" {
|
|
t.Error("Remaining cluster should be from pve-2")
|
|
}
|
|
}
|
|
|
|
func TestCephClusterStatePrefersProxmoxAPIByFSID(t *testing.T) {
|
|
state := NewState()
|
|
|
|
state.UpsertCephCluster(CephCluster{
|
|
ID: "fsid-123",
|
|
Instance: "pve5",
|
|
Source: CephClusterSourceHostAgent,
|
|
Name: "pve5 Ceph",
|
|
FSID: "fsid-123",
|
|
Pools: []CephPool{{
|
|
ID: 2,
|
|
Name: "data_replication",
|
|
StoredBytes: 70,
|
|
AvailableBytes: 30,
|
|
PercentUsed: 70,
|
|
}},
|
|
})
|
|
|
|
stored := state.UpdateCephClustersForInstance("prod", []CephCluster{{
|
|
ID: "prod-fsid-123",
|
|
Instance: "prod",
|
|
Source: CephClusterSourceProxmoxAPI,
|
|
Name: "Ceph",
|
|
FSID: "fsid-123",
|
|
Health: "HEALTH_OK",
|
|
}})
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.CephClusters) != 1 {
|
|
t.Fatalf("expected API and agent reports for same FSID to reconcile to one cluster, got %#v", snapshot.CephClusters)
|
|
}
|
|
cluster := snapshot.CephClusters[0]
|
|
if cluster.Source != CephClusterSourceProxmoxAPI || cluster.Instance != "prod" || cluster.ID != "prod-fsid-123" {
|
|
t.Fatalf("expected Proxmox API cluster to be canonical, got %+v", cluster)
|
|
}
|
|
if !containsString(cluster.InstanceAliases, "pve5") {
|
|
t.Fatalf("expected agent instance alias to be preserved, got %#v", cluster.InstanceAliases)
|
|
}
|
|
if len(cluster.Pools) != 1 || cluster.Pools[0].Name != "data_replication" {
|
|
t.Fatalf("expected agent pool data to supplement API status-only cluster, got %#v", cluster.Pools)
|
|
}
|
|
if len(stored) != 1 || stored[0].Instance != "prod" || !containsString(stored[0].InstanceAliases, "pve5") {
|
|
t.Fatalf("stored clusters did not return reconciled canonical cluster: %#v", stored)
|
|
}
|
|
}
|
|
|
|
func TestUpsertCephClusterKeepsProxmoxAPIWhenAgentRefreshArrives(t *testing.T) {
|
|
state := NewState()
|
|
|
|
state.UpdateCephClustersForInstance("prod", []CephCluster{{
|
|
ID: "prod-fsid-123",
|
|
Instance: "prod",
|
|
Source: CephClusterSourceProxmoxAPI,
|
|
Name: "Ceph",
|
|
FSID: "fsid-123",
|
|
Health: "HEALTH_OK",
|
|
Pools: []CephPool{{
|
|
ID: 2,
|
|
Name: "data_replication",
|
|
StoredBytes: 60,
|
|
AvailableBytes: 40,
|
|
PercentUsed: 60,
|
|
}},
|
|
}})
|
|
|
|
stored := state.UpsertCephCluster(CephCluster{
|
|
ID: "fsid-123",
|
|
Instance: "pve5",
|
|
Source: CephClusterSourceHostAgent,
|
|
Name: "pve5 Ceph",
|
|
FSID: "fsid-123",
|
|
})
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.CephClusters) != 1 {
|
|
t.Fatalf("expected agent refresh to merge with API cluster, got %#v", snapshot.CephClusters)
|
|
}
|
|
cluster := snapshot.CephClusters[0]
|
|
if cluster.Source != CephClusterSourceProxmoxAPI || cluster.Instance != "prod" {
|
|
t.Fatalf("expected API cluster to remain canonical, got %+v", cluster)
|
|
}
|
|
if !containsString(cluster.InstanceAliases, "pve5") {
|
|
t.Fatalf("expected agent instance alias to be preserved, got %#v", cluster.InstanceAliases)
|
|
}
|
|
if stored.Source != CephClusterSourceProxmoxAPI || stored.Instance != "prod" {
|
|
t.Fatalf("expected upsert to return canonical API cluster, got %+v", stored)
|
|
}
|
|
}
|
|
|
|
func TestUpdateBackupTasksForInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
|
|
// Add tasks from first instance
|
|
tasks1 := []BackupTask{
|
|
{ID: "pve-1-task-1", Instance: "pve-1", StartTime: now},
|
|
{ID: "pve-1-task-2", Instance: "pve-1", StartTime: now.Add(-time.Hour)},
|
|
}
|
|
state.UpdateBackupTasksForInstance("pve-1", tasks1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.BackupTasks) != 2 {
|
|
t.Fatalf("Expected 2 tasks, got %d", len(snapshot.PVEBackups.BackupTasks))
|
|
}
|
|
|
|
// Tasks should be sorted by start time descending
|
|
if snapshot.PVEBackups.BackupTasks[0].StartTime.Before(snapshot.PVEBackups.BackupTasks[1].StartTime) {
|
|
t.Error("Tasks should be sorted by start time descending")
|
|
}
|
|
|
|
// Add tasks from second instance
|
|
tasks2 := []BackupTask{
|
|
{ID: "pve-2-task-1", Instance: "pve-2", StartTime: now.Add(-30 * time.Minute)},
|
|
}
|
|
state.UpdateBackupTasksForInstance("pve-2", tasks2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.BackupTasks) != 3 {
|
|
t.Fatalf("Expected 3 tasks, got %d", len(snapshot.PVEBackups.BackupTasks))
|
|
}
|
|
|
|
// Update first instance (should replace its tasks)
|
|
tasks1Updated := []BackupTask{
|
|
{ID: "pve-1-task-3", Instance: "pve-1", StartTime: now.Add(time.Hour)},
|
|
}
|
|
state.UpdateBackupTasksForInstance("pve-1", tasks1Updated)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.BackupTasks) != 2 {
|
|
t.Fatalf("Expected 2 tasks after update, got %d", len(snapshot.PVEBackups.BackupTasks))
|
|
}
|
|
}
|
|
|
|
func TestUpdateReplicationJobsForInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Add jobs from first instance
|
|
jobs1 := []ReplicationJob{
|
|
{ID: "job-1", Instance: "pve-1", GuestID: 100, JobNumber: 0},
|
|
{ID: "job-2", Instance: "pve-1", GuestID: 101, JobNumber: 0},
|
|
}
|
|
state.UpdateReplicationJobsForInstance("pve-1", jobs1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.ReplicationJobs) != 2 {
|
|
t.Fatalf("Expected 2 jobs, got %d", len(snapshot.ReplicationJobs))
|
|
}
|
|
|
|
// Add jobs from second instance
|
|
jobs2 := []ReplicationJob{
|
|
{ID: "job-3", Instance: "pve-2", GuestID: 200, JobNumber: 0},
|
|
}
|
|
state.UpdateReplicationJobsForInstance("pve-2", jobs2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.ReplicationJobs) != 3 {
|
|
t.Fatalf("Expected 3 jobs, got %d", len(snapshot.ReplicationJobs))
|
|
}
|
|
|
|
// Update first instance with empty (should remove its jobs)
|
|
state.UpdateReplicationJobsForInstance("pve-1", []ReplicationJob{})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.ReplicationJobs) != 1 {
|
|
t.Fatalf("Expected 1 job after clearing pve-1, got %d", len(snapshot.ReplicationJobs))
|
|
}
|
|
if snapshot.ReplicationJobs[0].Instance != "pve-2" {
|
|
t.Error("Remaining job should be from pve-2")
|
|
}
|
|
}
|
|
|
|
func TestUpdateGuestSnapshotsForInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
|
|
// Add snapshots from first instance
|
|
snapshots1 := []GuestSnapshot{
|
|
{ID: "pve-1-snap-1", Instance: "pve-1", VMID: 100, Name: "snapshot1", Time: now},
|
|
{ID: "pve-1-snap-2", Instance: "pve-1", VMID: 100, Name: "snapshot2", Time: now.Add(-time.Hour)},
|
|
}
|
|
state.UpdateGuestSnapshotsForInstance("pve-1", snapshots1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.GuestSnapshots) != 2 {
|
|
t.Fatalf("Expected 2 snapshots, got %d", len(snapshot.PVEBackups.GuestSnapshots))
|
|
}
|
|
|
|
// Add snapshots from second instance
|
|
snapshots2 := []GuestSnapshot{
|
|
{ID: "pve-2-snap-1", Instance: "pve-2", VMID: 200, Name: "snapshot1", Time: now.Add(-30 * time.Minute)},
|
|
}
|
|
state.UpdateGuestSnapshotsForInstance("pve-2", snapshots2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.GuestSnapshots) != 3 {
|
|
t.Fatalf("Expected 3 snapshots, got %d", len(snapshot.PVEBackups.GuestSnapshots))
|
|
}
|
|
|
|
// Update first instance (should replace its snapshots)
|
|
snapshots1Updated := []GuestSnapshot{
|
|
{ID: "pve-1-snap-3", Instance: "pve-1", VMID: 100, Name: "new-snapshot", Time: now.Add(time.Hour)},
|
|
}
|
|
state.UpdateGuestSnapshotsForInstance("pve-1", snapshots1Updated)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.GuestSnapshots) != 2 {
|
|
t.Fatalf("Expected 2 snapshots after update, got %d", len(snapshot.PVEBackups.GuestSnapshots))
|
|
}
|
|
}
|
|
|
|
func TestSyncGuestBackupTimes(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
oldBackup := now.Add(-24 * time.Hour)
|
|
newBackup := now.Add(-1 * time.Hour)
|
|
|
|
// Set up VMs and containers
|
|
state.UpdateVMs([]VM{
|
|
{VMID: 100, Name: "vm-100", Instance: "pve-1"},
|
|
{VMID: 101, Name: "vm-101", Instance: "pve-1"},
|
|
})
|
|
state.UpdateContainers([]Container{
|
|
{VMID: 200, Name: "ct-200", Instance: "pve-1"},
|
|
})
|
|
|
|
// Add storage backups for VM 100 (must include Instance for proper matching)
|
|
state.mu.Lock()
|
|
state.PVEBackups.StorageBackups = []StorageBackup{
|
|
{ID: "pve-1-backup-1", VMID: 100, Instance: "pve-1", Time: oldBackup},
|
|
{ID: "pve-1-backup-2", VMID: 100, Instance: "pve-1", Time: newBackup}, // newer
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
// Add PBS backup for container 200
|
|
state.UpdatePBSBackups("pbs-1", []PBSBackup{
|
|
{ID: "pbs-backup-1", VMID: "200", BackupType: "ct", BackupTime: newBackup},
|
|
})
|
|
|
|
// Sync backup times
|
|
state.SyncGuestBackupTimes()
|
|
|
|
snapshot := state.GetSnapshot()
|
|
|
|
// VM 100 should have the newer backup time
|
|
var vm100 *VM
|
|
for i := range snapshot.VMs {
|
|
if snapshot.VMs[i].VMID == 100 {
|
|
vm100 = &snapshot.VMs[i]
|
|
break
|
|
}
|
|
}
|
|
if vm100 == nil {
|
|
t.Fatal("VM 100 not found")
|
|
}
|
|
if !vm100.LastBackup.Equal(newBackup) {
|
|
t.Errorf("VM 100 LastBackup = %v, expected %v", vm100.LastBackup, newBackup)
|
|
}
|
|
|
|
// VM 101 should have no backup time (zero)
|
|
var vm101 *VM
|
|
for i := range snapshot.VMs {
|
|
if snapshot.VMs[i].VMID == 101 {
|
|
vm101 = &snapshot.VMs[i]
|
|
break
|
|
}
|
|
}
|
|
if vm101 == nil {
|
|
t.Fatal("VM 101 not found")
|
|
}
|
|
if !vm101.LastBackup.IsZero() {
|
|
t.Errorf("VM 101 LastBackup should be zero, got %v", vm101.LastBackup)
|
|
}
|
|
|
|
// Container 200 should have backup time from PBS
|
|
var ct200 *Container
|
|
for i := range snapshot.Containers {
|
|
if snapshot.Containers[i].VMID == 200 {
|
|
ct200 = &snapshot.Containers[i]
|
|
break
|
|
}
|
|
}
|
|
if ct200 == nil {
|
|
t.Fatal("Container 200 not found")
|
|
}
|
|
if !ct200.LastBackup.Equal(newBackup) {
|
|
t.Errorf("Container 200 LastBackup = %v, expected %v", ct200.LastBackup, newBackup)
|
|
}
|
|
}
|
|
|
|
// TestSyncGuestBackupTimesCrossInstance verifies that backup matching uses instance+VMID.
|
|
// This prevents a newly created container on one instance from incorrectly showing
|
|
// backup time from a different container with the same VMID on another instance.
|
|
func TestSyncGuestBackupTimesCrossInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
threeMonthsAgo := now.Add(-90 * 24 * time.Hour)
|
|
|
|
// Set up containers with the SAME VMID on DIFFERENT instances
|
|
// This simulates: pve-1 has VMID 100 with old backup, pve-2 has newly created VMID 100
|
|
state.UpdateContainers([]Container{
|
|
{VMID: 100, Name: "old-container", Instance: "pve-1"},
|
|
{VMID: 100, Name: "new-container", Instance: "pve-2"}, // newly created, no backup
|
|
})
|
|
|
|
// Add a 3-month-old backup for pve-1's container (VMID 100)
|
|
state.mu.Lock()
|
|
state.PVEBackups.StorageBackups = []StorageBackup{
|
|
{ID: "pve-1-backup-old", VMID: 100, Instance: "pve-1", Time: threeMonthsAgo},
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
// Sync backup times
|
|
state.SyncGuestBackupTimes()
|
|
|
|
snapshot := state.GetSnapshot()
|
|
|
|
// Find both containers
|
|
var oldContainer, newContainer *Container
|
|
for i := range snapshot.Containers {
|
|
if snapshot.Containers[i].Instance == "pve-1" && snapshot.Containers[i].VMID == 100 {
|
|
oldContainer = &snapshot.Containers[i]
|
|
}
|
|
if snapshot.Containers[i].Instance == "pve-2" && snapshot.Containers[i].VMID == 100 {
|
|
newContainer = &snapshot.Containers[i]
|
|
}
|
|
}
|
|
|
|
if oldContainer == nil {
|
|
t.Fatal("pve-1 container not found")
|
|
}
|
|
if newContainer == nil {
|
|
t.Fatal("pve-2 container not found")
|
|
}
|
|
|
|
// The old container on pve-1 SHOULD have the backup time
|
|
if !oldContainer.LastBackup.Equal(threeMonthsAgo) {
|
|
t.Errorf("pve-1 container LastBackup = %v, expected %v", oldContainer.LastBackup, threeMonthsAgo)
|
|
}
|
|
|
|
// The NEW container on pve-2 should NOT have any backup time (it's a different container!)
|
|
if !newContainer.LastBackup.IsZero() {
|
|
t.Errorf("pve-2 container (newly created) should have no backup, got %v", newContainer.LastBackup)
|
|
}
|
|
}
|
|
|
|
// TestSyncGuestBackupTimesNamespaceDisambiguation verifies that PBS namespace is used to
|
|
// disambiguate backups when multiple VMs have the same VMID across different PVE instances.
|
|
// This addresses issue #1095 where users have multiple PVE instances with overlapping VMIDs.
|
|
func TestSyncGuestBackupTimesNamespaceDisambiguation(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
pveBackupTime := now.Add(-1 * time.Hour)
|
|
pveNatBackupTime := now.Add(-2 * time.Hour)
|
|
|
|
// Set up VMs with the SAME VMID on DIFFERENT instances
|
|
state.UpdateVMs([]VM{
|
|
{VMID: 100, Name: "webserver-pve", Instance: "pve", Node: "node1"},
|
|
{VMID: 100, Name: "webserver-nat", Instance: "pve-nat", Node: "node2"},
|
|
})
|
|
|
|
// Add PBS backups with namespaces that correspond to the PVE instances
|
|
state.mu.Lock()
|
|
state.PBSBackups = []PBSBackup{
|
|
{
|
|
ID: "pbs-pve-100",
|
|
VMID: "100",
|
|
Namespace: "pve",
|
|
BackupType: "vm",
|
|
BackupTime: pveBackupTime,
|
|
Instance: "pbs-main",
|
|
},
|
|
{
|
|
ID: "pbs-nat-100",
|
|
VMID: "100",
|
|
Namespace: "nat", // Should match "pve-nat" instance
|
|
BackupType: "vm",
|
|
BackupTime: pveNatBackupTime,
|
|
Instance: "pbs-main",
|
|
},
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
// Sync backup times
|
|
state.SyncGuestBackupTimes()
|
|
|
|
snapshot := state.GetSnapshot()
|
|
|
|
// Find both VMs
|
|
var vmPVE, vmNAT *VM
|
|
for i := range snapshot.VMs {
|
|
if snapshot.VMs[i].Instance == "pve" && snapshot.VMs[i].VMID == 100 {
|
|
vmPVE = &snapshot.VMs[i]
|
|
}
|
|
if snapshot.VMs[i].Instance == "pve-nat" && snapshot.VMs[i].VMID == 100 {
|
|
vmNAT = &snapshot.VMs[i]
|
|
}
|
|
}
|
|
|
|
if vmPVE == nil {
|
|
t.Fatal("pve VM not found")
|
|
}
|
|
if vmNAT == nil {
|
|
t.Fatal("pve-nat VM not found")
|
|
}
|
|
|
|
// The pve VM should have the backup with namespace "pve"
|
|
if !vmPVE.LastBackup.Equal(pveBackupTime) {
|
|
t.Errorf("pve VM LastBackup = %v, expected %v (from namespace 'pve')", vmPVE.LastBackup, pveBackupTime)
|
|
}
|
|
|
|
// The pve-nat VM should have the backup with namespace "nat"
|
|
if !vmNAT.LastBackup.Equal(pveNatBackupTime) {
|
|
t.Errorf("pve-nat VM LastBackup = %v, expected %v (from namespace 'nat')", vmNAT.LastBackup, pveNatBackupTime)
|
|
}
|
|
}
|
|
|
|
// TestSyncGuestBackupTimesClusterEntrypointUsesGuestNodeNamespace verifies that
|
|
// a cluster API entrypoint name does not shadow the guest's actual node namespace.
|
|
func TestSyncGuestBackupTimesClusterEntrypointUsesGuestNodeNamespace(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
oldBackupTime := now.Add(-180 * 24 * time.Hour)
|
|
freshBackupTime := now.Add(-1 * time.Hour)
|
|
|
|
state.UpdateContainers([]Container{
|
|
{VMID: 112, Name: "debian-go", Instance: "delly", Node: "minipc"},
|
|
})
|
|
|
|
state.mu.Lock()
|
|
state.PBSBackups = []PBSBackup{
|
|
{
|
|
ID: "pbs-delly-112-old",
|
|
VMID: "112",
|
|
Namespace: "delly",
|
|
BackupType: "ct",
|
|
Comment: "112",
|
|
BackupTime: oldBackupTime,
|
|
Instance: "pbs-main",
|
|
},
|
|
{
|
|
ID: "pbs-minipc-112-fresh",
|
|
VMID: "112",
|
|
Namespace: "minipc",
|
|
BackupType: "ct",
|
|
Comment: "debian-go",
|
|
BackupTime: freshBackupTime,
|
|
Instance: "pbs-main",
|
|
},
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
state.SyncGuestBackupTimes()
|
|
snapshot := state.GetSnapshot()
|
|
|
|
var found *Container
|
|
for i := range snapshot.Containers {
|
|
if snapshot.Containers[i].VMID == 112 {
|
|
found = &snapshot.Containers[i]
|
|
break
|
|
}
|
|
}
|
|
if found == nil {
|
|
t.Fatal("container 112 not found")
|
|
}
|
|
if !found.LastBackup.Equal(freshBackupTime) {
|
|
t.Errorf("cluster entrypoint guest LastBackup = %v, want fresh node-namespaced backup %v",
|
|
found.LastBackup, freshBackupTime)
|
|
}
|
|
}
|
|
|
|
func TestSyncGuestBackupTimesClearsStaleBackupWhenCurrentEvidenceDisappears(t *testing.T) {
|
|
state := NewState()
|
|
|
|
staleBackupTime := time.Now().Add(-30 * 24 * time.Hour)
|
|
state.UpdateVMs([]VM{
|
|
{VMID: 101, Name: "unbacked-vm", Instance: "pve-1", Node: "node1", LastBackup: staleBackupTime},
|
|
})
|
|
state.UpdateContainers([]Container{
|
|
{VMID: 201, Name: "unbacked-ct", Instance: "pve-1", Node: "node1", LastBackup: staleBackupTime},
|
|
})
|
|
|
|
state.SyncGuestBackupTimes()
|
|
snapshot := state.GetSnapshot()
|
|
|
|
for _, vm := range snapshot.VMs {
|
|
if vm.VMID == 101 && !vm.LastBackup.IsZero() {
|
|
t.Errorf("VM stale LastBackup should be cleared, got %v", vm.LastBackup)
|
|
}
|
|
}
|
|
for _, ct := range snapshot.Containers {
|
|
if ct.VMID == 201 && !ct.LastBackup.IsZero() {
|
|
t.Errorf("container stale LastBackup should be cleared, got %v", ct.LastBackup)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestSyncGuestBackupTimesVMIDCollisionNonMatchingNamespace verifies that when the same VMID
|
|
// exists on multiple PVE instances and a PBS backup namespace matches neither, both guests
|
|
// get zero LastBackup instead of a false positive. The backup's submission
|
|
// source (owner/datastore) carries no learned attribution and no PVE-side
|
|
// storage listing confirms it, so the #1639 evidence paths must not fire.
|
|
func TestSyncGuestBackupTimesVMIDCollisionNonMatchingNamespace(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
backupTime := now.Add(-1 * time.Hour)
|
|
|
|
// Two VMs with the same VMID on different instances
|
|
state.UpdateVMs([]VM{
|
|
{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
|
|
{VMID: 100, Name: "vm-pve2", Instance: "pve2", Node: "node2"},
|
|
})
|
|
|
|
// PBS backup with a namespace that matches neither instance
|
|
state.mu.Lock()
|
|
state.PBSBackups = []PBSBackup{
|
|
{
|
|
ID: "pbs-100",
|
|
VMID: "100",
|
|
Namespace: "staging",
|
|
BackupType: "vm",
|
|
BackupTime: backupTime,
|
|
Instance: "pbs-main",
|
|
Datastore: "backups",
|
|
Owner: "unlearned@pbs!token",
|
|
},
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
state.SyncGuestBackupTimes()
|
|
snapshot := state.GetSnapshot()
|
|
|
|
for _, vm := range snapshot.VMs {
|
|
if vm.VMID == 100 && !vm.LastBackup.IsZero() {
|
|
t.Errorf("VM %q on %s should have zero LastBackup (ambiguous VMID, non-matching namespace), got %v",
|
|
vm.Name, vm.Instance, vm.LastBackup)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestSyncGuestBackupTimesVMIDCollisionEmptyNamespace verifies that when the same VMID
|
|
// exists on multiple PVE instances and a PBS backup has no namespace, both guests
|
|
// get zero LastBackup. This is the genuinely unattributable core of #1639:
|
|
// with a single snapshot, no learnable submission source, and no PVE-side
|
|
// confirmation, guessing either guest would be a false positive. The
|
|
// resolvable variants live in issue1639_pbs_collision_test.go.
|
|
func TestSyncGuestBackupTimesVMIDCollisionEmptyNamespace(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
backupTime := now.Add(-1 * time.Hour)
|
|
|
|
state.UpdateVMs([]VM{
|
|
{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
|
|
{VMID: 100, Name: "vm-pve2", Instance: "pve2", Node: "node2"},
|
|
})
|
|
|
|
// PBS backup with empty namespace
|
|
state.mu.Lock()
|
|
state.PBSBackups = []PBSBackup{
|
|
{
|
|
ID: "pbs-100",
|
|
VMID: "100",
|
|
Namespace: "",
|
|
BackupType: "vm",
|
|
BackupTime: backupTime,
|
|
Instance: "pbs-main",
|
|
},
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
state.SyncGuestBackupTimes()
|
|
snapshot := state.GetSnapshot()
|
|
|
|
for _, vm := range snapshot.VMs {
|
|
if vm.VMID == 100 && !vm.LastBackup.IsZero() {
|
|
t.Errorf("VM %q on %s should have zero LastBackup (ambiguous VMID, empty namespace), got %v",
|
|
vm.Name, vm.Instance, vm.LastBackup)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestSyncGuestBackupTimesUniqueVMIDFallback verifies that a unique VMID still gets
|
|
// the PBS backup via fallback even when the namespace doesn't match.
|
|
func TestSyncGuestBackupTimesUniqueVMIDFallback(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
backupTime := now.Add(-1 * time.Hour)
|
|
|
|
// Single VM — VMID is unique
|
|
state.UpdateVMs([]VM{
|
|
{VMID: 100, Name: "my-vm", Instance: "pve1", Node: "node1"},
|
|
})
|
|
|
|
// PBS backup with a namespace that does NOT match the instance
|
|
state.mu.Lock()
|
|
state.PBSBackups = []PBSBackup{
|
|
{
|
|
ID: "pbs-100",
|
|
VMID: "100",
|
|
Namespace: "daily",
|
|
BackupType: "vm",
|
|
BackupTime: backupTime,
|
|
Instance: "pbs-main",
|
|
},
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
state.SyncGuestBackupTimes()
|
|
snapshot := state.GetSnapshot()
|
|
|
|
var found *VM
|
|
for i := range snapshot.VMs {
|
|
if snapshot.VMs[i].VMID == 100 {
|
|
found = &snapshot.VMs[i]
|
|
}
|
|
}
|
|
if found == nil {
|
|
t.Fatal("VM not found")
|
|
}
|
|
if !found.LastBackup.Equal(backupTime) {
|
|
t.Errorf("unique VMID should fall back to PBS backup; got LastBackup = %v, want %v",
|
|
found.LastBackup, backupTime)
|
|
}
|
|
}
|
|
|
|
// TestSyncGuestBackupTimesVMContainerCollision verifies that PBS vm/ID and ct/ID
|
|
// subjects are matched independently even when a VM and container share the same
|
|
// numeric ID.
|
|
func TestSyncGuestBackupTimesVMContainerCollision(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
backupTime := now.Add(-1 * time.Hour)
|
|
|
|
state.UpdateVMs([]VM{
|
|
{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
|
|
})
|
|
state.UpdateContainers([]Container{
|
|
{VMID: 100, Name: "ct-pve2", Instance: "pve2", Node: "node2"},
|
|
})
|
|
|
|
state.mu.Lock()
|
|
state.PBSBackups = []PBSBackup{
|
|
{
|
|
ID: "pbs-100",
|
|
VMID: "100",
|
|
Namespace: "node1",
|
|
BackupType: "vm",
|
|
BackupTime: backupTime,
|
|
Instance: "pbs-main",
|
|
},
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
state.SyncGuestBackupTimes()
|
|
snapshot := state.GetSnapshot()
|
|
|
|
for _, vm := range snapshot.VMs {
|
|
if vm.VMID == 100 && !vm.LastBackup.Equal(backupTime) {
|
|
t.Errorf("VM %q LastBackup = %v, want typed vm/100 backup %v",
|
|
vm.Name, vm.LastBackup, backupTime)
|
|
}
|
|
}
|
|
for _, ct := range snapshot.Containers {
|
|
if ct.VMID == 100 && !ct.LastBackup.IsZero() {
|
|
t.Errorf("Container %q should have zero LastBackup because the PBS subject is vm/100, got %v",
|
|
ct.Name, ct.LastBackup)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestUpdateGuestsForInstancePublishesCoherentGeneration(t *testing.T) {
|
|
state := NewState()
|
|
previousBackup := time.Now().Add(-2 * time.Hour).UTC()
|
|
|
|
state.UpdateVMs([]VM{
|
|
{ID: "lab-a:node-a:101", VMID: 101, Instance: "lab-a", Node: "node-a", LastBackup: previousBackup},
|
|
{ID: "lab-b:node-b:201", VMID: 201, Instance: "lab-b", Node: "node-b"},
|
|
})
|
|
state.UpdateContainers([]Container{
|
|
{ID: "lab-a:node-a:102", VMID: 102, Instance: "lab-a", Node: "node-a", LastBackup: previousBackup},
|
|
{ID: "lab-b:node-b:202", VMID: 202, Instance: "lab-b", Node: "node-b"},
|
|
})
|
|
before := state.GetSnapshot().LastUpdate
|
|
|
|
state.UpdateGuestsForInstance(
|
|
"lab-a",
|
|
[]VM{{ID: "lab-a:node-a:101", VMID: 101, Instance: "lab-a", Node: "node-a", Status: "running"}},
|
|
[]Container{{ID: "lab-a:node-a:103", VMID: 103, Instance: "lab-a", Node: "node-a", Status: "running"}},
|
|
)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if !snapshot.LastUpdate.After(before) {
|
|
t.Fatalf("LastUpdate did not advance: before=%v after=%v", before, snapshot.LastUpdate)
|
|
}
|
|
if len(snapshot.VMs) != 2 || len(snapshot.Containers) != 2 {
|
|
t.Fatalf("unexpected coherent generation sizes: vms=%d containers=%d", len(snapshot.VMs), len(snapshot.Containers))
|
|
}
|
|
if snapshot.VMs[0].ID != "lab-a:node-a:101" || !snapshot.VMs[0].LastBackup.Equal(previousBackup) {
|
|
t.Fatalf("updated VM did not retain backup state: %+v", snapshot.VMs[0])
|
|
}
|
|
if snapshot.Containers[0].ID != "lab-a:node-a:103" {
|
|
t.Fatalf("authoritatively deleted container remained in state: %+v", snapshot.Containers)
|
|
}
|
|
if snapshot.VMs[1].Instance != "lab-b" || snapshot.Containers[1].Instance != "lab-b" {
|
|
t.Fatalf("other instance was not isolated: vms=%+v containers=%+v", snapshot.VMs, snapshot.Containers)
|
|
}
|
|
}
|
|
|
|
func TestUpdateStorageBackupsForInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
|
|
// Set up a VM so node normalization has something to work with
|
|
state.UpdateVMsForInstance("pve-1", []VM{
|
|
{VMID: 100, Instance: "pve-1", Node: "node1"},
|
|
})
|
|
|
|
// Add backups from first instance
|
|
backups1 := []StorageBackup{
|
|
{ID: "pve-1-backup-1", Instance: "pve-1", VMID: 100, Time: now, Node: "node1"},
|
|
{ID: "pve-1-backup-2", Instance: "pve-1", VMID: 100, Time: now.Add(-time.Hour), Node: "node1"},
|
|
}
|
|
state.UpdateStorageBackupsForInstance("pve-1", backups1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.StorageBackups) != 2 {
|
|
t.Fatalf("Expected 2 backups, got %d", len(snapshot.PVEBackups.StorageBackups))
|
|
}
|
|
|
|
// Backups should be sorted by time descending
|
|
if snapshot.PVEBackups.StorageBackups[0].Time.Before(snapshot.PVEBackups.StorageBackups[1].Time) {
|
|
t.Error("Backups should be sorted by time descending")
|
|
}
|
|
|
|
// Add backups from second instance
|
|
backups2 := []StorageBackup{
|
|
{ID: "pve-2-backup-1", Instance: "pve-2", VMID: 200, Time: now.Add(-30 * time.Minute), Node: "node2"},
|
|
}
|
|
state.UpdateStorageBackupsForInstance("pve-2", backups2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.StorageBackups) != 3 {
|
|
t.Fatalf("Expected 3 backups, got %d", len(snapshot.PVEBackups.StorageBackups))
|
|
}
|
|
|
|
// Update first instance (should replace its backups)
|
|
backups1Updated := []StorageBackup{
|
|
{ID: "pve-1-backup-3", Instance: "pve-1", VMID: 100, Time: now.Add(time.Hour), Node: "node1"},
|
|
}
|
|
state.UpdateStorageBackupsForInstance("pve-1", backups1Updated)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.StorageBackups) != 2 {
|
|
t.Fatalf("Expected 2 backups after update, got %d", len(snapshot.PVEBackups.StorageBackups))
|
|
}
|
|
|
|
// Verify old pve-1 backups were replaced
|
|
for _, b := range snapshot.PVEBackups.StorageBackups {
|
|
if b.ID == "pve-1-backup-1" || b.ID == "pve-1-backup-2" {
|
|
t.Error("Old pve-1 backups should have been replaced")
|
|
}
|
|
}
|
|
}
|
|
|
|
// Host.IntegrationSource is fabric-only provenance: it must survive host
|
|
// upserts, serialize when set, and stay entirely absent from JSON for real
|
|
// agent-report hosts so existing payloads do not change shape.
|
|
func TestHostIntegrationSourceRoundTrip(t *testing.T) {
|
|
state := NewState()
|
|
state.UpsertHost(Host{ID: "host-esxi", Hostname: "esxi-01", IntegrationSource: "vmware"})
|
|
state.UpsertHost(Host{ID: "host-agent", Hostname: "apollo-114"})
|
|
|
|
var esxi, agent *Host
|
|
hosts := state.GetHosts()
|
|
for i := range hosts {
|
|
switch hosts[i].ID {
|
|
case "host-esxi":
|
|
esxi = &hosts[i]
|
|
case "host-agent":
|
|
agent = &hosts[i]
|
|
}
|
|
}
|
|
if esxi == nil || esxi.IntegrationSource != "vmware" {
|
|
t.Fatalf("expected upserted host to preserve IntegrationSource %q, got %+v", "vmware", esxi)
|
|
}
|
|
if agent == nil || agent.IntegrationSource != "" {
|
|
t.Fatalf("expected agent host to keep empty IntegrationSource, got %+v", agent)
|
|
}
|
|
|
|
marked, err := json.Marshal(esxi)
|
|
if err != nil {
|
|
t.Fatalf("marshal integration host: %v", err)
|
|
}
|
|
if !strings.Contains(string(marked), `"integrationSource":"vmware"`) {
|
|
t.Fatalf("expected integrationSource in payload, got %s", marked)
|
|
}
|
|
|
|
unmarked, err := json.Marshal(agent)
|
|
if err != nil {
|
|
t.Fatalf("marshal agent host: %v", err)
|
|
}
|
|
if strings.Contains(string(unmarked), "integrationSource") {
|
|
t.Fatalf("expected integrationSource omitted for agent hosts, got %s", unmarked)
|
|
}
|
|
}
|
|
|
|
// Two clusters reusing the same node names (pve01/pve02 on different subnets)
|
|
// must stay apart: the second cluster's node must not steal the first
|
|
// cluster's agent link by bare hostname, and the shared name must not merge
|
|
// the two nodes into one slot.
|
|
func TestUpdateNodesForInstanceKeepsClustersWithSharedNodeNamesApart(t *testing.T) {
|
|
state := &State{
|
|
Hosts: []Host{
|
|
{
|
|
ID: "host-enacon-pve01",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.168.16.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
|
|
},
|
|
},
|
|
{
|
|
ID: "host-enacon-pve02",
|
|
Hostname: "pve02",
|
|
ReportIP: "192.168.16.12",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.16.12/24"}},
|
|
},
|
|
},
|
|
},
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.16.11:8006",
|
|
LinkedAgentID: "host-enacon-pve01",
|
|
Status: "online",
|
|
},
|
|
{
|
|
ID: "enacon-pve02",
|
|
Name: "pve02",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.16.12:8006",
|
|
LinkedAgentID: "host-enacon-pve02",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("rewo", []Node{
|
|
{
|
|
ID: "rewo-pve01",
|
|
Name: "pve01",
|
|
Instance: "rewo",
|
|
ClusterName: "rewo",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 3 {
|
|
t.Fatalf("nodes = %#v, want 3 (enacon pve01+pve02 and rewo pve01)", state.Nodes)
|
|
}
|
|
|
|
byID := make(map[string]Node)
|
|
for _, node := range state.Nodes {
|
|
byID[node.ID] = node
|
|
}
|
|
|
|
enacon, ok := byID["enacon-pve01"]
|
|
if !ok {
|
|
t.Fatalf("enacon-pve01 was clobbered, nodes = %#v", state.Nodes)
|
|
}
|
|
if enacon.ClusterName != "enacon" {
|
|
t.Fatalf("enacon-pve01 ClusterName = %q, want enacon", enacon.ClusterName)
|
|
}
|
|
if enacon.LinkedAgentID != "host-enacon-pve01" {
|
|
t.Fatalf("enacon-pve01 LinkedAgentID = %q, want host-enacon-pve01", enacon.LinkedAgentID)
|
|
}
|
|
|
|
rewo, ok := byID["rewo-pve01"]
|
|
if !ok {
|
|
t.Fatalf("rewo-pve01 missing, nodes = %#v", state.Nodes)
|
|
}
|
|
if rewo.ClusterName != "rewo" {
|
|
t.Fatalf("rewo-pve01 ClusterName = %q, want rewo", rewo.ClusterName)
|
|
}
|
|
if rewo.LinkedAgentID != "" {
|
|
t.Fatalf("rewo-pve01 LinkedAgentID = %q, want none (its host runs no agent)", rewo.LinkedAgentID)
|
|
}
|
|
}
|
|
|
|
// Standalone PVE connections have no cluster metadata, so sequential setup is
|
|
// the dangerous variant of the shared-name case: while only the first site's
|
|
// agent exists, a second pve01 must not borrow that agent through the short
|
|
// endpoint alias and collapse both provider instances into one node slot.
|
|
// Refs #1753.
|
|
func TestUpdateNodesForInstanceKeepsSequentialSameNameStandaloneSitesApart(t *testing.T) {
|
|
state := &State{
|
|
Hosts: []Host{
|
|
{
|
|
ID: "agent-staging-pve01",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.0.2.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.0.2.11/24"}},
|
|
},
|
|
},
|
|
},
|
|
Nodes: []Node{
|
|
{
|
|
ID: "staging-pve01",
|
|
Name: "pve01",
|
|
Instance: "staging",
|
|
Host: "https://pve01.staging.example:8006",
|
|
LinkedAgentID: "agent-staging-pve01",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("production", []Node{
|
|
{
|
|
ID: "production-pve01",
|
|
Name: "pve01",
|
|
Instance: "production",
|
|
Host: "https://pve01.production.example:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 2 {
|
|
t.Fatalf("nodes = %#v, want 2 standalone sites", state.Nodes)
|
|
}
|
|
byID := make(map[string]Node, len(state.Nodes))
|
|
for _, node := range state.Nodes {
|
|
byID[node.ID] = node
|
|
}
|
|
if byID["staging-pve01"].LinkedAgentID != "agent-staging-pve01" {
|
|
t.Fatalf("staging link = %q, want agent-staging-pve01", byID["staging-pve01"].LinkedAgentID)
|
|
}
|
|
if byID["production-pve01"].LinkedAgentID != "" {
|
|
t.Fatalf("production link = %q, want no guessed cross-site agent", byID["production-pve01"].LinkedAgentID)
|
|
}
|
|
|
|
// Existing installations may already hold the bad many-nodes-to-one-agent
|
|
// link in memory. The next provider poll must discard it, not preserve it.
|
|
for i := range state.Nodes {
|
|
if state.Nodes[i].ID == "production-pve01" {
|
|
state.Nodes[i].LinkedAgentID = "agent-staging-pve01"
|
|
}
|
|
}
|
|
state.UpdateNodesForInstance("production", []Node{
|
|
{
|
|
ID: "production-pve01",
|
|
Name: "pve01",
|
|
Instance: "production",
|
|
Host: "https://pve01.production.example:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
if len(state.Nodes) != 2 {
|
|
t.Fatalf("self-heal nodes = %#v, want 2 standalone sites", state.Nodes)
|
|
}
|
|
for _, node := range state.Nodes {
|
|
if node.ID == "production-pve01" && node.LinkedAgentID != "" {
|
|
t.Fatalf("self-healed production link = %q, want empty", node.LinkedAgentID)
|
|
}
|
|
}
|
|
}
|
|
|
|
// When TLS settings degrade node endpoints to the bare node name, the
|
|
// endpoint-host merge alias is identical for both clusters. The alias must
|
|
// not merge nodes whose named clusters contradict each other.
|
|
func TestUpdateNodesForInstanceEndpointHostAliasDoesNotMergeAcrossClusters(t *testing.T) {
|
|
state := &State{
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://pve01:8006",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("rewo", []Node{
|
|
{
|
|
ID: "rewo-pve01",
|
|
Name: "pve01",
|
|
Instance: "rewo",
|
|
ClusterName: "rewo",
|
|
IsClusterMember: true,
|
|
Host: "https://pve01:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 2 {
|
|
t.Fatalf("nodes = %#v, want 2 (one per cluster)", state.Nodes)
|
|
}
|
|
clusters := map[string]bool{}
|
|
for _, node := range state.Nodes {
|
|
clusters[node.ClusterName] = true
|
|
}
|
|
if !clusters["enacon"] || !clusters["rewo"] {
|
|
t.Fatalf("clusters = %#v, want both enacon and rewo", clusters)
|
|
}
|
|
}
|
|
|
|
// A standalone node whose endpoint IP is absent from a same-named agent's
|
|
// reported IPs must not link to that agent: the IP evidence contradicts the
|
|
// hostname match.
|
|
func TestUpdateNodesForInstanceHostnameMatchRejectedWhenIPContradicts(t *testing.T) {
|
|
state := &State{
|
|
Hosts: []Host{
|
|
{
|
|
ID: "host-a",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.168.16.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("steinboeck", []Node{
|
|
{
|
|
ID: "steinboeck-pve01",
|
|
Name: "pve01",
|
|
Instance: "steinboeck",
|
|
Host: "https://10.20.1.5:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 1 {
|
|
t.Fatalf("nodes = %#v, want 1", state.Nodes)
|
|
}
|
|
if state.Nodes[0].LinkedAgentID != "" {
|
|
t.Fatalf("LinkedAgentID = %q, want none (agent IPs contradict node endpoint)", state.Nodes[0].LinkedAgentID)
|
|
}
|
|
}
|
|
|
|
// Two different clusters at different sites can reuse the same corosync
|
|
// cluster name, the same member hostnames, and the same RFC1918 addressing
|
|
// (MSP support case: two "enacon" clusters both exposing pve01 on
|
|
// 192.168.1.11). The colliding endpoint-IP alias must not merge the two node
|
|
// slots, and the address match must not bind the second site's node to the
|
|
// first site's host agent when the TLS fingerprint evidence contradicts.
|
|
func TestUpdateNodesForInstanceSameNamedClustersWithCollidingIPsStayApart(t *testing.T) {
|
|
state := &State{
|
|
Hosts: []Host{
|
|
{
|
|
ID: "host-site-a-pve01",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.168.1.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
|
|
},
|
|
},
|
|
},
|
|
Nodes: []Node{
|
|
{
|
|
ID: "site-a-pve01",
|
|
Name: "pve01",
|
|
Instance: "site-a",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
TLSFingerprint: "AA:AA:AA:AA",
|
|
LinkedAgentID: "host-site-a-pve01",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("site-b", []Node{
|
|
{
|
|
ID: "site-b-pve01",
|
|
Name: "pve01",
|
|
Instance: "site-b",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
TLSFingerprint: "BB:BB:BB:BB",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 2 {
|
|
t.Fatalf("nodes = %#v, want 2 (one per site)", state.Nodes)
|
|
}
|
|
byID := make(map[string]Node)
|
|
for _, node := range state.Nodes {
|
|
byID[node.ID] = node
|
|
}
|
|
siteA, ok := byID["site-a-pve01"]
|
|
if !ok {
|
|
t.Fatalf("site-a-pve01 was clobbered, nodes = %#v", state.Nodes)
|
|
}
|
|
if siteA.LinkedAgentID != "host-site-a-pve01" {
|
|
t.Fatalf("site-a-pve01 LinkedAgentID = %q, want host-site-a-pve01", siteA.LinkedAgentID)
|
|
}
|
|
siteB, ok := byID["site-b-pve01"]
|
|
if !ok {
|
|
t.Fatalf("site-b-pve01 missing, nodes = %#v", state.Nodes)
|
|
}
|
|
if siteB.LinkedAgentID != "" {
|
|
t.Fatalf("site-b-pve01 LinkedAgentID = %q, want none (fingerprint contradicts the colliding-IP agent match)", siteB.LinkedAgentID)
|
|
}
|
|
}
|
|
|
|
// The endpoint-IP agent match must also respect the cluster-identity
|
|
// contradiction when no fingerprints are known: an agent whose linked nodes
|
|
// live in cluster "enacon" must not bind to a "rewo" node that presents the
|
|
// same RFC1918 address from another site.
|
|
func TestUpdateNodesForInstanceEndpointIPMatchRejectedWhenClusterContradicts(t *testing.T) {
|
|
state := &State{
|
|
Hosts: []Host{
|
|
{
|
|
ID: "host-enacon-pve01",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.168.1.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
|
|
},
|
|
},
|
|
},
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
LinkedAgentID: "host-enacon-pve01",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("rewo", []Node{
|
|
{
|
|
ID: "rewo-pve01",
|
|
Name: "pve01",
|
|
Instance: "rewo",
|
|
ClusterName: "rewo",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 2 {
|
|
t.Fatalf("nodes = %#v, want 2 (one per cluster)", state.Nodes)
|
|
}
|
|
for _, node := range state.Nodes {
|
|
if node.ClusterName == "rewo" && node.LinkedAgentID != "" {
|
|
t.Fatalf("rewo pve01 LinkedAgentID = %q, want none (agent belongs to the enacon cluster)", node.LinkedAgentID)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The same cluster added twice through different connection instances is the
|
|
// legitimate duplicate: when both views carry the same TOFU-captured TLS
|
|
// fingerprint for the same-named node, the endpoint alias must still fold
|
|
// them into one slot even though the instances differ.
|
|
func TestUpdateNodesForInstanceSameClusterAddedTwiceMergesWithMatchingFingerprints(t *testing.T) {
|
|
state := &State{
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-a-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon-a",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
TLSFingerprint: "AA:AA:AA:AA",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("enacon-b", []Node{
|
|
{
|
|
ID: "enacon-b-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon-b",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
TLSFingerprint: "aaaaaaaa",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 1 {
|
|
t.Fatalf("nodes = %#v, want 1 (matching fingerprints prove one machine)", state.Nodes)
|
|
}
|
|
if state.Nodes[0].Name != "pve01" {
|
|
t.Fatalf("merged node = %#v, want pve01", state.Nodes[0])
|
|
}
|
|
}
|
|
|
|
// Without fingerprint evidence the same-named cluster from another instance
|
|
// stays split: unknown identity must never fold two possibly-distinct
|
|
// clusters into one (fail-safe direction).
|
|
func TestUpdateNodesForInstanceSameNamedClustersStayApartWithoutFingerprints(t *testing.T) {
|
|
state := &State{
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-a-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon-a",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("enacon-b", []Node{
|
|
{
|
|
ID: "enacon-b-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon-b",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 2 {
|
|
t.Fatalf("nodes = %#v, want 2 (unknown identity must not merge)", state.Nodes)
|
|
}
|
|
}
|
|
|
|
// A shared agent link is the strongest cross-view merge signal, but for
|
|
// same-named clusters from different instances it must still be gated on
|
|
// fingerprint identity: matching fingerprints merge the duplicate views onto
|
|
// one slot, contradicting fingerprints never do.
|
|
func TestUpdateNodesForInstanceSharedAgentLinkMergeGatedOnFingerprint(t *testing.T) {
|
|
makeState := func(existingFP string) *State {
|
|
return &State{
|
|
Hosts: []Host{
|
|
{
|
|
ID: "host-pve01",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.168.1.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
|
|
},
|
|
},
|
|
},
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-a-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon-a",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://pve01.site-a.internal:8006",
|
|
TLSFingerprint: existingFP,
|
|
LinkedAgentID: "host-pve01",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
}
|
|
incoming := func(fp string) []Node {
|
|
return []Node{
|
|
{
|
|
ID: "enacon-b-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon-b",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
TLSFingerprint: fp,
|
|
LinkedAgentID: "host-pve01",
|
|
Status: "online",
|
|
},
|
|
}
|
|
}
|
|
|
|
merged := makeState("AA:AA:AA:AA")
|
|
merged.UpdateNodesForInstance("enacon-b", incoming("AA:AA:AA:AA"))
|
|
if len(merged.Nodes) != 1 {
|
|
t.Fatalf("matching fingerprints: nodes = %#v, want 1", merged.Nodes)
|
|
}
|
|
|
|
split := makeState("AA:AA:AA:AA")
|
|
split.UpdateNodesForInstance("enacon-b", incoming("BB:BB:BB:BB"))
|
|
if len(split.Nodes) != 2 {
|
|
t.Fatalf("contradicting fingerprints: nodes = %#v, want 2", split.Nodes)
|
|
}
|
|
}
|
|
|
|
// A cluster connection's first polls can commit nodes before membership
|
|
// detection classifies them (ClusterName ""), and cloned template
|
|
// deployments reuse /etc/machine-id, collapsing two sites' agents into one
|
|
// identity (MSP support case: clusters enacon and rewo, both with pve01, on
|
|
// reused RFC1918 ranges). During that window the shared agent identity must
|
|
// not fold the unclassified node into the established cluster's slot.
|
|
func TestUpdateNodesForInstanceUnclassifiedNodeSharedAgentMustNotStealClusterSlot(t *testing.T) {
|
|
state := &State{
|
|
Hosts: []Host{
|
|
{
|
|
// One host row standing in for two physical pve01 machines
|
|
// whose cloned machine-id collapsed them into one agent
|
|
// identity; it currently reports the second site's address.
|
|
ID: "host-cloned-machine-id",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.168.1.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
|
|
},
|
|
},
|
|
},
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.16.11:8006",
|
|
LinkedAgentID: "host-cloned-machine-id",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("rewo", []Node{
|
|
{
|
|
ID: "rewo-pve01",
|
|
Name: "pve01",
|
|
Instance: "rewo",
|
|
// Membership detection has not classified this connection yet.
|
|
ClusterName: "",
|
|
Host: "https://192.168.1.11:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 2 {
|
|
t.Fatalf("nodes = %#v, want 2 (unclassified node must not fold into the enacon slot)", state.Nodes)
|
|
}
|
|
byID := make(map[string]Node)
|
|
for _, node := range state.Nodes {
|
|
byID[node.ID] = node
|
|
}
|
|
enacon, ok := byID["enacon-pve01"]
|
|
if !ok {
|
|
t.Fatalf("enacon-pve01 was clobbered, nodes = %#v", state.Nodes)
|
|
}
|
|
if enacon.ClusterName != "enacon" {
|
|
t.Fatalf("enacon-pve01 ClusterName = %q, want enacon", enacon.ClusterName)
|
|
}
|
|
if _, ok := byID["rewo-pve01"]; !ok {
|
|
t.Fatalf("rewo-pve01 missing, nodes = %#v", state.Nodes)
|
|
}
|
|
}
|
|
|
|
// When TLS settings degrade both connections' node endpoints to the bare
|
|
// node name, the endpoint-host alias is identical across sites. An
|
|
// unclassified node (empty cluster name, e.g. during the first-poll
|
|
// membership window) must not ride that weak alias into another instance's
|
|
// established cluster slot.
|
|
func TestUpdateNodesForInstanceEndpointHostAliasRequiresProofForUnclassifiedNodes(t *testing.T) {
|
|
state := &State{
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://pve01:8006",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("rewo", []Node{
|
|
{
|
|
ID: "rewo-pve01",
|
|
Name: "pve01",
|
|
Instance: "rewo",
|
|
ClusterName: "",
|
|
Host: "https://pve01:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 2 {
|
|
t.Fatalf("nodes = %#v, want 2 (bare-hostname alias must not fold an unclassified cross-instance node)", state.Nodes)
|
|
}
|
|
}
|
|
|
|
// The legitimate standalone-into-cluster fold must survive the weak-evidence
|
|
// tightening: a standalone connection view dialing the same address as the
|
|
// cluster's endpoint record still folds via the address alias, and a
|
|
// fingerprint-proven view still folds via the shared agent link.
|
|
func TestUpdateNodesForInstanceStandaloneViewStillFoldsIntoClusterNode(t *testing.T) {
|
|
byAddress := &State{
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.16.11:8006",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
byAddress.UpdateNodesForInstance("pve01-direct", []Node{
|
|
{
|
|
ID: "direct-pve01",
|
|
Name: "pve01",
|
|
Instance: "pve01-direct",
|
|
Host: "https://192.168.16.11:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
if len(byAddress.Nodes) != 1 {
|
|
t.Fatalf("address alias: nodes = %#v, want 1 (same-address standalone view folds)", byAddress.Nodes)
|
|
}
|
|
|
|
byAgent := &State{
|
|
Hosts: []Host{
|
|
{
|
|
ID: "host-pve01",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.168.16.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
|
|
},
|
|
},
|
|
},
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://pve01:8006",
|
|
TLSFingerprint: "AA:AA:AA:AA",
|
|
LinkedAgentID: "host-pve01",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
byAgent.UpdateNodesForInstance("pve01-direct", []Node{
|
|
{
|
|
ID: "direct-pve01",
|
|
Name: "pve01",
|
|
Instance: "pve01-direct",
|
|
Host: "https://192.168.16.11:8006",
|
|
TLSFingerprint: "aaaaaaaa",
|
|
Status: "online",
|
|
},
|
|
})
|
|
if len(byAgent.Nodes) != 1 {
|
|
t.Fatalf("agent link: nodes = %#v, want 1 (fingerprint-proven view folds via shared agent)", byAgent.Nodes)
|
|
}
|
|
}
|
|
|
|
// splitNodeDecider reports the pairs an operator split, by node and agent ID.
|
|
type splitNodeDecider map[[2]string]bool
|
|
|
|
func (d splitNodeDecider) ProxmoxNodeAgentSplit(node Node, host Host) bool {
|
|
return d[[2]string{node.ID, host.ID}]
|
|
}
|
|
|
|
// A split recorded after one connection's poll is applied to that connection's
|
|
// retained node before its link can fold the next connection's view into it.
|
|
func TestUpdateNodesForInstanceReadsASplitOfARetainedNodeBeforeItFolds(t *testing.T) {
|
|
agent := Host{ID: "agent-1", Hostname: "pve1.lan", ReportIP: "10.0.0.5", Status: "online"}
|
|
nodeA := Node{ID: "a-pve1", Name: "pve1", Instance: "a", Host: "https://10.0.0.5:8006", Status: "online"}
|
|
nodeB := Node{ID: "b-pve1", Name: "pve1", Instance: "b", Host: "https://pve1.lan:8006", Status: "online"}
|
|
|
|
decider := splitNodeDecider{}
|
|
state := NewState()
|
|
state.UpsertHost(agent)
|
|
state.SetNodeAgentSplitDecider(decider)
|
|
state.UpdateNodesForInstance("a", []Node{nodeA})
|
|
if linked := state.GetSnapshot().Nodes; len(linked) != 1 || linked[0].LinkedAgentID != agent.ID {
|
|
t.Fatalf("fixture: the first poll should link its node to the agent: %+v", linked)
|
|
}
|
|
|
|
// The operator splits the first node from the agent; nothing has written
|
|
// a link since, so the retained node still names the agent.
|
|
decider[[2]string{nodeA.ID, agent.ID}] = true
|
|
state.UpdateNodesForInstance("b", []Node{nodeB})
|
|
|
|
nodes := state.GetSnapshot().Nodes
|
|
if len(nodes) != 2 {
|
|
t.Fatalf("a retained node's split link still folded the next view into its slot: %+v", nodes)
|
|
}
|
|
for _, node := range nodes {
|
|
switch node.ID {
|
|
case nodeA.ID:
|
|
if node.LinkedAgentID != "" {
|
|
t.Fatalf("node %s kept the agent %q the operator split it from", node.ID, node.LinkedAgentID)
|
|
}
|
|
case nodeB.ID:
|
|
if node.LinkedAgentID != agent.ID {
|
|
t.Fatalf("node %s lost an agent link nobody split: %q", node.ID, node.LinkedAgentID)
|
|
}
|
|
default:
|
|
t.Fatalf("unexpected node %+v", node)
|
|
}
|
|
}
|
|
}
|
|
|
|
// One agent corroborates two standalone connections' nodes through the node's
|
|
// IP address and its full hostname, which is the proof that folds the second
|
|
// view into the first node's slot. When the operator has split the second
|
|
// node from that agent, the agent is no evidence for it, so the view is not
|
|
// folded, and neither a node nor the agent keeps a link the operator split.
|
|
func TestUpdateNodesForInstanceHoldsASplitAgentBackBeforeItFoldsNodeViews(t *testing.T) {
|
|
agent := Host{
|
|
ID: "agent-1",
|
|
Hostname: "pve1.lan",
|
|
ReportIP: "10.0.0.5",
|
|
Status: "online",
|
|
}
|
|
nodeA := Node{ID: "a-pve1", Name: "pve1", Instance: "a", Host: "https://10.0.0.5:8006", Status: "online"}
|
|
nodeB := Node{ID: "b-pve1", Name: "pve1", Instance: "b", Host: "https://pve1.lan:8006", Status: "online"}
|
|
|
|
poll := func(decider NodeAgentSplitDecider) []Node {
|
|
state := NewState()
|
|
state.UpsertHost(agent)
|
|
state.SetNodeAgentSplitDecider(decider)
|
|
state.UpdateNodesForInstance("a", []Node{nodeA})
|
|
state.UpdateNodesForInstance("b", []Node{nodeB})
|
|
return state.GetSnapshot().Nodes
|
|
}
|
|
|
|
folded := poll(nil)
|
|
if len(folded) != 1 || folded[0].LinkedAgentID != agent.ID {
|
|
t.Fatalf("fixture: the agent should fold both views into one linked node, got %+v", folded)
|
|
}
|
|
|
|
apart := poll(splitNodeDecider{{nodeB.ID, agent.ID}: true})
|
|
if len(apart) != 2 {
|
|
t.Fatalf("a node split from the agent folded into its slot anyway: %+v", apart)
|
|
}
|
|
for _, node := range apart {
|
|
switch node.ID {
|
|
case nodeA.ID:
|
|
if node.LinkedAgentID != agent.ID {
|
|
t.Fatalf("the split of the other node unlinked %s from the agent", node.ID)
|
|
}
|
|
case nodeB.ID:
|
|
if node.LinkedAgentID != "" {
|
|
t.Fatalf("node %s kept the agent %q the operator split it from", node.ID, node.LinkedAgentID)
|
|
}
|
|
default:
|
|
t.Fatalf("unexpected node %+v", node)
|
|
}
|
|
}
|
|
}
|