Files
pulse/internal/models/state_host_test.go
T
rcourtman 7d02f9f04e Stop monitoring linking a node to an agent the operator split off
An operator's split of a Proxmox node and its pulse-agent (report-merge
or unlink) split the registry's rows in #2741, but the monitor's own
link (Node.LinkedAgentID, Host.LinkedNodeID) stayed. Everything that
reads that link kept treating two machines as one: the agent's SMART
inventory as the node's disk fallback, its --disk-exclude patterns on
the node's disks, guest discovery through the agent, deployment
reporting the node as already running an agent, service discovery, the
agent's container filesystem readings, the node's temperatures and the
node/agent alert correlation.

The resource store's exclusion is now the one record of the split.
MonitorAdapter decides it with the registry's own rule for a node's link
and confirms a split against the store's latest decisions, and
models.State holds both sides of a split link back on every link write,
before a link can steer where node views fold. The agents API's node
link is the operator's newer decision, so it removes the pair's split
from the store (TakeExclusion) before recording manual intent, and
puts it back, unless the operator decided again meanwhile, if the
intent fails to persist (RestoreExclusion). A split recorded after a
manual link ends that intent on the agent's next report that finds the
split confirmed, so the node is no longer reserved for that agent.

A rebuild that cannot read the store's decisions still loads none, in
the registry as in monitoring; that needs a change in registry.go and
monitor_adapter.go and is left for a follow-up.
2026-10-10 10:00:09 +01:00

2268 lines
68 KiB
Go

package models
import (
"encoding/json"
"strings"
"testing"
"time"
)
func TestUpsertHost(t *testing.T) {
state := NewState()
// Test insert new host
host1 := Host{
ID: "host-1",
Hostname: "server-1",
Status: "online",
}
state.UpsertHost(host1)
hosts := state.GetHosts()
if len(hosts) != 1 {
t.Fatalf("Expected 1 host, got %d", len(hosts))
}
if hosts[0].ID != "host-1" {
t.Errorf("Expected host ID 'host-1', got %q", hosts[0].ID)
}
// Test update existing host
host1Updated := Host{
ID: "host-1",
Hostname: "server-1",
Status: "offline",
}
state.UpsertHost(host1Updated)
hosts = state.GetHosts()
if len(hosts) != 1 {
t.Fatalf("Expected 1 host after update, got %d", len(hosts))
}
if hosts[0].Status != "offline" {
t.Errorf("Expected status 'offline', got %q", hosts[0].Status)
}
// Test multiple hosts and sorting
state.UpsertHost(Host{ID: "host-2", Hostname: "alpha-server"})
state.UpsertHost(Host{ID: "host-3", Hostname: "zulu-server"})
hosts = state.GetHosts()
if len(hosts) != 3 {
t.Fatalf("Expected 3 hosts, got %d", len(hosts))
}
// Hosts should be sorted by hostname
if hosts[0].Hostname != "alpha-server" {
t.Errorf("First host should be 'alpha-server', got %q", hosts[0].Hostname)
}
}
func TestUpsertHostPreservesReportedCommandAuthority(t *testing.T) {
state := NewState()
state.UpsertHost(Host{
ID: "host-authority",
AgentPrivilege: &AgentPrivilegeStatus{
RunningAsRoot: true,
CommandAuthority: "monitoring-only",
},
})
hosts := state.GetHosts()
if len(hosts) != 1 || hosts[0].AgentPrivilege == nil {
t.Fatalf("reported privilege missing from state: %+v", hosts)
}
if got := hosts[0].AgentPrivilege.CommandAuthority; got != "monitoring-only" {
t.Fatalf("command authority = %q, want monitoring-only", got)
}
}
func TestGetHosts_Copy(t *testing.T) {
state := NewState()
state.UpsertHost(Host{ID: "host-1", Hostname: "server-1"})
hosts1 := state.GetHosts()
hosts2 := state.GetHosts()
// Modify one slice
if len(hosts1) > 0 {
hosts1[0].Hostname = "modified"
}
// Other slice should be unchanged
if len(hosts2) > 0 && hosts2[0].Hostname == "modified" {
t.Error("GetHosts should return a copy, not the same slice")
}
// Original state should be unchanged
hosts3 := state.GetHosts()
if hosts3[0].Hostname == "modified" {
t.Error("State should be unchanged by modifications to returned slice")
}
}
func TestRemoveHost(t *testing.T) {
state := NewState()
// Insert hosts
state.UpsertHost(Host{ID: "host-1", Hostname: "server1"})
state.UpsertHost(Host{ID: "host-2", Hostname: "server2"})
// Remove existing host
removed, ok := state.RemoveHost("host-1")
if !ok {
t.Error("Expected RemoveHost to return true for existing host")
}
if removed.ID != "host-1" {
t.Errorf("Expected removed host ID 'host-1', got %q", removed.ID)
}
hosts := state.GetHosts()
if len(hosts) != 1 {
t.Fatalf("Expected 1 host after removal, got %d", len(hosts))
}
if hosts[0].ID != "host-2" {
t.Errorf("Remaining host should be 'host-2', got %q", hosts[0].ID)
}
// Remove non-existing host
removed, ok = state.RemoveHost("non-existent")
if ok {
t.Error("Expected RemoveHost to return false for non-existent host")
}
if removed.ID != "" {
t.Errorf("Expected empty Host for non-existent removal, got ID %q", removed.ID)
}
}
func TestSetHostStatus(t *testing.T) {
state := NewState()
// Test with no hosts
changed := state.SetHostStatus("host-1", "online")
if changed {
t.Error("SetHostStatus should return false when host doesn't exist")
}
// Add host and set status
state.UpsertHost(Host{ID: "host-1", Hostname: "server1", Status: "offline"})
changed = state.SetHostStatus("host-1", "online")
if !changed {
t.Error("SetHostStatus should return true when host exists")
}
hosts := state.GetHosts()
if hosts[0].Status != "online" {
t.Errorf("Expected status 'online', got %q", hosts[0].Status)
}
// Set same status (no change)
changed = state.SetHostStatus("host-1", "online")
if !changed {
t.Error("SetHostStatus should return true even when status unchanged")
}
}
func TestTouchHost(t *testing.T) {
state := NewState()
now := time.Now()
// Test with non-existent host
ok := state.TouchHost("host-1", now)
if ok {
t.Error("TouchHost should return false for non-existent host")
}
// Add host and touch it
state.UpsertHost(Host{ID: "host-1", Hostname: "server1"})
later := now.Add(time.Hour)
ok = state.TouchHost("host-1", later)
if !ok {
t.Error("TouchHost should return true for existing host")
}
hosts := state.GetHosts()
if !hosts[0].LastSeen.Equal(later) {
t.Errorf("LastSeen should be updated to %v, got %v", later, hosts[0].LastSeen)
}
}
func TestSetConnectionHealth(t *testing.T) {
state := NewState()
// Set connection healthy
state.SetConnectionHealth("pve-cluster-1", true)
snapshot := state.GetSnapshot()
if healthy, ok := snapshot.ConnectionHealth["pve-cluster-1"]; !ok || !healthy {
t.Error("Expected connection to be healthy")
}
// Set connection unhealthy
state.SetConnectionHealth("pve-cluster-1", false)
snapshot = state.GetSnapshot()
if healthy, ok := snapshot.ConnectionHealth["pve-cluster-1"]; !ok || healthy {
t.Error("Expected connection to be unhealthy")
}
// Multiple connections
state.SetConnectionHealth("pve-cluster-2", true)
state.SetConnectionHealth("pbs-instance-1", true)
snapshot = state.GetSnapshot()
if len(snapshot.ConnectionHealth) != 3 {
t.Errorf("Expected 3 connection health entries, got %d", len(snapshot.ConnectionHealth))
}
}
func TestPVETagStyleSnapshotRebuildsAggregateByInstance(t *testing.T) {
state := NewState()
state.MergePVETagStyle("pve-a", PVETagStyle{
Colors: map[string]string{
"Production": "#00ff00",
},
CaseSensitive: true,
})
state.MergePVETagStyle("pve-b", PVETagStyle{
Colors: map[string]string{
"Backup": "#112233",
},
})
snapshot := state.GetSnapshot()
if got := snapshot.PVETagStyles["pve-a"].Colors["Production"]; got != "#00ff00" {
t.Fatalf("expected pve-a tag style to preserve case-sensitive color, got %q", got)
}
if !snapshot.PVETagStyles["pve-a"].CaseSensitive {
t.Fatal("expected pve-a tag style to preserve case-sensitive flag")
}
if got := snapshot.PVETagStyles["pve-b"].Colors["backup"]; got != "#112233" {
t.Fatalf("expected pve-b tag style to normalize non-case-sensitive tag, got %q", got)
}
state.MergePVETagStyle("pve-a", PVETagStyle{})
snapshot = state.GetSnapshot()
if _, ok := snapshot.PVETagColors["Production"]; ok {
t.Fatalf("expected stale pve-a aggregate color to be removed, got %#v", snapshot.PVETagColors)
}
if got := snapshot.PVETagColors["backup"]; got != "#112233" {
t.Fatalf("expected pve-b aggregate color to remain, got %q", got)
}
}
func TestRemoveConnectionHealth(t *testing.T) {
state := NewState()
// Set up connection health entries
state.SetConnectionHealth("pve-cluster-1", true)
state.SetConnectionHealth("pve-cluster-2", false)
// Remove one
state.RemoveConnectionHealth("pve-cluster-1")
snapshot := state.GetSnapshot()
if _, ok := snapshot.ConnectionHealth["pve-cluster-1"]; ok {
t.Error("Expected pve-cluster-1 to be removed")
}
if _, ok := snapshot.ConnectionHealth["pve-cluster-2"]; !ok {
t.Error("Expected pve-cluster-2 to still exist")
}
// Remove non-existent (should not panic)
state.RemoveConnectionHealth("non-existent")
// Remove remaining
state.RemoveConnectionHealth("pve-cluster-2")
snapshot = state.GetSnapshot()
if len(snapshot.ConnectionHealth) != 0 {
t.Errorf("Expected empty connection health map, got %d entries", len(snapshot.ConnectionHealth))
}
}
func TestUpdatePBSBackups(t *testing.T) {
state := NewState()
now := time.Now()
// Add backups from first instance
backups1 := []PBSBackup{
{ID: "backup-1", Instance: "pbs-1", BackupTime: now},
{ID: "backup-2", Instance: "pbs-1", BackupTime: now.Add(-time.Hour)},
}
state.UpdatePBSBackups("pbs-1", backups1)
snapshot := state.GetSnapshot()
if len(snapshot.PBSBackups) != 2 {
t.Fatalf("Expected 2 backups, got %d", len(snapshot.PBSBackups))
}
// Add backups from second instance
backups2 := []PBSBackup{
{ID: "backup-3", Instance: "pbs-2", BackupTime: now.Add(-30 * time.Minute)},
}
state.UpdatePBSBackups("pbs-2", backups2)
snapshot = state.GetSnapshot()
if len(snapshot.PBSBackups) != 3 {
t.Fatalf("Expected 3 backups, got %d", len(snapshot.PBSBackups))
}
// Update first instance (should replace its backups)
backups1Updated := []PBSBackup{
{ID: "backup-4", Instance: "pbs-1", BackupTime: now.Add(time.Hour)},
}
state.UpdatePBSBackups("pbs-1", backups1Updated)
snapshot = state.GetSnapshot()
if len(snapshot.PBSBackups) != 2 {
t.Fatalf("Expected 2 backups after update, got %d", len(snapshot.PBSBackups))
}
// Verify pbs-1 backups were replaced
hasOldBackup := false
for _, b := range snapshot.PBSBackups {
if b.ID == "backup-1" || b.ID == "backup-2" {
hasOldBackup = true
break
}
}
if hasOldBackup {
t.Error("Old pbs-1 backups should have been replaced")
}
}
func TestUpdatePMGBackups(t *testing.T) {
state := NewState()
now := time.Now()
// Add backups from first instance
backups1 := []PMGBackup{
{Filename: "backup-1.conf", Instance: "pmg-1", BackupTime: now},
{Filename: "backup-2.conf", Instance: "pmg-1", BackupTime: now.Add(-time.Hour)},
}
state.UpdatePMGBackups("pmg-1", backups1)
snapshot := state.GetSnapshot()
if len(snapshot.PMGBackups) != 2 {
t.Fatalf("Expected 2 backups, got %d", len(snapshot.PMGBackups))
}
// Backups should be sorted by time (newest first)
if snapshot.PMGBackups[0].BackupTime.Before(snapshot.PMGBackups[1].BackupTime) {
t.Error("Backups should be sorted by time descending")
}
// Add backups from second instance
backups2 := []PMGBackup{
{Filename: "backup-3.conf", Instance: "pmg-2", BackupTime: now.Add(-30 * time.Minute)},
}
state.UpdatePMGBackups("pmg-2", backups2)
snapshot = state.GetSnapshot()
if len(snapshot.PMGBackups) != 3 {
t.Fatalf("Expected 3 backups, got %d", len(snapshot.PMGBackups))
}
// Update first instance with empty (should remove its backups)
state.UpdatePMGBackups("pmg-1", []PMGBackup{})
snapshot = state.GetSnapshot()
if len(snapshot.PMGBackups) != 1 {
t.Fatalf("Expected 1 backup after clearing pmg-1, got %d", len(snapshot.PMGBackups))
}
if snapshot.PMGBackups[0].Instance != "pmg-2" {
t.Error("Remaining backup should be from pmg-2")
}
}
func TestUpdatePhysicalDisks(t *testing.T) {
state := NewState()
// Add disks from first instance
disks1 := []PhysicalDisk{
{ID: "disk-1", Instance: "pve-1", Node: "node1", DevPath: "/dev/sda"},
{ID: "disk-2", Instance: "pve-1", Node: "node1", DevPath: "/dev/sdb"},
}
state.UpdatePhysicalDisks("pve-1", disks1)
snapshot := state.GetSnapshot()
if len(snapshot.PhysicalDisks) != 2 {
t.Fatalf("Expected 2 disks, got %d", len(snapshot.PhysicalDisks))
}
// Add disks from second instance
disks2 := []PhysicalDisk{
{ID: "disk-3", Instance: "pve-2", Node: "node2", DevPath: "/dev/sda"},
}
state.UpdatePhysicalDisks("pve-2", disks2)
snapshot = state.GetSnapshot()
if len(snapshot.PhysicalDisks) != 3 {
t.Fatalf("Expected 3 disks, got %d", len(snapshot.PhysicalDisks))
}
// Update first instance (should replace its disks)
disks1Updated := []PhysicalDisk{
{ID: "disk-4", Instance: "pve-1", Node: "node1", DevPath: "/dev/nvme0n1"},
}
state.UpdatePhysicalDisks("pve-1", disks1Updated)
snapshot = state.GetSnapshot()
if len(snapshot.PhysicalDisks) != 2 {
t.Fatalf("Expected 2 disks after update, got %d", len(snapshot.PhysicalDisks))
}
// Verify sorting by node then devpath
if snapshot.PhysicalDisks[0].Node > snapshot.PhysicalDisks[1].Node {
t.Error("Disks should be sorted by node")
}
}
func TestUpdateStorageForInstance(t *testing.T) {
state := NewState()
// Add storage from first instance
storage1 := []Storage{
{ID: "storage-1", Instance: "pve-1", Name: "local"},
{ID: "storage-2", Instance: "pve-1", Name: "ceph-pool"},
}
state.UpdateStorageForInstance("pve-1", storage1)
snapshot := state.GetSnapshot()
if len(snapshot.Storage) != 2 {
t.Fatalf("Expected 2 storage entries, got %d", len(snapshot.Storage))
}
// Add storage from second instance
storage2 := []Storage{
{ID: "storage-3", Instance: "pve-2", Name: "local"},
}
state.UpdateStorageForInstance("pve-2", storage2)
snapshot = state.GetSnapshot()
if len(snapshot.Storage) != 3 {
t.Fatalf("Expected 3 storage entries, got %d", len(snapshot.Storage))
}
// Update first instance with empty (should remove its storage)
state.UpdateStorageForInstance("pve-1", []Storage{})
snapshot = state.GetSnapshot()
if len(snapshot.Storage) != 1 {
t.Fatalf("Expected 1 storage after clearing pve-1, got %d", len(snapshot.Storage))
}
}
func TestUpdatePBSInstances(t *testing.T) {
state := NewState()
instances := []PBSInstance{
{ID: "pbs-1", Name: "Backup Server 1"},
{ID: "pbs-2", Name: "Backup Server 2"},
}
state.UpdatePBSInstances(instances)
snapshot := state.GetSnapshot()
if len(snapshot.PBSInstances) != 2 {
t.Fatalf("Expected 2 PBS instances, got %d", len(snapshot.PBSInstances))
}
// Replace with different instances
newInstances := []PBSInstance{
{ID: "pbs-3", Name: "New Backup Server"},
}
state.UpdatePBSInstances(newInstances)
snapshot = state.GetSnapshot()
if len(snapshot.PBSInstances) != 1 {
t.Fatalf("Expected 1 PBS instance after replacement, got %d", len(snapshot.PBSInstances))
}
if snapshot.PBSInstances[0].ID != "pbs-3" {
t.Error("Expected pbs-3 instance")
}
}
func TestUpdatePBSInstance(t *testing.T) {
state := NewState()
// Add first instance
state.UpdatePBSInstance(PBSInstance{ID: "pbs-1", Name: "Server 1"})
snapshot := state.GetSnapshot()
if len(snapshot.PBSInstances) != 1 {
t.Fatalf("Expected 1 PBS instance, got %d", len(snapshot.PBSInstances))
}
// Add second instance
state.UpdatePBSInstance(PBSInstance{ID: "pbs-2", Name: "Server 2"})
snapshot = state.GetSnapshot()
if len(snapshot.PBSInstances) != 2 {
t.Fatalf("Expected 2 PBS instances, got %d", len(snapshot.PBSInstances))
}
// Update existing instance
state.UpdatePBSInstance(PBSInstance{ID: "pbs-1", Name: "Server 1 Updated"})
snapshot = state.GetSnapshot()
if len(snapshot.PBSInstances) != 2 {
t.Fatalf("Expected 2 PBS instances after update, got %d", len(snapshot.PBSInstances))
}
// Find updated instance
var found bool
for _, inst := range snapshot.PBSInstances {
if inst.ID == "pbs-1" && inst.Name == "Server 1 Updated" {
found = true
break
}
}
if !found {
t.Error("Expected pbs-1 to be updated with new name")
}
}
func TestPBSInstanceJobHealthEvidenceNormalizeAndClone(t *testing.T) {
state := NewState()
observedAt := time.Unix(1700000000, 0).UTC()
state.UpdatePBSInstance(PBSInstance{
ID: "pbs-1",
Name: "pbs-main",
JobHealthEvidence: []PBSJobHealthEvidence{{
ID: "sync-1",
Family: "sync",
Store: "fast",
LastRunState: "OK",
LastRunUPID: "UPID:sync:1",
LastRunEndtime: 1699999900,
NextRun: 1700003600,
Confidence: "direct-task-match",
EvidenceSource: "pbs-job-config",
EvidenceScope: "configured-job",
Freshness: PBSJobHealthFreshness{
ObservedAt: observedAt,
State: "observed",
},
Posture: "healthy",
}},
})
snapshot := state.GetSnapshot()
if len(snapshot.PBSInstances) != 1 || len(snapshot.PBSInstances[0].JobHealthEvidence) != 1 {
t.Fatalf("expected cloned PBS job health evidence, got %+v", snapshot.PBSInstances)
}
snapshot.PBSInstances[0].JobHealthEvidence[0].LastRunState = "changed"
fresh := state.GetSnapshot()
if got := fresh.PBSInstances[0].JobHealthEvidence[0].LastRunState; got != "OK" {
t.Fatalf("state clone leaked mutation, got last-run-state %q", got)
}
if got := fresh.PBSInstances[0].JobHealthEvidence[0]; got.EvidenceSource != "pbs-job-config" || got.EvidenceScope != "configured-job" {
t.Fatalf("state clone did not preserve evidence source/scope: %+v", got)
}
normalized := (PBSInstance{}).NormalizeCollections()
if normalized.JobHealthEvidence == nil {
t.Fatal("JobHealthEvidence should normalize to an initialized empty slice")
}
}
func TestUpdatePMGInstances(t *testing.T) {
state := NewState()
instances := []PMGInstance{
{ID: "pmg-1", Name: "Mail Gateway 1"},
{ID: "pmg-2", Name: "Mail Gateway 2"},
}
state.UpdatePMGInstances(instances)
snapshot := state.GetSnapshot()
if len(snapshot.PMGInstances) != 2 {
t.Fatalf("Expected 2 PMG instances, got %d", len(snapshot.PMGInstances))
}
// Replace with empty
state.UpdatePMGInstances([]PMGInstance{})
snapshot = state.GetSnapshot()
if len(snapshot.PMGInstances) != 0 {
t.Fatalf("Expected 0 PMG instances after clearing, got %d", len(snapshot.PMGInstances))
}
}
func TestUpdatePMGInstance(t *testing.T) {
state := NewState()
// Add first instance
state.UpdatePMGInstance(PMGInstance{ID: "pmg-1", Name: "Gateway 1"})
snapshot := state.GetSnapshot()
if len(snapshot.PMGInstances) != 1 {
t.Fatalf("Expected 1 PMG instance, got %d", len(snapshot.PMGInstances))
}
// Add second instance
state.UpdatePMGInstance(PMGInstance{ID: "pmg-2", Name: "Gateway 2"})
snapshot = state.GetSnapshot()
if len(snapshot.PMGInstances) != 2 {
t.Fatalf("Expected 2 PMG instances, got %d", len(snapshot.PMGInstances))
}
// Update existing instance
state.UpdatePMGInstance(PMGInstance{ID: "pmg-1", Name: "Gateway 1 Updated"})
snapshot = state.GetSnapshot()
if len(snapshot.PMGInstances) != 2 {
t.Fatalf("Expected 2 PMG instances after update, got %d", len(snapshot.PMGInstances))
}
// Verify update
var found bool
for _, inst := range snapshot.PMGInstances {
if inst.ID == "pmg-1" && inst.Name == "Gateway 1 Updated" {
found = true
break
}
}
if !found {
t.Error("Expected pmg-1 to be updated with new name")
}
}
func TestUpdateCephClustersForInstance(t *testing.T) {
state := NewState()
// Add clusters from first instance
clusters1 := []CephCluster{
{ID: "ceph-1", Instance: "pve-1", Name: "ceph-pool-1"},
{ID: "ceph-2", Instance: "pve-1", Name: "ceph-pool-2"},
}
state.UpdateCephClustersForInstance("pve-1", clusters1)
snapshot := state.GetSnapshot()
if len(snapshot.CephClusters) != 2 {
t.Fatalf("Expected 2 clusters, got %d", len(snapshot.CephClusters))
}
// Add clusters from second instance
clusters2 := []CephCluster{
{ID: "ceph-3", Instance: "pve-2", Name: "ceph-pool-1"},
}
state.UpdateCephClustersForInstance("pve-2", clusters2)
snapshot = state.GetSnapshot()
if len(snapshot.CephClusters) != 3 {
t.Fatalf("Expected 3 clusters, got %d", len(snapshot.CephClusters))
}
// Update first instance with empty (should remove its clusters)
state.UpdateCephClustersForInstance("pve-1", []CephCluster{})
snapshot = state.GetSnapshot()
if len(snapshot.CephClusters) != 1 {
t.Fatalf("Expected 1 cluster after clearing pve-1, got %d", len(snapshot.CephClusters))
}
if snapshot.CephClusters[0].Instance != "pve-2" {
t.Error("Remaining cluster should be from pve-2")
}
}
func TestCephClusterStatePrefersProxmoxAPIByFSID(t *testing.T) {
state := NewState()
state.UpsertCephCluster(CephCluster{
ID: "fsid-123",
Instance: "pve5",
Source: CephClusterSourceHostAgent,
Name: "pve5 Ceph",
FSID: "fsid-123",
Pools: []CephPool{{
ID: 2,
Name: "data_replication",
StoredBytes: 70,
AvailableBytes: 30,
PercentUsed: 70,
}},
})
stored := state.UpdateCephClustersForInstance("prod", []CephCluster{{
ID: "prod-fsid-123",
Instance: "prod",
Source: CephClusterSourceProxmoxAPI,
Name: "Ceph",
FSID: "fsid-123",
Health: "HEALTH_OK",
}})
snapshot := state.GetSnapshot()
if len(snapshot.CephClusters) != 1 {
t.Fatalf("expected API and agent reports for same FSID to reconcile to one cluster, got %#v", snapshot.CephClusters)
}
cluster := snapshot.CephClusters[0]
if cluster.Source != CephClusterSourceProxmoxAPI || cluster.Instance != "prod" || cluster.ID != "prod-fsid-123" {
t.Fatalf("expected Proxmox API cluster to be canonical, got %+v", cluster)
}
if !containsString(cluster.InstanceAliases, "pve5") {
t.Fatalf("expected agent instance alias to be preserved, got %#v", cluster.InstanceAliases)
}
if len(cluster.Pools) != 1 || cluster.Pools[0].Name != "data_replication" {
t.Fatalf("expected agent pool data to supplement API status-only cluster, got %#v", cluster.Pools)
}
if len(stored) != 1 || stored[0].Instance != "prod" || !containsString(stored[0].InstanceAliases, "pve5") {
t.Fatalf("stored clusters did not return reconciled canonical cluster: %#v", stored)
}
}
func TestUpsertCephClusterKeepsProxmoxAPIWhenAgentRefreshArrives(t *testing.T) {
state := NewState()
state.UpdateCephClustersForInstance("prod", []CephCluster{{
ID: "prod-fsid-123",
Instance: "prod",
Source: CephClusterSourceProxmoxAPI,
Name: "Ceph",
FSID: "fsid-123",
Health: "HEALTH_OK",
Pools: []CephPool{{
ID: 2,
Name: "data_replication",
StoredBytes: 60,
AvailableBytes: 40,
PercentUsed: 60,
}},
}})
stored := state.UpsertCephCluster(CephCluster{
ID: "fsid-123",
Instance: "pve5",
Source: CephClusterSourceHostAgent,
Name: "pve5 Ceph",
FSID: "fsid-123",
})
snapshot := state.GetSnapshot()
if len(snapshot.CephClusters) != 1 {
t.Fatalf("expected agent refresh to merge with API cluster, got %#v", snapshot.CephClusters)
}
cluster := snapshot.CephClusters[0]
if cluster.Source != CephClusterSourceProxmoxAPI || cluster.Instance != "prod" {
t.Fatalf("expected API cluster to remain canonical, got %+v", cluster)
}
if !containsString(cluster.InstanceAliases, "pve5") {
t.Fatalf("expected agent instance alias to be preserved, got %#v", cluster.InstanceAliases)
}
if stored.Source != CephClusterSourceProxmoxAPI || stored.Instance != "prod" {
t.Fatalf("expected upsert to return canonical API cluster, got %+v", stored)
}
}
func TestUpdateBackupTasksForInstance(t *testing.T) {
state := NewState()
now := time.Now()
// Add tasks from first instance
tasks1 := []BackupTask{
{ID: "pve-1-task-1", Instance: "pve-1", StartTime: now},
{ID: "pve-1-task-2", Instance: "pve-1", StartTime: now.Add(-time.Hour)},
}
state.UpdateBackupTasksForInstance("pve-1", tasks1)
snapshot := state.GetSnapshot()
if len(snapshot.PVEBackups.BackupTasks) != 2 {
t.Fatalf("Expected 2 tasks, got %d", len(snapshot.PVEBackups.BackupTasks))
}
// Tasks should be sorted by start time descending
if snapshot.PVEBackups.BackupTasks[0].StartTime.Before(snapshot.PVEBackups.BackupTasks[1].StartTime) {
t.Error("Tasks should be sorted by start time descending")
}
// Add tasks from second instance
tasks2 := []BackupTask{
{ID: "pve-2-task-1", Instance: "pve-2", StartTime: now.Add(-30 * time.Minute)},
}
state.UpdateBackupTasksForInstance("pve-2", tasks2)
snapshot = state.GetSnapshot()
if len(snapshot.PVEBackups.BackupTasks) != 3 {
t.Fatalf("Expected 3 tasks, got %d", len(snapshot.PVEBackups.BackupTasks))
}
// Update first instance (should replace its tasks)
tasks1Updated := []BackupTask{
{ID: "pve-1-task-3", Instance: "pve-1", StartTime: now.Add(time.Hour)},
}
state.UpdateBackupTasksForInstance("pve-1", tasks1Updated)
snapshot = state.GetSnapshot()
if len(snapshot.PVEBackups.BackupTasks) != 2 {
t.Fatalf("Expected 2 tasks after update, got %d", len(snapshot.PVEBackups.BackupTasks))
}
}
func TestUpdateReplicationJobsForInstance(t *testing.T) {
state := NewState()
// Add jobs from first instance
jobs1 := []ReplicationJob{
{ID: "job-1", Instance: "pve-1", GuestID: 100, JobNumber: 0},
{ID: "job-2", Instance: "pve-1", GuestID: 101, JobNumber: 0},
}
state.UpdateReplicationJobsForInstance("pve-1", jobs1)
snapshot := state.GetSnapshot()
if len(snapshot.ReplicationJobs) != 2 {
t.Fatalf("Expected 2 jobs, got %d", len(snapshot.ReplicationJobs))
}
// Add jobs from second instance
jobs2 := []ReplicationJob{
{ID: "job-3", Instance: "pve-2", GuestID: 200, JobNumber: 0},
}
state.UpdateReplicationJobsForInstance("pve-2", jobs2)
snapshot = state.GetSnapshot()
if len(snapshot.ReplicationJobs) != 3 {
t.Fatalf("Expected 3 jobs, got %d", len(snapshot.ReplicationJobs))
}
// Update first instance with empty (should remove its jobs)
state.UpdateReplicationJobsForInstance("pve-1", []ReplicationJob{})
snapshot = state.GetSnapshot()
if len(snapshot.ReplicationJobs) != 1 {
t.Fatalf("Expected 1 job after clearing pve-1, got %d", len(snapshot.ReplicationJobs))
}
if snapshot.ReplicationJobs[0].Instance != "pve-2" {
t.Error("Remaining job should be from pve-2")
}
}
func TestUpdateGuestSnapshotsForInstance(t *testing.T) {
state := NewState()
now := time.Now()
// Add snapshots from first instance
snapshots1 := []GuestSnapshot{
{ID: "pve-1-snap-1", Instance: "pve-1", VMID: 100, Name: "snapshot1", Time: now},
{ID: "pve-1-snap-2", Instance: "pve-1", VMID: 100, Name: "snapshot2", Time: now.Add(-time.Hour)},
}
state.UpdateGuestSnapshotsForInstance("pve-1", snapshots1)
snapshot := state.GetSnapshot()
if len(snapshot.PVEBackups.GuestSnapshots) != 2 {
t.Fatalf("Expected 2 snapshots, got %d", len(snapshot.PVEBackups.GuestSnapshots))
}
// Add snapshots from second instance
snapshots2 := []GuestSnapshot{
{ID: "pve-2-snap-1", Instance: "pve-2", VMID: 200, Name: "snapshot1", Time: now.Add(-30 * time.Minute)},
}
state.UpdateGuestSnapshotsForInstance("pve-2", snapshots2)
snapshot = state.GetSnapshot()
if len(snapshot.PVEBackups.GuestSnapshots) != 3 {
t.Fatalf("Expected 3 snapshots, got %d", len(snapshot.PVEBackups.GuestSnapshots))
}
// Update first instance (should replace its snapshots)
snapshots1Updated := []GuestSnapshot{
{ID: "pve-1-snap-3", Instance: "pve-1", VMID: 100, Name: "new-snapshot", Time: now.Add(time.Hour)},
}
state.UpdateGuestSnapshotsForInstance("pve-1", snapshots1Updated)
snapshot = state.GetSnapshot()
if len(snapshot.PVEBackups.GuestSnapshots) != 2 {
t.Fatalf("Expected 2 snapshots after update, got %d", len(snapshot.PVEBackups.GuestSnapshots))
}
}
func TestSyncGuestBackupTimes(t *testing.T) {
state := NewState()
now := time.Now()
oldBackup := now.Add(-24 * time.Hour)
newBackup := now.Add(-1 * time.Hour)
// Set up VMs and containers
state.UpdateVMs([]VM{
{VMID: 100, Name: "vm-100", Instance: "pve-1"},
{VMID: 101, Name: "vm-101", Instance: "pve-1"},
})
state.UpdateContainers([]Container{
{VMID: 200, Name: "ct-200", Instance: "pve-1"},
})
// Add storage backups for VM 100 (must include Instance for proper matching)
state.mu.Lock()
state.PVEBackups.StorageBackups = []StorageBackup{
{ID: "pve-1-backup-1", VMID: 100, Instance: "pve-1", Time: oldBackup},
{ID: "pve-1-backup-2", VMID: 100, Instance: "pve-1", Time: newBackup}, // newer
}
state.mu.Unlock()
// Add PBS backup for container 200
state.UpdatePBSBackups("pbs-1", []PBSBackup{
{ID: "pbs-backup-1", VMID: "200", BackupType: "ct", BackupTime: newBackup},
})
// Sync backup times
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
// VM 100 should have the newer backup time
var vm100 *VM
for i := range snapshot.VMs {
if snapshot.VMs[i].VMID == 100 {
vm100 = &snapshot.VMs[i]
break
}
}
if vm100 == nil {
t.Fatal("VM 100 not found")
}
if !vm100.LastBackup.Equal(newBackup) {
t.Errorf("VM 100 LastBackup = %v, expected %v", vm100.LastBackup, newBackup)
}
// VM 101 should have no backup time (zero)
var vm101 *VM
for i := range snapshot.VMs {
if snapshot.VMs[i].VMID == 101 {
vm101 = &snapshot.VMs[i]
break
}
}
if vm101 == nil {
t.Fatal("VM 101 not found")
}
if !vm101.LastBackup.IsZero() {
t.Errorf("VM 101 LastBackup should be zero, got %v", vm101.LastBackup)
}
// Container 200 should have backup time from PBS
var ct200 *Container
for i := range snapshot.Containers {
if snapshot.Containers[i].VMID == 200 {
ct200 = &snapshot.Containers[i]
break
}
}
if ct200 == nil {
t.Fatal("Container 200 not found")
}
if !ct200.LastBackup.Equal(newBackup) {
t.Errorf("Container 200 LastBackup = %v, expected %v", ct200.LastBackup, newBackup)
}
}
// TestSyncGuestBackupTimesCrossInstance verifies that backup matching uses instance+VMID.
// This prevents a newly created container on one instance from incorrectly showing
// backup time from a different container with the same VMID on another instance.
func TestSyncGuestBackupTimesCrossInstance(t *testing.T) {
state := NewState()
now := time.Now()
threeMonthsAgo := now.Add(-90 * 24 * time.Hour)
// Set up containers with the SAME VMID on DIFFERENT instances
// This simulates: pve-1 has VMID 100 with old backup, pve-2 has newly created VMID 100
state.UpdateContainers([]Container{
{VMID: 100, Name: "old-container", Instance: "pve-1"},
{VMID: 100, Name: "new-container", Instance: "pve-2"}, // newly created, no backup
})
// Add a 3-month-old backup for pve-1's container (VMID 100)
state.mu.Lock()
state.PVEBackups.StorageBackups = []StorageBackup{
{ID: "pve-1-backup-old", VMID: 100, Instance: "pve-1", Time: threeMonthsAgo},
}
state.mu.Unlock()
// Sync backup times
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
// Find both containers
var oldContainer, newContainer *Container
for i := range snapshot.Containers {
if snapshot.Containers[i].Instance == "pve-1" && snapshot.Containers[i].VMID == 100 {
oldContainer = &snapshot.Containers[i]
}
if snapshot.Containers[i].Instance == "pve-2" && snapshot.Containers[i].VMID == 100 {
newContainer = &snapshot.Containers[i]
}
}
if oldContainer == nil {
t.Fatal("pve-1 container not found")
}
if newContainer == nil {
t.Fatal("pve-2 container not found")
}
// The old container on pve-1 SHOULD have the backup time
if !oldContainer.LastBackup.Equal(threeMonthsAgo) {
t.Errorf("pve-1 container LastBackup = %v, expected %v", oldContainer.LastBackup, threeMonthsAgo)
}
// The NEW container on pve-2 should NOT have any backup time (it's a different container!)
if !newContainer.LastBackup.IsZero() {
t.Errorf("pve-2 container (newly created) should have no backup, got %v", newContainer.LastBackup)
}
}
// TestSyncGuestBackupTimesNamespaceDisambiguation verifies that PBS namespace is used to
// disambiguate backups when multiple VMs have the same VMID across different PVE instances.
// This addresses issue #1095 where users have multiple PVE instances with overlapping VMIDs.
func TestSyncGuestBackupTimesNamespaceDisambiguation(t *testing.T) {
state := NewState()
now := time.Now()
pveBackupTime := now.Add(-1 * time.Hour)
pveNatBackupTime := now.Add(-2 * time.Hour)
// Set up VMs with the SAME VMID on DIFFERENT instances
state.UpdateVMs([]VM{
{VMID: 100, Name: "webserver-pve", Instance: "pve", Node: "node1"},
{VMID: 100, Name: "webserver-nat", Instance: "pve-nat", Node: "node2"},
})
// Add PBS backups with namespaces that correspond to the PVE instances
state.mu.Lock()
state.PBSBackups = []PBSBackup{
{
ID: "pbs-pve-100",
VMID: "100",
Namespace: "pve",
BackupType: "vm",
BackupTime: pveBackupTime,
Instance: "pbs-main",
},
{
ID: "pbs-nat-100",
VMID: "100",
Namespace: "nat", // Should match "pve-nat" instance
BackupType: "vm",
BackupTime: pveNatBackupTime,
Instance: "pbs-main",
},
}
state.mu.Unlock()
// Sync backup times
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
// Find both VMs
var vmPVE, vmNAT *VM
for i := range snapshot.VMs {
if snapshot.VMs[i].Instance == "pve" && snapshot.VMs[i].VMID == 100 {
vmPVE = &snapshot.VMs[i]
}
if snapshot.VMs[i].Instance == "pve-nat" && snapshot.VMs[i].VMID == 100 {
vmNAT = &snapshot.VMs[i]
}
}
if vmPVE == nil {
t.Fatal("pve VM not found")
}
if vmNAT == nil {
t.Fatal("pve-nat VM not found")
}
// The pve VM should have the backup with namespace "pve"
if !vmPVE.LastBackup.Equal(pveBackupTime) {
t.Errorf("pve VM LastBackup = %v, expected %v (from namespace 'pve')", vmPVE.LastBackup, pveBackupTime)
}
// The pve-nat VM should have the backup with namespace "nat"
if !vmNAT.LastBackup.Equal(pveNatBackupTime) {
t.Errorf("pve-nat VM LastBackup = %v, expected %v (from namespace 'nat')", vmNAT.LastBackup, pveNatBackupTime)
}
}
// TestSyncGuestBackupTimesClusterEntrypointUsesGuestNodeNamespace verifies that
// a cluster API entrypoint name does not shadow the guest's actual node namespace.
func TestSyncGuestBackupTimesClusterEntrypointUsesGuestNodeNamespace(t *testing.T) {
state := NewState()
now := time.Now()
oldBackupTime := now.Add(-180 * 24 * time.Hour)
freshBackupTime := now.Add(-1 * time.Hour)
state.UpdateContainers([]Container{
{VMID: 112, Name: "debian-go", Instance: "delly", Node: "minipc"},
})
state.mu.Lock()
state.PBSBackups = []PBSBackup{
{
ID: "pbs-delly-112-old",
VMID: "112",
Namespace: "delly",
BackupType: "ct",
Comment: "112",
BackupTime: oldBackupTime,
Instance: "pbs-main",
},
{
ID: "pbs-minipc-112-fresh",
VMID: "112",
Namespace: "minipc",
BackupType: "ct",
Comment: "debian-go",
BackupTime: freshBackupTime,
Instance: "pbs-main",
},
}
state.mu.Unlock()
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
var found *Container
for i := range snapshot.Containers {
if snapshot.Containers[i].VMID == 112 {
found = &snapshot.Containers[i]
break
}
}
if found == nil {
t.Fatal("container 112 not found")
}
if !found.LastBackup.Equal(freshBackupTime) {
t.Errorf("cluster entrypoint guest LastBackup = %v, want fresh node-namespaced backup %v",
found.LastBackup, freshBackupTime)
}
}
func TestSyncGuestBackupTimesClearsStaleBackupWhenCurrentEvidenceDisappears(t *testing.T) {
state := NewState()
staleBackupTime := time.Now().Add(-30 * 24 * time.Hour)
state.UpdateVMs([]VM{
{VMID: 101, Name: "unbacked-vm", Instance: "pve-1", Node: "node1", LastBackup: staleBackupTime},
})
state.UpdateContainers([]Container{
{VMID: 201, Name: "unbacked-ct", Instance: "pve-1", Node: "node1", LastBackup: staleBackupTime},
})
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
for _, vm := range snapshot.VMs {
if vm.VMID == 101 && !vm.LastBackup.IsZero() {
t.Errorf("VM stale LastBackup should be cleared, got %v", vm.LastBackup)
}
}
for _, ct := range snapshot.Containers {
if ct.VMID == 201 && !ct.LastBackup.IsZero() {
t.Errorf("container stale LastBackup should be cleared, got %v", ct.LastBackup)
}
}
}
// TestSyncGuestBackupTimesVMIDCollisionNonMatchingNamespace verifies that when the same VMID
// exists on multiple PVE instances and a PBS backup namespace matches neither, both guests
// get zero LastBackup instead of a false positive. The backup's submission
// source (owner/datastore) carries no learned attribution and no PVE-side
// storage listing confirms it, so the #1639 evidence paths must not fire.
func TestSyncGuestBackupTimesVMIDCollisionNonMatchingNamespace(t *testing.T) {
state := NewState()
now := time.Now()
backupTime := now.Add(-1 * time.Hour)
// Two VMs with the same VMID on different instances
state.UpdateVMs([]VM{
{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
{VMID: 100, Name: "vm-pve2", Instance: "pve2", Node: "node2"},
})
// PBS backup with a namespace that matches neither instance
state.mu.Lock()
state.PBSBackups = []PBSBackup{
{
ID: "pbs-100",
VMID: "100",
Namespace: "staging",
BackupType: "vm",
BackupTime: backupTime,
Instance: "pbs-main",
Datastore: "backups",
Owner: "unlearned@pbs!token",
},
}
state.mu.Unlock()
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
for _, vm := range snapshot.VMs {
if vm.VMID == 100 && !vm.LastBackup.IsZero() {
t.Errorf("VM %q on %s should have zero LastBackup (ambiguous VMID, non-matching namespace), got %v",
vm.Name, vm.Instance, vm.LastBackup)
}
}
}
// TestSyncGuestBackupTimesVMIDCollisionEmptyNamespace verifies that when the same VMID
// exists on multiple PVE instances and a PBS backup has no namespace, both guests
// get zero LastBackup. This is the genuinely unattributable core of #1639:
// with a single snapshot, no learnable submission source, and no PVE-side
// confirmation, guessing either guest would be a false positive. The
// resolvable variants live in issue1639_pbs_collision_test.go.
func TestSyncGuestBackupTimesVMIDCollisionEmptyNamespace(t *testing.T) {
state := NewState()
now := time.Now()
backupTime := now.Add(-1 * time.Hour)
state.UpdateVMs([]VM{
{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
{VMID: 100, Name: "vm-pve2", Instance: "pve2", Node: "node2"},
})
// PBS backup with empty namespace
state.mu.Lock()
state.PBSBackups = []PBSBackup{
{
ID: "pbs-100",
VMID: "100",
Namespace: "",
BackupType: "vm",
BackupTime: backupTime,
Instance: "pbs-main",
},
}
state.mu.Unlock()
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
for _, vm := range snapshot.VMs {
if vm.VMID == 100 && !vm.LastBackup.IsZero() {
t.Errorf("VM %q on %s should have zero LastBackup (ambiguous VMID, empty namespace), got %v",
vm.Name, vm.Instance, vm.LastBackup)
}
}
}
// TestSyncGuestBackupTimesUniqueVMIDFallback verifies that a unique VMID still gets
// the PBS backup via fallback even when the namespace doesn't match.
func TestSyncGuestBackupTimesUniqueVMIDFallback(t *testing.T) {
state := NewState()
now := time.Now()
backupTime := now.Add(-1 * time.Hour)
// Single VM — VMID is unique
state.UpdateVMs([]VM{
{VMID: 100, Name: "my-vm", Instance: "pve1", Node: "node1"},
})
// PBS backup with a namespace that does NOT match the instance
state.mu.Lock()
state.PBSBackups = []PBSBackup{
{
ID: "pbs-100",
VMID: "100",
Namespace: "daily",
BackupType: "vm",
BackupTime: backupTime,
Instance: "pbs-main",
},
}
state.mu.Unlock()
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
var found *VM
for i := range snapshot.VMs {
if snapshot.VMs[i].VMID == 100 {
found = &snapshot.VMs[i]
}
}
if found == nil {
t.Fatal("VM not found")
}
if !found.LastBackup.Equal(backupTime) {
t.Errorf("unique VMID should fall back to PBS backup; got LastBackup = %v, want %v",
found.LastBackup, backupTime)
}
}
// TestSyncGuestBackupTimesVMContainerCollision verifies that PBS vm/ID and ct/ID
// subjects are matched independently even when a VM and container share the same
// numeric ID.
func TestSyncGuestBackupTimesVMContainerCollision(t *testing.T) {
state := NewState()
now := time.Now()
backupTime := now.Add(-1 * time.Hour)
state.UpdateVMs([]VM{
{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
})
state.UpdateContainers([]Container{
{VMID: 100, Name: "ct-pve2", Instance: "pve2", Node: "node2"},
})
state.mu.Lock()
state.PBSBackups = []PBSBackup{
{
ID: "pbs-100",
VMID: "100",
Namespace: "node1",
BackupType: "vm",
BackupTime: backupTime,
Instance: "pbs-main",
},
}
state.mu.Unlock()
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
for _, vm := range snapshot.VMs {
if vm.VMID == 100 && !vm.LastBackup.Equal(backupTime) {
t.Errorf("VM %q LastBackup = %v, want typed vm/100 backup %v",
vm.Name, vm.LastBackup, backupTime)
}
}
for _, ct := range snapshot.Containers {
if ct.VMID == 100 && !ct.LastBackup.IsZero() {
t.Errorf("Container %q should have zero LastBackup because the PBS subject is vm/100, got %v",
ct.Name, ct.LastBackup)
}
}
}
func TestUpdateGuestsForInstancePublishesCoherentGeneration(t *testing.T) {
state := NewState()
previousBackup := time.Now().Add(-2 * time.Hour).UTC()
state.UpdateVMs([]VM{
{ID: "lab-a:node-a:101", VMID: 101, Instance: "lab-a", Node: "node-a", LastBackup: previousBackup},
{ID: "lab-b:node-b:201", VMID: 201, Instance: "lab-b", Node: "node-b"},
})
state.UpdateContainers([]Container{
{ID: "lab-a:node-a:102", VMID: 102, Instance: "lab-a", Node: "node-a", LastBackup: previousBackup},
{ID: "lab-b:node-b:202", VMID: 202, Instance: "lab-b", Node: "node-b"},
})
before := state.GetSnapshot().LastUpdate
state.UpdateGuestsForInstance(
"lab-a",
[]VM{{ID: "lab-a:node-a:101", VMID: 101, Instance: "lab-a", Node: "node-a", Status: "running"}},
[]Container{{ID: "lab-a:node-a:103", VMID: 103, Instance: "lab-a", Node: "node-a", Status: "running"}},
)
snapshot := state.GetSnapshot()
if !snapshot.LastUpdate.After(before) {
t.Fatalf("LastUpdate did not advance: before=%v after=%v", before, snapshot.LastUpdate)
}
if len(snapshot.VMs) != 2 || len(snapshot.Containers) != 2 {
t.Fatalf("unexpected coherent generation sizes: vms=%d containers=%d", len(snapshot.VMs), len(snapshot.Containers))
}
if snapshot.VMs[0].ID != "lab-a:node-a:101" || !snapshot.VMs[0].LastBackup.Equal(previousBackup) {
t.Fatalf("updated VM did not retain backup state: %+v", snapshot.VMs[0])
}
if snapshot.Containers[0].ID != "lab-a:node-a:103" {
t.Fatalf("authoritatively deleted container remained in state: %+v", snapshot.Containers)
}
if snapshot.VMs[1].Instance != "lab-b" || snapshot.Containers[1].Instance != "lab-b" {
t.Fatalf("other instance was not isolated: vms=%+v containers=%+v", snapshot.VMs, snapshot.Containers)
}
}
func TestUpdateStorageBackupsForInstance(t *testing.T) {
state := NewState()
now := time.Now()
// Set up a VM so node normalization has something to work with
state.UpdateVMsForInstance("pve-1", []VM{
{VMID: 100, Instance: "pve-1", Node: "node1"},
})
// Add backups from first instance
backups1 := []StorageBackup{
{ID: "pve-1-backup-1", Instance: "pve-1", VMID: 100, Time: now, Node: "node1"},
{ID: "pve-1-backup-2", Instance: "pve-1", VMID: 100, Time: now.Add(-time.Hour), Node: "node1"},
}
state.UpdateStorageBackupsForInstance("pve-1", backups1)
snapshot := state.GetSnapshot()
if len(snapshot.PVEBackups.StorageBackups) != 2 {
t.Fatalf("Expected 2 backups, got %d", len(snapshot.PVEBackups.StorageBackups))
}
// Backups should be sorted by time descending
if snapshot.PVEBackups.StorageBackups[0].Time.Before(snapshot.PVEBackups.StorageBackups[1].Time) {
t.Error("Backups should be sorted by time descending")
}
// Add backups from second instance
backups2 := []StorageBackup{
{ID: "pve-2-backup-1", Instance: "pve-2", VMID: 200, Time: now.Add(-30 * time.Minute), Node: "node2"},
}
state.UpdateStorageBackupsForInstance("pve-2", backups2)
snapshot = state.GetSnapshot()
if len(snapshot.PVEBackups.StorageBackups) != 3 {
t.Fatalf("Expected 3 backups, got %d", len(snapshot.PVEBackups.StorageBackups))
}
// Update first instance (should replace its backups)
backups1Updated := []StorageBackup{
{ID: "pve-1-backup-3", Instance: "pve-1", VMID: 100, Time: now.Add(time.Hour), Node: "node1"},
}
state.UpdateStorageBackupsForInstance("pve-1", backups1Updated)
snapshot = state.GetSnapshot()
if len(snapshot.PVEBackups.StorageBackups) != 2 {
t.Fatalf("Expected 2 backups after update, got %d", len(snapshot.PVEBackups.StorageBackups))
}
// Verify old pve-1 backups were replaced
for _, b := range snapshot.PVEBackups.StorageBackups {
if b.ID == "pve-1-backup-1" || b.ID == "pve-1-backup-2" {
t.Error("Old pve-1 backups should have been replaced")
}
}
}
// Host.IntegrationSource is fabric-only provenance: it must survive host
// upserts, serialize when set, and stay entirely absent from JSON for real
// agent-report hosts so existing payloads do not change shape.
func TestHostIntegrationSourceRoundTrip(t *testing.T) {
state := NewState()
state.UpsertHost(Host{ID: "host-esxi", Hostname: "esxi-01", IntegrationSource: "vmware"})
state.UpsertHost(Host{ID: "host-agent", Hostname: "apollo-114"})
var esxi, agent *Host
hosts := state.GetHosts()
for i := range hosts {
switch hosts[i].ID {
case "host-esxi":
esxi = &hosts[i]
case "host-agent":
agent = &hosts[i]
}
}
if esxi == nil || esxi.IntegrationSource != "vmware" {
t.Fatalf("expected upserted host to preserve IntegrationSource %q, got %+v", "vmware", esxi)
}
if agent == nil || agent.IntegrationSource != "" {
t.Fatalf("expected agent host to keep empty IntegrationSource, got %+v", agent)
}
marked, err := json.Marshal(esxi)
if err != nil {
t.Fatalf("marshal integration host: %v", err)
}
if !strings.Contains(string(marked), `"integrationSource":"vmware"`) {
t.Fatalf("expected integrationSource in payload, got %s", marked)
}
unmarked, err := json.Marshal(agent)
if err != nil {
t.Fatalf("marshal agent host: %v", err)
}
if strings.Contains(string(unmarked), "integrationSource") {
t.Fatalf("expected integrationSource omitted for agent hosts, got %s", unmarked)
}
}
// Two clusters reusing the same node names (pve01/pve02 on different subnets)
// must stay apart: the second cluster's node must not steal the first
// cluster's agent link by bare hostname, and the shared name must not merge
// the two nodes into one slot.
func TestUpdateNodesForInstanceKeepsClustersWithSharedNodeNamesApart(t *testing.T) {
state := &State{
Hosts: []Host{
{
ID: "host-enacon-pve01",
Hostname: "pve01",
ReportIP: "192.168.16.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
},
},
{
ID: "host-enacon-pve02",
Hostname: "pve02",
ReportIP: "192.168.16.12",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.16.12/24"}},
},
},
},
Nodes: []Node{
{
ID: "enacon-pve01",
Name: "pve01",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.16.11:8006",
LinkedAgentID: "host-enacon-pve01",
Status: "online",
},
{
ID: "enacon-pve02",
Name: "pve02",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.16.12:8006",
LinkedAgentID: "host-enacon-pve02",
Status: "online",
},
},
}
state.UpdateNodesForInstance("rewo", []Node{
{
ID: "rewo-pve01",
Name: "pve01",
Instance: "rewo",
ClusterName: "rewo",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
Status: "online",
},
})
if len(state.Nodes) != 3 {
t.Fatalf("nodes = %#v, want 3 (enacon pve01+pve02 and rewo pve01)", state.Nodes)
}
byID := make(map[string]Node)
for _, node := range state.Nodes {
byID[node.ID] = node
}
enacon, ok := byID["enacon-pve01"]
if !ok {
t.Fatalf("enacon-pve01 was clobbered, nodes = %#v", state.Nodes)
}
if enacon.ClusterName != "enacon" {
t.Fatalf("enacon-pve01 ClusterName = %q, want enacon", enacon.ClusterName)
}
if enacon.LinkedAgentID != "host-enacon-pve01" {
t.Fatalf("enacon-pve01 LinkedAgentID = %q, want host-enacon-pve01", enacon.LinkedAgentID)
}
rewo, ok := byID["rewo-pve01"]
if !ok {
t.Fatalf("rewo-pve01 missing, nodes = %#v", state.Nodes)
}
if rewo.ClusterName != "rewo" {
t.Fatalf("rewo-pve01 ClusterName = %q, want rewo", rewo.ClusterName)
}
if rewo.LinkedAgentID != "" {
t.Fatalf("rewo-pve01 LinkedAgentID = %q, want none (its host runs no agent)", rewo.LinkedAgentID)
}
}
// Standalone PVE connections have no cluster metadata, so sequential setup is
// the dangerous variant of the shared-name case: while only the first site's
// agent exists, a second pve01 must not borrow that agent through the short
// endpoint alias and collapse both provider instances into one node slot.
// Refs #1753.
func TestUpdateNodesForInstanceKeepsSequentialSameNameStandaloneSitesApart(t *testing.T) {
state := &State{
Hosts: []Host{
{
ID: "agent-staging-pve01",
Hostname: "pve01",
ReportIP: "192.0.2.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.0.2.11/24"}},
},
},
},
Nodes: []Node{
{
ID: "staging-pve01",
Name: "pve01",
Instance: "staging",
Host: "https://pve01.staging.example:8006",
LinkedAgentID: "agent-staging-pve01",
Status: "online",
},
},
}
state.UpdateNodesForInstance("production", []Node{
{
ID: "production-pve01",
Name: "pve01",
Instance: "production",
Host: "https://pve01.production.example:8006",
Status: "online",
},
})
if len(state.Nodes) != 2 {
t.Fatalf("nodes = %#v, want 2 standalone sites", state.Nodes)
}
byID := make(map[string]Node, len(state.Nodes))
for _, node := range state.Nodes {
byID[node.ID] = node
}
if byID["staging-pve01"].LinkedAgentID != "agent-staging-pve01" {
t.Fatalf("staging link = %q, want agent-staging-pve01", byID["staging-pve01"].LinkedAgentID)
}
if byID["production-pve01"].LinkedAgentID != "" {
t.Fatalf("production link = %q, want no guessed cross-site agent", byID["production-pve01"].LinkedAgentID)
}
// Existing installations may already hold the bad many-nodes-to-one-agent
// link in memory. The next provider poll must discard it, not preserve it.
for i := range state.Nodes {
if state.Nodes[i].ID == "production-pve01" {
state.Nodes[i].LinkedAgentID = "agent-staging-pve01"
}
}
state.UpdateNodesForInstance("production", []Node{
{
ID: "production-pve01",
Name: "pve01",
Instance: "production",
Host: "https://pve01.production.example:8006",
Status: "online",
},
})
if len(state.Nodes) != 2 {
t.Fatalf("self-heal nodes = %#v, want 2 standalone sites", state.Nodes)
}
for _, node := range state.Nodes {
if node.ID == "production-pve01" && node.LinkedAgentID != "" {
t.Fatalf("self-healed production link = %q, want empty", node.LinkedAgentID)
}
}
}
// When TLS settings degrade node endpoints to the bare node name, the
// endpoint-host merge alias is identical for both clusters. The alias must
// not merge nodes whose named clusters contradict each other.
func TestUpdateNodesForInstanceEndpointHostAliasDoesNotMergeAcrossClusters(t *testing.T) {
state := &State{
Nodes: []Node{
{
ID: "enacon-pve01",
Name: "pve01",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://pve01:8006",
Status: "online",
},
},
}
state.UpdateNodesForInstance("rewo", []Node{
{
ID: "rewo-pve01",
Name: "pve01",
Instance: "rewo",
ClusterName: "rewo",
IsClusterMember: true,
Host: "https://pve01:8006",
Status: "online",
},
})
if len(state.Nodes) != 2 {
t.Fatalf("nodes = %#v, want 2 (one per cluster)", state.Nodes)
}
clusters := map[string]bool{}
for _, node := range state.Nodes {
clusters[node.ClusterName] = true
}
if !clusters["enacon"] || !clusters["rewo"] {
t.Fatalf("clusters = %#v, want both enacon and rewo", clusters)
}
}
// A standalone node whose endpoint IP is absent from a same-named agent's
// reported IPs must not link to that agent: the IP evidence contradicts the
// hostname match.
func TestUpdateNodesForInstanceHostnameMatchRejectedWhenIPContradicts(t *testing.T) {
state := &State{
Hosts: []Host{
{
ID: "host-a",
Hostname: "pve01",
ReportIP: "192.168.16.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
},
},
},
}
state.UpdateNodesForInstance("steinboeck", []Node{
{
ID: "steinboeck-pve01",
Name: "pve01",
Instance: "steinboeck",
Host: "https://10.20.1.5:8006",
Status: "online",
},
})
if len(state.Nodes) != 1 {
t.Fatalf("nodes = %#v, want 1", state.Nodes)
}
if state.Nodes[0].LinkedAgentID != "" {
t.Fatalf("LinkedAgentID = %q, want none (agent IPs contradict node endpoint)", state.Nodes[0].LinkedAgentID)
}
}
// Two different clusters at different sites can reuse the same corosync
// cluster name, the same member hostnames, and the same RFC1918 addressing
// (MSP support case: two "enacon" clusters both exposing pve01 on
// 192.168.1.11). The colliding endpoint-IP alias must not merge the two node
// slots, and the address match must not bind the second site's node to the
// first site's host agent when the TLS fingerprint evidence contradicts.
func TestUpdateNodesForInstanceSameNamedClustersWithCollidingIPsStayApart(t *testing.T) {
state := &State{
Hosts: []Host{
{
ID: "host-site-a-pve01",
Hostname: "pve01",
ReportIP: "192.168.1.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
},
},
},
Nodes: []Node{
{
ID: "site-a-pve01",
Name: "pve01",
Instance: "site-a",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
TLSFingerprint: "AA:AA:AA:AA",
LinkedAgentID: "host-site-a-pve01",
Status: "online",
},
},
}
state.UpdateNodesForInstance("site-b", []Node{
{
ID: "site-b-pve01",
Name: "pve01",
Instance: "site-b",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
TLSFingerprint: "BB:BB:BB:BB",
Status: "online",
},
})
if len(state.Nodes) != 2 {
t.Fatalf("nodes = %#v, want 2 (one per site)", state.Nodes)
}
byID := make(map[string]Node)
for _, node := range state.Nodes {
byID[node.ID] = node
}
siteA, ok := byID["site-a-pve01"]
if !ok {
t.Fatalf("site-a-pve01 was clobbered, nodes = %#v", state.Nodes)
}
if siteA.LinkedAgentID != "host-site-a-pve01" {
t.Fatalf("site-a-pve01 LinkedAgentID = %q, want host-site-a-pve01", siteA.LinkedAgentID)
}
siteB, ok := byID["site-b-pve01"]
if !ok {
t.Fatalf("site-b-pve01 missing, nodes = %#v", state.Nodes)
}
if siteB.LinkedAgentID != "" {
t.Fatalf("site-b-pve01 LinkedAgentID = %q, want none (fingerprint contradicts the colliding-IP agent match)", siteB.LinkedAgentID)
}
}
// The endpoint-IP agent match must also respect the cluster-identity
// contradiction when no fingerprints are known: an agent whose linked nodes
// live in cluster "enacon" must not bind to a "rewo" node that presents the
// same RFC1918 address from another site.
func TestUpdateNodesForInstanceEndpointIPMatchRejectedWhenClusterContradicts(t *testing.T) {
state := &State{
Hosts: []Host{
{
ID: "host-enacon-pve01",
Hostname: "pve01",
ReportIP: "192.168.1.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
},
},
},
Nodes: []Node{
{
ID: "enacon-pve01",
Name: "pve01",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
LinkedAgentID: "host-enacon-pve01",
Status: "online",
},
},
}
state.UpdateNodesForInstance("rewo", []Node{
{
ID: "rewo-pve01",
Name: "pve01",
Instance: "rewo",
ClusterName: "rewo",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
Status: "online",
},
})
if len(state.Nodes) != 2 {
t.Fatalf("nodes = %#v, want 2 (one per cluster)", state.Nodes)
}
for _, node := range state.Nodes {
if node.ClusterName == "rewo" && node.LinkedAgentID != "" {
t.Fatalf("rewo pve01 LinkedAgentID = %q, want none (agent belongs to the enacon cluster)", node.LinkedAgentID)
}
}
}
// The same cluster added twice through different connection instances is the
// legitimate duplicate: when both views carry the same TOFU-captured TLS
// fingerprint for the same-named node, the endpoint alias must still fold
// them into one slot even though the instances differ.
func TestUpdateNodesForInstanceSameClusterAddedTwiceMergesWithMatchingFingerprints(t *testing.T) {
state := &State{
Nodes: []Node{
{
ID: "enacon-a-pve01",
Name: "pve01",
Instance: "enacon-a",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
TLSFingerprint: "AA:AA:AA:AA",
Status: "online",
},
},
}
state.UpdateNodesForInstance("enacon-b", []Node{
{
ID: "enacon-b-pve01",
Name: "pve01",
Instance: "enacon-b",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
TLSFingerprint: "aaaaaaaa",
Status: "online",
},
})
if len(state.Nodes) != 1 {
t.Fatalf("nodes = %#v, want 1 (matching fingerprints prove one machine)", state.Nodes)
}
if state.Nodes[0].Name != "pve01" {
t.Fatalf("merged node = %#v, want pve01", state.Nodes[0])
}
}
// Without fingerprint evidence the same-named cluster from another instance
// stays split: unknown identity must never fold two possibly-distinct
// clusters into one (fail-safe direction).
func TestUpdateNodesForInstanceSameNamedClustersStayApartWithoutFingerprints(t *testing.T) {
state := &State{
Nodes: []Node{
{
ID: "enacon-a-pve01",
Name: "pve01",
Instance: "enacon-a",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
Status: "online",
},
},
}
state.UpdateNodesForInstance("enacon-b", []Node{
{
ID: "enacon-b-pve01",
Name: "pve01",
Instance: "enacon-b",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
Status: "online",
},
})
if len(state.Nodes) != 2 {
t.Fatalf("nodes = %#v, want 2 (unknown identity must not merge)", state.Nodes)
}
}
// A shared agent link is the strongest cross-view merge signal, but for
// same-named clusters from different instances it must still be gated on
// fingerprint identity: matching fingerprints merge the duplicate views onto
// one slot, contradicting fingerprints never do.
func TestUpdateNodesForInstanceSharedAgentLinkMergeGatedOnFingerprint(t *testing.T) {
makeState := func(existingFP string) *State {
return &State{
Hosts: []Host{
{
ID: "host-pve01",
Hostname: "pve01",
ReportIP: "192.168.1.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
},
},
},
Nodes: []Node{
{
ID: "enacon-a-pve01",
Name: "pve01",
Instance: "enacon-a",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://pve01.site-a.internal:8006",
TLSFingerprint: existingFP,
LinkedAgentID: "host-pve01",
Status: "online",
},
},
}
}
incoming := func(fp string) []Node {
return []Node{
{
ID: "enacon-b-pve01",
Name: "pve01",
Instance: "enacon-b",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
TLSFingerprint: fp,
LinkedAgentID: "host-pve01",
Status: "online",
},
}
}
merged := makeState("AA:AA:AA:AA")
merged.UpdateNodesForInstance("enacon-b", incoming("AA:AA:AA:AA"))
if len(merged.Nodes) != 1 {
t.Fatalf("matching fingerprints: nodes = %#v, want 1", merged.Nodes)
}
split := makeState("AA:AA:AA:AA")
split.UpdateNodesForInstance("enacon-b", incoming("BB:BB:BB:BB"))
if len(split.Nodes) != 2 {
t.Fatalf("contradicting fingerprints: nodes = %#v, want 2", split.Nodes)
}
}
// A cluster connection's first polls can commit nodes before membership
// detection classifies them (ClusterName ""), and cloned template
// deployments reuse /etc/machine-id, collapsing two sites' agents into one
// identity (MSP support case: clusters enacon and rewo, both with pve01, on
// reused RFC1918 ranges). During that window the shared agent identity must
// not fold the unclassified node into the established cluster's slot.
func TestUpdateNodesForInstanceUnclassifiedNodeSharedAgentMustNotStealClusterSlot(t *testing.T) {
state := &State{
Hosts: []Host{
{
// One host row standing in for two physical pve01 machines
// whose cloned machine-id collapsed them into one agent
// identity; it currently reports the second site's address.
ID: "host-cloned-machine-id",
Hostname: "pve01",
ReportIP: "192.168.1.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
},
},
},
Nodes: []Node{
{
ID: "enacon-pve01",
Name: "pve01",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.16.11:8006",
LinkedAgentID: "host-cloned-machine-id",
Status: "online",
},
},
}
state.UpdateNodesForInstance("rewo", []Node{
{
ID: "rewo-pve01",
Name: "pve01",
Instance: "rewo",
// Membership detection has not classified this connection yet.
ClusterName: "",
Host: "https://192.168.1.11:8006",
Status: "online",
},
})
if len(state.Nodes) != 2 {
t.Fatalf("nodes = %#v, want 2 (unclassified node must not fold into the enacon slot)", state.Nodes)
}
byID := make(map[string]Node)
for _, node := range state.Nodes {
byID[node.ID] = node
}
enacon, ok := byID["enacon-pve01"]
if !ok {
t.Fatalf("enacon-pve01 was clobbered, nodes = %#v", state.Nodes)
}
if enacon.ClusterName != "enacon" {
t.Fatalf("enacon-pve01 ClusterName = %q, want enacon", enacon.ClusterName)
}
if _, ok := byID["rewo-pve01"]; !ok {
t.Fatalf("rewo-pve01 missing, nodes = %#v", state.Nodes)
}
}
// When TLS settings degrade both connections' node endpoints to the bare
// node name, the endpoint-host alias is identical across sites. An
// unclassified node (empty cluster name, e.g. during the first-poll
// membership window) must not ride that weak alias into another instance's
// established cluster slot.
func TestUpdateNodesForInstanceEndpointHostAliasRequiresProofForUnclassifiedNodes(t *testing.T) {
state := &State{
Nodes: []Node{
{
ID: "enacon-pve01",
Name: "pve01",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://pve01:8006",
Status: "online",
},
},
}
state.UpdateNodesForInstance("rewo", []Node{
{
ID: "rewo-pve01",
Name: "pve01",
Instance: "rewo",
ClusterName: "",
Host: "https://pve01:8006",
Status: "online",
},
})
if len(state.Nodes) != 2 {
t.Fatalf("nodes = %#v, want 2 (bare-hostname alias must not fold an unclassified cross-instance node)", state.Nodes)
}
}
// The legitimate standalone-into-cluster fold must survive the weak-evidence
// tightening: a standalone connection view dialing the same address as the
// cluster's endpoint record still folds via the address alias, and a
// fingerprint-proven view still folds via the shared agent link.
func TestUpdateNodesForInstanceStandaloneViewStillFoldsIntoClusterNode(t *testing.T) {
byAddress := &State{
Nodes: []Node{
{
ID: "enacon-pve01",
Name: "pve01",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.16.11:8006",
Status: "online",
},
},
}
byAddress.UpdateNodesForInstance("pve01-direct", []Node{
{
ID: "direct-pve01",
Name: "pve01",
Instance: "pve01-direct",
Host: "https://192.168.16.11:8006",
Status: "online",
},
})
if len(byAddress.Nodes) != 1 {
t.Fatalf("address alias: nodes = %#v, want 1 (same-address standalone view folds)", byAddress.Nodes)
}
byAgent := &State{
Hosts: []Host{
{
ID: "host-pve01",
Hostname: "pve01",
ReportIP: "192.168.16.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
},
},
},
Nodes: []Node{
{
ID: "enacon-pve01",
Name: "pve01",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://pve01:8006",
TLSFingerprint: "AA:AA:AA:AA",
LinkedAgentID: "host-pve01",
Status: "online",
},
},
}
byAgent.UpdateNodesForInstance("pve01-direct", []Node{
{
ID: "direct-pve01",
Name: "pve01",
Instance: "pve01-direct",
Host: "https://192.168.16.11:8006",
TLSFingerprint: "aaaaaaaa",
Status: "online",
},
})
if len(byAgent.Nodes) != 1 {
t.Fatalf("agent link: nodes = %#v, want 1 (fingerprint-proven view folds via shared agent)", byAgent.Nodes)
}
}
// splitNodeDecider reports the pairs an operator split, by node and agent ID.
type splitNodeDecider map[[2]string]bool
func (d splitNodeDecider) ProxmoxNodeAgentSplit(node Node, host Host) bool {
return d[[2]string{node.ID, host.ID}]
}
// A split recorded after one connection's poll is applied to that connection's
// retained node before its link can fold the next connection's view into it.
func TestUpdateNodesForInstanceReadsASplitOfARetainedNodeBeforeItFolds(t *testing.T) {
agent := Host{ID: "agent-1", Hostname: "pve1.lan", ReportIP: "10.0.0.5", Status: "online"}
nodeA := Node{ID: "a-pve1", Name: "pve1", Instance: "a", Host: "https://10.0.0.5:8006", Status: "online"}
nodeB := Node{ID: "b-pve1", Name: "pve1", Instance: "b", Host: "https://pve1.lan:8006", Status: "online"}
decider := splitNodeDecider{}
state := NewState()
state.UpsertHost(agent)
state.SetNodeAgentSplitDecider(decider)
state.UpdateNodesForInstance("a", []Node{nodeA})
if linked := state.GetSnapshot().Nodes; len(linked) != 1 || linked[0].LinkedAgentID != agent.ID {
t.Fatalf("fixture: the first poll should link its node to the agent: %+v", linked)
}
// The operator splits the first node from the agent; nothing has written
// a link since, so the retained node still names the agent.
decider[[2]string{nodeA.ID, agent.ID}] = true
state.UpdateNodesForInstance("b", []Node{nodeB})
nodes := state.GetSnapshot().Nodes
if len(nodes) != 2 {
t.Fatalf("a retained node's split link still folded the next view into its slot: %+v", nodes)
}
for _, node := range nodes {
switch node.ID {
case nodeA.ID:
if node.LinkedAgentID != "" {
t.Fatalf("node %s kept the agent %q the operator split it from", node.ID, node.LinkedAgentID)
}
case nodeB.ID:
if node.LinkedAgentID != agent.ID {
t.Fatalf("node %s lost an agent link nobody split: %q", node.ID, node.LinkedAgentID)
}
default:
t.Fatalf("unexpected node %+v", node)
}
}
}
// One agent corroborates two standalone connections' nodes through the node's
// IP address and its full hostname, which is the proof that folds the second
// view into the first node's slot. When the operator has split the second
// node from that agent, the agent is no evidence for it, so the view is not
// folded, and neither a node nor the agent keeps a link the operator split.
func TestUpdateNodesForInstanceHoldsASplitAgentBackBeforeItFoldsNodeViews(t *testing.T) {
agent := Host{
ID: "agent-1",
Hostname: "pve1.lan",
ReportIP: "10.0.0.5",
Status: "online",
}
nodeA := Node{ID: "a-pve1", Name: "pve1", Instance: "a", Host: "https://10.0.0.5:8006", Status: "online"}
nodeB := Node{ID: "b-pve1", Name: "pve1", Instance: "b", Host: "https://pve1.lan:8006", Status: "online"}
poll := func(decider NodeAgentSplitDecider) []Node {
state := NewState()
state.UpsertHost(agent)
state.SetNodeAgentSplitDecider(decider)
state.UpdateNodesForInstance("a", []Node{nodeA})
state.UpdateNodesForInstance("b", []Node{nodeB})
return state.GetSnapshot().Nodes
}
folded := poll(nil)
if len(folded) != 1 || folded[0].LinkedAgentID != agent.ID {
t.Fatalf("fixture: the agent should fold both views into one linked node, got %+v", folded)
}
apart := poll(splitNodeDecider{{nodeB.ID, agent.ID}: true})
if len(apart) != 2 {
t.Fatalf("a node split from the agent folded into its slot anyway: %+v", apart)
}
for _, node := range apart {
switch node.ID {
case nodeA.ID:
if node.LinkedAgentID != agent.ID {
t.Fatalf("the split of the other node unlinked %s from the agent", node.ID)
}
case nodeB.ID:
if node.LinkedAgentID != "" {
t.Fatalf("node %s kept the agent %q the operator split it from", node.ID, node.LinkedAgentID)
}
default:
t.Fatalf("unexpected node %+v", node)
}
}
}