Files
pulse/internal/models/state_host_test.go
T
courtmanr@gmail.com 84dba861b5 Fix PBS backup attribution for VMIDs shared across clusters
v6.1.0's identity rework (eab73d245) made the VMID-only fallback refuse
to fire whenever a typed VMID exists on more than one PVE location.
Root-namespace snapshots with no matching comment then score zero for
every guest, so on setups with two clusters and overlapping VMIDs most
guests showed no backup at all - while PVE itself listed the backups
fine, because monitoring discards pbs-type storage contents entirely
whenever a direct PBS connection is configured.

Attribution is now evidence-driven instead of dropped:

Storage backup polling keeps a per-connection record of every snapshot
its own pbs-type storage listed (type, VMID, backup time) even though
the raw entries stay out of the PVE backup list. Which cluster listed a
snapshot is deterministic attribution, and it survives fully mirrored
clusters that share one datastore and token. The evidence is
monitoring-internal, cleared on instance retirement or when the storage
poll stops seeing pbs content, and never serialized into state payloads
or snapshots.

Guest backup-time sync and the recovery-point mapper additionally learn
each PBS submission source's cluster (owner token, datastore, PBS
instance - strongest first, scoped to the PBS instance) from the poll's
attributable snapshots, then resolve collision VMIDs whose snapshots
carry no evidence of their own. A source seen from several clusters is
not a discriminator, an unfamiliar component stops resolution rather
than deferring to weaker ones, and a snapshot decisively attributed to
another cluster is kept away from this one. Unattributable snapshots
still drop rather than guess.

Backup-age alert attribution no longer suffix-matches the subject ref's
connection label against guest locations. The label there is a PVE or
PBS instance name, not a PBS namespace, and loose matching could
cross-attribute clusters sharing a VMID; it now requires exact
normalized equality.

Reported in #1639 (two PVE clusters with PBS 4.0/4.1, VM 173 shown 974
days overdue despite valid verified backups).

Fixes #1639

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-28 11:24:01 +01:00

2073 lines
61 KiB
Go

package models
import (
"encoding/json"
"strings"
"testing"
"time"
)
func TestUpsertHost(t *testing.T) {
state := NewState()
// Test insert new host
host1 := Host{
ID: "host-1",
Hostname: "server-1",
Status: "online",
}
state.UpsertHost(host1)
hosts := state.GetHosts()
if len(hosts) != 1 {
t.Fatalf("Expected 1 host, got %d", len(hosts))
}
if hosts[0].ID != "host-1" {
t.Errorf("Expected host ID 'host-1', got %q", hosts[0].ID)
}
// Test update existing host
host1Updated := Host{
ID: "host-1",
Hostname: "server-1",
Status: "offline",
}
state.UpsertHost(host1Updated)
hosts = state.GetHosts()
if len(hosts) != 1 {
t.Fatalf("Expected 1 host after update, got %d", len(hosts))
}
if hosts[0].Status != "offline" {
t.Errorf("Expected status 'offline', got %q", hosts[0].Status)
}
// Test multiple hosts and sorting
state.UpsertHost(Host{ID: "host-2", Hostname: "alpha-server"})
state.UpsertHost(Host{ID: "host-3", Hostname: "zulu-server"})
hosts = state.GetHosts()
if len(hosts) != 3 {
t.Fatalf("Expected 3 hosts, got %d", len(hosts))
}
// Hosts should be sorted by hostname
if hosts[0].Hostname != "alpha-server" {
t.Errorf("First host should be 'alpha-server', got %q", hosts[0].Hostname)
}
}
func TestGetHosts_Copy(t *testing.T) {
state := NewState()
state.UpsertHost(Host{ID: "host-1", Hostname: "server-1"})
hosts1 := state.GetHosts()
hosts2 := state.GetHosts()
// Modify one slice
if len(hosts1) > 0 {
hosts1[0].Hostname = "modified"
}
// Other slice should be unchanged
if len(hosts2) > 0 && hosts2[0].Hostname == "modified" {
t.Error("GetHosts should return a copy, not the same slice")
}
// Original state should be unchanged
hosts3 := state.GetHosts()
if hosts3[0].Hostname == "modified" {
t.Error("State should be unchanged by modifications to returned slice")
}
}
func TestRemoveHost(t *testing.T) {
state := NewState()
// Insert hosts
state.UpsertHost(Host{ID: "host-1", Hostname: "server1"})
state.UpsertHost(Host{ID: "host-2", Hostname: "server2"})
// Remove existing host
removed, ok := state.RemoveHost("host-1")
if !ok {
t.Error("Expected RemoveHost to return true for existing host")
}
if removed.ID != "host-1" {
t.Errorf("Expected removed host ID 'host-1', got %q", removed.ID)
}
hosts := state.GetHosts()
if len(hosts) != 1 {
t.Fatalf("Expected 1 host after removal, got %d", len(hosts))
}
if hosts[0].ID != "host-2" {
t.Errorf("Remaining host should be 'host-2', got %q", hosts[0].ID)
}
// Remove non-existing host
removed, ok = state.RemoveHost("non-existent")
if ok {
t.Error("Expected RemoveHost to return false for non-existent host")
}
if removed.ID != "" {
t.Errorf("Expected empty Host for non-existent removal, got ID %q", removed.ID)
}
}
func TestSetHostStatus(t *testing.T) {
state := NewState()
// Test with no hosts
changed := state.SetHostStatus("host-1", "online")
if changed {
t.Error("SetHostStatus should return false when host doesn't exist")
}
// Add host and set status
state.UpsertHost(Host{ID: "host-1", Hostname: "server1", Status: "offline"})
changed = state.SetHostStatus("host-1", "online")
if !changed {
t.Error("SetHostStatus should return true when host exists")
}
hosts := state.GetHosts()
if hosts[0].Status != "online" {
t.Errorf("Expected status 'online', got %q", hosts[0].Status)
}
// Set same status (no change)
changed = state.SetHostStatus("host-1", "online")
if !changed {
t.Error("SetHostStatus should return true even when status unchanged")
}
}
func TestTouchHost(t *testing.T) {
state := NewState()
now := time.Now()
// Test with non-existent host
ok := state.TouchHost("host-1", now)
if ok {
t.Error("TouchHost should return false for non-existent host")
}
// Add host and touch it
state.UpsertHost(Host{ID: "host-1", Hostname: "server1"})
later := now.Add(time.Hour)
ok = state.TouchHost("host-1", later)
if !ok {
t.Error("TouchHost should return true for existing host")
}
hosts := state.GetHosts()
if !hosts[0].LastSeen.Equal(later) {
t.Errorf("LastSeen should be updated to %v, got %v", later, hosts[0].LastSeen)
}
}
func TestSetConnectionHealth(t *testing.T) {
state := NewState()
// Set connection healthy
state.SetConnectionHealth("pve-cluster-1", true)
snapshot := state.GetSnapshot()
if healthy, ok := snapshot.ConnectionHealth["pve-cluster-1"]; !ok || !healthy {
t.Error("Expected connection to be healthy")
}
// Set connection unhealthy
state.SetConnectionHealth("pve-cluster-1", false)
snapshot = state.GetSnapshot()
if healthy, ok := snapshot.ConnectionHealth["pve-cluster-1"]; !ok || healthy {
t.Error("Expected connection to be unhealthy")
}
// Multiple connections
state.SetConnectionHealth("pve-cluster-2", true)
state.SetConnectionHealth("pbs-instance-1", true)
snapshot = state.GetSnapshot()
if len(snapshot.ConnectionHealth) != 3 {
t.Errorf("Expected 3 connection health entries, got %d", len(snapshot.ConnectionHealth))
}
}
func TestPVETagStyleSnapshotRebuildsAggregateByInstance(t *testing.T) {
state := NewState()
state.MergePVETagStyle("pve-a", PVETagStyle{
Colors: map[string]string{
"Production": "#00ff00",
},
CaseSensitive: true,
})
state.MergePVETagStyle("pve-b", PVETagStyle{
Colors: map[string]string{
"Backup": "#112233",
},
})
snapshot := state.GetSnapshot()
if got := snapshot.PVETagStyles["pve-a"].Colors["Production"]; got != "#00ff00" {
t.Fatalf("expected pve-a tag style to preserve case-sensitive color, got %q", got)
}
if !snapshot.PVETagStyles["pve-a"].CaseSensitive {
t.Fatal("expected pve-a tag style to preserve case-sensitive flag")
}
if got := snapshot.PVETagStyles["pve-b"].Colors["backup"]; got != "#112233" {
t.Fatalf("expected pve-b tag style to normalize non-case-sensitive tag, got %q", got)
}
state.MergePVETagStyle("pve-a", PVETagStyle{})
snapshot = state.GetSnapshot()
if _, ok := snapshot.PVETagColors["Production"]; ok {
t.Fatalf("expected stale pve-a aggregate color to be removed, got %#v", snapshot.PVETagColors)
}
if got := snapshot.PVETagColors["backup"]; got != "#112233" {
t.Fatalf("expected pve-b aggregate color to remain, got %q", got)
}
}
func TestRemoveConnectionHealth(t *testing.T) {
state := NewState()
// Set up connection health entries
state.SetConnectionHealth("pve-cluster-1", true)
state.SetConnectionHealth("pve-cluster-2", false)
// Remove one
state.RemoveConnectionHealth("pve-cluster-1")
snapshot := state.GetSnapshot()
if _, ok := snapshot.ConnectionHealth["pve-cluster-1"]; ok {
t.Error("Expected pve-cluster-1 to be removed")
}
if _, ok := snapshot.ConnectionHealth["pve-cluster-2"]; !ok {
t.Error("Expected pve-cluster-2 to still exist")
}
// Remove non-existent (should not panic)
state.RemoveConnectionHealth("non-existent")
// Remove remaining
state.RemoveConnectionHealth("pve-cluster-2")
snapshot = state.GetSnapshot()
if len(snapshot.ConnectionHealth) != 0 {
t.Errorf("Expected empty connection health map, got %d entries", len(snapshot.ConnectionHealth))
}
}
func TestUpdatePBSBackups(t *testing.T) {
state := NewState()
now := time.Now()
// Add backups from first instance
backups1 := []PBSBackup{
{ID: "backup-1", Instance: "pbs-1", BackupTime: now},
{ID: "backup-2", Instance: "pbs-1", BackupTime: now.Add(-time.Hour)},
}
state.UpdatePBSBackups("pbs-1", backups1)
snapshot := state.GetSnapshot()
if len(snapshot.PBSBackups) != 2 {
t.Fatalf("Expected 2 backups, got %d", len(snapshot.PBSBackups))
}
// Add backups from second instance
backups2 := []PBSBackup{
{ID: "backup-3", Instance: "pbs-2", BackupTime: now.Add(-30 * time.Minute)},
}
state.UpdatePBSBackups("pbs-2", backups2)
snapshot = state.GetSnapshot()
if len(snapshot.PBSBackups) != 3 {
t.Fatalf("Expected 3 backups, got %d", len(snapshot.PBSBackups))
}
// Update first instance (should replace its backups)
backups1Updated := []PBSBackup{
{ID: "backup-4", Instance: "pbs-1", BackupTime: now.Add(time.Hour)},
}
state.UpdatePBSBackups("pbs-1", backups1Updated)
snapshot = state.GetSnapshot()
if len(snapshot.PBSBackups) != 2 {
t.Fatalf("Expected 2 backups after update, got %d", len(snapshot.PBSBackups))
}
// Verify pbs-1 backups were replaced
hasOldBackup := false
for _, b := range snapshot.PBSBackups {
if b.ID == "backup-1" || b.ID == "backup-2" {
hasOldBackup = true
break
}
}
if hasOldBackup {
t.Error("Old pbs-1 backups should have been replaced")
}
}
func TestUpdatePMGBackups(t *testing.T) {
state := NewState()
now := time.Now()
// Add backups from first instance
backups1 := []PMGBackup{
{Filename: "backup-1.conf", Instance: "pmg-1", BackupTime: now},
{Filename: "backup-2.conf", Instance: "pmg-1", BackupTime: now.Add(-time.Hour)},
}
state.UpdatePMGBackups("pmg-1", backups1)
snapshot := state.GetSnapshot()
if len(snapshot.PMGBackups) != 2 {
t.Fatalf("Expected 2 backups, got %d", len(snapshot.PMGBackups))
}
// Backups should be sorted by time (newest first)
if snapshot.PMGBackups[0].BackupTime.Before(snapshot.PMGBackups[1].BackupTime) {
t.Error("Backups should be sorted by time descending")
}
// Add backups from second instance
backups2 := []PMGBackup{
{Filename: "backup-3.conf", Instance: "pmg-2", BackupTime: now.Add(-30 * time.Minute)},
}
state.UpdatePMGBackups("pmg-2", backups2)
snapshot = state.GetSnapshot()
if len(snapshot.PMGBackups) != 3 {
t.Fatalf("Expected 3 backups, got %d", len(snapshot.PMGBackups))
}
// Update first instance with empty (should remove its backups)
state.UpdatePMGBackups("pmg-1", []PMGBackup{})
snapshot = state.GetSnapshot()
if len(snapshot.PMGBackups) != 1 {
t.Fatalf("Expected 1 backup after clearing pmg-1, got %d", len(snapshot.PMGBackups))
}
if snapshot.PMGBackups[0].Instance != "pmg-2" {
t.Error("Remaining backup should be from pmg-2")
}
}
func TestUpdatePhysicalDisks(t *testing.T) {
state := NewState()
// Add disks from first instance
disks1 := []PhysicalDisk{
{ID: "disk-1", Instance: "pve-1", Node: "node1", DevPath: "/dev/sda"},
{ID: "disk-2", Instance: "pve-1", Node: "node1", DevPath: "/dev/sdb"},
}
state.UpdatePhysicalDisks("pve-1", disks1)
snapshot := state.GetSnapshot()
if len(snapshot.PhysicalDisks) != 2 {
t.Fatalf("Expected 2 disks, got %d", len(snapshot.PhysicalDisks))
}
// Add disks from second instance
disks2 := []PhysicalDisk{
{ID: "disk-3", Instance: "pve-2", Node: "node2", DevPath: "/dev/sda"},
}
state.UpdatePhysicalDisks("pve-2", disks2)
snapshot = state.GetSnapshot()
if len(snapshot.PhysicalDisks) != 3 {
t.Fatalf("Expected 3 disks, got %d", len(snapshot.PhysicalDisks))
}
// Update first instance (should replace its disks)
disks1Updated := []PhysicalDisk{
{ID: "disk-4", Instance: "pve-1", Node: "node1", DevPath: "/dev/nvme0n1"},
}
state.UpdatePhysicalDisks("pve-1", disks1Updated)
snapshot = state.GetSnapshot()
if len(snapshot.PhysicalDisks) != 2 {
t.Fatalf("Expected 2 disks after update, got %d", len(snapshot.PhysicalDisks))
}
// Verify sorting by node then devpath
if snapshot.PhysicalDisks[0].Node > snapshot.PhysicalDisks[1].Node {
t.Error("Disks should be sorted by node")
}
}
func TestUpdateStorageForInstance(t *testing.T) {
state := NewState()
// Add storage from first instance
storage1 := []Storage{
{ID: "storage-1", Instance: "pve-1", Name: "local"},
{ID: "storage-2", Instance: "pve-1", Name: "ceph-pool"},
}
state.UpdateStorageForInstance("pve-1", storage1)
snapshot := state.GetSnapshot()
if len(snapshot.Storage) != 2 {
t.Fatalf("Expected 2 storage entries, got %d", len(snapshot.Storage))
}
// Add storage from second instance
storage2 := []Storage{
{ID: "storage-3", Instance: "pve-2", Name: "local"},
}
state.UpdateStorageForInstance("pve-2", storage2)
snapshot = state.GetSnapshot()
if len(snapshot.Storage) != 3 {
t.Fatalf("Expected 3 storage entries, got %d", len(snapshot.Storage))
}
// Update first instance with empty (should remove its storage)
state.UpdateStorageForInstance("pve-1", []Storage{})
snapshot = state.GetSnapshot()
if len(snapshot.Storage) != 1 {
t.Fatalf("Expected 1 storage after clearing pve-1, got %d", len(snapshot.Storage))
}
}
func TestUpdatePBSInstances(t *testing.T) {
state := NewState()
instances := []PBSInstance{
{ID: "pbs-1", Name: "Backup Server 1"},
{ID: "pbs-2", Name: "Backup Server 2"},
}
state.UpdatePBSInstances(instances)
snapshot := state.GetSnapshot()
if len(snapshot.PBSInstances) != 2 {
t.Fatalf("Expected 2 PBS instances, got %d", len(snapshot.PBSInstances))
}
// Replace with different instances
newInstances := []PBSInstance{
{ID: "pbs-3", Name: "New Backup Server"},
}
state.UpdatePBSInstances(newInstances)
snapshot = state.GetSnapshot()
if len(snapshot.PBSInstances) != 1 {
t.Fatalf("Expected 1 PBS instance after replacement, got %d", len(snapshot.PBSInstances))
}
if snapshot.PBSInstances[0].ID != "pbs-3" {
t.Error("Expected pbs-3 instance")
}
}
func TestUpdatePBSInstance(t *testing.T) {
state := NewState()
// Add first instance
state.UpdatePBSInstance(PBSInstance{ID: "pbs-1", Name: "Server 1"})
snapshot := state.GetSnapshot()
if len(snapshot.PBSInstances) != 1 {
t.Fatalf("Expected 1 PBS instance, got %d", len(snapshot.PBSInstances))
}
// Add second instance
state.UpdatePBSInstance(PBSInstance{ID: "pbs-2", Name: "Server 2"})
snapshot = state.GetSnapshot()
if len(snapshot.PBSInstances) != 2 {
t.Fatalf("Expected 2 PBS instances, got %d", len(snapshot.PBSInstances))
}
// Update existing instance
state.UpdatePBSInstance(PBSInstance{ID: "pbs-1", Name: "Server 1 Updated"})
snapshot = state.GetSnapshot()
if len(snapshot.PBSInstances) != 2 {
t.Fatalf("Expected 2 PBS instances after update, got %d", len(snapshot.PBSInstances))
}
// Find updated instance
var found bool
for _, inst := range snapshot.PBSInstances {
if inst.ID == "pbs-1" && inst.Name == "Server 1 Updated" {
found = true
break
}
}
if !found {
t.Error("Expected pbs-1 to be updated with new name")
}
}
func TestPBSInstanceJobHealthEvidenceNormalizeAndClone(t *testing.T) {
state := NewState()
observedAt := time.Unix(1700000000, 0).UTC()
state.UpdatePBSInstance(PBSInstance{
ID: "pbs-1",
Name: "pbs-main",
JobHealthEvidence: []PBSJobHealthEvidence{{
ID: "sync-1",
Family: "sync",
Store: "fast",
LastRunState: "OK",
LastRunUPID: "UPID:sync:1",
LastRunEndtime: 1699999900,
NextRun: 1700003600,
Confidence: "direct-task-match",
EvidenceSource: "pbs-job-config",
EvidenceScope: "configured-job",
Freshness: PBSJobHealthFreshness{
ObservedAt: observedAt,
State: "observed",
},
Posture: "healthy",
}},
})
snapshot := state.GetSnapshot()
if len(snapshot.PBSInstances) != 1 || len(snapshot.PBSInstances[0].JobHealthEvidence) != 1 {
t.Fatalf("expected cloned PBS job health evidence, got %+v", snapshot.PBSInstances)
}
snapshot.PBSInstances[0].JobHealthEvidence[0].LastRunState = "changed"
fresh := state.GetSnapshot()
if got := fresh.PBSInstances[0].JobHealthEvidence[0].LastRunState; got != "OK" {
t.Fatalf("state clone leaked mutation, got last-run-state %q", got)
}
if got := fresh.PBSInstances[0].JobHealthEvidence[0]; got.EvidenceSource != "pbs-job-config" || got.EvidenceScope != "configured-job" {
t.Fatalf("state clone did not preserve evidence source/scope: %+v", got)
}
normalized := (PBSInstance{}).NormalizeCollections()
if normalized.JobHealthEvidence == nil {
t.Fatal("JobHealthEvidence should normalize to an initialized empty slice")
}
}
func TestUpdatePMGInstances(t *testing.T) {
state := NewState()
instances := []PMGInstance{
{ID: "pmg-1", Name: "Mail Gateway 1"},
{ID: "pmg-2", Name: "Mail Gateway 2"},
}
state.UpdatePMGInstances(instances)
snapshot := state.GetSnapshot()
if len(snapshot.PMGInstances) != 2 {
t.Fatalf("Expected 2 PMG instances, got %d", len(snapshot.PMGInstances))
}
// Replace with empty
state.UpdatePMGInstances([]PMGInstance{})
snapshot = state.GetSnapshot()
if len(snapshot.PMGInstances) != 0 {
t.Fatalf("Expected 0 PMG instances after clearing, got %d", len(snapshot.PMGInstances))
}
}
func TestUpdatePMGInstance(t *testing.T) {
state := NewState()
// Add first instance
state.UpdatePMGInstance(PMGInstance{ID: "pmg-1", Name: "Gateway 1"})
snapshot := state.GetSnapshot()
if len(snapshot.PMGInstances) != 1 {
t.Fatalf("Expected 1 PMG instance, got %d", len(snapshot.PMGInstances))
}
// Add second instance
state.UpdatePMGInstance(PMGInstance{ID: "pmg-2", Name: "Gateway 2"})
snapshot = state.GetSnapshot()
if len(snapshot.PMGInstances) != 2 {
t.Fatalf("Expected 2 PMG instances, got %d", len(snapshot.PMGInstances))
}
// Update existing instance
state.UpdatePMGInstance(PMGInstance{ID: "pmg-1", Name: "Gateway 1 Updated"})
snapshot = state.GetSnapshot()
if len(snapshot.PMGInstances) != 2 {
t.Fatalf("Expected 2 PMG instances after update, got %d", len(snapshot.PMGInstances))
}
// Verify update
var found bool
for _, inst := range snapshot.PMGInstances {
if inst.ID == "pmg-1" && inst.Name == "Gateway 1 Updated" {
found = true
break
}
}
if !found {
t.Error("Expected pmg-1 to be updated with new name")
}
}
func TestUpdateCephClustersForInstance(t *testing.T) {
state := NewState()
// Add clusters from first instance
clusters1 := []CephCluster{
{ID: "ceph-1", Instance: "pve-1", Name: "ceph-pool-1"},
{ID: "ceph-2", Instance: "pve-1", Name: "ceph-pool-2"},
}
state.UpdateCephClustersForInstance("pve-1", clusters1)
snapshot := state.GetSnapshot()
if len(snapshot.CephClusters) != 2 {
t.Fatalf("Expected 2 clusters, got %d", len(snapshot.CephClusters))
}
// Add clusters from second instance
clusters2 := []CephCluster{
{ID: "ceph-3", Instance: "pve-2", Name: "ceph-pool-1"},
}
state.UpdateCephClustersForInstance("pve-2", clusters2)
snapshot = state.GetSnapshot()
if len(snapshot.CephClusters) != 3 {
t.Fatalf("Expected 3 clusters, got %d", len(snapshot.CephClusters))
}
// Update first instance with empty (should remove its clusters)
state.UpdateCephClustersForInstance("pve-1", []CephCluster{})
snapshot = state.GetSnapshot()
if len(snapshot.CephClusters) != 1 {
t.Fatalf("Expected 1 cluster after clearing pve-1, got %d", len(snapshot.CephClusters))
}
if snapshot.CephClusters[0].Instance != "pve-2" {
t.Error("Remaining cluster should be from pve-2")
}
}
func TestCephClusterStatePrefersProxmoxAPIByFSID(t *testing.T) {
state := NewState()
state.UpsertCephCluster(CephCluster{
ID: "fsid-123",
Instance: "pve5",
Source: CephClusterSourceHostAgent,
Name: "pve5 Ceph",
FSID: "fsid-123",
Pools: []CephPool{{
ID: 2,
Name: "data_replication",
StoredBytes: 70,
AvailableBytes: 30,
PercentUsed: 70,
}},
})
stored := state.UpdateCephClustersForInstance("prod", []CephCluster{{
ID: "prod-fsid-123",
Instance: "prod",
Source: CephClusterSourceProxmoxAPI,
Name: "Ceph",
FSID: "fsid-123",
Health: "HEALTH_OK",
}})
snapshot := state.GetSnapshot()
if len(snapshot.CephClusters) != 1 {
t.Fatalf("expected API and agent reports for same FSID to reconcile to one cluster, got %#v", snapshot.CephClusters)
}
cluster := snapshot.CephClusters[0]
if cluster.Source != CephClusterSourceProxmoxAPI || cluster.Instance != "prod" || cluster.ID != "prod-fsid-123" {
t.Fatalf("expected Proxmox API cluster to be canonical, got %+v", cluster)
}
if !containsString(cluster.InstanceAliases, "pve5") {
t.Fatalf("expected agent instance alias to be preserved, got %#v", cluster.InstanceAliases)
}
if len(cluster.Pools) != 1 || cluster.Pools[0].Name != "data_replication" {
t.Fatalf("expected agent pool data to supplement API status-only cluster, got %#v", cluster.Pools)
}
if len(stored) != 1 || stored[0].Instance != "prod" || !containsString(stored[0].InstanceAliases, "pve5") {
t.Fatalf("stored clusters did not return reconciled canonical cluster: %#v", stored)
}
}
func TestUpsertCephClusterKeepsProxmoxAPIWhenAgentRefreshArrives(t *testing.T) {
state := NewState()
state.UpdateCephClustersForInstance("prod", []CephCluster{{
ID: "prod-fsid-123",
Instance: "prod",
Source: CephClusterSourceProxmoxAPI,
Name: "Ceph",
FSID: "fsid-123",
Health: "HEALTH_OK",
Pools: []CephPool{{
ID: 2,
Name: "data_replication",
StoredBytes: 60,
AvailableBytes: 40,
PercentUsed: 60,
}},
}})
stored := state.UpsertCephCluster(CephCluster{
ID: "fsid-123",
Instance: "pve5",
Source: CephClusterSourceHostAgent,
Name: "pve5 Ceph",
FSID: "fsid-123",
})
snapshot := state.GetSnapshot()
if len(snapshot.CephClusters) != 1 {
t.Fatalf("expected agent refresh to merge with API cluster, got %#v", snapshot.CephClusters)
}
cluster := snapshot.CephClusters[0]
if cluster.Source != CephClusterSourceProxmoxAPI || cluster.Instance != "prod" {
t.Fatalf("expected API cluster to remain canonical, got %+v", cluster)
}
if !containsString(cluster.InstanceAliases, "pve5") {
t.Fatalf("expected agent instance alias to be preserved, got %#v", cluster.InstanceAliases)
}
if stored.Source != CephClusterSourceProxmoxAPI || stored.Instance != "prod" {
t.Fatalf("expected upsert to return canonical API cluster, got %+v", stored)
}
}
func TestUpdateBackupTasksForInstance(t *testing.T) {
state := NewState()
now := time.Now()
// Add tasks from first instance
tasks1 := []BackupTask{
{ID: "pve-1-task-1", Instance: "pve-1", StartTime: now},
{ID: "pve-1-task-2", Instance: "pve-1", StartTime: now.Add(-time.Hour)},
}
state.UpdateBackupTasksForInstance("pve-1", tasks1)
snapshot := state.GetSnapshot()
if len(snapshot.PVEBackups.BackupTasks) != 2 {
t.Fatalf("Expected 2 tasks, got %d", len(snapshot.PVEBackups.BackupTasks))
}
// Tasks should be sorted by start time descending
if snapshot.PVEBackups.BackupTasks[0].StartTime.Before(snapshot.PVEBackups.BackupTasks[1].StartTime) {
t.Error("Tasks should be sorted by start time descending")
}
// Add tasks from second instance
tasks2 := []BackupTask{
{ID: "pve-2-task-1", Instance: "pve-2", StartTime: now.Add(-30 * time.Minute)},
}
state.UpdateBackupTasksForInstance("pve-2", tasks2)
snapshot = state.GetSnapshot()
if len(snapshot.PVEBackups.BackupTasks) != 3 {
t.Fatalf("Expected 3 tasks, got %d", len(snapshot.PVEBackups.BackupTasks))
}
// Update first instance (should replace its tasks)
tasks1Updated := []BackupTask{
{ID: "pve-1-task-3", Instance: "pve-1", StartTime: now.Add(time.Hour)},
}
state.UpdateBackupTasksForInstance("pve-1", tasks1Updated)
snapshot = state.GetSnapshot()
if len(snapshot.PVEBackups.BackupTasks) != 2 {
t.Fatalf("Expected 2 tasks after update, got %d", len(snapshot.PVEBackups.BackupTasks))
}
}
func TestUpdateReplicationJobsForInstance(t *testing.T) {
state := NewState()
// Add jobs from first instance
jobs1 := []ReplicationJob{
{ID: "job-1", Instance: "pve-1", GuestID: 100, JobNumber: 0},
{ID: "job-2", Instance: "pve-1", GuestID: 101, JobNumber: 0},
}
state.UpdateReplicationJobsForInstance("pve-1", jobs1)
snapshot := state.GetSnapshot()
if len(snapshot.ReplicationJobs) != 2 {
t.Fatalf("Expected 2 jobs, got %d", len(snapshot.ReplicationJobs))
}
// Add jobs from second instance
jobs2 := []ReplicationJob{
{ID: "job-3", Instance: "pve-2", GuestID: 200, JobNumber: 0},
}
state.UpdateReplicationJobsForInstance("pve-2", jobs2)
snapshot = state.GetSnapshot()
if len(snapshot.ReplicationJobs) != 3 {
t.Fatalf("Expected 3 jobs, got %d", len(snapshot.ReplicationJobs))
}
// Update first instance with empty (should remove its jobs)
state.UpdateReplicationJobsForInstance("pve-1", []ReplicationJob{})
snapshot = state.GetSnapshot()
if len(snapshot.ReplicationJobs) != 1 {
t.Fatalf("Expected 1 job after clearing pve-1, got %d", len(snapshot.ReplicationJobs))
}
if snapshot.ReplicationJobs[0].Instance != "pve-2" {
t.Error("Remaining job should be from pve-2")
}
}
func TestUpdateGuestSnapshotsForInstance(t *testing.T) {
state := NewState()
now := time.Now()
// Add snapshots from first instance
snapshots1 := []GuestSnapshot{
{ID: "pve-1-snap-1", Instance: "pve-1", VMID: 100, Name: "snapshot1", Time: now},
{ID: "pve-1-snap-2", Instance: "pve-1", VMID: 100, Name: "snapshot2", Time: now.Add(-time.Hour)},
}
state.UpdateGuestSnapshotsForInstance("pve-1", snapshots1)
snapshot := state.GetSnapshot()
if len(snapshot.PVEBackups.GuestSnapshots) != 2 {
t.Fatalf("Expected 2 snapshots, got %d", len(snapshot.PVEBackups.GuestSnapshots))
}
// Add snapshots from second instance
snapshots2 := []GuestSnapshot{
{ID: "pve-2-snap-1", Instance: "pve-2", VMID: 200, Name: "snapshot1", Time: now.Add(-30 * time.Minute)},
}
state.UpdateGuestSnapshotsForInstance("pve-2", snapshots2)
snapshot = state.GetSnapshot()
if len(snapshot.PVEBackups.GuestSnapshots) != 3 {
t.Fatalf("Expected 3 snapshots, got %d", len(snapshot.PVEBackups.GuestSnapshots))
}
// Update first instance (should replace its snapshots)
snapshots1Updated := []GuestSnapshot{
{ID: "pve-1-snap-3", Instance: "pve-1", VMID: 100, Name: "new-snapshot", Time: now.Add(time.Hour)},
}
state.UpdateGuestSnapshotsForInstance("pve-1", snapshots1Updated)
snapshot = state.GetSnapshot()
if len(snapshot.PVEBackups.GuestSnapshots) != 2 {
t.Fatalf("Expected 2 snapshots after update, got %d", len(snapshot.PVEBackups.GuestSnapshots))
}
}
func TestSyncGuestBackupTimes(t *testing.T) {
state := NewState()
now := time.Now()
oldBackup := now.Add(-24 * time.Hour)
newBackup := now.Add(-1 * time.Hour)
// Set up VMs and containers
state.UpdateVMs([]VM{
{VMID: 100, Name: "vm-100", Instance: "pve-1"},
{VMID: 101, Name: "vm-101", Instance: "pve-1"},
})
state.UpdateContainers([]Container{
{VMID: 200, Name: "ct-200", Instance: "pve-1"},
})
// Add storage backups for VM 100 (must include Instance for proper matching)
state.mu.Lock()
state.PVEBackups.StorageBackups = []StorageBackup{
{ID: "pve-1-backup-1", VMID: 100, Instance: "pve-1", Time: oldBackup},
{ID: "pve-1-backup-2", VMID: 100, Instance: "pve-1", Time: newBackup}, // newer
}
state.mu.Unlock()
// Add PBS backup for container 200
state.UpdatePBSBackups("pbs-1", []PBSBackup{
{ID: "pbs-backup-1", VMID: "200", BackupType: "ct", BackupTime: newBackup},
})
// Sync backup times
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
// VM 100 should have the newer backup time
var vm100 *VM
for i := range snapshot.VMs {
if snapshot.VMs[i].VMID == 100 {
vm100 = &snapshot.VMs[i]
break
}
}
if vm100 == nil {
t.Fatal("VM 100 not found")
}
if !vm100.LastBackup.Equal(newBackup) {
t.Errorf("VM 100 LastBackup = %v, expected %v", vm100.LastBackup, newBackup)
}
// VM 101 should have no backup time (zero)
var vm101 *VM
for i := range snapshot.VMs {
if snapshot.VMs[i].VMID == 101 {
vm101 = &snapshot.VMs[i]
break
}
}
if vm101 == nil {
t.Fatal("VM 101 not found")
}
if !vm101.LastBackup.IsZero() {
t.Errorf("VM 101 LastBackup should be zero, got %v", vm101.LastBackup)
}
// Container 200 should have backup time from PBS
var ct200 *Container
for i := range snapshot.Containers {
if snapshot.Containers[i].VMID == 200 {
ct200 = &snapshot.Containers[i]
break
}
}
if ct200 == nil {
t.Fatal("Container 200 not found")
}
if !ct200.LastBackup.Equal(newBackup) {
t.Errorf("Container 200 LastBackup = %v, expected %v", ct200.LastBackup, newBackup)
}
}
// TestSyncGuestBackupTimesCrossInstance verifies that backup matching uses instance+VMID.
// This prevents a newly created container on one instance from incorrectly showing
// backup time from a different container with the same VMID on another instance.
func TestSyncGuestBackupTimesCrossInstance(t *testing.T) {
state := NewState()
now := time.Now()
threeMonthsAgo := now.Add(-90 * 24 * time.Hour)
// Set up containers with the SAME VMID on DIFFERENT instances
// This simulates: pve-1 has VMID 100 with old backup, pve-2 has newly created VMID 100
state.UpdateContainers([]Container{
{VMID: 100, Name: "old-container", Instance: "pve-1"},
{VMID: 100, Name: "new-container", Instance: "pve-2"}, // newly created, no backup
})
// Add a 3-month-old backup for pve-1's container (VMID 100)
state.mu.Lock()
state.PVEBackups.StorageBackups = []StorageBackup{
{ID: "pve-1-backup-old", VMID: 100, Instance: "pve-1", Time: threeMonthsAgo},
}
state.mu.Unlock()
// Sync backup times
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
// Find both containers
var oldContainer, newContainer *Container
for i := range snapshot.Containers {
if snapshot.Containers[i].Instance == "pve-1" && snapshot.Containers[i].VMID == 100 {
oldContainer = &snapshot.Containers[i]
}
if snapshot.Containers[i].Instance == "pve-2" && snapshot.Containers[i].VMID == 100 {
newContainer = &snapshot.Containers[i]
}
}
if oldContainer == nil {
t.Fatal("pve-1 container not found")
}
if newContainer == nil {
t.Fatal("pve-2 container not found")
}
// The old container on pve-1 SHOULD have the backup time
if !oldContainer.LastBackup.Equal(threeMonthsAgo) {
t.Errorf("pve-1 container LastBackup = %v, expected %v", oldContainer.LastBackup, threeMonthsAgo)
}
// The NEW container on pve-2 should NOT have any backup time (it's a different container!)
if !newContainer.LastBackup.IsZero() {
t.Errorf("pve-2 container (newly created) should have no backup, got %v", newContainer.LastBackup)
}
}
// TestSyncGuestBackupTimesNamespaceDisambiguation verifies that PBS namespace is used to
// disambiguate backups when multiple VMs have the same VMID across different PVE instances.
// This addresses issue #1095 where users have multiple PVE instances with overlapping VMIDs.
func TestSyncGuestBackupTimesNamespaceDisambiguation(t *testing.T) {
state := NewState()
now := time.Now()
pveBackupTime := now.Add(-1 * time.Hour)
pveNatBackupTime := now.Add(-2 * time.Hour)
// Set up VMs with the SAME VMID on DIFFERENT instances
state.UpdateVMs([]VM{
{VMID: 100, Name: "webserver-pve", Instance: "pve", Node: "node1"},
{VMID: 100, Name: "webserver-nat", Instance: "pve-nat", Node: "node2"},
})
// Add PBS backups with namespaces that correspond to the PVE instances
state.mu.Lock()
state.PBSBackups = []PBSBackup{
{
ID: "pbs-pve-100",
VMID: "100",
Namespace: "pve",
BackupType: "vm",
BackupTime: pveBackupTime,
Instance: "pbs-main",
},
{
ID: "pbs-nat-100",
VMID: "100",
Namespace: "nat", // Should match "pve-nat" instance
BackupType: "vm",
BackupTime: pveNatBackupTime,
Instance: "pbs-main",
},
}
state.mu.Unlock()
// Sync backup times
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
// Find both VMs
var vmPVE, vmNAT *VM
for i := range snapshot.VMs {
if snapshot.VMs[i].Instance == "pve" && snapshot.VMs[i].VMID == 100 {
vmPVE = &snapshot.VMs[i]
}
if snapshot.VMs[i].Instance == "pve-nat" && snapshot.VMs[i].VMID == 100 {
vmNAT = &snapshot.VMs[i]
}
}
if vmPVE == nil {
t.Fatal("pve VM not found")
}
if vmNAT == nil {
t.Fatal("pve-nat VM not found")
}
// The pve VM should have the backup with namespace "pve"
if !vmPVE.LastBackup.Equal(pveBackupTime) {
t.Errorf("pve VM LastBackup = %v, expected %v (from namespace 'pve')", vmPVE.LastBackup, pveBackupTime)
}
// The pve-nat VM should have the backup with namespace "nat"
if !vmNAT.LastBackup.Equal(pveNatBackupTime) {
t.Errorf("pve-nat VM LastBackup = %v, expected %v (from namespace 'nat')", vmNAT.LastBackup, pveNatBackupTime)
}
}
// TestSyncGuestBackupTimesClusterEntrypointUsesGuestNodeNamespace verifies that
// a cluster API entrypoint name does not shadow the guest's actual node namespace.
func TestSyncGuestBackupTimesClusterEntrypointUsesGuestNodeNamespace(t *testing.T) {
state := NewState()
now := time.Now()
oldBackupTime := now.Add(-180 * 24 * time.Hour)
freshBackupTime := now.Add(-1 * time.Hour)
state.UpdateContainers([]Container{
{VMID: 112, Name: "debian-go", Instance: "delly", Node: "minipc"},
})
state.mu.Lock()
state.PBSBackups = []PBSBackup{
{
ID: "pbs-delly-112-old",
VMID: "112",
Namespace: "delly",
BackupType: "ct",
Comment: "112",
BackupTime: oldBackupTime,
Instance: "pbs-main",
},
{
ID: "pbs-minipc-112-fresh",
VMID: "112",
Namespace: "minipc",
BackupType: "ct",
Comment: "debian-go",
BackupTime: freshBackupTime,
Instance: "pbs-main",
},
}
state.mu.Unlock()
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
var found *Container
for i := range snapshot.Containers {
if snapshot.Containers[i].VMID == 112 {
found = &snapshot.Containers[i]
break
}
}
if found == nil {
t.Fatal("container 112 not found")
}
if !found.LastBackup.Equal(freshBackupTime) {
t.Errorf("cluster entrypoint guest LastBackup = %v, want fresh node-namespaced backup %v",
found.LastBackup, freshBackupTime)
}
}
func TestSyncGuestBackupTimesClearsStaleBackupWhenCurrentEvidenceDisappears(t *testing.T) {
state := NewState()
staleBackupTime := time.Now().Add(-30 * 24 * time.Hour)
state.UpdateVMs([]VM{
{VMID: 101, Name: "unbacked-vm", Instance: "pve-1", Node: "node1", LastBackup: staleBackupTime},
})
state.UpdateContainers([]Container{
{VMID: 201, Name: "unbacked-ct", Instance: "pve-1", Node: "node1", LastBackup: staleBackupTime},
})
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
for _, vm := range snapshot.VMs {
if vm.VMID == 101 && !vm.LastBackup.IsZero() {
t.Errorf("VM stale LastBackup should be cleared, got %v", vm.LastBackup)
}
}
for _, ct := range snapshot.Containers {
if ct.VMID == 201 && !ct.LastBackup.IsZero() {
t.Errorf("container stale LastBackup should be cleared, got %v", ct.LastBackup)
}
}
}
// TestSyncGuestBackupTimesVMIDCollisionNonMatchingNamespace verifies that when the same VMID
// exists on multiple PVE instances and a PBS backup namespace matches neither, both guests
// get zero LastBackup instead of a false positive. The backup's submission
// source (owner/datastore) carries no learned attribution and no PVE-side
// storage listing confirms it, so the #1639 evidence paths must not fire.
func TestSyncGuestBackupTimesVMIDCollisionNonMatchingNamespace(t *testing.T) {
state := NewState()
now := time.Now()
backupTime := now.Add(-1 * time.Hour)
// Two VMs with the same VMID on different instances
state.UpdateVMs([]VM{
{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
{VMID: 100, Name: "vm-pve2", Instance: "pve2", Node: "node2"},
})
// PBS backup with a namespace that matches neither instance
state.mu.Lock()
state.PBSBackups = []PBSBackup{
{
ID: "pbs-100",
VMID: "100",
Namespace: "staging",
BackupType: "vm",
BackupTime: backupTime,
Instance: "pbs-main",
Datastore: "backups",
Owner: "unlearned@pbs!token",
},
}
state.mu.Unlock()
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
for _, vm := range snapshot.VMs {
if vm.VMID == 100 && !vm.LastBackup.IsZero() {
t.Errorf("VM %q on %s should have zero LastBackup (ambiguous VMID, non-matching namespace), got %v",
vm.Name, vm.Instance, vm.LastBackup)
}
}
}
// TestSyncGuestBackupTimesVMIDCollisionEmptyNamespace verifies that when the same VMID
// exists on multiple PVE instances and a PBS backup has no namespace, both guests
// get zero LastBackup. This is the genuinely unattributable core of #1639:
// with a single snapshot, no learnable submission source, and no PVE-side
// confirmation, guessing either guest would be a false positive. The
// resolvable variants live in issue1639_pbs_collision_test.go.
func TestSyncGuestBackupTimesVMIDCollisionEmptyNamespace(t *testing.T) {
state := NewState()
now := time.Now()
backupTime := now.Add(-1 * time.Hour)
state.UpdateVMs([]VM{
{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
{VMID: 100, Name: "vm-pve2", Instance: "pve2", Node: "node2"},
})
// PBS backup with empty namespace
state.mu.Lock()
state.PBSBackups = []PBSBackup{
{
ID: "pbs-100",
VMID: "100",
Namespace: "",
BackupType: "vm",
BackupTime: backupTime,
Instance: "pbs-main",
},
}
state.mu.Unlock()
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
for _, vm := range snapshot.VMs {
if vm.VMID == 100 && !vm.LastBackup.IsZero() {
t.Errorf("VM %q on %s should have zero LastBackup (ambiguous VMID, empty namespace), got %v",
vm.Name, vm.Instance, vm.LastBackup)
}
}
}
// TestSyncGuestBackupTimesUniqueVMIDFallback verifies that a unique VMID still gets
// the PBS backup via fallback even when the namespace doesn't match.
func TestSyncGuestBackupTimesUniqueVMIDFallback(t *testing.T) {
state := NewState()
now := time.Now()
backupTime := now.Add(-1 * time.Hour)
// Single VM — VMID is unique
state.UpdateVMs([]VM{
{VMID: 100, Name: "my-vm", Instance: "pve1", Node: "node1"},
})
// PBS backup with a namespace that does NOT match the instance
state.mu.Lock()
state.PBSBackups = []PBSBackup{
{
ID: "pbs-100",
VMID: "100",
Namespace: "daily",
BackupType: "vm",
BackupTime: backupTime,
Instance: "pbs-main",
},
}
state.mu.Unlock()
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
var found *VM
for i := range snapshot.VMs {
if snapshot.VMs[i].VMID == 100 {
found = &snapshot.VMs[i]
}
}
if found == nil {
t.Fatal("VM not found")
}
if !found.LastBackup.Equal(backupTime) {
t.Errorf("unique VMID should fall back to PBS backup; got LastBackup = %v, want %v",
found.LastBackup, backupTime)
}
}
// TestSyncGuestBackupTimesVMContainerCollision verifies that PBS vm/ID and ct/ID
// subjects are matched independently even when a VM and container share the same
// numeric ID.
func TestSyncGuestBackupTimesVMContainerCollision(t *testing.T) {
state := NewState()
now := time.Now()
backupTime := now.Add(-1 * time.Hour)
state.UpdateVMs([]VM{
{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
})
state.UpdateContainers([]Container{
{VMID: 100, Name: "ct-pve2", Instance: "pve2", Node: "node2"},
})
state.mu.Lock()
state.PBSBackups = []PBSBackup{
{
ID: "pbs-100",
VMID: "100",
Namespace: "node1",
BackupType: "vm",
BackupTime: backupTime,
Instance: "pbs-main",
},
}
state.mu.Unlock()
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
for _, vm := range snapshot.VMs {
if vm.VMID == 100 && !vm.LastBackup.Equal(backupTime) {
t.Errorf("VM %q LastBackup = %v, want typed vm/100 backup %v",
vm.Name, vm.LastBackup, backupTime)
}
}
for _, ct := range snapshot.Containers {
if ct.VMID == 100 && !ct.LastBackup.IsZero() {
t.Errorf("Container %q should have zero LastBackup because the PBS subject is vm/100, got %v",
ct.Name, ct.LastBackup)
}
}
}
func TestUpdateGuestsForInstancePublishesCoherentGeneration(t *testing.T) {
state := NewState()
previousBackup := time.Now().Add(-2 * time.Hour).UTC()
state.UpdateVMs([]VM{
{ID: "lab-a:node-a:101", VMID: 101, Instance: "lab-a", Node: "node-a", LastBackup: previousBackup},
{ID: "lab-b:node-b:201", VMID: 201, Instance: "lab-b", Node: "node-b"},
})
state.UpdateContainers([]Container{
{ID: "lab-a:node-a:102", VMID: 102, Instance: "lab-a", Node: "node-a", LastBackup: previousBackup},
{ID: "lab-b:node-b:202", VMID: 202, Instance: "lab-b", Node: "node-b"},
})
before := state.GetSnapshot().LastUpdate
state.UpdateGuestsForInstance(
"lab-a",
[]VM{{ID: "lab-a:node-a:101", VMID: 101, Instance: "lab-a", Node: "node-a", Status: "running"}},
[]Container{{ID: "lab-a:node-a:103", VMID: 103, Instance: "lab-a", Node: "node-a", Status: "running"}},
)
snapshot := state.GetSnapshot()
if !snapshot.LastUpdate.After(before) {
t.Fatalf("LastUpdate did not advance: before=%v after=%v", before, snapshot.LastUpdate)
}
if len(snapshot.VMs) != 2 || len(snapshot.Containers) != 2 {
t.Fatalf("unexpected coherent generation sizes: vms=%d containers=%d", len(snapshot.VMs), len(snapshot.Containers))
}
if snapshot.VMs[0].ID != "lab-a:node-a:101" || !snapshot.VMs[0].LastBackup.Equal(previousBackup) {
t.Fatalf("updated VM did not retain backup state: %+v", snapshot.VMs[0])
}
if snapshot.Containers[0].ID != "lab-a:node-a:103" {
t.Fatalf("authoritatively deleted container remained in state: %+v", snapshot.Containers)
}
if snapshot.VMs[1].Instance != "lab-b" || snapshot.Containers[1].Instance != "lab-b" {
t.Fatalf("other instance was not isolated: vms=%+v containers=%+v", snapshot.VMs, snapshot.Containers)
}
}
func TestUpdateStorageBackupsForInstance(t *testing.T) {
state := NewState()
now := time.Now()
// Set up a VM so node normalization has something to work with
state.UpdateVMsForInstance("pve-1", []VM{
{VMID: 100, Instance: "pve-1", Node: "node1"},
})
// Add backups from first instance
backups1 := []StorageBackup{
{ID: "pve-1-backup-1", Instance: "pve-1", VMID: 100, Time: now, Node: "node1"},
{ID: "pve-1-backup-2", Instance: "pve-1", VMID: 100, Time: now.Add(-time.Hour), Node: "node1"},
}
state.UpdateStorageBackupsForInstance("pve-1", backups1)
snapshot := state.GetSnapshot()
if len(snapshot.PVEBackups.StorageBackups) != 2 {
t.Fatalf("Expected 2 backups, got %d", len(snapshot.PVEBackups.StorageBackups))
}
// Backups should be sorted by time descending
if snapshot.PVEBackups.StorageBackups[0].Time.Before(snapshot.PVEBackups.StorageBackups[1].Time) {
t.Error("Backups should be sorted by time descending")
}
// Add backups from second instance
backups2 := []StorageBackup{
{ID: "pve-2-backup-1", Instance: "pve-2", VMID: 200, Time: now.Add(-30 * time.Minute), Node: "node2"},
}
state.UpdateStorageBackupsForInstance("pve-2", backups2)
snapshot = state.GetSnapshot()
if len(snapshot.PVEBackups.StorageBackups) != 3 {
t.Fatalf("Expected 3 backups, got %d", len(snapshot.PVEBackups.StorageBackups))
}
// Update first instance (should replace its backups)
backups1Updated := []StorageBackup{
{ID: "pve-1-backup-3", Instance: "pve-1", VMID: 100, Time: now.Add(time.Hour), Node: "node1"},
}
state.UpdateStorageBackupsForInstance("pve-1", backups1Updated)
snapshot = state.GetSnapshot()
if len(snapshot.PVEBackups.StorageBackups) != 2 {
t.Fatalf("Expected 2 backups after update, got %d", len(snapshot.PVEBackups.StorageBackups))
}
// Verify old pve-1 backups were replaced
for _, b := range snapshot.PVEBackups.StorageBackups {
if b.ID == "pve-1-backup-1" || b.ID == "pve-1-backup-2" {
t.Error("Old pve-1 backups should have been replaced")
}
}
}
// Host.IntegrationSource is fabric-only provenance: it must survive host
// upserts, serialize when set, and stay entirely absent from JSON for real
// agent-report hosts so existing payloads do not change shape.
func TestHostIntegrationSourceRoundTrip(t *testing.T) {
state := NewState()
state.UpsertHost(Host{ID: "host-esxi", Hostname: "esxi-01", IntegrationSource: "vmware"})
state.UpsertHost(Host{ID: "host-agent", Hostname: "apollo-114"})
var esxi, agent *Host
hosts := state.GetHosts()
for i := range hosts {
switch hosts[i].ID {
case "host-esxi":
esxi = &hosts[i]
case "host-agent":
agent = &hosts[i]
}
}
if esxi == nil || esxi.IntegrationSource != "vmware" {
t.Fatalf("expected upserted host to preserve IntegrationSource %q, got %+v", "vmware", esxi)
}
if agent == nil || agent.IntegrationSource != "" {
t.Fatalf("expected agent host to keep empty IntegrationSource, got %+v", agent)
}
marked, err := json.Marshal(esxi)
if err != nil {
t.Fatalf("marshal integration host: %v", err)
}
if !strings.Contains(string(marked), `"integrationSource":"vmware"`) {
t.Fatalf("expected integrationSource in payload, got %s", marked)
}
unmarked, err := json.Marshal(agent)
if err != nil {
t.Fatalf("marshal agent host: %v", err)
}
if strings.Contains(string(unmarked), "integrationSource") {
t.Fatalf("expected integrationSource omitted for agent hosts, got %s", unmarked)
}
}
// Two clusters reusing the same node names (pve01/pve02 on different subnets)
// must stay apart: the second cluster's node must not steal the first
// cluster's agent link by bare hostname, and the shared name must not merge
// the two nodes into one slot.
func TestUpdateNodesForInstanceKeepsClustersWithSharedNodeNamesApart(t *testing.T) {
state := &State{
Hosts: []Host{
{
ID: "host-enacon-pve01",
Hostname: "pve01",
ReportIP: "192.168.16.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
},
},
{
ID: "host-enacon-pve02",
Hostname: "pve02",
ReportIP: "192.168.16.12",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.16.12/24"}},
},
},
},
Nodes: []Node{
{
ID: "enacon-pve01",
Name: "pve01",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.16.11:8006",
LinkedAgentID: "host-enacon-pve01",
Status: "online",
},
{
ID: "enacon-pve02",
Name: "pve02",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.16.12:8006",
LinkedAgentID: "host-enacon-pve02",
Status: "online",
},
},
}
state.UpdateNodesForInstance("rewo", []Node{
{
ID: "rewo-pve01",
Name: "pve01",
Instance: "rewo",
ClusterName: "rewo",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
Status: "online",
},
})
if len(state.Nodes) != 3 {
t.Fatalf("nodes = %#v, want 3 (enacon pve01+pve02 and rewo pve01)", state.Nodes)
}
byID := make(map[string]Node)
for _, node := range state.Nodes {
byID[node.ID] = node
}
enacon, ok := byID["enacon-pve01"]
if !ok {
t.Fatalf("enacon-pve01 was clobbered, nodes = %#v", state.Nodes)
}
if enacon.ClusterName != "enacon" {
t.Fatalf("enacon-pve01 ClusterName = %q, want enacon", enacon.ClusterName)
}
if enacon.LinkedAgentID != "host-enacon-pve01" {
t.Fatalf("enacon-pve01 LinkedAgentID = %q, want host-enacon-pve01", enacon.LinkedAgentID)
}
rewo, ok := byID["rewo-pve01"]
if !ok {
t.Fatalf("rewo-pve01 missing, nodes = %#v", state.Nodes)
}
if rewo.ClusterName != "rewo" {
t.Fatalf("rewo-pve01 ClusterName = %q, want rewo", rewo.ClusterName)
}
if rewo.LinkedAgentID != "" {
t.Fatalf("rewo-pve01 LinkedAgentID = %q, want none (its host runs no agent)", rewo.LinkedAgentID)
}
}
// When TLS settings degrade node endpoints to the bare node name, the
// endpoint-host merge alias is identical for both clusters. The alias must
// not merge nodes whose named clusters contradict each other.
func TestUpdateNodesForInstanceEndpointHostAliasDoesNotMergeAcrossClusters(t *testing.T) {
state := &State{
Nodes: []Node{
{
ID: "enacon-pve01",
Name: "pve01",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://pve01:8006",
Status: "online",
},
},
}
state.UpdateNodesForInstance("rewo", []Node{
{
ID: "rewo-pve01",
Name: "pve01",
Instance: "rewo",
ClusterName: "rewo",
IsClusterMember: true,
Host: "https://pve01:8006",
Status: "online",
},
})
if len(state.Nodes) != 2 {
t.Fatalf("nodes = %#v, want 2 (one per cluster)", state.Nodes)
}
clusters := map[string]bool{}
for _, node := range state.Nodes {
clusters[node.ClusterName] = true
}
if !clusters["enacon"] || !clusters["rewo"] {
t.Fatalf("clusters = %#v, want both enacon and rewo", clusters)
}
}
// A standalone node whose endpoint IP is absent from a same-named agent's
// reported IPs must not link to that agent: the IP evidence contradicts the
// hostname match.
func TestUpdateNodesForInstanceHostnameMatchRejectedWhenIPContradicts(t *testing.T) {
state := &State{
Hosts: []Host{
{
ID: "host-a",
Hostname: "pve01",
ReportIP: "192.168.16.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
},
},
},
}
state.UpdateNodesForInstance("steinboeck", []Node{
{
ID: "steinboeck-pve01",
Name: "pve01",
Instance: "steinboeck",
Host: "https://10.20.1.5:8006",
Status: "online",
},
})
if len(state.Nodes) != 1 {
t.Fatalf("nodes = %#v, want 1", state.Nodes)
}
if state.Nodes[0].LinkedAgentID != "" {
t.Fatalf("LinkedAgentID = %q, want none (agent IPs contradict node endpoint)", state.Nodes[0].LinkedAgentID)
}
}
// Two different clusters at different sites can reuse the same corosync
// cluster name, the same member hostnames, and the same RFC1918 addressing
// (MSP support case: two "enacon" clusters both exposing pve01 on
// 192.168.1.11). The colliding endpoint-IP alias must not merge the two node
// slots, and the address match must not bind the second site's node to the
// first site's host agent when the TLS fingerprint evidence contradicts.
func TestUpdateNodesForInstanceSameNamedClustersWithCollidingIPsStayApart(t *testing.T) {
state := &State{
Hosts: []Host{
{
ID: "host-site-a-pve01",
Hostname: "pve01",
ReportIP: "192.168.1.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
},
},
},
Nodes: []Node{
{
ID: "site-a-pve01",
Name: "pve01",
Instance: "site-a",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
TLSFingerprint: "AA:AA:AA:AA",
LinkedAgentID: "host-site-a-pve01",
Status: "online",
},
},
}
state.UpdateNodesForInstance("site-b", []Node{
{
ID: "site-b-pve01",
Name: "pve01",
Instance: "site-b",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
TLSFingerprint: "BB:BB:BB:BB",
Status: "online",
},
})
if len(state.Nodes) != 2 {
t.Fatalf("nodes = %#v, want 2 (one per site)", state.Nodes)
}
byID := make(map[string]Node)
for _, node := range state.Nodes {
byID[node.ID] = node
}
siteA, ok := byID["site-a-pve01"]
if !ok {
t.Fatalf("site-a-pve01 was clobbered, nodes = %#v", state.Nodes)
}
if siteA.LinkedAgentID != "host-site-a-pve01" {
t.Fatalf("site-a-pve01 LinkedAgentID = %q, want host-site-a-pve01", siteA.LinkedAgentID)
}
siteB, ok := byID["site-b-pve01"]
if !ok {
t.Fatalf("site-b-pve01 missing, nodes = %#v", state.Nodes)
}
if siteB.LinkedAgentID != "" {
t.Fatalf("site-b-pve01 LinkedAgentID = %q, want none (fingerprint contradicts the colliding-IP agent match)", siteB.LinkedAgentID)
}
}
// The endpoint-IP agent match must also respect the cluster-identity
// contradiction when no fingerprints are known: an agent whose linked nodes
// live in cluster "enacon" must not bind to a "rewo" node that presents the
// same RFC1918 address from another site.
func TestUpdateNodesForInstanceEndpointIPMatchRejectedWhenClusterContradicts(t *testing.T) {
state := &State{
Hosts: []Host{
{
ID: "host-enacon-pve01",
Hostname: "pve01",
ReportIP: "192.168.1.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
},
},
},
Nodes: []Node{
{
ID: "enacon-pve01",
Name: "pve01",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
LinkedAgentID: "host-enacon-pve01",
Status: "online",
},
},
}
state.UpdateNodesForInstance("rewo", []Node{
{
ID: "rewo-pve01",
Name: "pve01",
Instance: "rewo",
ClusterName: "rewo",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
Status: "online",
},
})
if len(state.Nodes) != 2 {
t.Fatalf("nodes = %#v, want 2 (one per cluster)", state.Nodes)
}
for _, node := range state.Nodes {
if node.ClusterName == "rewo" && node.LinkedAgentID != "" {
t.Fatalf("rewo pve01 LinkedAgentID = %q, want none (agent belongs to the enacon cluster)", node.LinkedAgentID)
}
}
}
// The same cluster added twice through different connection instances is the
// legitimate duplicate: when both views carry the same TOFU-captured TLS
// fingerprint for the same-named node, the endpoint alias must still fold
// them into one slot even though the instances differ.
func TestUpdateNodesForInstanceSameClusterAddedTwiceMergesWithMatchingFingerprints(t *testing.T) {
state := &State{
Nodes: []Node{
{
ID: "enacon-a-pve01",
Name: "pve01",
Instance: "enacon-a",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
TLSFingerprint: "AA:AA:AA:AA",
Status: "online",
},
},
}
state.UpdateNodesForInstance("enacon-b", []Node{
{
ID: "enacon-b-pve01",
Name: "pve01",
Instance: "enacon-b",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
TLSFingerprint: "aaaaaaaa",
Status: "online",
},
})
if len(state.Nodes) != 1 {
t.Fatalf("nodes = %#v, want 1 (matching fingerprints prove one machine)", state.Nodes)
}
if state.Nodes[0].Name != "pve01" {
t.Fatalf("merged node = %#v, want pve01", state.Nodes[0])
}
}
// Without fingerprint evidence the same-named cluster from another instance
// stays split: unknown identity must never fold two possibly-distinct
// clusters into one (fail-safe direction).
func TestUpdateNodesForInstanceSameNamedClustersStayApartWithoutFingerprints(t *testing.T) {
state := &State{
Nodes: []Node{
{
ID: "enacon-a-pve01",
Name: "pve01",
Instance: "enacon-a",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
Status: "online",
},
},
}
state.UpdateNodesForInstance("enacon-b", []Node{
{
ID: "enacon-b-pve01",
Name: "pve01",
Instance: "enacon-b",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
Status: "online",
},
})
if len(state.Nodes) != 2 {
t.Fatalf("nodes = %#v, want 2 (unknown identity must not merge)", state.Nodes)
}
}
// A shared agent link is the strongest cross-view merge signal, but for
// same-named clusters from different instances it must still be gated on
// fingerprint identity: matching fingerprints merge the duplicate views onto
// one slot, contradicting fingerprints never do.
func TestUpdateNodesForInstanceSharedAgentLinkMergeGatedOnFingerprint(t *testing.T) {
makeState := func(existingFP string) *State {
return &State{
Hosts: []Host{
{
ID: "host-pve01",
Hostname: "pve01",
ReportIP: "192.168.1.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
},
},
},
Nodes: []Node{
{
ID: "enacon-a-pve01",
Name: "pve01",
Instance: "enacon-a",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://pve01.site-a.internal:8006",
TLSFingerprint: existingFP,
LinkedAgentID: "host-pve01",
Status: "online",
},
},
}
}
incoming := func(fp string) []Node {
return []Node{
{
ID: "enacon-b-pve01",
Name: "pve01",
Instance: "enacon-b",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.1.11:8006",
TLSFingerprint: fp,
LinkedAgentID: "host-pve01",
Status: "online",
},
}
}
merged := makeState("AA:AA:AA:AA")
merged.UpdateNodesForInstance("enacon-b", incoming("AA:AA:AA:AA"))
if len(merged.Nodes) != 1 {
t.Fatalf("matching fingerprints: nodes = %#v, want 1", merged.Nodes)
}
split := makeState("AA:AA:AA:AA")
split.UpdateNodesForInstance("enacon-b", incoming("BB:BB:BB:BB"))
if len(split.Nodes) != 2 {
t.Fatalf("contradicting fingerprints: nodes = %#v, want 2", split.Nodes)
}
}
// A cluster connection's first polls can commit nodes before membership
// detection classifies them (ClusterName ""), and cloned template
// deployments reuse /etc/machine-id, collapsing two sites' agents into one
// identity (MSP support case: clusters enacon and rewo, both with pve01, on
// reused RFC1918 ranges). During that window the shared agent identity must
// not fold the unclassified node into the established cluster's slot.
func TestUpdateNodesForInstanceUnclassifiedNodeSharedAgentMustNotStealClusterSlot(t *testing.T) {
state := &State{
Hosts: []Host{
{
// One host row standing in for two physical pve01 machines
// whose cloned machine-id collapsed them into one agent
// identity; it currently reports the second site's address.
ID: "host-cloned-machine-id",
Hostname: "pve01",
ReportIP: "192.168.1.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
},
},
},
Nodes: []Node{
{
ID: "enacon-pve01",
Name: "pve01",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.16.11:8006",
LinkedAgentID: "host-cloned-machine-id",
Status: "online",
},
},
}
state.UpdateNodesForInstance("rewo", []Node{
{
ID: "rewo-pve01",
Name: "pve01",
Instance: "rewo",
// Membership detection has not classified this connection yet.
ClusterName: "",
Host: "https://192.168.1.11:8006",
Status: "online",
},
})
if len(state.Nodes) != 2 {
t.Fatalf("nodes = %#v, want 2 (unclassified node must not fold into the enacon slot)", state.Nodes)
}
byID := make(map[string]Node)
for _, node := range state.Nodes {
byID[node.ID] = node
}
enacon, ok := byID["enacon-pve01"]
if !ok {
t.Fatalf("enacon-pve01 was clobbered, nodes = %#v", state.Nodes)
}
if enacon.ClusterName != "enacon" {
t.Fatalf("enacon-pve01 ClusterName = %q, want enacon", enacon.ClusterName)
}
if _, ok := byID["rewo-pve01"]; !ok {
t.Fatalf("rewo-pve01 missing, nodes = %#v", state.Nodes)
}
}
// When TLS settings degrade both connections' node endpoints to the bare
// node name, the endpoint-host alias is identical across sites. An
// unclassified node (empty cluster name, e.g. during the first-poll
// membership window) must not ride that weak alias into another instance's
// established cluster slot.
func TestUpdateNodesForInstanceEndpointHostAliasRequiresProofForUnclassifiedNodes(t *testing.T) {
state := &State{
Nodes: []Node{
{
ID: "enacon-pve01",
Name: "pve01",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://pve01:8006",
Status: "online",
},
},
}
state.UpdateNodesForInstance("rewo", []Node{
{
ID: "rewo-pve01",
Name: "pve01",
Instance: "rewo",
ClusterName: "",
Host: "https://pve01:8006",
Status: "online",
},
})
if len(state.Nodes) != 2 {
t.Fatalf("nodes = %#v, want 2 (bare-hostname alias must not fold an unclassified cross-instance node)", state.Nodes)
}
}
// The legitimate standalone-into-cluster fold must survive the weak-evidence
// tightening: a standalone connection view dialing the same address as the
// cluster's endpoint record still folds via the address alias, and a
// fingerprint-proven view still folds via the shared agent link.
func TestUpdateNodesForInstanceStandaloneViewStillFoldsIntoClusterNode(t *testing.T) {
byAddress := &State{
Nodes: []Node{
{
ID: "enacon-pve01",
Name: "pve01",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://192.168.16.11:8006",
Status: "online",
},
},
}
byAddress.UpdateNodesForInstance("pve01-direct", []Node{
{
ID: "direct-pve01",
Name: "pve01",
Instance: "pve01-direct",
Host: "https://192.168.16.11:8006",
Status: "online",
},
})
if len(byAddress.Nodes) != 1 {
t.Fatalf("address alias: nodes = %#v, want 1 (same-address standalone view folds)", byAddress.Nodes)
}
byAgent := &State{
Hosts: []Host{
{
ID: "host-pve01",
Hostname: "pve01",
ReportIP: "192.168.16.11",
NetworkInterfaces: []HostNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
},
},
},
Nodes: []Node{
{
ID: "enacon-pve01",
Name: "pve01",
Instance: "enacon",
ClusterName: "enacon",
IsClusterMember: true,
Host: "https://pve01:8006",
TLSFingerprint: "AA:AA:AA:AA",
LinkedAgentID: "host-pve01",
Status: "online",
},
},
}
byAgent.UpdateNodesForInstance("pve01-direct", []Node{
{
ID: "direct-pve01",
Name: "pve01",
Instance: "pve01-direct",
Host: "https://192.168.16.11:8006",
TLSFingerprint: "aaaaaaaa",
Status: "online",
},
})
if len(byAgent.Nodes) != 1 {
t.Fatalf("agent link: nodes = %#v, want 1 (fingerprint-proven view folds via shared agent)", byAgent.Nodes)
}
}