mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-24 20:22:53 +00:00
84dba861b5
v6.1.0's identity rework (eab73d245) made the VMID-only fallback refuse
to fire whenever a typed VMID exists on more than one PVE location.
Root-namespace snapshots with no matching comment then score zero for
every guest, so on setups with two clusters and overlapping VMIDs most
guests showed no backup at all - while PVE itself listed the backups
fine, because monitoring discards pbs-type storage contents entirely
whenever a direct PBS connection is configured.
Attribution is now evidence-driven instead of dropped:
Storage backup polling keeps a per-connection record of every snapshot
its own pbs-type storage listed (type, VMID, backup time) even though
the raw entries stay out of the PVE backup list. Which cluster listed a
snapshot is deterministic attribution, and it survives fully mirrored
clusters that share one datastore and token. The evidence is
monitoring-internal, cleared on instance retirement or when the storage
poll stops seeing pbs content, and never serialized into state payloads
or snapshots.
Guest backup-time sync and the recovery-point mapper additionally learn
each PBS submission source's cluster (owner token, datastore, PBS
instance - strongest first, scoped to the PBS instance) from the poll's
attributable snapshots, then resolve collision VMIDs whose snapshots
carry no evidence of their own. A source seen from several clusters is
not a discriminator, an unfamiliar component stops resolution rather
than deferring to weaker ones, and a snapshot decisively attributed to
another cluster is kept away from this one. Unattributable snapshots
still drop rather than guess.
Backup-age alert attribution no longer suffix-matches the subject ref's
connection label against guest locations. The label there is a PVE or
PBS instance name, not a PBS namespace, and loose matching could
cross-attribute clusters sharing a VMID; it now requires exact
normalized equality.
Reported in #1639 (two PVE clusters with PBS 4.0/4.1, VM 173 shown 974
days overdue despite valid verified backups).
Fixes #1639
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2073 lines
61 KiB
Go
2073 lines
61 KiB
Go
package models
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestUpsertHost(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Test insert new host
|
|
host1 := Host{
|
|
ID: "host-1",
|
|
Hostname: "server-1",
|
|
Status: "online",
|
|
}
|
|
state.UpsertHost(host1)
|
|
|
|
hosts := state.GetHosts()
|
|
if len(hosts) != 1 {
|
|
t.Fatalf("Expected 1 host, got %d", len(hosts))
|
|
}
|
|
if hosts[0].ID != "host-1" {
|
|
t.Errorf("Expected host ID 'host-1', got %q", hosts[0].ID)
|
|
}
|
|
|
|
// Test update existing host
|
|
host1Updated := Host{
|
|
ID: "host-1",
|
|
Hostname: "server-1",
|
|
Status: "offline",
|
|
}
|
|
state.UpsertHost(host1Updated)
|
|
|
|
hosts = state.GetHosts()
|
|
if len(hosts) != 1 {
|
|
t.Fatalf("Expected 1 host after update, got %d", len(hosts))
|
|
}
|
|
if hosts[0].Status != "offline" {
|
|
t.Errorf("Expected status 'offline', got %q", hosts[0].Status)
|
|
}
|
|
|
|
// Test multiple hosts and sorting
|
|
state.UpsertHost(Host{ID: "host-2", Hostname: "alpha-server"})
|
|
state.UpsertHost(Host{ID: "host-3", Hostname: "zulu-server"})
|
|
|
|
hosts = state.GetHosts()
|
|
if len(hosts) != 3 {
|
|
t.Fatalf("Expected 3 hosts, got %d", len(hosts))
|
|
}
|
|
|
|
// Hosts should be sorted by hostname
|
|
if hosts[0].Hostname != "alpha-server" {
|
|
t.Errorf("First host should be 'alpha-server', got %q", hosts[0].Hostname)
|
|
}
|
|
}
|
|
|
|
func TestGetHosts_Copy(t *testing.T) {
|
|
state := NewState()
|
|
|
|
state.UpsertHost(Host{ID: "host-1", Hostname: "server-1"})
|
|
|
|
hosts1 := state.GetHosts()
|
|
hosts2 := state.GetHosts()
|
|
|
|
// Modify one slice
|
|
if len(hosts1) > 0 {
|
|
hosts1[0].Hostname = "modified"
|
|
}
|
|
|
|
// Other slice should be unchanged
|
|
if len(hosts2) > 0 && hosts2[0].Hostname == "modified" {
|
|
t.Error("GetHosts should return a copy, not the same slice")
|
|
}
|
|
|
|
// Original state should be unchanged
|
|
hosts3 := state.GetHosts()
|
|
if hosts3[0].Hostname == "modified" {
|
|
t.Error("State should be unchanged by modifications to returned slice")
|
|
}
|
|
}
|
|
|
|
func TestRemoveHost(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Insert hosts
|
|
state.UpsertHost(Host{ID: "host-1", Hostname: "server1"})
|
|
state.UpsertHost(Host{ID: "host-2", Hostname: "server2"})
|
|
|
|
// Remove existing host
|
|
removed, ok := state.RemoveHost("host-1")
|
|
if !ok {
|
|
t.Error("Expected RemoveHost to return true for existing host")
|
|
}
|
|
if removed.ID != "host-1" {
|
|
t.Errorf("Expected removed host ID 'host-1', got %q", removed.ID)
|
|
}
|
|
|
|
hosts := state.GetHosts()
|
|
if len(hosts) != 1 {
|
|
t.Fatalf("Expected 1 host after removal, got %d", len(hosts))
|
|
}
|
|
if hosts[0].ID != "host-2" {
|
|
t.Errorf("Remaining host should be 'host-2', got %q", hosts[0].ID)
|
|
}
|
|
|
|
// Remove non-existing host
|
|
removed, ok = state.RemoveHost("non-existent")
|
|
if ok {
|
|
t.Error("Expected RemoveHost to return false for non-existent host")
|
|
}
|
|
if removed.ID != "" {
|
|
t.Errorf("Expected empty Host for non-existent removal, got ID %q", removed.ID)
|
|
}
|
|
}
|
|
|
|
func TestSetHostStatus(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Test with no hosts
|
|
changed := state.SetHostStatus("host-1", "online")
|
|
if changed {
|
|
t.Error("SetHostStatus should return false when host doesn't exist")
|
|
}
|
|
|
|
// Add host and set status
|
|
state.UpsertHost(Host{ID: "host-1", Hostname: "server1", Status: "offline"})
|
|
|
|
changed = state.SetHostStatus("host-1", "online")
|
|
if !changed {
|
|
t.Error("SetHostStatus should return true when host exists")
|
|
}
|
|
|
|
hosts := state.GetHosts()
|
|
if hosts[0].Status != "online" {
|
|
t.Errorf("Expected status 'online', got %q", hosts[0].Status)
|
|
}
|
|
|
|
// Set same status (no change)
|
|
changed = state.SetHostStatus("host-1", "online")
|
|
if !changed {
|
|
t.Error("SetHostStatus should return true even when status unchanged")
|
|
}
|
|
}
|
|
|
|
func TestTouchHost(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
|
|
// Test with non-existent host
|
|
ok := state.TouchHost("host-1", now)
|
|
if ok {
|
|
t.Error("TouchHost should return false for non-existent host")
|
|
}
|
|
|
|
// Add host and touch it
|
|
state.UpsertHost(Host{ID: "host-1", Hostname: "server1"})
|
|
|
|
later := now.Add(time.Hour)
|
|
ok = state.TouchHost("host-1", later)
|
|
if !ok {
|
|
t.Error("TouchHost should return true for existing host")
|
|
}
|
|
|
|
hosts := state.GetHosts()
|
|
if !hosts[0].LastSeen.Equal(later) {
|
|
t.Errorf("LastSeen should be updated to %v, got %v", later, hosts[0].LastSeen)
|
|
}
|
|
}
|
|
|
|
func TestSetConnectionHealth(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Set connection healthy
|
|
state.SetConnectionHealth("pve-cluster-1", true)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if healthy, ok := snapshot.ConnectionHealth["pve-cluster-1"]; !ok || !healthy {
|
|
t.Error("Expected connection to be healthy")
|
|
}
|
|
|
|
// Set connection unhealthy
|
|
state.SetConnectionHealth("pve-cluster-1", false)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if healthy, ok := snapshot.ConnectionHealth["pve-cluster-1"]; !ok || healthy {
|
|
t.Error("Expected connection to be unhealthy")
|
|
}
|
|
|
|
// Multiple connections
|
|
state.SetConnectionHealth("pve-cluster-2", true)
|
|
state.SetConnectionHealth("pbs-instance-1", true)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.ConnectionHealth) != 3 {
|
|
t.Errorf("Expected 3 connection health entries, got %d", len(snapshot.ConnectionHealth))
|
|
}
|
|
}
|
|
|
|
func TestPVETagStyleSnapshotRebuildsAggregateByInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
state.MergePVETagStyle("pve-a", PVETagStyle{
|
|
Colors: map[string]string{
|
|
"Production": "#00ff00",
|
|
},
|
|
CaseSensitive: true,
|
|
})
|
|
state.MergePVETagStyle("pve-b", PVETagStyle{
|
|
Colors: map[string]string{
|
|
"Backup": "#112233",
|
|
},
|
|
})
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if got := snapshot.PVETagStyles["pve-a"].Colors["Production"]; got != "#00ff00" {
|
|
t.Fatalf("expected pve-a tag style to preserve case-sensitive color, got %q", got)
|
|
}
|
|
if !snapshot.PVETagStyles["pve-a"].CaseSensitive {
|
|
t.Fatal("expected pve-a tag style to preserve case-sensitive flag")
|
|
}
|
|
if got := snapshot.PVETagStyles["pve-b"].Colors["backup"]; got != "#112233" {
|
|
t.Fatalf("expected pve-b tag style to normalize non-case-sensitive tag, got %q", got)
|
|
}
|
|
|
|
state.MergePVETagStyle("pve-a", PVETagStyle{})
|
|
snapshot = state.GetSnapshot()
|
|
if _, ok := snapshot.PVETagColors["Production"]; ok {
|
|
t.Fatalf("expected stale pve-a aggregate color to be removed, got %#v", snapshot.PVETagColors)
|
|
}
|
|
if got := snapshot.PVETagColors["backup"]; got != "#112233" {
|
|
t.Fatalf("expected pve-b aggregate color to remain, got %q", got)
|
|
}
|
|
}
|
|
|
|
func TestRemoveConnectionHealth(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Set up connection health entries
|
|
state.SetConnectionHealth("pve-cluster-1", true)
|
|
state.SetConnectionHealth("pve-cluster-2", false)
|
|
|
|
// Remove one
|
|
state.RemoveConnectionHealth("pve-cluster-1")
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if _, ok := snapshot.ConnectionHealth["pve-cluster-1"]; ok {
|
|
t.Error("Expected pve-cluster-1 to be removed")
|
|
}
|
|
if _, ok := snapshot.ConnectionHealth["pve-cluster-2"]; !ok {
|
|
t.Error("Expected pve-cluster-2 to still exist")
|
|
}
|
|
|
|
// Remove non-existent (should not panic)
|
|
state.RemoveConnectionHealth("non-existent")
|
|
|
|
// Remove remaining
|
|
state.RemoveConnectionHealth("pve-cluster-2")
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.ConnectionHealth) != 0 {
|
|
t.Errorf("Expected empty connection health map, got %d entries", len(snapshot.ConnectionHealth))
|
|
}
|
|
}
|
|
|
|
func TestUpdatePBSBackups(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
|
|
// Add backups from first instance
|
|
backups1 := []PBSBackup{
|
|
{ID: "backup-1", Instance: "pbs-1", BackupTime: now},
|
|
{ID: "backup-2", Instance: "pbs-1", BackupTime: now.Add(-time.Hour)},
|
|
}
|
|
state.UpdatePBSBackups("pbs-1", backups1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PBSBackups) != 2 {
|
|
t.Fatalf("Expected 2 backups, got %d", len(snapshot.PBSBackups))
|
|
}
|
|
|
|
// Add backups from second instance
|
|
backups2 := []PBSBackup{
|
|
{ID: "backup-3", Instance: "pbs-2", BackupTime: now.Add(-30 * time.Minute)},
|
|
}
|
|
state.UpdatePBSBackups("pbs-2", backups2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PBSBackups) != 3 {
|
|
t.Fatalf("Expected 3 backups, got %d", len(snapshot.PBSBackups))
|
|
}
|
|
|
|
// Update first instance (should replace its backups)
|
|
backups1Updated := []PBSBackup{
|
|
{ID: "backup-4", Instance: "pbs-1", BackupTime: now.Add(time.Hour)},
|
|
}
|
|
state.UpdatePBSBackups("pbs-1", backups1Updated)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PBSBackups) != 2 {
|
|
t.Fatalf("Expected 2 backups after update, got %d", len(snapshot.PBSBackups))
|
|
}
|
|
|
|
// Verify pbs-1 backups were replaced
|
|
hasOldBackup := false
|
|
for _, b := range snapshot.PBSBackups {
|
|
if b.ID == "backup-1" || b.ID == "backup-2" {
|
|
hasOldBackup = true
|
|
break
|
|
}
|
|
}
|
|
if hasOldBackup {
|
|
t.Error("Old pbs-1 backups should have been replaced")
|
|
}
|
|
}
|
|
|
|
func TestUpdatePMGBackups(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
|
|
// Add backups from first instance
|
|
backups1 := []PMGBackup{
|
|
{Filename: "backup-1.conf", Instance: "pmg-1", BackupTime: now},
|
|
{Filename: "backup-2.conf", Instance: "pmg-1", BackupTime: now.Add(-time.Hour)},
|
|
}
|
|
state.UpdatePMGBackups("pmg-1", backups1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PMGBackups) != 2 {
|
|
t.Fatalf("Expected 2 backups, got %d", len(snapshot.PMGBackups))
|
|
}
|
|
|
|
// Backups should be sorted by time (newest first)
|
|
if snapshot.PMGBackups[0].BackupTime.Before(snapshot.PMGBackups[1].BackupTime) {
|
|
t.Error("Backups should be sorted by time descending")
|
|
}
|
|
|
|
// Add backups from second instance
|
|
backups2 := []PMGBackup{
|
|
{Filename: "backup-3.conf", Instance: "pmg-2", BackupTime: now.Add(-30 * time.Minute)},
|
|
}
|
|
state.UpdatePMGBackups("pmg-2", backups2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PMGBackups) != 3 {
|
|
t.Fatalf("Expected 3 backups, got %d", len(snapshot.PMGBackups))
|
|
}
|
|
|
|
// Update first instance with empty (should remove its backups)
|
|
state.UpdatePMGBackups("pmg-1", []PMGBackup{})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PMGBackups) != 1 {
|
|
t.Fatalf("Expected 1 backup after clearing pmg-1, got %d", len(snapshot.PMGBackups))
|
|
}
|
|
if snapshot.PMGBackups[0].Instance != "pmg-2" {
|
|
t.Error("Remaining backup should be from pmg-2")
|
|
}
|
|
}
|
|
|
|
func TestUpdatePhysicalDisks(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Add disks from first instance
|
|
disks1 := []PhysicalDisk{
|
|
{ID: "disk-1", Instance: "pve-1", Node: "node1", DevPath: "/dev/sda"},
|
|
{ID: "disk-2", Instance: "pve-1", Node: "node1", DevPath: "/dev/sdb"},
|
|
}
|
|
state.UpdatePhysicalDisks("pve-1", disks1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PhysicalDisks) != 2 {
|
|
t.Fatalf("Expected 2 disks, got %d", len(snapshot.PhysicalDisks))
|
|
}
|
|
|
|
// Add disks from second instance
|
|
disks2 := []PhysicalDisk{
|
|
{ID: "disk-3", Instance: "pve-2", Node: "node2", DevPath: "/dev/sda"},
|
|
}
|
|
state.UpdatePhysicalDisks("pve-2", disks2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PhysicalDisks) != 3 {
|
|
t.Fatalf("Expected 3 disks, got %d", len(snapshot.PhysicalDisks))
|
|
}
|
|
|
|
// Update first instance (should replace its disks)
|
|
disks1Updated := []PhysicalDisk{
|
|
{ID: "disk-4", Instance: "pve-1", Node: "node1", DevPath: "/dev/nvme0n1"},
|
|
}
|
|
state.UpdatePhysicalDisks("pve-1", disks1Updated)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PhysicalDisks) != 2 {
|
|
t.Fatalf("Expected 2 disks after update, got %d", len(snapshot.PhysicalDisks))
|
|
}
|
|
|
|
// Verify sorting by node then devpath
|
|
if snapshot.PhysicalDisks[0].Node > snapshot.PhysicalDisks[1].Node {
|
|
t.Error("Disks should be sorted by node")
|
|
}
|
|
}
|
|
|
|
func TestUpdateStorageForInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Add storage from first instance
|
|
storage1 := []Storage{
|
|
{ID: "storage-1", Instance: "pve-1", Name: "local"},
|
|
{ID: "storage-2", Instance: "pve-1", Name: "ceph-pool"},
|
|
}
|
|
state.UpdateStorageForInstance("pve-1", storage1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.Storage) != 2 {
|
|
t.Fatalf("Expected 2 storage entries, got %d", len(snapshot.Storage))
|
|
}
|
|
|
|
// Add storage from second instance
|
|
storage2 := []Storage{
|
|
{ID: "storage-3", Instance: "pve-2", Name: "local"},
|
|
}
|
|
state.UpdateStorageForInstance("pve-2", storage2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.Storage) != 3 {
|
|
t.Fatalf("Expected 3 storage entries, got %d", len(snapshot.Storage))
|
|
}
|
|
|
|
// Update first instance with empty (should remove its storage)
|
|
state.UpdateStorageForInstance("pve-1", []Storage{})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.Storage) != 1 {
|
|
t.Fatalf("Expected 1 storage after clearing pve-1, got %d", len(snapshot.Storage))
|
|
}
|
|
}
|
|
|
|
func TestUpdatePBSInstances(t *testing.T) {
|
|
state := NewState()
|
|
|
|
instances := []PBSInstance{
|
|
{ID: "pbs-1", Name: "Backup Server 1"},
|
|
{ID: "pbs-2", Name: "Backup Server 2"},
|
|
}
|
|
state.UpdatePBSInstances(instances)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PBSInstances) != 2 {
|
|
t.Fatalf("Expected 2 PBS instances, got %d", len(snapshot.PBSInstances))
|
|
}
|
|
|
|
// Replace with different instances
|
|
newInstances := []PBSInstance{
|
|
{ID: "pbs-3", Name: "New Backup Server"},
|
|
}
|
|
state.UpdatePBSInstances(newInstances)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PBSInstances) != 1 {
|
|
t.Fatalf("Expected 1 PBS instance after replacement, got %d", len(snapshot.PBSInstances))
|
|
}
|
|
if snapshot.PBSInstances[0].ID != "pbs-3" {
|
|
t.Error("Expected pbs-3 instance")
|
|
}
|
|
}
|
|
|
|
func TestUpdatePBSInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Add first instance
|
|
state.UpdatePBSInstance(PBSInstance{ID: "pbs-1", Name: "Server 1"})
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PBSInstances) != 1 {
|
|
t.Fatalf("Expected 1 PBS instance, got %d", len(snapshot.PBSInstances))
|
|
}
|
|
|
|
// Add second instance
|
|
state.UpdatePBSInstance(PBSInstance{ID: "pbs-2", Name: "Server 2"})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PBSInstances) != 2 {
|
|
t.Fatalf("Expected 2 PBS instances, got %d", len(snapshot.PBSInstances))
|
|
}
|
|
|
|
// Update existing instance
|
|
state.UpdatePBSInstance(PBSInstance{ID: "pbs-1", Name: "Server 1 Updated"})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PBSInstances) != 2 {
|
|
t.Fatalf("Expected 2 PBS instances after update, got %d", len(snapshot.PBSInstances))
|
|
}
|
|
|
|
// Find updated instance
|
|
var found bool
|
|
for _, inst := range snapshot.PBSInstances {
|
|
if inst.ID == "pbs-1" && inst.Name == "Server 1 Updated" {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
t.Error("Expected pbs-1 to be updated with new name")
|
|
}
|
|
}
|
|
|
|
func TestPBSInstanceJobHealthEvidenceNormalizeAndClone(t *testing.T) {
|
|
state := NewState()
|
|
observedAt := time.Unix(1700000000, 0).UTC()
|
|
state.UpdatePBSInstance(PBSInstance{
|
|
ID: "pbs-1",
|
|
Name: "pbs-main",
|
|
JobHealthEvidence: []PBSJobHealthEvidence{{
|
|
ID: "sync-1",
|
|
Family: "sync",
|
|
Store: "fast",
|
|
LastRunState: "OK",
|
|
LastRunUPID: "UPID:sync:1",
|
|
LastRunEndtime: 1699999900,
|
|
NextRun: 1700003600,
|
|
Confidence: "direct-task-match",
|
|
EvidenceSource: "pbs-job-config",
|
|
EvidenceScope: "configured-job",
|
|
Freshness: PBSJobHealthFreshness{
|
|
ObservedAt: observedAt,
|
|
State: "observed",
|
|
},
|
|
Posture: "healthy",
|
|
}},
|
|
})
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PBSInstances) != 1 || len(snapshot.PBSInstances[0].JobHealthEvidence) != 1 {
|
|
t.Fatalf("expected cloned PBS job health evidence, got %+v", snapshot.PBSInstances)
|
|
}
|
|
snapshot.PBSInstances[0].JobHealthEvidence[0].LastRunState = "changed"
|
|
|
|
fresh := state.GetSnapshot()
|
|
if got := fresh.PBSInstances[0].JobHealthEvidence[0].LastRunState; got != "OK" {
|
|
t.Fatalf("state clone leaked mutation, got last-run-state %q", got)
|
|
}
|
|
if got := fresh.PBSInstances[0].JobHealthEvidence[0]; got.EvidenceSource != "pbs-job-config" || got.EvidenceScope != "configured-job" {
|
|
t.Fatalf("state clone did not preserve evidence source/scope: %+v", got)
|
|
}
|
|
|
|
normalized := (PBSInstance{}).NormalizeCollections()
|
|
if normalized.JobHealthEvidence == nil {
|
|
t.Fatal("JobHealthEvidence should normalize to an initialized empty slice")
|
|
}
|
|
}
|
|
|
|
func TestUpdatePMGInstances(t *testing.T) {
|
|
state := NewState()
|
|
|
|
instances := []PMGInstance{
|
|
{ID: "pmg-1", Name: "Mail Gateway 1"},
|
|
{ID: "pmg-2", Name: "Mail Gateway 2"},
|
|
}
|
|
state.UpdatePMGInstances(instances)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PMGInstances) != 2 {
|
|
t.Fatalf("Expected 2 PMG instances, got %d", len(snapshot.PMGInstances))
|
|
}
|
|
|
|
// Replace with empty
|
|
state.UpdatePMGInstances([]PMGInstance{})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PMGInstances) != 0 {
|
|
t.Fatalf("Expected 0 PMG instances after clearing, got %d", len(snapshot.PMGInstances))
|
|
}
|
|
}
|
|
|
|
func TestUpdatePMGInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Add first instance
|
|
state.UpdatePMGInstance(PMGInstance{ID: "pmg-1", Name: "Gateway 1"})
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PMGInstances) != 1 {
|
|
t.Fatalf("Expected 1 PMG instance, got %d", len(snapshot.PMGInstances))
|
|
}
|
|
|
|
// Add second instance
|
|
state.UpdatePMGInstance(PMGInstance{ID: "pmg-2", Name: "Gateway 2"})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PMGInstances) != 2 {
|
|
t.Fatalf("Expected 2 PMG instances, got %d", len(snapshot.PMGInstances))
|
|
}
|
|
|
|
// Update existing instance
|
|
state.UpdatePMGInstance(PMGInstance{ID: "pmg-1", Name: "Gateway 1 Updated"})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PMGInstances) != 2 {
|
|
t.Fatalf("Expected 2 PMG instances after update, got %d", len(snapshot.PMGInstances))
|
|
}
|
|
|
|
// Verify update
|
|
var found bool
|
|
for _, inst := range snapshot.PMGInstances {
|
|
if inst.ID == "pmg-1" && inst.Name == "Gateway 1 Updated" {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
t.Error("Expected pmg-1 to be updated with new name")
|
|
}
|
|
}
|
|
|
|
func TestUpdateCephClustersForInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Add clusters from first instance
|
|
clusters1 := []CephCluster{
|
|
{ID: "ceph-1", Instance: "pve-1", Name: "ceph-pool-1"},
|
|
{ID: "ceph-2", Instance: "pve-1", Name: "ceph-pool-2"},
|
|
}
|
|
state.UpdateCephClustersForInstance("pve-1", clusters1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.CephClusters) != 2 {
|
|
t.Fatalf("Expected 2 clusters, got %d", len(snapshot.CephClusters))
|
|
}
|
|
|
|
// Add clusters from second instance
|
|
clusters2 := []CephCluster{
|
|
{ID: "ceph-3", Instance: "pve-2", Name: "ceph-pool-1"},
|
|
}
|
|
state.UpdateCephClustersForInstance("pve-2", clusters2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.CephClusters) != 3 {
|
|
t.Fatalf("Expected 3 clusters, got %d", len(snapshot.CephClusters))
|
|
}
|
|
|
|
// Update first instance with empty (should remove its clusters)
|
|
state.UpdateCephClustersForInstance("pve-1", []CephCluster{})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.CephClusters) != 1 {
|
|
t.Fatalf("Expected 1 cluster after clearing pve-1, got %d", len(snapshot.CephClusters))
|
|
}
|
|
if snapshot.CephClusters[0].Instance != "pve-2" {
|
|
t.Error("Remaining cluster should be from pve-2")
|
|
}
|
|
}
|
|
|
|
func TestCephClusterStatePrefersProxmoxAPIByFSID(t *testing.T) {
|
|
state := NewState()
|
|
|
|
state.UpsertCephCluster(CephCluster{
|
|
ID: "fsid-123",
|
|
Instance: "pve5",
|
|
Source: CephClusterSourceHostAgent,
|
|
Name: "pve5 Ceph",
|
|
FSID: "fsid-123",
|
|
Pools: []CephPool{{
|
|
ID: 2,
|
|
Name: "data_replication",
|
|
StoredBytes: 70,
|
|
AvailableBytes: 30,
|
|
PercentUsed: 70,
|
|
}},
|
|
})
|
|
|
|
stored := state.UpdateCephClustersForInstance("prod", []CephCluster{{
|
|
ID: "prod-fsid-123",
|
|
Instance: "prod",
|
|
Source: CephClusterSourceProxmoxAPI,
|
|
Name: "Ceph",
|
|
FSID: "fsid-123",
|
|
Health: "HEALTH_OK",
|
|
}})
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.CephClusters) != 1 {
|
|
t.Fatalf("expected API and agent reports for same FSID to reconcile to one cluster, got %#v", snapshot.CephClusters)
|
|
}
|
|
cluster := snapshot.CephClusters[0]
|
|
if cluster.Source != CephClusterSourceProxmoxAPI || cluster.Instance != "prod" || cluster.ID != "prod-fsid-123" {
|
|
t.Fatalf("expected Proxmox API cluster to be canonical, got %+v", cluster)
|
|
}
|
|
if !containsString(cluster.InstanceAliases, "pve5") {
|
|
t.Fatalf("expected agent instance alias to be preserved, got %#v", cluster.InstanceAliases)
|
|
}
|
|
if len(cluster.Pools) != 1 || cluster.Pools[0].Name != "data_replication" {
|
|
t.Fatalf("expected agent pool data to supplement API status-only cluster, got %#v", cluster.Pools)
|
|
}
|
|
if len(stored) != 1 || stored[0].Instance != "prod" || !containsString(stored[0].InstanceAliases, "pve5") {
|
|
t.Fatalf("stored clusters did not return reconciled canonical cluster: %#v", stored)
|
|
}
|
|
}
|
|
|
|
func TestUpsertCephClusterKeepsProxmoxAPIWhenAgentRefreshArrives(t *testing.T) {
|
|
state := NewState()
|
|
|
|
state.UpdateCephClustersForInstance("prod", []CephCluster{{
|
|
ID: "prod-fsid-123",
|
|
Instance: "prod",
|
|
Source: CephClusterSourceProxmoxAPI,
|
|
Name: "Ceph",
|
|
FSID: "fsid-123",
|
|
Health: "HEALTH_OK",
|
|
Pools: []CephPool{{
|
|
ID: 2,
|
|
Name: "data_replication",
|
|
StoredBytes: 60,
|
|
AvailableBytes: 40,
|
|
PercentUsed: 60,
|
|
}},
|
|
}})
|
|
|
|
stored := state.UpsertCephCluster(CephCluster{
|
|
ID: "fsid-123",
|
|
Instance: "pve5",
|
|
Source: CephClusterSourceHostAgent,
|
|
Name: "pve5 Ceph",
|
|
FSID: "fsid-123",
|
|
})
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.CephClusters) != 1 {
|
|
t.Fatalf("expected agent refresh to merge with API cluster, got %#v", snapshot.CephClusters)
|
|
}
|
|
cluster := snapshot.CephClusters[0]
|
|
if cluster.Source != CephClusterSourceProxmoxAPI || cluster.Instance != "prod" {
|
|
t.Fatalf("expected API cluster to remain canonical, got %+v", cluster)
|
|
}
|
|
if !containsString(cluster.InstanceAliases, "pve5") {
|
|
t.Fatalf("expected agent instance alias to be preserved, got %#v", cluster.InstanceAliases)
|
|
}
|
|
if stored.Source != CephClusterSourceProxmoxAPI || stored.Instance != "prod" {
|
|
t.Fatalf("expected upsert to return canonical API cluster, got %+v", stored)
|
|
}
|
|
}
|
|
|
|
func TestUpdateBackupTasksForInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
|
|
// Add tasks from first instance
|
|
tasks1 := []BackupTask{
|
|
{ID: "pve-1-task-1", Instance: "pve-1", StartTime: now},
|
|
{ID: "pve-1-task-2", Instance: "pve-1", StartTime: now.Add(-time.Hour)},
|
|
}
|
|
state.UpdateBackupTasksForInstance("pve-1", tasks1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.BackupTasks) != 2 {
|
|
t.Fatalf("Expected 2 tasks, got %d", len(snapshot.PVEBackups.BackupTasks))
|
|
}
|
|
|
|
// Tasks should be sorted by start time descending
|
|
if snapshot.PVEBackups.BackupTasks[0].StartTime.Before(snapshot.PVEBackups.BackupTasks[1].StartTime) {
|
|
t.Error("Tasks should be sorted by start time descending")
|
|
}
|
|
|
|
// Add tasks from second instance
|
|
tasks2 := []BackupTask{
|
|
{ID: "pve-2-task-1", Instance: "pve-2", StartTime: now.Add(-30 * time.Minute)},
|
|
}
|
|
state.UpdateBackupTasksForInstance("pve-2", tasks2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.BackupTasks) != 3 {
|
|
t.Fatalf("Expected 3 tasks, got %d", len(snapshot.PVEBackups.BackupTasks))
|
|
}
|
|
|
|
// Update first instance (should replace its tasks)
|
|
tasks1Updated := []BackupTask{
|
|
{ID: "pve-1-task-3", Instance: "pve-1", StartTime: now.Add(time.Hour)},
|
|
}
|
|
state.UpdateBackupTasksForInstance("pve-1", tasks1Updated)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.BackupTasks) != 2 {
|
|
t.Fatalf("Expected 2 tasks after update, got %d", len(snapshot.PVEBackups.BackupTasks))
|
|
}
|
|
}
|
|
|
|
func TestUpdateReplicationJobsForInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
// Add jobs from first instance
|
|
jobs1 := []ReplicationJob{
|
|
{ID: "job-1", Instance: "pve-1", GuestID: 100, JobNumber: 0},
|
|
{ID: "job-2", Instance: "pve-1", GuestID: 101, JobNumber: 0},
|
|
}
|
|
state.UpdateReplicationJobsForInstance("pve-1", jobs1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.ReplicationJobs) != 2 {
|
|
t.Fatalf("Expected 2 jobs, got %d", len(snapshot.ReplicationJobs))
|
|
}
|
|
|
|
// Add jobs from second instance
|
|
jobs2 := []ReplicationJob{
|
|
{ID: "job-3", Instance: "pve-2", GuestID: 200, JobNumber: 0},
|
|
}
|
|
state.UpdateReplicationJobsForInstance("pve-2", jobs2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.ReplicationJobs) != 3 {
|
|
t.Fatalf("Expected 3 jobs, got %d", len(snapshot.ReplicationJobs))
|
|
}
|
|
|
|
// Update first instance with empty (should remove its jobs)
|
|
state.UpdateReplicationJobsForInstance("pve-1", []ReplicationJob{})
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.ReplicationJobs) != 1 {
|
|
t.Fatalf("Expected 1 job after clearing pve-1, got %d", len(snapshot.ReplicationJobs))
|
|
}
|
|
if snapshot.ReplicationJobs[0].Instance != "pve-2" {
|
|
t.Error("Remaining job should be from pve-2")
|
|
}
|
|
}
|
|
|
|
func TestUpdateGuestSnapshotsForInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
|
|
// Add snapshots from first instance
|
|
snapshots1 := []GuestSnapshot{
|
|
{ID: "pve-1-snap-1", Instance: "pve-1", VMID: 100, Name: "snapshot1", Time: now},
|
|
{ID: "pve-1-snap-2", Instance: "pve-1", VMID: 100, Name: "snapshot2", Time: now.Add(-time.Hour)},
|
|
}
|
|
state.UpdateGuestSnapshotsForInstance("pve-1", snapshots1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.GuestSnapshots) != 2 {
|
|
t.Fatalf("Expected 2 snapshots, got %d", len(snapshot.PVEBackups.GuestSnapshots))
|
|
}
|
|
|
|
// Add snapshots from second instance
|
|
snapshots2 := []GuestSnapshot{
|
|
{ID: "pve-2-snap-1", Instance: "pve-2", VMID: 200, Name: "snapshot1", Time: now.Add(-30 * time.Minute)},
|
|
}
|
|
state.UpdateGuestSnapshotsForInstance("pve-2", snapshots2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.GuestSnapshots) != 3 {
|
|
t.Fatalf("Expected 3 snapshots, got %d", len(snapshot.PVEBackups.GuestSnapshots))
|
|
}
|
|
|
|
// Update first instance (should replace its snapshots)
|
|
snapshots1Updated := []GuestSnapshot{
|
|
{ID: "pve-1-snap-3", Instance: "pve-1", VMID: 100, Name: "new-snapshot", Time: now.Add(time.Hour)},
|
|
}
|
|
state.UpdateGuestSnapshotsForInstance("pve-1", snapshots1Updated)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.GuestSnapshots) != 2 {
|
|
t.Fatalf("Expected 2 snapshots after update, got %d", len(snapshot.PVEBackups.GuestSnapshots))
|
|
}
|
|
}
|
|
|
|
func TestSyncGuestBackupTimes(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
oldBackup := now.Add(-24 * time.Hour)
|
|
newBackup := now.Add(-1 * time.Hour)
|
|
|
|
// Set up VMs and containers
|
|
state.UpdateVMs([]VM{
|
|
{VMID: 100, Name: "vm-100", Instance: "pve-1"},
|
|
{VMID: 101, Name: "vm-101", Instance: "pve-1"},
|
|
})
|
|
state.UpdateContainers([]Container{
|
|
{VMID: 200, Name: "ct-200", Instance: "pve-1"},
|
|
})
|
|
|
|
// Add storage backups for VM 100 (must include Instance for proper matching)
|
|
state.mu.Lock()
|
|
state.PVEBackups.StorageBackups = []StorageBackup{
|
|
{ID: "pve-1-backup-1", VMID: 100, Instance: "pve-1", Time: oldBackup},
|
|
{ID: "pve-1-backup-2", VMID: 100, Instance: "pve-1", Time: newBackup}, // newer
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
// Add PBS backup for container 200
|
|
state.UpdatePBSBackups("pbs-1", []PBSBackup{
|
|
{ID: "pbs-backup-1", VMID: "200", BackupType: "ct", BackupTime: newBackup},
|
|
})
|
|
|
|
// Sync backup times
|
|
state.SyncGuestBackupTimes()
|
|
|
|
snapshot := state.GetSnapshot()
|
|
|
|
// VM 100 should have the newer backup time
|
|
var vm100 *VM
|
|
for i := range snapshot.VMs {
|
|
if snapshot.VMs[i].VMID == 100 {
|
|
vm100 = &snapshot.VMs[i]
|
|
break
|
|
}
|
|
}
|
|
if vm100 == nil {
|
|
t.Fatal("VM 100 not found")
|
|
}
|
|
if !vm100.LastBackup.Equal(newBackup) {
|
|
t.Errorf("VM 100 LastBackup = %v, expected %v", vm100.LastBackup, newBackup)
|
|
}
|
|
|
|
// VM 101 should have no backup time (zero)
|
|
var vm101 *VM
|
|
for i := range snapshot.VMs {
|
|
if snapshot.VMs[i].VMID == 101 {
|
|
vm101 = &snapshot.VMs[i]
|
|
break
|
|
}
|
|
}
|
|
if vm101 == nil {
|
|
t.Fatal("VM 101 not found")
|
|
}
|
|
if !vm101.LastBackup.IsZero() {
|
|
t.Errorf("VM 101 LastBackup should be zero, got %v", vm101.LastBackup)
|
|
}
|
|
|
|
// Container 200 should have backup time from PBS
|
|
var ct200 *Container
|
|
for i := range snapshot.Containers {
|
|
if snapshot.Containers[i].VMID == 200 {
|
|
ct200 = &snapshot.Containers[i]
|
|
break
|
|
}
|
|
}
|
|
if ct200 == nil {
|
|
t.Fatal("Container 200 not found")
|
|
}
|
|
if !ct200.LastBackup.Equal(newBackup) {
|
|
t.Errorf("Container 200 LastBackup = %v, expected %v", ct200.LastBackup, newBackup)
|
|
}
|
|
}
|
|
|
|
// TestSyncGuestBackupTimesCrossInstance verifies that backup matching uses instance+VMID.
|
|
// This prevents a newly created container on one instance from incorrectly showing
|
|
// backup time from a different container with the same VMID on another instance.
|
|
func TestSyncGuestBackupTimesCrossInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
threeMonthsAgo := now.Add(-90 * 24 * time.Hour)
|
|
|
|
// Set up containers with the SAME VMID on DIFFERENT instances
|
|
// This simulates: pve-1 has VMID 100 with old backup, pve-2 has newly created VMID 100
|
|
state.UpdateContainers([]Container{
|
|
{VMID: 100, Name: "old-container", Instance: "pve-1"},
|
|
{VMID: 100, Name: "new-container", Instance: "pve-2"}, // newly created, no backup
|
|
})
|
|
|
|
// Add a 3-month-old backup for pve-1's container (VMID 100)
|
|
state.mu.Lock()
|
|
state.PVEBackups.StorageBackups = []StorageBackup{
|
|
{ID: "pve-1-backup-old", VMID: 100, Instance: "pve-1", Time: threeMonthsAgo},
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
// Sync backup times
|
|
state.SyncGuestBackupTimes()
|
|
|
|
snapshot := state.GetSnapshot()
|
|
|
|
// Find both containers
|
|
var oldContainer, newContainer *Container
|
|
for i := range snapshot.Containers {
|
|
if snapshot.Containers[i].Instance == "pve-1" && snapshot.Containers[i].VMID == 100 {
|
|
oldContainer = &snapshot.Containers[i]
|
|
}
|
|
if snapshot.Containers[i].Instance == "pve-2" && snapshot.Containers[i].VMID == 100 {
|
|
newContainer = &snapshot.Containers[i]
|
|
}
|
|
}
|
|
|
|
if oldContainer == nil {
|
|
t.Fatal("pve-1 container not found")
|
|
}
|
|
if newContainer == nil {
|
|
t.Fatal("pve-2 container not found")
|
|
}
|
|
|
|
// The old container on pve-1 SHOULD have the backup time
|
|
if !oldContainer.LastBackup.Equal(threeMonthsAgo) {
|
|
t.Errorf("pve-1 container LastBackup = %v, expected %v", oldContainer.LastBackup, threeMonthsAgo)
|
|
}
|
|
|
|
// The NEW container on pve-2 should NOT have any backup time (it's a different container!)
|
|
if !newContainer.LastBackup.IsZero() {
|
|
t.Errorf("pve-2 container (newly created) should have no backup, got %v", newContainer.LastBackup)
|
|
}
|
|
}
|
|
|
|
// TestSyncGuestBackupTimesNamespaceDisambiguation verifies that PBS namespace is used to
|
|
// disambiguate backups when multiple VMs have the same VMID across different PVE instances.
|
|
// This addresses issue #1095 where users have multiple PVE instances with overlapping VMIDs.
|
|
func TestSyncGuestBackupTimesNamespaceDisambiguation(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
pveBackupTime := now.Add(-1 * time.Hour)
|
|
pveNatBackupTime := now.Add(-2 * time.Hour)
|
|
|
|
// Set up VMs with the SAME VMID on DIFFERENT instances
|
|
state.UpdateVMs([]VM{
|
|
{VMID: 100, Name: "webserver-pve", Instance: "pve", Node: "node1"},
|
|
{VMID: 100, Name: "webserver-nat", Instance: "pve-nat", Node: "node2"},
|
|
})
|
|
|
|
// Add PBS backups with namespaces that correspond to the PVE instances
|
|
state.mu.Lock()
|
|
state.PBSBackups = []PBSBackup{
|
|
{
|
|
ID: "pbs-pve-100",
|
|
VMID: "100",
|
|
Namespace: "pve",
|
|
BackupType: "vm",
|
|
BackupTime: pveBackupTime,
|
|
Instance: "pbs-main",
|
|
},
|
|
{
|
|
ID: "pbs-nat-100",
|
|
VMID: "100",
|
|
Namespace: "nat", // Should match "pve-nat" instance
|
|
BackupType: "vm",
|
|
BackupTime: pveNatBackupTime,
|
|
Instance: "pbs-main",
|
|
},
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
// Sync backup times
|
|
state.SyncGuestBackupTimes()
|
|
|
|
snapshot := state.GetSnapshot()
|
|
|
|
// Find both VMs
|
|
var vmPVE, vmNAT *VM
|
|
for i := range snapshot.VMs {
|
|
if snapshot.VMs[i].Instance == "pve" && snapshot.VMs[i].VMID == 100 {
|
|
vmPVE = &snapshot.VMs[i]
|
|
}
|
|
if snapshot.VMs[i].Instance == "pve-nat" && snapshot.VMs[i].VMID == 100 {
|
|
vmNAT = &snapshot.VMs[i]
|
|
}
|
|
}
|
|
|
|
if vmPVE == nil {
|
|
t.Fatal("pve VM not found")
|
|
}
|
|
if vmNAT == nil {
|
|
t.Fatal("pve-nat VM not found")
|
|
}
|
|
|
|
// The pve VM should have the backup with namespace "pve"
|
|
if !vmPVE.LastBackup.Equal(pveBackupTime) {
|
|
t.Errorf("pve VM LastBackup = %v, expected %v (from namespace 'pve')", vmPVE.LastBackup, pveBackupTime)
|
|
}
|
|
|
|
// The pve-nat VM should have the backup with namespace "nat"
|
|
if !vmNAT.LastBackup.Equal(pveNatBackupTime) {
|
|
t.Errorf("pve-nat VM LastBackup = %v, expected %v (from namespace 'nat')", vmNAT.LastBackup, pveNatBackupTime)
|
|
}
|
|
}
|
|
|
|
// TestSyncGuestBackupTimesClusterEntrypointUsesGuestNodeNamespace verifies that
|
|
// a cluster API entrypoint name does not shadow the guest's actual node namespace.
|
|
func TestSyncGuestBackupTimesClusterEntrypointUsesGuestNodeNamespace(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
oldBackupTime := now.Add(-180 * 24 * time.Hour)
|
|
freshBackupTime := now.Add(-1 * time.Hour)
|
|
|
|
state.UpdateContainers([]Container{
|
|
{VMID: 112, Name: "debian-go", Instance: "delly", Node: "minipc"},
|
|
})
|
|
|
|
state.mu.Lock()
|
|
state.PBSBackups = []PBSBackup{
|
|
{
|
|
ID: "pbs-delly-112-old",
|
|
VMID: "112",
|
|
Namespace: "delly",
|
|
BackupType: "ct",
|
|
Comment: "112",
|
|
BackupTime: oldBackupTime,
|
|
Instance: "pbs-main",
|
|
},
|
|
{
|
|
ID: "pbs-minipc-112-fresh",
|
|
VMID: "112",
|
|
Namespace: "minipc",
|
|
BackupType: "ct",
|
|
Comment: "debian-go",
|
|
BackupTime: freshBackupTime,
|
|
Instance: "pbs-main",
|
|
},
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
state.SyncGuestBackupTimes()
|
|
snapshot := state.GetSnapshot()
|
|
|
|
var found *Container
|
|
for i := range snapshot.Containers {
|
|
if snapshot.Containers[i].VMID == 112 {
|
|
found = &snapshot.Containers[i]
|
|
break
|
|
}
|
|
}
|
|
if found == nil {
|
|
t.Fatal("container 112 not found")
|
|
}
|
|
if !found.LastBackup.Equal(freshBackupTime) {
|
|
t.Errorf("cluster entrypoint guest LastBackup = %v, want fresh node-namespaced backup %v",
|
|
found.LastBackup, freshBackupTime)
|
|
}
|
|
}
|
|
|
|
func TestSyncGuestBackupTimesClearsStaleBackupWhenCurrentEvidenceDisappears(t *testing.T) {
|
|
state := NewState()
|
|
|
|
staleBackupTime := time.Now().Add(-30 * 24 * time.Hour)
|
|
state.UpdateVMs([]VM{
|
|
{VMID: 101, Name: "unbacked-vm", Instance: "pve-1", Node: "node1", LastBackup: staleBackupTime},
|
|
})
|
|
state.UpdateContainers([]Container{
|
|
{VMID: 201, Name: "unbacked-ct", Instance: "pve-1", Node: "node1", LastBackup: staleBackupTime},
|
|
})
|
|
|
|
state.SyncGuestBackupTimes()
|
|
snapshot := state.GetSnapshot()
|
|
|
|
for _, vm := range snapshot.VMs {
|
|
if vm.VMID == 101 && !vm.LastBackup.IsZero() {
|
|
t.Errorf("VM stale LastBackup should be cleared, got %v", vm.LastBackup)
|
|
}
|
|
}
|
|
for _, ct := range snapshot.Containers {
|
|
if ct.VMID == 201 && !ct.LastBackup.IsZero() {
|
|
t.Errorf("container stale LastBackup should be cleared, got %v", ct.LastBackup)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestSyncGuestBackupTimesVMIDCollisionNonMatchingNamespace verifies that when the same VMID
|
|
// exists on multiple PVE instances and a PBS backup namespace matches neither, both guests
|
|
// get zero LastBackup instead of a false positive. The backup's submission
|
|
// source (owner/datastore) carries no learned attribution and no PVE-side
|
|
// storage listing confirms it, so the #1639 evidence paths must not fire.
|
|
func TestSyncGuestBackupTimesVMIDCollisionNonMatchingNamespace(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
backupTime := now.Add(-1 * time.Hour)
|
|
|
|
// Two VMs with the same VMID on different instances
|
|
state.UpdateVMs([]VM{
|
|
{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
|
|
{VMID: 100, Name: "vm-pve2", Instance: "pve2", Node: "node2"},
|
|
})
|
|
|
|
// PBS backup with a namespace that matches neither instance
|
|
state.mu.Lock()
|
|
state.PBSBackups = []PBSBackup{
|
|
{
|
|
ID: "pbs-100",
|
|
VMID: "100",
|
|
Namespace: "staging",
|
|
BackupType: "vm",
|
|
BackupTime: backupTime,
|
|
Instance: "pbs-main",
|
|
Datastore: "backups",
|
|
Owner: "unlearned@pbs!token",
|
|
},
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
state.SyncGuestBackupTimes()
|
|
snapshot := state.GetSnapshot()
|
|
|
|
for _, vm := range snapshot.VMs {
|
|
if vm.VMID == 100 && !vm.LastBackup.IsZero() {
|
|
t.Errorf("VM %q on %s should have zero LastBackup (ambiguous VMID, non-matching namespace), got %v",
|
|
vm.Name, vm.Instance, vm.LastBackup)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestSyncGuestBackupTimesVMIDCollisionEmptyNamespace verifies that when the same VMID
|
|
// exists on multiple PVE instances and a PBS backup has no namespace, both guests
|
|
// get zero LastBackup. This is the genuinely unattributable core of #1639:
|
|
// with a single snapshot, no learnable submission source, and no PVE-side
|
|
// confirmation, guessing either guest would be a false positive. The
|
|
// resolvable variants live in issue1639_pbs_collision_test.go.
|
|
func TestSyncGuestBackupTimesVMIDCollisionEmptyNamespace(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
backupTime := now.Add(-1 * time.Hour)
|
|
|
|
state.UpdateVMs([]VM{
|
|
{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
|
|
{VMID: 100, Name: "vm-pve2", Instance: "pve2", Node: "node2"},
|
|
})
|
|
|
|
// PBS backup with empty namespace
|
|
state.mu.Lock()
|
|
state.PBSBackups = []PBSBackup{
|
|
{
|
|
ID: "pbs-100",
|
|
VMID: "100",
|
|
Namespace: "",
|
|
BackupType: "vm",
|
|
BackupTime: backupTime,
|
|
Instance: "pbs-main",
|
|
},
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
state.SyncGuestBackupTimes()
|
|
snapshot := state.GetSnapshot()
|
|
|
|
for _, vm := range snapshot.VMs {
|
|
if vm.VMID == 100 && !vm.LastBackup.IsZero() {
|
|
t.Errorf("VM %q on %s should have zero LastBackup (ambiguous VMID, empty namespace), got %v",
|
|
vm.Name, vm.Instance, vm.LastBackup)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestSyncGuestBackupTimesUniqueVMIDFallback verifies that a unique VMID still gets
|
|
// the PBS backup via fallback even when the namespace doesn't match.
|
|
func TestSyncGuestBackupTimesUniqueVMIDFallback(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
backupTime := now.Add(-1 * time.Hour)
|
|
|
|
// Single VM — VMID is unique
|
|
state.UpdateVMs([]VM{
|
|
{VMID: 100, Name: "my-vm", Instance: "pve1", Node: "node1"},
|
|
})
|
|
|
|
// PBS backup with a namespace that does NOT match the instance
|
|
state.mu.Lock()
|
|
state.PBSBackups = []PBSBackup{
|
|
{
|
|
ID: "pbs-100",
|
|
VMID: "100",
|
|
Namespace: "daily",
|
|
BackupType: "vm",
|
|
BackupTime: backupTime,
|
|
Instance: "pbs-main",
|
|
},
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
state.SyncGuestBackupTimes()
|
|
snapshot := state.GetSnapshot()
|
|
|
|
var found *VM
|
|
for i := range snapshot.VMs {
|
|
if snapshot.VMs[i].VMID == 100 {
|
|
found = &snapshot.VMs[i]
|
|
}
|
|
}
|
|
if found == nil {
|
|
t.Fatal("VM not found")
|
|
}
|
|
if !found.LastBackup.Equal(backupTime) {
|
|
t.Errorf("unique VMID should fall back to PBS backup; got LastBackup = %v, want %v",
|
|
found.LastBackup, backupTime)
|
|
}
|
|
}
|
|
|
|
// TestSyncGuestBackupTimesVMContainerCollision verifies that PBS vm/ID and ct/ID
|
|
// subjects are matched independently even when a VM and container share the same
|
|
// numeric ID.
|
|
func TestSyncGuestBackupTimesVMContainerCollision(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
backupTime := now.Add(-1 * time.Hour)
|
|
|
|
state.UpdateVMs([]VM{
|
|
{VMID: 100, Name: "vm-pve1", Instance: "pve1", Node: "node1"},
|
|
})
|
|
state.UpdateContainers([]Container{
|
|
{VMID: 100, Name: "ct-pve2", Instance: "pve2", Node: "node2"},
|
|
})
|
|
|
|
state.mu.Lock()
|
|
state.PBSBackups = []PBSBackup{
|
|
{
|
|
ID: "pbs-100",
|
|
VMID: "100",
|
|
Namespace: "node1",
|
|
BackupType: "vm",
|
|
BackupTime: backupTime,
|
|
Instance: "pbs-main",
|
|
},
|
|
}
|
|
state.mu.Unlock()
|
|
|
|
state.SyncGuestBackupTimes()
|
|
snapshot := state.GetSnapshot()
|
|
|
|
for _, vm := range snapshot.VMs {
|
|
if vm.VMID == 100 && !vm.LastBackup.Equal(backupTime) {
|
|
t.Errorf("VM %q LastBackup = %v, want typed vm/100 backup %v",
|
|
vm.Name, vm.LastBackup, backupTime)
|
|
}
|
|
}
|
|
for _, ct := range snapshot.Containers {
|
|
if ct.VMID == 100 && !ct.LastBackup.IsZero() {
|
|
t.Errorf("Container %q should have zero LastBackup because the PBS subject is vm/100, got %v",
|
|
ct.Name, ct.LastBackup)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestUpdateGuestsForInstancePublishesCoherentGeneration(t *testing.T) {
|
|
state := NewState()
|
|
previousBackup := time.Now().Add(-2 * time.Hour).UTC()
|
|
|
|
state.UpdateVMs([]VM{
|
|
{ID: "lab-a:node-a:101", VMID: 101, Instance: "lab-a", Node: "node-a", LastBackup: previousBackup},
|
|
{ID: "lab-b:node-b:201", VMID: 201, Instance: "lab-b", Node: "node-b"},
|
|
})
|
|
state.UpdateContainers([]Container{
|
|
{ID: "lab-a:node-a:102", VMID: 102, Instance: "lab-a", Node: "node-a", LastBackup: previousBackup},
|
|
{ID: "lab-b:node-b:202", VMID: 202, Instance: "lab-b", Node: "node-b"},
|
|
})
|
|
before := state.GetSnapshot().LastUpdate
|
|
|
|
state.UpdateGuestsForInstance(
|
|
"lab-a",
|
|
[]VM{{ID: "lab-a:node-a:101", VMID: 101, Instance: "lab-a", Node: "node-a", Status: "running"}},
|
|
[]Container{{ID: "lab-a:node-a:103", VMID: 103, Instance: "lab-a", Node: "node-a", Status: "running"}},
|
|
)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if !snapshot.LastUpdate.After(before) {
|
|
t.Fatalf("LastUpdate did not advance: before=%v after=%v", before, snapshot.LastUpdate)
|
|
}
|
|
if len(snapshot.VMs) != 2 || len(snapshot.Containers) != 2 {
|
|
t.Fatalf("unexpected coherent generation sizes: vms=%d containers=%d", len(snapshot.VMs), len(snapshot.Containers))
|
|
}
|
|
if snapshot.VMs[0].ID != "lab-a:node-a:101" || !snapshot.VMs[0].LastBackup.Equal(previousBackup) {
|
|
t.Fatalf("updated VM did not retain backup state: %+v", snapshot.VMs[0])
|
|
}
|
|
if snapshot.Containers[0].ID != "lab-a:node-a:103" {
|
|
t.Fatalf("authoritatively deleted container remained in state: %+v", snapshot.Containers)
|
|
}
|
|
if snapshot.VMs[1].Instance != "lab-b" || snapshot.Containers[1].Instance != "lab-b" {
|
|
t.Fatalf("other instance was not isolated: vms=%+v containers=%+v", snapshot.VMs, snapshot.Containers)
|
|
}
|
|
}
|
|
|
|
func TestUpdateStorageBackupsForInstance(t *testing.T) {
|
|
state := NewState()
|
|
|
|
now := time.Now()
|
|
|
|
// Set up a VM so node normalization has something to work with
|
|
state.UpdateVMsForInstance("pve-1", []VM{
|
|
{VMID: 100, Instance: "pve-1", Node: "node1"},
|
|
})
|
|
|
|
// Add backups from first instance
|
|
backups1 := []StorageBackup{
|
|
{ID: "pve-1-backup-1", Instance: "pve-1", VMID: 100, Time: now, Node: "node1"},
|
|
{ID: "pve-1-backup-2", Instance: "pve-1", VMID: 100, Time: now.Add(-time.Hour), Node: "node1"},
|
|
}
|
|
state.UpdateStorageBackupsForInstance("pve-1", backups1)
|
|
|
|
snapshot := state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.StorageBackups) != 2 {
|
|
t.Fatalf("Expected 2 backups, got %d", len(snapshot.PVEBackups.StorageBackups))
|
|
}
|
|
|
|
// Backups should be sorted by time descending
|
|
if snapshot.PVEBackups.StorageBackups[0].Time.Before(snapshot.PVEBackups.StorageBackups[1].Time) {
|
|
t.Error("Backups should be sorted by time descending")
|
|
}
|
|
|
|
// Add backups from second instance
|
|
backups2 := []StorageBackup{
|
|
{ID: "pve-2-backup-1", Instance: "pve-2", VMID: 200, Time: now.Add(-30 * time.Minute), Node: "node2"},
|
|
}
|
|
state.UpdateStorageBackupsForInstance("pve-2", backups2)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.StorageBackups) != 3 {
|
|
t.Fatalf("Expected 3 backups, got %d", len(snapshot.PVEBackups.StorageBackups))
|
|
}
|
|
|
|
// Update first instance (should replace its backups)
|
|
backups1Updated := []StorageBackup{
|
|
{ID: "pve-1-backup-3", Instance: "pve-1", VMID: 100, Time: now.Add(time.Hour), Node: "node1"},
|
|
}
|
|
state.UpdateStorageBackupsForInstance("pve-1", backups1Updated)
|
|
|
|
snapshot = state.GetSnapshot()
|
|
if len(snapshot.PVEBackups.StorageBackups) != 2 {
|
|
t.Fatalf("Expected 2 backups after update, got %d", len(snapshot.PVEBackups.StorageBackups))
|
|
}
|
|
|
|
// Verify old pve-1 backups were replaced
|
|
for _, b := range snapshot.PVEBackups.StorageBackups {
|
|
if b.ID == "pve-1-backup-1" || b.ID == "pve-1-backup-2" {
|
|
t.Error("Old pve-1 backups should have been replaced")
|
|
}
|
|
}
|
|
}
|
|
|
|
// Host.IntegrationSource is fabric-only provenance: it must survive host
|
|
// upserts, serialize when set, and stay entirely absent from JSON for real
|
|
// agent-report hosts so existing payloads do not change shape.
|
|
func TestHostIntegrationSourceRoundTrip(t *testing.T) {
|
|
state := NewState()
|
|
state.UpsertHost(Host{ID: "host-esxi", Hostname: "esxi-01", IntegrationSource: "vmware"})
|
|
state.UpsertHost(Host{ID: "host-agent", Hostname: "apollo-114"})
|
|
|
|
var esxi, agent *Host
|
|
hosts := state.GetHosts()
|
|
for i := range hosts {
|
|
switch hosts[i].ID {
|
|
case "host-esxi":
|
|
esxi = &hosts[i]
|
|
case "host-agent":
|
|
agent = &hosts[i]
|
|
}
|
|
}
|
|
if esxi == nil || esxi.IntegrationSource != "vmware" {
|
|
t.Fatalf("expected upserted host to preserve IntegrationSource %q, got %+v", "vmware", esxi)
|
|
}
|
|
if agent == nil || agent.IntegrationSource != "" {
|
|
t.Fatalf("expected agent host to keep empty IntegrationSource, got %+v", agent)
|
|
}
|
|
|
|
marked, err := json.Marshal(esxi)
|
|
if err != nil {
|
|
t.Fatalf("marshal integration host: %v", err)
|
|
}
|
|
if !strings.Contains(string(marked), `"integrationSource":"vmware"`) {
|
|
t.Fatalf("expected integrationSource in payload, got %s", marked)
|
|
}
|
|
|
|
unmarked, err := json.Marshal(agent)
|
|
if err != nil {
|
|
t.Fatalf("marshal agent host: %v", err)
|
|
}
|
|
if strings.Contains(string(unmarked), "integrationSource") {
|
|
t.Fatalf("expected integrationSource omitted for agent hosts, got %s", unmarked)
|
|
}
|
|
}
|
|
|
|
// Two clusters reusing the same node names (pve01/pve02 on different subnets)
|
|
// must stay apart: the second cluster's node must not steal the first
|
|
// cluster's agent link by bare hostname, and the shared name must not merge
|
|
// the two nodes into one slot.
|
|
func TestUpdateNodesForInstanceKeepsClustersWithSharedNodeNamesApart(t *testing.T) {
|
|
state := &State{
|
|
Hosts: []Host{
|
|
{
|
|
ID: "host-enacon-pve01",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.168.16.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
|
|
},
|
|
},
|
|
{
|
|
ID: "host-enacon-pve02",
|
|
Hostname: "pve02",
|
|
ReportIP: "192.168.16.12",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.16.12/24"}},
|
|
},
|
|
},
|
|
},
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.16.11:8006",
|
|
LinkedAgentID: "host-enacon-pve01",
|
|
Status: "online",
|
|
},
|
|
{
|
|
ID: "enacon-pve02",
|
|
Name: "pve02",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.16.12:8006",
|
|
LinkedAgentID: "host-enacon-pve02",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("rewo", []Node{
|
|
{
|
|
ID: "rewo-pve01",
|
|
Name: "pve01",
|
|
Instance: "rewo",
|
|
ClusterName: "rewo",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 3 {
|
|
t.Fatalf("nodes = %#v, want 3 (enacon pve01+pve02 and rewo pve01)", state.Nodes)
|
|
}
|
|
|
|
byID := make(map[string]Node)
|
|
for _, node := range state.Nodes {
|
|
byID[node.ID] = node
|
|
}
|
|
|
|
enacon, ok := byID["enacon-pve01"]
|
|
if !ok {
|
|
t.Fatalf("enacon-pve01 was clobbered, nodes = %#v", state.Nodes)
|
|
}
|
|
if enacon.ClusterName != "enacon" {
|
|
t.Fatalf("enacon-pve01 ClusterName = %q, want enacon", enacon.ClusterName)
|
|
}
|
|
if enacon.LinkedAgentID != "host-enacon-pve01" {
|
|
t.Fatalf("enacon-pve01 LinkedAgentID = %q, want host-enacon-pve01", enacon.LinkedAgentID)
|
|
}
|
|
|
|
rewo, ok := byID["rewo-pve01"]
|
|
if !ok {
|
|
t.Fatalf("rewo-pve01 missing, nodes = %#v", state.Nodes)
|
|
}
|
|
if rewo.ClusterName != "rewo" {
|
|
t.Fatalf("rewo-pve01 ClusterName = %q, want rewo", rewo.ClusterName)
|
|
}
|
|
if rewo.LinkedAgentID != "" {
|
|
t.Fatalf("rewo-pve01 LinkedAgentID = %q, want none (its host runs no agent)", rewo.LinkedAgentID)
|
|
}
|
|
}
|
|
|
|
// When TLS settings degrade node endpoints to the bare node name, the
|
|
// endpoint-host merge alias is identical for both clusters. The alias must
|
|
// not merge nodes whose named clusters contradict each other.
|
|
func TestUpdateNodesForInstanceEndpointHostAliasDoesNotMergeAcrossClusters(t *testing.T) {
|
|
state := &State{
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://pve01:8006",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("rewo", []Node{
|
|
{
|
|
ID: "rewo-pve01",
|
|
Name: "pve01",
|
|
Instance: "rewo",
|
|
ClusterName: "rewo",
|
|
IsClusterMember: true,
|
|
Host: "https://pve01:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 2 {
|
|
t.Fatalf("nodes = %#v, want 2 (one per cluster)", state.Nodes)
|
|
}
|
|
clusters := map[string]bool{}
|
|
for _, node := range state.Nodes {
|
|
clusters[node.ClusterName] = true
|
|
}
|
|
if !clusters["enacon"] || !clusters["rewo"] {
|
|
t.Fatalf("clusters = %#v, want both enacon and rewo", clusters)
|
|
}
|
|
}
|
|
|
|
// A standalone node whose endpoint IP is absent from a same-named agent's
|
|
// reported IPs must not link to that agent: the IP evidence contradicts the
|
|
// hostname match.
|
|
func TestUpdateNodesForInstanceHostnameMatchRejectedWhenIPContradicts(t *testing.T) {
|
|
state := &State{
|
|
Hosts: []Host{
|
|
{
|
|
ID: "host-a",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.168.16.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("steinboeck", []Node{
|
|
{
|
|
ID: "steinboeck-pve01",
|
|
Name: "pve01",
|
|
Instance: "steinboeck",
|
|
Host: "https://10.20.1.5:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 1 {
|
|
t.Fatalf("nodes = %#v, want 1", state.Nodes)
|
|
}
|
|
if state.Nodes[0].LinkedAgentID != "" {
|
|
t.Fatalf("LinkedAgentID = %q, want none (agent IPs contradict node endpoint)", state.Nodes[0].LinkedAgentID)
|
|
}
|
|
}
|
|
|
|
// Two different clusters at different sites can reuse the same corosync
|
|
// cluster name, the same member hostnames, and the same RFC1918 addressing
|
|
// (MSP support case: two "enacon" clusters both exposing pve01 on
|
|
// 192.168.1.11). The colliding endpoint-IP alias must not merge the two node
|
|
// slots, and the address match must not bind the second site's node to the
|
|
// first site's host agent when the TLS fingerprint evidence contradicts.
|
|
func TestUpdateNodesForInstanceSameNamedClustersWithCollidingIPsStayApart(t *testing.T) {
|
|
state := &State{
|
|
Hosts: []Host{
|
|
{
|
|
ID: "host-site-a-pve01",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.168.1.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
|
|
},
|
|
},
|
|
},
|
|
Nodes: []Node{
|
|
{
|
|
ID: "site-a-pve01",
|
|
Name: "pve01",
|
|
Instance: "site-a",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
TLSFingerprint: "AA:AA:AA:AA",
|
|
LinkedAgentID: "host-site-a-pve01",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("site-b", []Node{
|
|
{
|
|
ID: "site-b-pve01",
|
|
Name: "pve01",
|
|
Instance: "site-b",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
TLSFingerprint: "BB:BB:BB:BB",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 2 {
|
|
t.Fatalf("nodes = %#v, want 2 (one per site)", state.Nodes)
|
|
}
|
|
byID := make(map[string]Node)
|
|
for _, node := range state.Nodes {
|
|
byID[node.ID] = node
|
|
}
|
|
siteA, ok := byID["site-a-pve01"]
|
|
if !ok {
|
|
t.Fatalf("site-a-pve01 was clobbered, nodes = %#v", state.Nodes)
|
|
}
|
|
if siteA.LinkedAgentID != "host-site-a-pve01" {
|
|
t.Fatalf("site-a-pve01 LinkedAgentID = %q, want host-site-a-pve01", siteA.LinkedAgentID)
|
|
}
|
|
siteB, ok := byID["site-b-pve01"]
|
|
if !ok {
|
|
t.Fatalf("site-b-pve01 missing, nodes = %#v", state.Nodes)
|
|
}
|
|
if siteB.LinkedAgentID != "" {
|
|
t.Fatalf("site-b-pve01 LinkedAgentID = %q, want none (fingerprint contradicts the colliding-IP agent match)", siteB.LinkedAgentID)
|
|
}
|
|
}
|
|
|
|
// The endpoint-IP agent match must also respect the cluster-identity
|
|
// contradiction when no fingerprints are known: an agent whose linked nodes
|
|
// live in cluster "enacon" must not bind to a "rewo" node that presents the
|
|
// same RFC1918 address from another site.
|
|
func TestUpdateNodesForInstanceEndpointIPMatchRejectedWhenClusterContradicts(t *testing.T) {
|
|
state := &State{
|
|
Hosts: []Host{
|
|
{
|
|
ID: "host-enacon-pve01",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.168.1.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
|
|
},
|
|
},
|
|
},
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
LinkedAgentID: "host-enacon-pve01",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("rewo", []Node{
|
|
{
|
|
ID: "rewo-pve01",
|
|
Name: "pve01",
|
|
Instance: "rewo",
|
|
ClusterName: "rewo",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 2 {
|
|
t.Fatalf("nodes = %#v, want 2 (one per cluster)", state.Nodes)
|
|
}
|
|
for _, node := range state.Nodes {
|
|
if node.ClusterName == "rewo" && node.LinkedAgentID != "" {
|
|
t.Fatalf("rewo pve01 LinkedAgentID = %q, want none (agent belongs to the enacon cluster)", node.LinkedAgentID)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The same cluster added twice through different connection instances is the
|
|
// legitimate duplicate: when both views carry the same TOFU-captured TLS
|
|
// fingerprint for the same-named node, the endpoint alias must still fold
|
|
// them into one slot even though the instances differ.
|
|
func TestUpdateNodesForInstanceSameClusterAddedTwiceMergesWithMatchingFingerprints(t *testing.T) {
|
|
state := &State{
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-a-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon-a",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
TLSFingerprint: "AA:AA:AA:AA",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("enacon-b", []Node{
|
|
{
|
|
ID: "enacon-b-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon-b",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
TLSFingerprint: "aaaaaaaa",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 1 {
|
|
t.Fatalf("nodes = %#v, want 1 (matching fingerprints prove one machine)", state.Nodes)
|
|
}
|
|
if state.Nodes[0].Name != "pve01" {
|
|
t.Fatalf("merged node = %#v, want pve01", state.Nodes[0])
|
|
}
|
|
}
|
|
|
|
// Without fingerprint evidence the same-named cluster from another instance
|
|
// stays split: unknown identity must never fold two possibly-distinct
|
|
// clusters into one (fail-safe direction).
|
|
func TestUpdateNodesForInstanceSameNamedClustersStayApartWithoutFingerprints(t *testing.T) {
|
|
state := &State{
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-a-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon-a",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("enacon-b", []Node{
|
|
{
|
|
ID: "enacon-b-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon-b",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 2 {
|
|
t.Fatalf("nodes = %#v, want 2 (unknown identity must not merge)", state.Nodes)
|
|
}
|
|
}
|
|
|
|
// A shared agent link is the strongest cross-view merge signal, but for
|
|
// same-named clusters from different instances it must still be gated on
|
|
// fingerprint identity: matching fingerprints merge the duplicate views onto
|
|
// one slot, contradicting fingerprints never do.
|
|
func TestUpdateNodesForInstanceSharedAgentLinkMergeGatedOnFingerprint(t *testing.T) {
|
|
makeState := func(existingFP string) *State {
|
|
return &State{
|
|
Hosts: []Host{
|
|
{
|
|
ID: "host-pve01",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.168.1.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
|
|
},
|
|
},
|
|
},
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-a-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon-a",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://pve01.site-a.internal:8006",
|
|
TLSFingerprint: existingFP,
|
|
LinkedAgentID: "host-pve01",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
}
|
|
incoming := func(fp string) []Node {
|
|
return []Node{
|
|
{
|
|
ID: "enacon-b-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon-b",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.1.11:8006",
|
|
TLSFingerprint: fp,
|
|
LinkedAgentID: "host-pve01",
|
|
Status: "online",
|
|
},
|
|
}
|
|
}
|
|
|
|
merged := makeState("AA:AA:AA:AA")
|
|
merged.UpdateNodesForInstance("enacon-b", incoming("AA:AA:AA:AA"))
|
|
if len(merged.Nodes) != 1 {
|
|
t.Fatalf("matching fingerprints: nodes = %#v, want 1", merged.Nodes)
|
|
}
|
|
|
|
split := makeState("AA:AA:AA:AA")
|
|
split.UpdateNodesForInstance("enacon-b", incoming("BB:BB:BB:BB"))
|
|
if len(split.Nodes) != 2 {
|
|
t.Fatalf("contradicting fingerprints: nodes = %#v, want 2", split.Nodes)
|
|
}
|
|
}
|
|
|
|
// A cluster connection's first polls can commit nodes before membership
|
|
// detection classifies them (ClusterName ""), and cloned template
|
|
// deployments reuse /etc/machine-id, collapsing two sites' agents into one
|
|
// identity (MSP support case: clusters enacon and rewo, both with pve01, on
|
|
// reused RFC1918 ranges). During that window the shared agent identity must
|
|
// not fold the unclassified node into the established cluster's slot.
|
|
func TestUpdateNodesForInstanceUnclassifiedNodeSharedAgentMustNotStealClusterSlot(t *testing.T) {
|
|
state := &State{
|
|
Hosts: []Host{
|
|
{
|
|
// One host row standing in for two physical pve01 machines
|
|
// whose cloned machine-id collapsed them into one agent
|
|
// identity; it currently reports the second site's address.
|
|
ID: "host-cloned-machine-id",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.168.1.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.1.11/24"}},
|
|
},
|
|
},
|
|
},
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.16.11:8006",
|
|
LinkedAgentID: "host-cloned-machine-id",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("rewo", []Node{
|
|
{
|
|
ID: "rewo-pve01",
|
|
Name: "pve01",
|
|
Instance: "rewo",
|
|
// Membership detection has not classified this connection yet.
|
|
ClusterName: "",
|
|
Host: "https://192.168.1.11:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 2 {
|
|
t.Fatalf("nodes = %#v, want 2 (unclassified node must not fold into the enacon slot)", state.Nodes)
|
|
}
|
|
byID := make(map[string]Node)
|
|
for _, node := range state.Nodes {
|
|
byID[node.ID] = node
|
|
}
|
|
enacon, ok := byID["enacon-pve01"]
|
|
if !ok {
|
|
t.Fatalf("enacon-pve01 was clobbered, nodes = %#v", state.Nodes)
|
|
}
|
|
if enacon.ClusterName != "enacon" {
|
|
t.Fatalf("enacon-pve01 ClusterName = %q, want enacon", enacon.ClusterName)
|
|
}
|
|
if _, ok := byID["rewo-pve01"]; !ok {
|
|
t.Fatalf("rewo-pve01 missing, nodes = %#v", state.Nodes)
|
|
}
|
|
}
|
|
|
|
// When TLS settings degrade both connections' node endpoints to the bare
|
|
// node name, the endpoint-host alias is identical across sites. An
|
|
// unclassified node (empty cluster name, e.g. during the first-poll
|
|
// membership window) must not ride that weak alias into another instance's
|
|
// established cluster slot.
|
|
func TestUpdateNodesForInstanceEndpointHostAliasRequiresProofForUnclassifiedNodes(t *testing.T) {
|
|
state := &State{
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://pve01:8006",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
|
|
state.UpdateNodesForInstance("rewo", []Node{
|
|
{
|
|
ID: "rewo-pve01",
|
|
Name: "pve01",
|
|
Instance: "rewo",
|
|
ClusterName: "",
|
|
Host: "https://pve01:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
|
|
if len(state.Nodes) != 2 {
|
|
t.Fatalf("nodes = %#v, want 2 (bare-hostname alias must not fold an unclassified cross-instance node)", state.Nodes)
|
|
}
|
|
}
|
|
|
|
// The legitimate standalone-into-cluster fold must survive the weak-evidence
|
|
// tightening: a standalone connection view dialing the same address as the
|
|
// cluster's endpoint record still folds via the address alias, and a
|
|
// fingerprint-proven view still folds via the shared agent link.
|
|
func TestUpdateNodesForInstanceStandaloneViewStillFoldsIntoClusterNode(t *testing.T) {
|
|
byAddress := &State{
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://192.168.16.11:8006",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
byAddress.UpdateNodesForInstance("pve01-direct", []Node{
|
|
{
|
|
ID: "direct-pve01",
|
|
Name: "pve01",
|
|
Instance: "pve01-direct",
|
|
Host: "https://192.168.16.11:8006",
|
|
Status: "online",
|
|
},
|
|
})
|
|
if len(byAddress.Nodes) != 1 {
|
|
t.Fatalf("address alias: nodes = %#v, want 1 (same-address standalone view folds)", byAddress.Nodes)
|
|
}
|
|
|
|
byAgent := &State{
|
|
Hosts: []Host{
|
|
{
|
|
ID: "host-pve01",
|
|
Hostname: "pve01",
|
|
ReportIP: "192.168.16.11",
|
|
NetworkInterfaces: []HostNetworkInterface{
|
|
{Name: "eth0", Addresses: []string{"192.168.16.11/24"}},
|
|
},
|
|
},
|
|
},
|
|
Nodes: []Node{
|
|
{
|
|
ID: "enacon-pve01",
|
|
Name: "pve01",
|
|
Instance: "enacon",
|
|
ClusterName: "enacon",
|
|
IsClusterMember: true,
|
|
Host: "https://pve01:8006",
|
|
TLSFingerprint: "AA:AA:AA:AA",
|
|
LinkedAgentID: "host-pve01",
|
|
Status: "online",
|
|
},
|
|
},
|
|
}
|
|
byAgent.UpdateNodesForInstance("pve01-direct", []Node{
|
|
{
|
|
ID: "direct-pve01",
|
|
Name: "pve01",
|
|
Instance: "pve01-direct",
|
|
Host: "https://192.168.16.11:8006",
|
|
TLSFingerprint: "aaaaaaaa",
|
|
Status: "online",
|
|
},
|
|
})
|
|
if len(byAgent.Nodes) != 1 {
|
|
t.Fatalf("agent link: nodes = %#v, want 1 (fingerprint-proven view folds via shared agent)", byAgent.Nodes)
|
|
}
|
|
}
|