Fix PBS backup identity matching

This commit is contained in:
rcourtman
2026-06-03 11:43:20 +01:00
parent 458ffd351b
commit eab73d2453
16 changed files with 636 additions and 236 deletions
@@ -93,7 +93,7 @@ operator-facing alert routing behavior for live runtime alerts.
## Shared Boundaries
1. None.
1. `internal/proxmoxidentity/backup_identity.go` shared with `monitoring`, `storage-recovery`: Proxmox PBS backup subject identity is a shared runtime boundary for monitoring backup freshness, backup-age alert attribution, and recovery-point guest mapping.
## Extension Points
@@ -76,7 +76,7 @@ truth for live infrastructure data.
## Shared Boundaries
1. None.
1. `internal/proxmoxidentity/backup_identity.go` shared with `alerts`, `storage-recovery`: Proxmox PBS backup subject identity is a shared runtime boundary for monitoring backup freshness, backup-age alert attribution, and recovery-point guest mapping.
## Extension Points
@@ -658,6 +658,15 @@
"deployment-installability"
]
},
{
"path": "internal/proxmoxidentity/backup_identity.go",
"rationale": "Proxmox PBS backup subject identity is a shared runtime boundary for monitoring backup freshness, backup-age alert attribution, and recovery-point guest mapping",
"subsystems": [
"alerts",
"monitoring",
"storage-recovery"
]
},
{
"path": "scripts/install.ps1",
"rationale": "the Windows installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary",
@@ -1265,7 +1274,8 @@
"frontend-modern/src/utils/alertThresholdsPresentation.ts",
"frontend-modern/src/utils/alertThresholdsSectionPresentation.ts",
"frontend-modern/src/utils/alertWebhookPresentation.ts",
"frontend-modern/src/utils/metricThresholds.ts"
"frontend-modern/src/utils/metricThresholds.ts",
"internal/proxmoxidentity/backup_identity.go"
],
"verification": {
"allow_same_subsystem_tests": true,
@@ -1497,6 +1507,20 @@
"internal/alerts/threshold_resolution_shared_test.go",
"internal/alerts/update_alerts_test.go"
]
},
{
"id": "proxmox-backup-identity-alerts",
"label": "Proxmox backup identity alert proof",
"match_prefixes": [],
"match_files": [
"internal/proxmoxidentity/backup_identity.go"
],
"allow_same_subsystem_tests": false,
"test_prefixes": [],
"exact_files": [
"internal/alerts/alerts_test.go",
"internal/proxmoxidentity/backup_identity_test.go"
]
}
],
"match_files": null
@@ -3572,6 +3596,7 @@
"internal/models/deepcopy.go",
"internal/models/models.go",
"internal/models/models_frontend.go",
"internal/proxmoxidentity/backup_identity.go",
"pkg/agents/docker/report.go",
"pkg/agents/kubernetes/report.go",
"pkg/proxmox/ceph.go",
@@ -3775,6 +3800,7 @@
"test_prefixes": [],
"exact_files": [
"internal/dockeragent/swarm_coverage_test.go",
"internal/models/state_host_test.go",
"internal/monitoring/monitor_host_agents_test.go",
"internal/unifiedresources/adapter_coverage_test.go",
"internal/unifiedresources/registry_test.go"
@@ -3808,6 +3834,20 @@
"pkg/proxmox/cluster_client_api_test.go"
]
},
{
"id": "proxmox-backup-identity-monitoring",
"label": "Proxmox backup identity monitoring proof",
"match_prefixes": [],
"match_files": [
"internal/proxmoxidentity/backup_identity.go"
],
"allow_same_subsystem_tests": false,
"test_prefixes": [],
"exact_files": [
"internal/models/state_host_test.go",
"internal/proxmoxidentity/backup_identity_test.go"
]
},
{
"id": "container-entrypoint-runtime",
"label": "container entrypoint runtime proof",
@@ -3852,6 +3892,7 @@
"test_prefixes": [],
"exact_files": [
"internal/monitoring/canonical_guardrails_test.go",
"internal/monitoring/monitor_backups_readstate_test.go",
"internal/monitoring/monitor_host_agents_test.go",
"internal/unifiedresources/code_standards_test.go"
]
@@ -4892,7 +4933,8 @@
"frontend-modern/src/utils/recoveryOutcomePresentation.ts",
"frontend-modern/src/utils/recoveryTimelineChartPresentation.ts",
"frontend-modern/src/utils/recoveryTimelinePresentation.ts",
"internal/api/setup_script_render.go"
"internal/api/setup_script_render.go",
"internal/proxmoxidentity/backup_identity.go"
],
"verification": {
"allow_same_subsystem_tests": false,
@@ -4915,6 +4957,7 @@
"exact_files": [
"internal/recovery/idgen/idgen_test.go",
"internal/recovery/manager/manager_test.go",
"internal/recovery/mapper/proxmox/mapper_test.go",
"internal/recovery/recovery_test.go",
"internal/recovery/store/store_queryplan_test.go",
"internal/recovery/store/store_rollups_test.go",
@@ -5003,6 +5046,20 @@
"exact_files": [
"internal/api/contract_test.go"
]
},
{
"id": "proxmox-backup-identity-recovery",
"label": "Proxmox backup identity recovery proof",
"match_prefixes": [],
"match_files": [
"internal/proxmoxidentity/backup_identity.go"
],
"allow_same_subsystem_tests": false,
"test_prefixes": [],
"exact_files": [
"internal/proxmoxidentity/backup_identity_test.go",
"internal/recovery/mapper/proxmox/mapper_test.go"
]
}
],
"match_files": null
@@ -55,6 +55,7 @@ state.
## Shared Boundaries
1. `internal/api/setup_script_render.go` shared with `agent-lifecycle`, `api-contracts`: the generated Proxmox setup-script is a shared boundary across agent lifecycle (forced-command keys, install/uninstall edits), API contracts (rendered token shape and encoded rerun URL), and storage/recovery (backup visibility grants, Pulse-managed temperature SSH keys, and SMART disk-temperature collection).
2. `internal/proxmoxidentity/backup_identity.go` shared with `alerts`, `monitoring`: Proxmox PBS backup subject identity is a shared runtime boundary for monitoring backup freshness, backup-age alert attribution, and recovery-point guest mapping.
## Extension Points
+90 -10
View File
@@ -14,6 +14,7 @@ import (
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/proxmoxidentity"
"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
@@ -1608,7 +1609,7 @@ func TestCheckBackupsDisambiguatesWithNamespace(t *testing.T) {
RollupID: "ext:pbs-nat-100",
SubjectRef: &recovery.ExternalRef{
Type: "proxmox-vm",
Namespace: "nat", // namespaceMatchesInstance("nat", "pve-nat") should match
Namespace: "nat", // namespace "nat" should match the pve-nat location
Name: "100",
ID: "100",
},
@@ -1729,6 +1730,85 @@ func TestCheckBackupsVMIDCollisionNonMatchingNamespace(t *testing.T) {
}
}
func TestCheckBackupsDisambiguatesClusterEntrypointByNodeNamespace(t *testing.T) {
m := newTestManager(t)
m.ClearActiveAlerts()
m.mu.Lock()
m.config.Enabled = true
m.config.BackupDefaults = BackupAlertConfig{
Enabled: true,
WarningDays: 3,
CriticalDays: 5,
}
m.mu.Unlock()
now := time.Now()
guestsByKey := map[string]GuestLookup{
"delly-delly-112": {
ResourceID: "lxc/112",
Name: "old-debian-go",
Instance: "delly",
Node: "delly",
Type: "lxc",
VMID: 112,
},
"delly-minipc-112": {
ResourceID: "lxc/112",
Name: "debian-go",
Instance: "delly",
Node: "minipc",
Type: "lxc",
VMID: 112,
},
}
guestsByVMID := map[string][]GuestLookup{
"112": {
guestsByKey["delly-delly-112"],
guestsByKey["delly-minipc-112"],
},
}
rollups := []recovery.ProtectionRollup{
{
RollupID: "ext:pbs-minipc-112",
SubjectRef: &recovery.ExternalRef{
Type: "proxmox-lxc",
Namespace: "minipc",
Name: "debian-go",
ID: "112",
},
LastSuccessAt: ptrTime(now.Add(-6 * 24 * time.Hour)),
LastOutcome: recovery.OutcomeSuccess,
Providers: []recovery.Provider{recovery.ProviderProxmoxPBS},
},
}
m.CheckBackups(rollups, guestsByKey, guestsByVMID)
m.mu.RLock()
defer m.mu.RUnlock()
alert, exists := testLookupActiveAlert(t, m, "backup-age-delly-minipc-112")
if !exists {
var keys []string
for storageKey, active := range m.activeAlerts {
keys = append(keys, effectiveAlertID(active, storageKey))
}
t.Fatalf("expected minipc backup alert; found keys: %v", keys)
}
if alert.ResourceName != "debian-go backup" {
t.Errorf("ResourceName = %q, want %q", alert.ResourceName, "debian-go backup")
}
if alert.Instance != "delly" {
t.Errorf("Instance = %q, want %q", alert.Instance, "delly")
}
if alert.Node != "minipc" {
t.Errorf("Node = %q, want %q", alert.Node, "minipc")
}
}
// TestCheckBackupsVMIDCollisionNoNamespace verifies that when multiple guests
// share a VMID and the PBS backup has no namespace, the alert uses the generic PBS key.
func TestCheckBackupsVMIDCollisionNoNamespace(t *testing.T) {
@@ -18346,22 +18426,22 @@ func TestLoadActiveAlertsHardensExistingFilePermissions(t *testing.T) {
}
}
func TestNamespaceMatchesInstance(t *testing.T) {
func TestNamespaceMatchesLocation(t *testing.T) {
tests := []struct {
name string
namespace string
instance string
location string
expected bool
}{
// Exact matches
{"exact match", "pve", "pve", true},
{"exact match with numbers", "pve1", "pve1", true},
// Suffix matches (namespace is suffix of instance)
{"namespace suffix of instance", "nat", "pve-nat", true},
{"namespace suffix of instance no dash", "nat", "pvenat", true},
// Suffix matches (namespace is suffix of location)
{"namespace suffix of location", "nat", "pve-nat", true},
{"namespace suffix of location no dash", "nat", "pvenat", true},
// Suffix matches (instance is suffix of namespace)
// Suffix matches (location is suffix of namespace)
{"instance suffix of namespace", "pvebackups", "pve", false}, // "pve" is not suffix of "pvebackups"
{"instance suffix of namespace 2", "backupspve", "pve", true}, // "pve" IS suffix of "backupspve"
@@ -18396,10 +18476,10 @@ func TestNamespaceMatchesInstance(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := namespaceMatchesInstance(tt.namespace, tt.instance)
result := proxmoxidentity.NamespaceMatchesLocation(tt.namespace, tt.location)
if result != tt.expected {
t.Errorf("namespaceMatchesInstance(%q, %q) = %v, want %v",
tt.namespace, tt.instance, result, tt.expected)
t.Errorf("NamespaceMatchesLocation(%q, %q) = %v, want %v",
tt.namespace, tt.location, result, tt.expected)
}
})
}
+19 -51
View File
@@ -9,6 +9,7 @@ import (
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/proxmoxidentity"
"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog/log"
@@ -612,12 +613,28 @@ func (m *Manager) CheckBackupsWithInventory(
if len(guests) == 1 {
info = guests[0]
} else if len(guests) > 1 && strings.TrimSpace(ref.Namespace) != "" {
bestScore := 0
matchedMultiple := false
for _, g := range guests {
if namespaceMatchesInstance(ref.Namespace, g.Instance) {
score := proxmoxidentity.BackupGuestMatchScore(
ref.Namespace,
ref.Name,
vmidStr,
g.Name,
g.Instance,
g.Node,
)
if score > bestScore {
bestScore = score
info = g
break
matchedMultiple = false
} else if score > 0 && score == bestScore {
matchedMultiple = true
}
}
if matchedMultiple {
info = GuestLookup{}
}
}
if info.Instance != "" && info.Node != "" {
key = BuildGuestKey(info.Instance, info.Node, info.VMID)
@@ -891,55 +908,6 @@ func parseGuestID(raw string) (instance string, node string, vmid int, ok bool)
return strings.TrimSpace(inst), strings.TrimSpace(prev), n, true
}
// namespaceMatchesInstance checks if a PBS namespace likely corresponds to a PVE instance.
// This helps disambiguate backups when multiple PVE instances have VMs with the same VMID.
// Examples: namespace "pve1" matches instance "pve1", namespace "nat" matches instance "pve-nat"
func namespaceMatchesInstance(namespace, instance string) bool {
if namespace == "" || instance == "" {
return false
}
// Normalize both strings: lowercase and keep only alphanumeric
normalize := func(s string) string {
var b strings.Builder
for _, r := range strings.ToLower(s) {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
b.WriteRune(r)
}
}
return b.String()
}
ns := normalize(namespace)
inst := normalize(instance)
if ns == "" || inst == "" {
return false
}
// Exact match after normalization
if ns == inst {
return true
}
// Check if namespace is a suffix of instance
// e.g., namespace "nat" matches instance "pvenat" (normalized from "pve-nat")
// This is more precise than substring matching because:
// - "nat" should match "pve-nat" but not "natpve"
// - "pve" should match "pve" but not "pve-nat" (handled by exact match above)
if strings.HasSuffix(inst, ns) {
return true
}
// Check if instance is a suffix of namespace (reverse case)
// e.g., namespace "pvebackups" could match instance "pve"
if strings.HasSuffix(ns, inst) {
return true
}
return false
}
func (m *Manager) clearSnapshotAlertsForInstance(instance string) {
m.mu.Lock()
m.clearSnapshotAlertsForInstanceLocked(instance)
+68 -102
View File
@@ -10,6 +10,8 @@ import (
"strings"
"sync"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/proxmoxidentity"
)
// State represents the current state of all monitored resources
@@ -3599,63 +3601,20 @@ func backupKey(instance string, vmid int) string {
return instance + "-" + strconv.Itoa(vmid)
}
// namespaceMatchesInstance checks if a PBS namespace likely corresponds to a PVE instance.
// This helps disambiguate backups when multiple PVE instances have VMs with the same VMID.
// Examples: namespace "pve1" matches instance "pve1", namespace "nat" matches instance "pve-nat"
func namespaceMatchesInstance(namespace, instance string) bool {
if namespace == "" || instance == "" {
return false
}
// Normalize both strings: lowercase and keep only alphanumeric
normalize := func(s string) string {
var b strings.Builder
for _, r := range strings.ToLower(s) {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
b.WriteRune(r)
}
}
return b.String()
}
ns := normalize(namespace)
inst := normalize(instance)
if ns == "" || inst == "" {
return false
}
// Exact match after normalization
if ns == inst {
return true
}
// Check if namespace is a suffix of instance
// e.g., namespace "nat" matches instance "pvenat" (normalized from "pve-nat")
// This is more precise than substring matching because:
// - "nat" should match "pve-nat" but not "natpve"
// - "pve" should match "pve" but not "pve-nat" (handled by exact match above)
if strings.HasSuffix(inst, ns) {
return true
}
// Check if instance is a suffix of namespace (reverse case)
// e.g., namespace "pvebackups" could match instance "pve"
if strings.HasSuffix(ns, inst) {
return true
}
return false
}
// SyncGuestBackupTimes updates LastBackup on VMs and Containers from storage backups and PBS backups.
// Call this after updating storage backups or PBS backups to ensure guest backup indicators are accurate.
// Matching is done by instance+VMID to prevent cross-instance VMID collisions.
// For PBS backups with namespaces, namespace matching is used to disambiguate.
// For PBS backups with namespaces, namespace matching is ranked against the guest's node and
// connection instance so clustered API entrypoints do not shadow the guest's actual placement.
func (s *State) SyncGuestBackupTimes() {
s.mu.Lock()
defer s.mu.Unlock()
type pbsSubjectKey struct {
backupType string
vmid int
}
// Build a map of instance+VMID -> latest backup time from all backup sources
// Using composite key prevents cross-instance VMID collision issues
latestBackup := make(map[string]time.Time)
@@ -3671,113 +3630,120 @@ func (s *State) SyncGuestBackupTimes() {
}
}
// Process PBS backups (VMID is string, BackupTime is the timestamp)
// PBS backups can have a Namespace field that often corresponds to the PVE instance.
// We use namespace matching to associate PBS backups with the correct PVE instance.
// Structure: map[vmid][]PBSBackup to handle multiple backups per VMID
pbsBackupsByVMID := make(map[int][]PBSBackup)
// Process PBS backups (VMID is string, BackupTime is the timestamp).
// Group by PBS subject type and VMID so VM and CT IDs do not cross-match.
pbsBackupsBySubject := make(map[pbsSubjectKey][]PBSBackup)
for _, backup := range s.PBSBackups {
vmid, err := strconv.Atoi(backup.VMID)
if err != nil || vmid <= 0 {
continue
}
pbsBackupsByVMID[vmid] = append(pbsBackupsByVMID[vmid], backup)
backupType := strings.ToLower(strings.TrimSpace(backup.BackupType))
if backupType != "vm" && backupType != "ct" {
continue
}
key := pbsSubjectKey{backupType: backupType, vmid: vmid}
pbsBackupsBySubject[key] = append(pbsBackupsBySubject[key], backup)
}
// Build a set of VMIDs that appear on more than one PVE instance.
// When a VMID is ambiguous, we must not fall back to VMID-only matching
// Build a set of typed VMIDs that appear on more than one PVE location.
// When a typed VMID is ambiguous, we must not fall back to VMID-only matching
// because we can't tell which guest the backup belongs to.
vmidInstances := make(map[int]map[string]struct{})
subjectLocations := make(map[pbsSubjectKey]map[string]struct{})
for i := range s.VMs {
m, ok := vmidInstances[s.VMs[i].VMID]
key := pbsSubjectKey{backupType: "vm", vmid: s.VMs[i].VMID}
m, ok := subjectLocations[key]
if !ok {
m = make(map[string]struct{})
vmidInstances[s.VMs[i].VMID] = m
subjectLocations[key] = m
}
m[s.VMs[i].Instance] = struct{}{}
m[backupKey(s.VMs[i].Instance, s.VMs[i].VMID)+"@"+s.VMs[i].Node] = struct{}{}
}
for i := range s.Containers {
m, ok := vmidInstances[s.Containers[i].VMID]
key := pbsSubjectKey{backupType: "ct", vmid: s.Containers[i].VMID}
m, ok := subjectLocations[key]
if !ok {
m = make(map[string]struct{})
vmidInstances[s.Containers[i].VMID] = m
subjectLocations[key] = m
}
m[s.Containers[i].Instance] = struct{}{}
m[backupKey(s.Containers[i].Instance, s.Containers[i].VMID)+"@"+s.Containers[i].Node] = struct{}{}
}
vmidIsAmbiguous := make(map[int]bool)
for vmid, instances := range vmidInstances {
if len(instances) > 1 {
vmidIsAmbiguous[vmid] = true
subjectIsAmbiguous := make(map[pbsSubjectKey]bool)
for key, locations := range subjectLocations {
if len(locations) > 1 {
subjectIsAmbiguous[key] = true
}
}
// findBestPBSBackup finds the most recent PBS backup for a given VMID and instance.
// If the backup has a namespace that matches the instance, it's preferred.
// findBestPBSBackup finds the best PBS backup for a given typed VMID and guest location.
// Placement and guest-name matches are preferred over VMID-only fallback.
// Returns zero time if no suitable backup found.
findBestPBSBackup := func(vmid int, instance string) time.Time {
backups, ok := pbsBackupsByVMID[vmid]
findBestPBSBackup := func(vmid int, backupType string, instance string, node string, name string) time.Time {
subjectKey := pbsSubjectKey{backupType: backupType, vmid: vmid}
backups, ok := pbsBackupsBySubject[subjectKey]
if !ok || len(backups) == 0 {
return time.Time{}
}
var bestTime time.Time
var bestMatchTime time.Time // Best time among namespace-matched backups
bestScore := -1
for _, backup := range backups {
// If namespace matches this instance, track it separately
if backup.Namespace != "" && namespaceMatchesInstance(backup.Namespace, instance) {
if backup.BackupTime.After(bestMatchTime) {
bestMatchTime = backup.BackupTime
}
score := proxmoxidentity.BackupGuestMatchScore(
backup.Namespace,
backup.Comment,
backup.VMID,
name,
instance,
node,
)
if score == 0 && !subjectIsAmbiguous[subjectKey] {
score = 1
}
// Track overall best time as fallback
if backup.BackupTime.After(bestTime) {
if score <= 0 {
continue
}
if score > bestScore || (score == bestScore && backup.BackupTime.After(bestTime)) {
bestScore = score
bestTime = backup.BackupTime
}
}
// Prefer namespace-matched backup if available
if !bestMatchTime.IsZero() {
return bestMatchTime
}
// Fall back to any backup with this VMID, but only when the VMID is
// unique across PVE instances. If the VMID exists on multiple instances,
// the match is ambiguous and we must not guess.
if vmidIsAmbiguous[vmid] {
return time.Time{}
}
return bestTime
}
// Update VMs - prefer instance-specific PVE backup, fall back to PBS
// Update VMs - recompute from current backup evidence instead of preserving stale values
for i := range s.VMs {
var lastBackup time.Time
key := backupKey(s.VMs[i].Instance, s.VMs[i].VMID)
if backupTime, ok := latestBackup[key]; ok {
s.VMs[i].LastBackup = backupTime
lastBackup = backupTime
}
// Check if PBS has a more recent backup
pbsTime := findBestPBSBackup(s.VMs[i].VMID, s.VMs[i].Instance)
pbsTime := findBestPBSBackup(s.VMs[i].VMID, "vm", s.VMs[i].Instance, s.VMs[i].Node, s.VMs[i].Name)
if !pbsTime.IsZero() {
if s.VMs[i].LastBackup.IsZero() || pbsTime.After(s.VMs[i].LastBackup) {
s.VMs[i].LastBackup = pbsTime
if lastBackup.IsZero() || pbsTime.After(lastBackup) {
lastBackup = pbsTime
}
}
s.VMs[i].LastBackup = lastBackup
}
// Update Containers - prefer instance-specific PVE backup, fall back to PBS
// Update Containers - recompute from current backup evidence instead of preserving stale values
for i := range s.Containers {
var lastBackup time.Time
key := backupKey(s.Containers[i].Instance, s.Containers[i].VMID)
if backupTime, ok := latestBackup[key]; ok {
s.Containers[i].LastBackup = backupTime
lastBackup = backupTime
}
// Check if PBS has a more recent backup
pbsTime := findBestPBSBackup(s.Containers[i].VMID, s.Containers[i].Instance)
pbsTime := findBestPBSBackup(s.Containers[i].VMID, "ct", s.Containers[i].Instance, s.Containers[i].Node, s.Containers[i].Name)
if !pbsTime.IsZero() {
if s.Containers[i].LastBackup.IsZero() || pbsTime.After(s.Containers[i].LastBackup) {
s.Containers[i].LastBackup = pbsTime
if lastBackup.IsZero() || pbsTime.After(lastBackup) {
lastBackup = pbsTime
}
}
s.Containers[i].LastBackup = lastBackup
}
s.LastUpdate = time.Now()
+9 -7
View File
@@ -3,6 +3,8 @@ package models
import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/proxmoxidentity"
)
// --- normalizeNodeIdentityPart ---
@@ -215,25 +217,25 @@ func TestBackupKey(t *testing.T) {
}
}
// --- namespaceMatchesInstance ---
// --- PBS namespace matching ---
func TestNamespaceMatchesInstance(t *testing.T) {
func TestNamespaceMatchesLocation(t *testing.T) {
tests := []struct {
namespace, instance string
namespace, location string
expected bool
}{
{"pve1", "pve1", true}, // Exact match
{"PVE1", "pve1", true}, // Case-insensitive
{"nat", "pve-nat", true}, // Namespace is suffix of normalized instance
{"nat", "pve-nat", true}, // Namespace is suffix of normalized location
{"pvebackups", "pve", false}, // "pve" is a prefix not suffix of "pvebackups"
{"", "pve1", false}, // Empty namespace
{"pve1", "", false}, // Empty instance
{"pve1", "", false}, // Empty location
{"completely-different", "pve1", false}, // No match
}
for _, tt := range tests {
got := namespaceMatchesInstance(tt.namespace, tt.instance)
got := proxmoxidentity.NamespaceMatchesLocation(tt.namespace, tt.location)
if got != tt.expected {
t.Errorf("namespaceMatchesInstance(%q, %q) = %v, want %v", tt.namespace, tt.instance, got, tt.expected)
t.Errorf("NamespaceMatchesLocation(%q, %q) = %v, want %v", tt.namespace, tt.location, got, tt.expected)
}
}
}
+90 -9
View File
@@ -855,7 +855,7 @@ func TestSyncGuestBackupTimes(t *testing.T) {
// Add PBS backup for container 200
state.UpdatePBSBackups("pbs-1", []PBSBackup{
{ID: "pbs-backup-1", VMID: "200", BackupTime: newBackup},
{ID: "pbs-backup-1", VMID: "200", BackupType: "ct", BackupTime: newBackup},
})
// Sync backup times
@@ -1038,6 +1038,87 @@ func TestSyncGuestBackupTimesNamespaceDisambiguation(t *testing.T) {
}
}
// TestSyncGuestBackupTimesClusterEntrypointUsesGuestNodeNamespace verifies that
// a cluster API entrypoint name does not shadow the guest's actual node namespace.
func TestSyncGuestBackupTimesClusterEntrypointUsesGuestNodeNamespace(t *testing.T) {
state := NewState()
now := time.Now()
oldBackupTime := now.Add(-180 * 24 * time.Hour)
freshBackupTime := now.Add(-1 * time.Hour)
state.UpdateContainers([]Container{
{VMID: 112, Name: "debian-go", Instance: "delly", Node: "minipc"},
})
state.mu.Lock()
state.PBSBackups = []PBSBackup{
{
ID: "pbs-delly-112-old",
VMID: "112",
Namespace: "delly",
BackupType: "ct",
Comment: "112",
BackupTime: oldBackupTime,
Instance: "pbs-main",
},
{
ID: "pbs-minipc-112-fresh",
VMID: "112",
Namespace: "minipc",
BackupType: "ct",
Comment: "debian-go",
BackupTime: freshBackupTime,
Instance: "pbs-main",
},
}
state.mu.Unlock()
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
var found *Container
for i := range snapshot.Containers {
if snapshot.Containers[i].VMID == 112 {
found = &snapshot.Containers[i]
break
}
}
if found == nil {
t.Fatal("container 112 not found")
}
if !found.LastBackup.Equal(freshBackupTime) {
t.Errorf("cluster entrypoint guest LastBackup = %v, want fresh node-namespaced backup %v",
found.LastBackup, freshBackupTime)
}
}
func TestSyncGuestBackupTimesClearsStaleBackupWhenCurrentEvidenceDisappears(t *testing.T) {
state := NewState()
staleBackupTime := time.Now().Add(-30 * 24 * time.Hour)
state.UpdateVMs([]VM{
{VMID: 101, Name: "unbacked-vm", Instance: "pve-1", Node: "node1", LastBackup: staleBackupTime},
})
state.UpdateContainers([]Container{
{VMID: 201, Name: "unbacked-ct", Instance: "pve-1", Node: "node1", LastBackup: staleBackupTime},
})
state.SyncGuestBackupTimes()
snapshot := state.GetSnapshot()
for _, vm := range snapshot.VMs {
if vm.VMID == 101 && !vm.LastBackup.IsZero() {
t.Errorf("VM stale LastBackup should be cleared, got %v", vm.LastBackup)
}
}
for _, ct := range snapshot.Containers {
if ct.VMID == 201 && !ct.LastBackup.IsZero() {
t.Errorf("container stale LastBackup should be cleared, got %v", ct.LastBackup)
}
}
}
// TestSyncGuestBackupTimesVMIDCollisionNonMatchingNamespace verifies that when the same VMID
// exists on multiple PVE instances and a PBS backup namespace matches neither, both guests
// get zero LastBackup instead of a false positive.
@@ -1162,8 +1243,9 @@ func TestSyncGuestBackupTimesUniqueVMIDFallback(t *testing.T) {
}
}
// TestSyncGuestBackupTimesVMContainerCollision verifies that when a VM and Container
// share the same VMID on different instances, neither gets an unmatched PBS backup.
// TestSyncGuestBackupTimesVMContainerCollision verifies that PBS vm/ID and ct/ID
// subjects are matched independently even when a VM and container share the same
// numeric ID.
func TestSyncGuestBackupTimesVMContainerCollision(t *testing.T) {
state := NewState()
@@ -1177,13 +1259,12 @@ func TestSyncGuestBackupTimesVMContainerCollision(t *testing.T) {
{VMID: 100, Name: "ct-pve2", Instance: "pve2", Node: "node2"},
})
// PBS backup with namespace matching neither
state.mu.Lock()
state.PBSBackups = []PBSBackup{
{
ID: "pbs-100",
VMID: "100",
Namespace: "other",
Namespace: "node1",
BackupType: "vm",
BackupTime: backupTime,
Instance: "pbs-main",
@@ -1195,14 +1276,14 @@ func TestSyncGuestBackupTimesVMContainerCollision(t *testing.T) {
snapshot := state.GetSnapshot()
for _, vm := range snapshot.VMs {
if vm.VMID == 100 && !vm.LastBackup.IsZero() {
t.Errorf("VM %q should have zero LastBackup (ambiguous VMID with container), got %v",
vm.Name, vm.LastBackup)
if vm.VMID == 100 && !vm.LastBackup.Equal(backupTime) {
t.Errorf("VM %q LastBackup = %v, want typed vm/100 backup %v",
vm.Name, vm.LastBackup, backupTime)
}
}
for _, ct := range snapshot.Containers {
if ct.VMID == 100 && !ct.LastBackup.IsZero() {
t.Errorf("Container %q should have zero LastBackup (ambiguous VMID with VM), got %v",
t.Errorf("Container %q should have zero LastBackup because the PBS subject is vm/100, got %v",
ct.Name, ct.LastBackup)
}
}
+13 -4
View File
@@ -369,8 +369,8 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
guestInfo := buildProxmoxGuestInfoIndex(readState)
m.ingestRecoveryPointsAsync(proxmoxrecoverymapper.FromPVEStorageBackups(allBackups, guestInfo))
// Sync backup times to VMs/Containers for backup status indicators
m.state.SyncGuestBackupTimes()
// Sync backup times to VMs/Containers and republish them to canonical resources.
m.syncGuestBackupTimesAndResourceStore()
if m.alertManager != nil {
guestsByKey, guestsByVMID := buildGuestLookupsFromReadState(m.GetUnifiedReadStateOrSnapshot(), m.guestMetadataStore)
@@ -416,6 +416,15 @@ func shouldPreservePBSBackups(datastoreCount, datastoreFetches int) bool {
return false
}
func (m *Monitor) syncGuestBackupTimesAndResourceStore() {
if m == nil || m.state == nil {
return
}
m.state.SyncGuestBackupTimes()
m.updateResourceStore(m.state.GetSnapshot())
}
func storageNamesForNode(readState unifiedresources.ReadState, instanceName, nodeName string) []string {
if readState == nil || nodeName == "" {
return nil
@@ -1441,8 +1450,8 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
candidates := buildPBSGuestCandidates(m.GetUnifiedReadStateOrSnapshot())
m.ingestRecoveryPointsAsync(proxmoxrecoverymapper.FromPBSBackups(allBackups, candidates))
// Sync backup times to VMs/Containers for backup status indicators
m.state.SyncGuestBackupTimes()
// Sync backup times to VMs/Containers and republish them to canonical resources.
m.syncGuestBackupTimesAndResourceStore()
if m.alertManager != nil {
guestsByKey, guestsByVMID := buildGuestLookupsFromReadState(m.GetUnifiedReadStateOrSnapshot(), m.guestMetadataStore)
@@ -230,6 +230,95 @@ func TestMonitorPollStorageBackupsWithNodes_UsesCanonicalReadStateForGuestNodeLo
}
}
func TestSyncGuestBackupTimesAndResourceStore_RefreshesCanonicalWorkloads(t *testing.T) {
stale := time.Date(2026, 1, 10, 2, 0, 0, 0, time.UTC)
fresh := time.Date(2026, 3, 11, 10, 0, 0, 0, time.UTC)
state := models.NewState()
state.UpdateVMsForInstance("homelab", []models.VM{{
ID: "homelab-minipc-vm-100",
VMID: 100,
Name: "docker",
Node: "minipc",
Instance: "homelab",
Status: "running",
LastBackup: stale,
LastSeen: fresh,
}})
state.UpdatePBSBackups("pbs-docker", []models.PBSBackup{{
ID: "pbs-docker/store/minipc/vm/100/2026-03-11T10:00:00Z",
Instance: "pbs-docker",
Datastore: "store",
Namespace: "minipc",
BackupType: "vm",
VMID: "100",
BackupTime: fresh,
Comment: "docker",
}})
registry := unifiedresources.NewRegistry(nil)
registry.IngestSnapshot(state.GetSnapshot())
adapter := unifiedresources.NewMonitorAdapter(registry)
m := &Monitor{
state: state,
resourceStore: adapter,
}
m.syncGuestBackupTimesAndResourceStore()
snapshot := state.GetSnapshot()
if len(snapshot.VMs) != 1 || !snapshot.VMs[0].LastBackup.Equal(fresh) {
t.Fatalf("expected state VM last backup %v, got %+v", fresh, snapshot.VMs)
}
vms := adapter.VMs()
if len(vms) != 1 {
t.Fatalf("expected one canonical VM, got %d", len(vms))
}
if got := vms[0].LastBackup(); !got.Equal(fresh) {
t.Fatalf("expected canonical VM last backup %v, got %v", fresh, got)
}
}
func TestSyncGuestBackupTimesAndResourceStore_ClearsCanonicalStaleBackup(t *testing.T) {
stale := time.Date(2026, 1, 10, 2, 0, 0, 0, time.UTC)
state := models.NewState()
state.UpdateVMsForInstance("homelab", []models.VM{{
ID: "homelab-minipc-vm-100",
VMID: 100,
Name: "docker",
Node: "minipc",
Instance: "homelab",
Status: "running",
LastBackup: stale,
LastSeen: stale,
}})
registry := unifiedresources.NewRegistry(nil)
registry.IngestSnapshot(state.GetSnapshot())
adapter := unifiedresources.NewMonitorAdapter(registry)
m := &Monitor{
state: state,
resourceStore: adapter,
}
m.syncGuestBackupTimesAndResourceStore()
snapshot := state.GetSnapshot()
if len(snapshot.VMs) != 1 || !snapshot.VMs[0].LastBackup.IsZero() {
t.Fatalf("expected state VM last backup to clear, got %+v", snapshot.VMs)
}
vms := adapter.VMs()
if len(vms) != 1 {
t.Fatalf("expected one canonical VM, got %d", len(vms))
}
if got := vms[0].LastBackup(); !got.IsZero() {
t.Fatalf("expected canonical VM last backup to clear, got %v", got)
}
}
func TestBuildPBSGuestCandidates_UsesCanonicalReadState(t *testing.T) {
readState := backupReadState([]unifiedresources.Resource{
{
@@ -0,0 +1,90 @@
package proxmoxidentity
import "strings"
const (
NamespaceNoLocationMatch = iota
NamespaceWeakInstanceMatch
NamespaceInstanceMatch
NamespaceNodeMatch
)
func normalizeLocationLabel(value string) string {
var b strings.Builder
for _, r := range strings.ToLower(strings.TrimSpace(value)) {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
b.WriteRune(r)
}
}
return b.String()
}
// NamespaceMatchesLocation reports whether a PBS namespace likely identifies a
// Proxmox location label such as a node name or a single-node connection name.
func NamespaceMatchesLocation(namespace, location string) bool {
ns := normalizeLocationLabel(namespace)
loc := normalizeLocationLabel(location)
if ns == "" || loc == "" {
return false
}
if ns == loc {
return true
}
return strings.HasSuffix(loc, ns) || strings.HasSuffix(ns, loc)
}
// NamespaceLocationScore ranks how strongly a PBS namespace identifies the
// current guest placement. Node matches are strongest because clustered PVE
// connections can use an API entrypoint name that is not the guest's node.
func NamespaceLocationScore(namespace, instanceName, nodeName string) int {
nodeMatches := NamespaceMatchesLocation(namespace, nodeName)
instanceMatches := NamespaceMatchesLocation(namespace, instanceName)
switch {
case nodeMatches:
return NamespaceNodeMatch
case instanceMatches && (normalizeLocationLabel(nodeName) == "" ||
normalizeLocationLabel(nodeName) == normalizeLocationLabel(instanceName)):
return NamespaceInstanceMatch
case instanceMatches:
return NamespaceWeakInstanceMatch
default:
return NamespaceNoLocationMatch
}
}
func PreferredPBSBackupSubjectName(comment, vmid string) string {
comment = strings.TrimSpace(comment)
vmid = strings.TrimSpace(vmid)
if comment == "" {
return ""
}
if vmid != "" {
if comment == vmid {
return ""
}
parts := strings.Split(comment, ",")
if len(parts) >= 2 {
last := strings.TrimSpace(parts[len(parts)-1])
first := strings.TrimSpace(parts[0])
if last == vmid && first != "" && first != vmid {
return first
}
}
}
return comment
}
func BackupCommentMatchesGuestName(comment, vmid, guestName string) bool {
subjectName := strings.ToLower(strings.TrimSpace(PreferredPBSBackupSubjectName(comment, vmid)))
guestName = strings.ToLower(strings.TrimSpace(guestName))
return subjectName != "" && guestName != "" && subjectName == guestName
}
func BackupGuestMatchScore(namespace, comment, vmid, guestName, instanceName, nodeName string) int {
score := NamespaceLocationScore(namespace, instanceName, nodeName) * 10
if BackupCommentMatchesGuestName(comment, vmid, guestName) {
score += 5
}
return score
}
@@ -0,0 +1,71 @@
package proxmoxidentity
import "testing"
func TestNamespaceMatchesLocation(t *testing.T) {
tests := []struct {
name string
namespace string
location string
expected bool
}{
{"exact match", "pve", "pve", true},
{"exact match with numbers", "pve1", "pve1", true},
{"case insensitive", "PVE1", "pve1", true},
{"namespace suffix of location", "nat", "pve-nat", true},
{"location suffix of namespace", "backupspve", "pve", true},
{"prefix is not enough", "pvebackups", "pve", false},
{"substring is not enough", "production", "my-production-server", false},
{"empty namespace", "", "pve1", false},
{"empty location", "pve1", "", false},
{"different names", "pve1", "pve2", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := NamespaceMatchesLocation(tt.namespace, tt.location)
if got != tt.expected {
t.Fatalf("NamespaceMatchesLocation(%q, %q) = %v, want %v",
tt.namespace, tt.location, got, tt.expected)
}
})
}
}
func TestNamespaceLocationScorePrefersGuestNodeOverClusterEntrypoint(t *testing.T) {
if got := NamespaceLocationScore("minipc", "delly", "minipc"); got != NamespaceNodeMatch {
t.Fatalf("NamespaceLocationScore(minipc, delly, minipc) = %d, want node match", got)
}
if got := NamespaceLocationScore("delly", "delly", "minipc"); got != NamespaceWeakInstanceMatch {
t.Fatalf("NamespaceLocationScore(delly, delly, minipc) = %d, want weak instance match", got)
}
if got := NamespaceLocationScore("delly", "delly", "delly"); got != NamespaceNodeMatch {
t.Fatalf("NamespaceLocationScore(delly, delly, delly) = %d, want node match", got)
}
}
func TestBackupCommentMatchesGuestName(t *testing.T) {
if !BackupCommentMatchesGuestName("debian-go", "112", "debian-go") {
t.Fatal("expected exact guest-name backup comment to match")
}
if !BackupCommentMatchesGuestName("debian-go, 112", "112", "debian-go") {
t.Fatal("expected notes-template style guest-name backup comment to match")
}
if BackupCommentMatchesGuestName("112", "112", "debian-go") {
t.Fatal("numeric-only backup comment should not match a guest name")
}
}
func TestBackupGuestMatchScoreRanksNodeMatchAboveClusterEntrypoint(t *testing.T) {
weakScore := BackupGuestMatchScore("delly", "112", "112", "debian-go", "delly", "minipc")
if weakScore <= 0 {
t.Fatalf("weak instance match score = %d, want positive", weakScore)
}
commentScore := BackupGuestMatchScore("delly", "debian-go", "112", "debian-go", "delly", "minipc")
if commentScore <= weakScore {
t.Fatalf("weak instance+comment score = %d, should outrank weak instance-only score %d", commentScore, weakScore)
}
if got := BackupGuestMatchScore("minipc", "112", "112", "debian-go", "delly", "minipc"); got <= commentScore {
t.Fatalf("node namespace score = %d, should outrank weak instance+comment match", got)
}
}
+3 -49
View File
@@ -6,6 +6,7 @@ import (
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/proxmoxidentity"
"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
@@ -36,54 +37,8 @@ func guestLookupKey(instanceName, nodeName string, vmid int) string {
return fmt.Sprintf("%s|%s|%d", strings.TrimSpace(instanceName), strings.TrimSpace(nodeName), vmid)
}
func namespaceLikelyMatchesProxmoxLocation(namespace, value string) bool {
namespace = strings.TrimSpace(namespace)
value = strings.TrimSpace(value)
if namespace == "" || value == "" {
return false
}
normalize := func(s string) string {
var b strings.Builder
for _, r := range strings.ToLower(s) {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
b.WriteRune(r)
}
}
return b.String()
}
ns := normalize(namespace)
location := normalize(value)
if ns == "" || location == "" {
return false
}
if ns == location {
return true
}
return strings.HasSuffix(location, ns) || strings.HasSuffix(ns, location)
}
func preferredPBSBackupSubjectName(comment, vmid string) string {
comment = strings.TrimSpace(comment)
vmid = strings.TrimSpace(vmid)
if comment == "" {
return ""
}
if vmid != "" {
if comment == vmid {
return ""
}
parts := strings.Split(comment, ",")
if len(parts) >= 2 {
last := strings.TrimSpace(parts[len(parts)-1])
first := strings.TrimSpace(parts[0])
if last == vmid && first != "" && first != vmid {
return first
}
}
}
return comment
return proxmoxidentity.PreferredPBSBackupSubjectName(comment, vmid)
}
func selectPBSGuestCandidate(backup models.PBSBackup, candidates []GuestCandidate) (GuestCandidate, bool) {
@@ -98,8 +53,7 @@ func selectPBSGuestCandidate(backup models.PBSBackup, candidates []GuestCandidat
if namespace := strings.TrimSpace(backup.Namespace); namespace != "" {
filtered := make([]GuestCandidate, 0, len(matched))
for _, candidate := range matched {
if namespaceLikelyMatchesProxmoxLocation(namespace, candidate.InstanceName) ||
namespaceLikelyMatchesProxmoxLocation(namespace, candidate.NodeName) {
if proxmoxidentity.NamespaceLocationScore(namespace, candidate.InstanceName, candidate.NodeName) > proxmoxidentity.NamespaceNoLocationMatch {
filtered = append(filtered, candidate)
}
}
@@ -279,6 +279,36 @@ func TestFromPBSBackups_PrefersCommentNameWhenGuestIsUnresolved(t *testing.T) {
}
}
func TestFromPBSBackups_IgnoresNumericOnlyCommentName(t *testing.T) {
backups := []models.PBSBackup{
{
ID: "pbs-backup-numeric-comment",
VMID: "112",
Instance: "pbs-docker",
Namespace: "minipc",
Datastore: "main",
BackupType: "ct",
BackupTime: time.Date(2026, 3, 29, 3, 3, 31, 0, time.UTC),
Comment: "112",
},
}
result := FromPBSBackups(backups, nil)
if len(result) != 1 {
t.Fatalf("expected 1 point, got %d", len(result))
}
if result[0].SubjectRef == nil {
t.Fatal("expected SubjectRef to be set")
}
if got := result[0].SubjectRef.Name; got != "112" {
t.Fatalf("SubjectRef.Name = %q, want VMID fallback", got)
}
if got := result[0].Details["comment"]; got != "112" {
t.Fatalf("Details[comment] = %#v, want raw numeric comment preserved", got)
}
}
func TestFromPBSBackups_WithCandidates(t *testing.T) {
backups := []models.PBSBackup{
{
@@ -224,6 +224,7 @@ class CanonicalCompletionGuardTest(unittest.TestCase):
"docker-runtime-report-model",
"proxmox-zfs-runtime",
"proxmox-ceph-runtime",
"proxmox-backup-identity-monitoring",
"container-entrypoint-runtime",
"mock-runtime-fixtures",
"monitoring-runtime",
@@ -240,6 +241,7 @@ class CanonicalCompletionGuardTest(unittest.TestCase):
"test_prefixes": [],
"exact_files": [
"internal/monitoring/canonical_guardrails_test.go",
"internal/monitoring/monitor_backups_readstate_test.go",
"internal/monitoring/monitor_host_agents_test.go",
"internal/unifiedresources/code_standards_test.go",
],