Preserve guest-agent continuity across transient PVE status failures

This commit is contained in:
rcourtman
2026-04-01 18:38:42 +01:00
parent 97da398d2b
commit 4ceb3ad433
13 changed files with 1255 additions and 53 deletions
@@ -576,3 +576,9 @@ metrics into the existing shared `agent` and `vm` history stores. Pulse must
not add a VMware-only charts cache, host history model, or VM metrics store
just because vSphere performance collection uses a different API family from
inventory and alarm reads.
That same monitoring boundary now also owns Proxmox guest-agent continuity
when `/status` is transiently missing. Recent guest-agent evidence and the
shared guest metadata cache must keep VM network and identity metadata alive
long enough to survive short Proxmox status failures, while incomplete
guest-agent metadata stays on a short retry cadence instead of freezing
partial VM summary data for minutes.
@@ -381,6 +381,55 @@ func TestProxmoxGuestPollersCarryPoolIntoCanonicalModels(t *testing.T) {
}
}
func TestProxmoxGuestAgentContinuityUsesCanonicalEvidenceAndRetryPaths(t *testing.T) {
requiredSnippets := map[string][]string{
"guest_agent_evidence.go": {
"func hasRecentGuestAgentEvidence(prev *models.VM, now time.Time) bool {",
`if prev == nil || prev.Type != "qemu" {`,
"func shouldQueryGuestAgent(vmStatus *proxmox.VMStatus, prev *models.VM, now time.Time) bool {",
"return hasRecentGuestAgentEvidence(prev, now)",
},
"guest_metadata.go": {
"guestMetadataEmptyTTL = 30 * time.Second",
"func guestMetadataCacheEntryTTL(entry guestMetadataCacheEntry) time.Duration {",
"if guestMetadataCacheHasCompleteNetworkData(entry) {",
"return guestMetadataEmptyTTL",
"func (m *Monitor) hasRecentGuestMetadataEvidence(instanceName, nodeName string, vmid int, now time.Time) bool {",
"func (m *Monitor) scheduleGuestMetadataFetchForEntry(key string, now time.Time, entry guestMetadataCacheEntry) {",
},
"monitor_previous_state.go": {
"vmsByID map[string]models.VM",
"ctx.vmsByID[modelVM.ID] = modelVM",
"guestID := makeGuestID(modelVM.Instance, modelVM.Node, modelVM.VMID)",
"ctx.vmsByID[guestID] = modelVM",
},
"monitor_pve_guest_builders.go": {
"guestAgentAvailable := shouldQueryGuestAgent(state.detailedStatus, prevVM, now) ||",
"m.hasRecentGuestMetadataEvidence(instanceName, res.Node, res.VMID, now)",
"if guestAgentAvailable && state.detailedStatus == nil {",
},
"monitor_polling_vm.go": {
"prevVMByID := prevGuests.vmsByID",
"guestAgentAvailable := vm.Status == \"running\" &&",
"m.hasRecentGuestMetadataEvidence(instanceName, n.Node, vm.VMID, now)",
"if guestAgentAvailable && diskTotal > 0 {",
},
}
for file, snippets := range requiredSnippets {
data, err := os.ReadFile(file)
if err != nil {
t.Fatalf("failed to read %s: %v", file, err)
}
source := string(data)
for _, snippet := range snippets {
if !strings.Contains(source, snippet) {
t.Fatalf("%s must contain %q", file, snippet)
}
}
}
}
func TestUnifiedAppContainerMetricsUseCanonicalGuestHistoryPath(t *testing.T) {
data, err := os.ReadFile("monitor.go")
if err != nil {
@@ -0,0 +1,44 @@
package monitoring
import (
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
const recentGuestAgentEvidenceMaxAge = 10 * time.Minute
func hasRecentGuestAgentEvidence(prev *models.VM, now time.Time) bool {
// Previous unified read-state snapshots normalize guest status to resource health
// (for example "online"), so continuity depends on recent guest-agent evidence
// rather than on replaying an exact prior runtime status string.
if prev == nil || prev.Type != "qemu" {
return false
}
if prev.LastSeen.IsZero() || now.Sub(prev.LastSeen) > recentGuestAgentEvidenceMaxAge {
return false
}
if prev.AgentVersion != "" ||
len(prev.IPAddresses) > 0 ||
len(prev.NetworkInterfaces) > 0 ||
prev.OSName != "" ||
prev.OSVersion != "" {
return true
}
if len(prev.Disks) > 0 && !strings.HasPrefix(prev.DiskStatusReason, "prev-") {
return true
}
return false
}
func shouldQueryGuestAgent(vmStatus *proxmox.VMStatus, prev *models.VM, now time.Time) bool {
if vmStatus != nil {
return vmStatus.Agent.Value > 0
}
return hasRecentGuestAgentEvidence(prev, now)
}
@@ -0,0 +1,40 @@
package monitoring
import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
func TestHasRecentGuestAgentEvidenceAcceptsUnifiedReadStateStatus(t *testing.T) {
now := time.Now()
prev := &models.VM{
Type: "qemu",
Status: "online",
IPAddresses: []string{"192.168.1.50"},
AgentVersion: "8.1.0",
LastSeen: now.Add(-time.Minute),
}
if !hasRecentGuestAgentEvidence(prev, now) {
t.Fatal("expected unified read-state VM snapshot to count as recent guest-agent evidence")
}
}
func TestHasRecentGuestAgentEvidenceRejectsStaleSnapshots(t *testing.T) {
now := time.Now()
prev := &models.VM{
Type: "qemu",
Status: "online",
IPAddresses: []string{"192.168.1.50"},
AgentVersion: "8.1.0",
LastSeen: now.Add(-recentGuestAgentEvidenceMaxAge - time.Second),
}
if hasRecentGuestAgentEvidence(prev, now) {
t.Fatal("expected stale guest-agent evidence to be rejected")
}
}
+94 -22
View File
@@ -18,6 +18,7 @@ import (
const (
guestMetadataCacheTTL = 5 * time.Minute
guestMetadataEmptyTTL = 30 * time.Second
defaultGuestMetadataHold = 15 * time.Second
// Guest agent timeout defaults (configurable via environment variables)
@@ -45,6 +46,48 @@ type guestMetadataCacheEntry struct {
osInfoSkip bool // Skip OS info calls after repeated failures (refs #692)
}
func guestMetadataCacheHasUsefulData(entry guestMetadataCacheEntry) bool {
return len(entry.ipAddresses) > 0 ||
len(entry.networkInterfaces) > 0 ||
entry.osName != "" ||
entry.osVersion != "" ||
entry.agentVersion != ""
}
func guestMetadataCacheHasCompleteNetworkData(entry guestMetadataCacheEntry) bool {
for _, iface := range entry.networkInterfaces {
if strings.TrimSpace(iface.Name) != "" || strings.TrimSpace(iface.MAC) != "" {
return true
}
}
return false
}
func guestMetadataCacheEntryTTL(entry guestMetadataCacheEntry) time.Duration {
// Treat identity-only and IP-only metadata as incomplete so VMs that answered
// guest-info/version or partial network calls but not full interface inventory
// are retried soon instead of freezing incomplete VM Summary data for minutes.
if guestMetadataCacheHasCompleteNetworkData(entry) {
return guestMetadataCacheTTL
}
return guestMetadataEmptyTTL
}
func (m *Monitor) hasRecentGuestMetadataEvidence(instanceName, nodeName string, vmid int, now time.Time) bool {
if m == nil {
return false
}
key := guestMetadataCacheKey(instanceName, nodeName, vmid)
m.guestMetadataMu.RLock()
entry, ok := m.guestMetadataCache[key]
m.guestMetadataMu.RUnlock()
if !ok || entry.fetchedAt.IsZero() || !guestMetadataCacheHasUsefulData(entry) {
return false
}
return now.Sub(entry.fetchedAt) < guestMetadataCacheEntryTTL(entry)
}
func (m *Monitor) tryReserveGuestMetadataFetch(key string, now time.Time) bool {
if m == nil {
return false
@@ -80,6 +123,19 @@ func (m *Monitor) scheduleNextGuestMetadataFetch(key string, now time.Time) {
m.guestMetadataLimiterMu.Unlock()
}
func (m *Monitor) scheduleGuestMetadataFetchForEntry(key string, now time.Time, entry guestMetadataCacheEntry) {
if m == nil {
return
}
if guestMetadataCacheEntryTTL(entry) == guestMetadataEmptyTTL {
m.guestMetadataLimiterMu.Lock()
m.guestMetadataLimiter[key] = now.Add(guestMetadataEmptyTTL)
m.guestMetadataLimiterMu.Unlock()
return
}
m.scheduleNextGuestMetadataFetch(key, now)
}
func (m *Monitor) deferGuestMetadataRetry(key string, now time.Time) {
if m == nil {
return
@@ -164,17 +220,7 @@ func (m *Monitor) retryGuestAgentCall(ctx context.Context, timeout time.Duration
return nil, lastErr
}
func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientInterface, instanceName, nodeName, vmName string, vmid int, vmStatus *proxmox.VMStatus) ([]string, []models.GuestNetworkInterface, string, string, string) {
if vmStatus == nil || client == nil {
m.clearGuestMetadataCache(instanceName, nodeName, vmid)
return nil, nil, "", "", ""
}
if vmStatus.Agent.Value <= 0 {
m.clearGuestMetadataCache(instanceName, nodeName, vmid)
return nil, nil, "", "", ""
}
func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientInterface, instanceName, nodeName, vmName string, vmid int, vmStatus *proxmox.VMStatus, allowWithoutStatus bool) ([]string, []models.GuestNetworkInterface, string, string, string) {
key := guestMetadataCacheKey(instanceName, nodeName, vmid)
now := time.Now()
@@ -182,11 +228,20 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
cached, ok := m.guestMetadataCache[key]
m.guestMetadataMu.RUnlock()
if ok && now.Sub(cached.fetchedAt) < guestMetadataCacheTTL {
agentAvailable := client != nil && ((vmStatus != nil && vmStatus.Agent.Value > 0) || allowWithoutStatus)
if !agentAvailable {
if ok && now.Sub(cached.fetchedAt) < guestMetadataCacheEntryTTL(cached) {
return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion
}
m.clearGuestMetadataCache(instanceName, nodeName, vmid)
return nil, nil, "", "", ""
}
if ok && now.Sub(cached.fetchedAt) < guestMetadataCacheEntryTTL(cached) {
return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion
}
needsFetch := !ok || now.Sub(cached.fetchedAt) >= guestMetadataCacheTTL
needsFetch := !ok || now.Sub(cached.fetchedAt) >= guestMetadataCacheEntryTTL(cached)
if !needsFetch {
return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion
}
@@ -212,9 +267,6 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
return ipAddresses, networkIfaces, osName, osVersion, agentVersion
}
defer m.releaseGuestMetadataSlot()
defer func() {
m.scheduleNextGuestMetadataFetch(key, time.Now())
}()
}
// Network interfaces with configurable timeout and retry (refs #592)
@@ -229,10 +281,24 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
Err(err).
Msg("Guest agent network interfaces unavailable")
} else if ifaces, ok := interfaces.([]proxmox.VMNetworkInterface); ok && len(ifaces) > 0 {
ipAddresses, networkIfaces = processGuestNetworkInterfaces(ifaces)
processedIPs, processedIfaces := processGuestNetworkInterfaces(ifaces)
if len(processedIPs) > 0 || len(processedIfaces) > 0 {
ipAddresses, networkIfaces = processedIPs, processedIfaces
} else if len(cached.ipAddresses) == 0 && len(cached.networkInterfaces) == 0 {
ipAddresses = nil
networkIfaces = nil
} else {
log.Debug().
Str("instance", instanceName).
Str("vm", vmName).
Int("vmid", vmid).
Msg("Guest agent returned empty network metadata; preserving last known interfaces")
}
} else {
ipAddresses = nil
networkIfaces = nil
if len(cached.ipAddresses) == 0 && len(cached.networkInterfaces) == 0 {
ipAddresses = nil
networkIfaces = nil
}
}
// OS info with configurable timeout and retry (refs #592)
@@ -275,10 +341,13 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
}
}
} else if agentInfo, ok := agentInfoRaw.(map[string]interface{}); ok && len(agentInfo) > 0 {
osName, osVersion = extractGuestOSInfo(agentInfo)
extractedOSName, extractedOSVersion := extractGuestOSInfo(agentInfo)
if extractedOSName != "" || extractedOSVersion != "" {
osName, osVersion = extractedOSName, extractedOSVersion
}
osInfoFailureCount = 0 // Reset on success
osInfoSkip = false
} else {
} else if cached.osName == "" && cached.osVersion == "" {
osName = ""
osVersion = ""
}
@@ -304,7 +373,7 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
Msg("Guest agent version unavailable")
} else if version, ok := versionRaw.(string); ok && version != "" {
agentVersion = version
} else {
} else if cached.agentVersion == "" {
agentVersion = ""
}
@@ -325,6 +394,9 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
}
m.guestMetadataCache[key] = entry
m.guestMetadataMu.Unlock()
if reserved {
m.scheduleGuestMetadataFetchForEntry(key, time.Now(), entry)
}
return ipAddresses, networkIfaces, osName, osVersion, agentVersion
}
+522
View File
@@ -11,6 +11,120 @@ import (
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
type emptyGuestMetadataClient struct {
mockPVEClient
}
func (emptyGuestMetadataClient) GetVMNetworkInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.VMNetworkInterface, error) {
return []proxmox.VMNetworkInterface{}, nil
}
func (emptyGuestMetadataClient) GetVMAgentInfo(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
return map[string]interface{}{}, nil
}
func (emptyGuestMetadataClient) GetVMAgentVersion(ctx context.Context, node string, vmid int) (string, error) {
return "", nil
}
type emptyThenPopulatedGuestMetadataClient struct {
mockPVEClient
networkCalls int
}
func (c *emptyThenPopulatedGuestMetadataClient) GetVMNetworkInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.VMNetworkInterface, error) {
c.networkCalls++
if c.networkCalls == 1 {
return []proxmox.VMNetworkInterface{}, nil
}
return []proxmox.VMNetworkInterface{
{
Name: "Ethernet0",
HardwareAddr: "00:11:22:33:44:55",
IPAddresses: []proxmox.VMIPAddress{
{Address: "192.168.1.50", Prefix: 24},
},
},
}, nil
}
func (*emptyThenPopulatedGuestMetadataClient) GetVMAgentInfo(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
return map[string]interface{}{}, nil
}
func (*emptyThenPopulatedGuestMetadataClient) GetVMAgentVersion(ctx context.Context, node string, vmid int) (string, error) {
return "", nil
}
type identityThenNetworkGuestMetadataClient struct {
mockPVEClient
networkCalls int
}
func (c *identityThenNetworkGuestMetadataClient) GetVMNetworkInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.VMNetworkInterface, error) {
c.networkCalls++
if c.networkCalls == 1 {
return []proxmox.VMNetworkInterface{}, nil
}
return []proxmox.VMNetworkInterface{
{
Name: "Ethernet0",
HardwareAddr: "00:11:22:33:44:55",
IPAddresses: []proxmox.VMIPAddress{
{Address: "192.168.1.50", Prefix: 24},
},
},
}, nil
}
func (*identityThenNetworkGuestMetadataClient) GetVMAgentInfo(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
return map[string]interface{}{
"result": map[string]interface{}{
"name": "Microsoft Windows",
"version": "11",
},
}, nil
}
func (*identityThenNetworkGuestMetadataClient) GetVMAgentVersion(ctx context.Context, node string, vmid int) (string, error) {
return "8.2.2", nil
}
type ipOnlyThenNetworkGuestMetadataClient struct {
mockPVEClient
networkCalls int
}
func (c *ipOnlyThenNetworkGuestMetadataClient) GetVMNetworkInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.VMNetworkInterface, error) {
c.networkCalls++
if c.networkCalls == 1 {
return []proxmox.VMNetworkInterface{
{
IPAddresses: []proxmox.VMIPAddress{
{Address: "192.168.1.60", Prefix: 24},
},
},
}, nil
}
return []proxmox.VMNetworkInterface{
{
Name: "Ethernet0",
HardwareAddr: "00:11:22:33:44:66",
IPAddresses: []proxmox.VMIPAddress{
{Address: "192.168.1.60", Prefix: 24},
},
},
}, nil
}
func (*ipOnlyThenNetworkGuestMetadataClient) GetVMAgentInfo(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
return map[string]interface{}{}, nil
}
func (*ipOnlyThenNetworkGuestMetadataClient) GetVMAgentVersion(ctx context.Context, node string, vmid int) (string, error) {
return "", nil
}
func TestGuestMetadataCacheKey(t *testing.T) {
t.Parallel()
@@ -771,6 +885,414 @@ func TestRetryGuestAgentCall(t *testing.T) {
})
}
func TestGuestMetadataCacheEntryTTL(t *testing.T) {
t.Parallel()
tests := []struct {
name string
entry guestMetadataCacheEntry
want time.Duration
}{
{
name: "empty entry retries quickly",
entry: guestMetadataCacheEntry{},
want: guestMetadataEmptyTTL,
},
{
name: "network metadata uses full ttl",
entry: guestMetadataCacheEntry{
ipAddresses: []string{"192.168.1.10"},
networkInterfaces: []models.GuestNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.1.10"}},
},
},
want: guestMetadataCacheTTL,
},
{
name: "ip-only metadata retries quickly",
entry: guestMetadataCacheEntry{
ipAddresses: []string{"192.168.1.10"},
},
want: guestMetadataEmptyTTL,
},
{
name: "identity-only metadata retries quickly",
entry: guestMetadataCacheEntry{
osName: "Ubuntu",
},
want: guestMetadataEmptyTTL,
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
if got := guestMetadataCacheEntryTTL(tc.entry); got != tc.want {
t.Fatalf("guestMetadataCacheEntryTTL() = %s, want %s", got, tc.want)
}
})
}
}
func TestScheduleGuestMetadataFetchForEntry(t *testing.T) {
t.Parallel()
t.Run("incomplete metadata retries sooner", func(t *testing.T) {
t.Parallel()
m := &Monitor{
guestMetadataLimiter: make(map[string]time.Time),
guestMetadataMinRefresh: 5 * time.Minute,
guestMetadataRefreshJitter: 0,
}
now := time.Now()
entry := guestMetadataCacheEntry{osName: "Ubuntu"}
m.scheduleGuestMetadataFetchForEntry("vm-1", now, entry)
next := m.guestMetadataLimiter["vm-1"]
expectedNext := now.Add(guestMetadataEmptyTTL)
if next.Sub(expectedNext) > time.Millisecond || expectedNext.Sub(next) > time.Millisecond {
t.Fatalf("expected next time ~%v, got %v", expectedNext, next)
}
})
t.Run("complete network metadata uses normal refresh cadence", func(t *testing.T) {
t.Parallel()
m := &Monitor{
guestMetadataLimiter: make(map[string]time.Time),
guestMetadataMinRefresh: 5 * time.Minute,
guestMetadataRefreshJitter: 0,
}
now := time.Now()
entry := guestMetadataCacheEntry{
networkInterfaces: []models.GuestNetworkInterface{
{Name: "eth0", Addresses: []string{"192.168.1.10"}},
},
}
m.scheduleGuestMetadataFetchForEntry("vm-1", now, entry)
next := m.guestMetadataLimiter["vm-1"]
expectedNext := now.Add(5 * time.Minute)
if next.Sub(expectedNext) > time.Millisecond || expectedNext.Sub(next) > time.Millisecond {
t.Fatalf("expected next time ~%v, got %v", expectedNext, next)
}
})
}
func TestFetchGuestAgentMetadataPreservesCachedValuesOnEmptyResponses(t *testing.T) {
t.Parallel()
key := guestMetadataCacheKey("pve", "node1", 100)
cachedFetchedAt := time.Now().Add(-guestMetadataCacheTTL - time.Minute)
monitor := &Monitor{
guestMetadataCache: map[string]guestMetadataCacheEntry{
key: {
ipAddresses: []string{"192.168.1.10"},
networkInterfaces: []models.GuestNetworkInterface{
{Name: "eth0", MAC: "00:11:22:33:44:55", Addresses: []string{"192.168.1.10"}},
},
osName: "Ubuntu",
osVersion: "24.04",
agentVersion: "8.2.0",
fetchedAt: cachedFetchedAt,
},
},
guestMetadataLimiter: make(map[string]time.Time),
}
gotIPs, gotIfaces, gotOSName, gotOSVersion, gotAgentVersion := monitor.fetchGuestAgentMetadata(
context.Background(),
&emptyGuestMetadataClient{},
"pve",
"node1",
"vm100",
100,
&proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}},
false,
)
if len(gotIPs) != 1 || gotIPs[0] != "192.168.1.10" {
t.Fatalf("expected cached IPs to be preserved, got %#v", gotIPs)
}
if len(gotIfaces) != 1 || gotIfaces[0].Name != "eth0" {
t.Fatalf("expected cached interfaces to be preserved, got %#v", gotIfaces)
}
if gotOSName != "Ubuntu" || gotOSVersion != "24.04" {
t.Fatalf("expected cached OS info to be preserved, got %q %q", gotOSName, gotOSVersion)
}
if gotAgentVersion != "8.2.0" {
t.Fatalf("expected cached agent version to be preserved, got %q", gotAgentVersion)
}
updated := monitor.guestMetadataCache[key]
if len(updated.ipAddresses) != 1 || updated.ipAddresses[0] != "192.168.1.10" {
t.Fatalf("expected cache IPs to remain populated, got %#v", updated.ipAddresses)
}
if len(updated.networkInterfaces) != 1 || updated.networkInterfaces[0].Name != "eth0" {
t.Fatalf("expected cache interfaces to remain populated, got %#v", updated.networkInterfaces)
}
if updated.fetchedAt.Before(cachedFetchedAt) {
t.Fatalf("expected cache timestamp to be refreshed, old=%v new=%v", cachedFetchedAt, updated.fetchedAt)
}
}
func TestFetchGuestAgentMetadataPreservesFreshCacheWhenAgentTemporarilyUnavailable(t *testing.T) {
t.Parallel()
key := guestMetadataCacheKey("pve", "node1", 100)
cachedFetchedAt := time.Now().Add(-time.Minute)
newMonitor := func() *Monitor {
return &Monitor{
guestMetadataCache: map[string]guestMetadataCacheEntry{
key: {
ipAddresses: []string{"192.168.1.10"},
networkInterfaces: []models.GuestNetworkInterface{
{Name: "eth0", MAC: "00:11:22:33:44:55", Addresses: []string{"192.168.1.10"}},
},
osName: "Ubuntu",
osVersion: "24.04",
agentVersion: "8.2.0",
fetchedAt: cachedFetchedAt,
},
},
guestMetadataLimiter: make(map[string]time.Time),
}
}
assertCachePreserved := func(t *testing.T, monitor *Monitor, gotIPs []string, gotIfaces []models.GuestNetworkInterface, gotOSName, gotOSVersion, gotAgentVersion string) {
t.Helper()
if len(gotIPs) != 1 || gotIPs[0] != "192.168.1.10" {
t.Fatalf("expected cached IPs to be preserved, got %#v", gotIPs)
}
if len(gotIfaces) != 1 || gotIfaces[0].Name != "eth0" {
t.Fatalf("expected cached interfaces to be preserved, got %#v", gotIfaces)
}
if gotOSName != "Ubuntu" || gotOSVersion != "24.04" {
t.Fatalf("expected cached OS info to be preserved, got %q %q", gotOSName, gotOSVersion)
}
if gotAgentVersion != "8.2.0" {
t.Fatalf("expected cached agent version to be preserved, got %q", gotAgentVersion)
}
entry, ok := monitor.guestMetadataCache[key]
if !ok {
t.Fatal("expected guest metadata cache entry to remain populated")
}
if len(entry.networkInterfaces) != 1 || entry.networkInterfaces[0].Name != "eth0" {
t.Fatalf("expected cached interfaces to remain populated, got %#v", entry.networkInterfaces)
}
}
t.Run("nil vm status keeps fresh cache", func(t *testing.T) {
t.Parallel()
monitor := newMonitor()
gotIPs, gotIfaces, gotOSName, gotOSVersion, gotAgentVersion := monitor.fetchGuestAgentMetadata(
context.Background(),
&emptyGuestMetadataClient{},
"pve",
"node1",
"vm100",
100,
nil,
false,
)
assertCachePreserved(t, monitor, gotIPs, gotIfaces, gotOSName, gotOSVersion, gotAgentVersion)
})
t.Run("agent temporarily unavailable keeps fresh cache", func(t *testing.T) {
t.Parallel()
monitor := newMonitor()
gotIPs, gotIfaces, gotOSName, gotOSVersion, gotAgentVersion := monitor.fetchGuestAgentMetadata(
context.Background(),
&emptyGuestMetadataClient{},
"pve",
"node1",
"vm100",
100,
&proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 0}},
false,
)
assertCachePreserved(t, monitor, gotIPs, gotIfaces, gotOSName, gotOSVersion, gotAgentVersion)
})
}
func TestFetchGuestAgentMetadataRetriesIdentityOnlyCacheSooner(t *testing.T) {
t.Parallel()
client := &identityThenNetworkGuestMetadataClient{}
monitor := &Monitor{
guestMetadataCache: make(map[string]guestMetadataCacheEntry),
guestMetadataLimiter: make(map[string]time.Time),
}
status := &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}
firstIPs, firstIfaces, firstOSName, firstOSVersion, firstAgentVersion := monitor.fetchGuestAgentMetadata(
context.Background(),
client,
"pve",
"node1",
"vm100",
100,
status,
false,
)
if len(firstIPs) != 0 || len(firstIfaces) != 0 {
t.Fatalf("expected first fetch to be missing network metadata, got ips=%#v ifaces=%#v", firstIPs, firstIfaces)
}
if firstOSName == "" || firstOSVersion == "" || firstAgentVersion == "" {
t.Fatalf("expected identity metadata on first fetch, got os=%q version=%q agent=%q", firstOSName, firstOSVersion, firstAgentVersion)
}
key := guestMetadataCacheKey("pve", "node1", 100)
entry := monitor.guestMetadataCache[key]
if got := guestMetadataCacheEntryTTL(entry); got != guestMetadataEmptyTTL {
t.Fatalf("guestMetadataCacheEntryTTL(identity-only) = %s, want %s", got, guestMetadataEmptyTTL)
}
entry.fetchedAt = time.Now().Add(-guestMetadataEmptyTTL - time.Second)
monitor.guestMetadataCache[key] = entry
monitor.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
secondIPs, secondIfaces, secondOSName, secondOSVersion, secondAgentVersion := monitor.fetchGuestAgentMetadata(
context.Background(),
client,
"pve",
"node1",
"vm100",
100,
status,
false,
)
if len(secondIPs) == 0 || len(secondIfaces) == 0 {
t.Fatalf("expected second fetch to populate network metadata, got ips=%#v ifaces=%#v", secondIPs, secondIfaces)
}
if secondOSName != firstOSName || secondOSVersion != firstOSVersion || secondAgentVersion != firstAgentVersion {
t.Fatalf("expected identity metadata to be preserved, got os=%q/%q agent=%q want os=%q/%q agent=%q",
secondOSName, secondOSVersion, secondAgentVersion, firstOSName, firstOSVersion, firstAgentVersion)
}
}
func TestFetchGuestAgentMetadataRetriesIPOnlyCacheSooner(t *testing.T) {
t.Parallel()
client := &ipOnlyThenNetworkGuestMetadataClient{}
monitor := &Monitor{
guestMetadataCache: make(map[string]guestMetadataCacheEntry),
guestMetadataLimiter: make(map[string]time.Time),
}
status := &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}
firstIPs, firstIfaces, _, _, _ := monitor.fetchGuestAgentMetadata(
context.Background(),
client,
"pve",
"node1",
"vm100",
100,
status,
false,
)
if len(firstIPs) != 1 || firstIPs[0] != "192.168.1.60" {
t.Fatalf("expected first fetch to preserve discovered IP, got %#v", firstIPs)
}
if len(firstIfaces) != 1 || firstIfaces[0].Name != "" || firstIfaces[0].MAC != "" {
t.Fatalf("expected first fetch to remain identity-incomplete, got %#v", firstIfaces)
}
key := guestMetadataCacheKey("pve", "node1", 100)
entry := monitor.guestMetadataCache[key]
if got := guestMetadataCacheEntryTTL(entry); got != guestMetadataEmptyTTL {
t.Fatalf("guestMetadataCacheEntryTTL(ip-only) = %s, want %s", got, guestMetadataEmptyTTL)
}
entry.fetchedAt = time.Now().Add(-guestMetadataEmptyTTL - time.Second)
monitor.guestMetadataCache[key] = entry
monitor.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
secondIPs, secondIfaces, _, _, _ := monitor.fetchGuestAgentMetadata(
context.Background(),
client,
"pve",
"node1",
"vm100",
100,
status,
false,
)
if len(secondIPs) != 1 || secondIPs[0] != "192.168.1.60" {
t.Fatalf("expected second fetch to preserve IP, got %#v", secondIPs)
}
if len(secondIfaces) != 1 || secondIfaces[0].Name != "Ethernet0" {
t.Fatalf("expected second fetch to populate interfaces, got %#v", secondIfaces)
}
if client.networkCalls != 2 {
t.Fatalf("expected network metadata to be fetched twice, got %d calls", client.networkCalls)
}
}
func TestFetchGuestAgentMetadataRetriesEmptyCacheSooner(t *testing.T) {
t.Parallel()
client := &emptyThenPopulatedGuestMetadataClient{}
monitor := &Monitor{
guestMetadataCache: make(map[string]guestMetadataCacheEntry),
guestMetadataLimiter: make(map[string]time.Time),
}
status := &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}
firstIPs, firstIfaces, _, _, _ := monitor.fetchGuestAgentMetadata(
context.Background(),
client,
"pve",
"node1",
"vm100",
100,
status,
false,
)
if len(firstIPs) != 0 || len(firstIfaces) != 0 {
t.Fatalf("expected first fetch to be empty, got ips=%#v ifaces=%#v", firstIPs, firstIfaces)
}
key := guestMetadataCacheKey("pve", "node1", 100)
entry := monitor.guestMetadataCache[key]
entry.fetchedAt = time.Now().Add(-guestMetadataEmptyTTL - time.Second)
monitor.guestMetadataCache[key] = entry
monitor.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
secondIPs, secondIfaces, _, _, _ := monitor.fetchGuestAgentMetadata(
context.Background(),
client,
"pve",
"node1",
"vm100",
100,
status,
false,
)
if len(secondIPs) != 1 || secondIPs[0] != "192.168.1.50" {
t.Fatalf("expected second fetch to refresh IPs, got %#v", secondIPs)
}
if len(secondIfaces) != 1 || secondIfaces[0].Name != "Ethernet0" {
t.Fatalf("expected second fetch to refresh interfaces, got %#v", secondIfaces)
}
if client.networkCalls != 2 {
t.Fatalf("expected network metadata to be fetched twice, got %d calls", client.networkCalls)
}
}
func TestAcquireGuestMetadataSlot(t *testing.T) {
t.Parallel()
@@ -300,7 +300,7 @@ func TestHandleClusterVMResourceMemoryTrustCharacterization(t *testing.T) {
MaxCPU: 4,
}
vm, ok := mon.handleClusterVMResource(context.Background(), "test", res, makeGuestID("test", "node1", 101), client, nil)
vm, ok := mon.handleClusterVMResource(context.Background(), "test", res, makeGuestID("test", "node1", 101), client, nil, nil)
if !ok {
t.Fatal("handleClusterVMResource() returned ok=false")
}
@@ -473,17 +473,28 @@ func TestMonitor_LinkNodeToHostAgent_Extra(t *testing.T) {
type mockPVEClientExtra struct {
mockPVEClient
resources []proxmox.ClusterResource
vmStatus *proxmox.VMStatus
fsInfo []proxmox.VMFileSystem
netIfaces []proxmox.VMNetworkInterface
vms []proxmox.VM
resources []proxmox.ClusterResource
vmStatus *proxmox.VMStatus
vmStatusErr error
fsInfo []proxmox.VMFileSystem
netIfaces []proxmox.VMNetworkInterface
agentInfo map[string]interface{}
agentVersion string
}
func (m *mockPVEClientExtra) GetClusterResources(ctx context.Context, resourceType string) ([]proxmox.ClusterResource, error) {
return m.resources, nil
}
func (m *mockPVEClientExtra) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
return m.vms, nil
}
func (m *mockPVEClientExtra) GetVMStatus(ctx context.Context, node string, vmid int) (*proxmox.VMStatus, error) {
if m.vmStatusErr != nil {
return nil, m.vmStatusErr
}
return m.vmStatus, nil
}
@@ -520,10 +531,16 @@ func (m *mockPVEClientExtra) GetContainerInterfaces(ctx context.Context, node st
}
func (m *mockPVEClientExtra) GetVMAgentInfo(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
if m.agentInfo != nil {
return m.agentInfo, nil
}
return map[string]interface{}{"os": "linux"}, nil
}
func (m *mockPVEClientExtra) GetVMAgentVersion(ctx context.Context, node string, vmid int) (string, error) {
if m.agentVersion != "" {
return m.agentVersion, nil
}
return "1.0", nil
}
@@ -1063,6 +1080,10 @@ func TestMonitor_PreviousGuestContextForInstance_Extra(t *testing.T) {
if len(prev.vms) != 1 || prev.vms[0].VMID != 101 || prev.vms[0].Instance != "pve1" || prev.vms[0].Name != "vm1" {
t.Fatalf("expected only pve1 VMs, got %#v", prev.vms)
}
canonicalID := prev.vms[0].ID
if len(prev.vmsByID) != 2 || prev.vmsByID[canonicalID].VMID != 101 || prev.vmsByID[makeGuestID("pve1", "", 101)].VMID != 101 {
t.Fatalf("expected previous VM lookup to be indexed by canonical and runtime guest IDs, got %#v", prev.vmsByID)
}
if len(prev.containers) != 2 {
t.Fatalf("expected only pve1 containers, got %#v", prev.containers)
}
@@ -1077,6 +1098,362 @@ func TestMonitor_PreviousGuestContextForInstance_Extra(t *testing.T) {
}
}
func TestBuildVMFromClusterResource_ContinuesGuestAgentQueriesAfterTransientStatusFailure(t *testing.T) {
monitor := &Monitor{
rateTracker: NewRateTracker(),
guestMetadataCache: make(map[string]guestMetadataCacheEntry),
guestMetadataLimiter: make(map[string]time.Time),
}
client := &mockPVEClientExtra{
vmStatusErr: fmt.Errorf("API error 500: timeout"),
fsInfo: []proxmox.VMFileSystem{
{Mountpoint: "/", Type: "ext4", TotalBytes: 100 * 1024 * 1024 * 1024, UsedBytes: 40 * 1024 * 1024 * 1024, Disk: "/dev/vda"},
},
netIfaces: []proxmox.VMNetworkInterface{
{Name: "eth0", HardwareAddr: "00:11:22:33:44:55", IPAddresses: []proxmox.VMIPAddress{{Address: "192.168.1.50", Prefix: 24}}},
},
agentInfo: map[string]interface{}{
"result": map[string]interface{}{
"name": "Debian GNU/Linux",
"version": "12",
},
},
agentVersion: "8.2.0",
}
guestID := makeGuestID("cluster-a", "node-a", 101)
prevVM := &models.VM{
ID: guestID,
Instance: "cluster-a",
Node: "node-a",
VMID: 101,
Name: "app-vm",
Type: "qemu",
Status: "running",
AgentVersion: "8.1.0",
NetworkInterfaces: []models.GuestNetworkInterface{
{Name: "eth0", MAC: "00:11:22:33:44:55", Addresses: []string{"192.168.1.50"}},
},
LastSeen: time.Now(),
}
vm, _, _, _, ok := monitor.buildVMFromClusterResource(
context.Background(),
"cluster-a",
proxmox.ClusterResource{
Type: "qemu",
Node: "node-a",
Name: "app-vm",
Status: "running",
VMID: 101,
MaxCPU: 4,
MaxMem: 8192,
MaxDisk: 100 * 1024 * 1024 * 1024,
},
client,
guestID,
nil,
prevVM,
)
if !ok {
t.Fatal("expected VM to be built")
}
if vm.Disk.Usage != 40 {
t.Fatalf("expected guest-agent disk usage after status failure, got %.2f", vm.Disk.Usage)
}
if len(vm.Disks) != 1 || vm.Disks[0].Device != "/dev/vda" {
t.Fatalf("expected guest-agent disk inventory, got %#v", vm.Disks)
}
if len(vm.NetworkInterfaces) != 1 || vm.NetworkInterfaces[0].Name != "eth0" {
t.Fatalf("expected guest-agent interfaces after status failure, got %#v", vm.NetworkInterfaces)
}
if vm.AgentVersion != "8.2.0" {
t.Fatalf("expected refreshed agent version, got %q", vm.AgentVersion)
}
if vm.OSName != "Debian GNU/Linux" || vm.OSVersion != "12" {
t.Fatalf("expected refreshed OS info, got %q %q", vm.OSName, vm.OSVersion)
}
}
func TestPollVMsAndContainersEfficient_ContinuesGuestAgentQueriesAfterTransientStatusFailure(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
m := &Monitor{
state: models.NewState(),
guestAgentFSInfoTimeout: time.Second,
guestAgentRetries: 1,
guestAgentNetworkTimeout: time.Second,
guestAgentOSInfoTimeout: time.Second,
guestAgentVersionTimeout: time.Second,
guestMetadataCache: make(map[string]guestMetadataCacheEntry),
guestMetadataLimiter: make(map[string]time.Time),
rateTracker: NewRateTracker(),
metricsHistory: NewMetricsHistory(100, time.Hour),
alertManager: alerts.NewManager(),
stalenessTracker: NewStalenessTracker(nil),
nodeRRDMemCache: make(map[string]rrdMemCacheEntry),
vmRRDMemCache: make(map[string]rrdMemCacheEntry),
}
defer m.alertManager.Stop()
registry := unifiedresources.NewRegistry(nil)
guestID := makeGuestID("pve1", "node1", 100)
registry.IngestSnapshot(models.StateSnapshot{
VMs: []models.VM{
{
ID: guestID,
VMID: 100,
Name: "vm100",
Node: "node1",
Instance: "pve1",
Type: "qemu",
Status: "running",
IPAddresses: []string{"192.168.1.50"},
AgentVersion: "8.1.0",
NetworkInterfaces: []models.GuestNetworkInterface{
{Name: "eth0", MAC: "00:11:22:33:44:55", Addresses: []string{"192.168.1.50"}},
},
Disks: []models.Disk{
{Total: 100 * 1024 * 1024 * 1024, Used: 40 * 1024 * 1024 * 1024, Free: 60 * 1024 * 1024 * 1024, Usage: 40, Mountpoint: "/", Type: "ext4", Device: "/dev/vda"},
},
LastSeen: time.Now(),
},
},
})
m.resourceStore = unifiedresources.NewMonitorAdapter(registry)
prev := m.previousGuestContextForInstance("pve1")
prevVM, ok := prev.vmsByID[guestID]
if !ok || !hasRecentGuestAgentEvidence(&prevVM, time.Now()) {
t.Fatalf("expected recent guest-agent evidence for %s, got %#v", guestID, prev.vmsByID)
}
client := &mockPVEClientExtra{
resources: []proxmox.ClusterResource{
{VMID: 100, Name: "vm100", Node: "node1", Status: "running", Type: "qemu", MaxMem: 8 * 1024, Mem: 4 * 1024, MaxDisk: 100 * 1024 * 1024 * 1024},
},
vmStatusErr: fmt.Errorf("API error 500: timeout"),
fsInfo: []proxmox.VMFileSystem{
{Mountpoint: "/", Type: "ext4", TotalBytes: 100 * 1024 * 1024 * 1024, UsedBytes: 40 * 1024 * 1024 * 1024, Disk: "/dev/vda"},
},
netIfaces: []proxmox.VMNetworkInterface{
{Name: "eth0", HardwareAddr: "00:11:22:33:44:55", IPAddresses: []proxmox.VMIPAddress{{Address: "192.168.1.50", Prefix: 24}}},
},
agentInfo: map[string]interface{}{
"result": map[string]interface{}{
"name": "Debian GNU/Linux",
"version": "12",
},
},
agentVersion: "8.2.0",
}
if ok := m.pollVMsAndContainersEfficient(
context.Background(),
"pve1",
"",
false,
client,
map[string]string{"node1": "online"},
); !ok {
t.Fatal("expected efficient polling path to succeed")
}
state := m.GetState()
if len(state.VMs) != 1 {
t.Fatalf("expected 1 VM, got %d", len(state.VMs))
}
vm := state.VMs[0]
if vm.Disk.Usage != 40 {
t.Fatalf("expected guest-agent disk usage after status failure, got %.2f", vm.Disk.Usage)
}
if len(vm.Disks) != 1 || vm.Disks[0].Device != "/dev/vda" {
t.Fatalf("expected guest-agent disk inventory, got %#v", vm.Disks)
}
if len(vm.NetworkInterfaces) != 1 || vm.NetworkInterfaces[0].Name != "eth0" {
t.Fatalf("expected guest-agent interfaces after status failure, got %#v", vm.NetworkInterfaces)
}
if vm.AgentVersion != "8.2.0" {
t.Fatalf("expected refreshed agent version, got %q", vm.AgentVersion)
}
}
func TestPollVMsWithNodes_PreservesCachedGuestMetadataWhenStatusUnavailable(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
m := &Monitor{
state: models.NewState(),
guestAgentFSInfoTimeout: time.Second,
guestAgentRetries: 1,
guestAgentNetworkTimeout: time.Second,
guestAgentOSInfoTimeout: time.Second,
guestAgentVersionTimeout: time.Second,
guestMetadataCache: make(map[string]guestMetadataCacheEntry),
guestMetadataLimiter: make(map[string]time.Time),
rateTracker: NewRateTracker(),
metricsHistory: NewMetricsHistory(100, time.Hour),
alertManager: alerts.NewManager(),
stalenessTracker: NewStalenessTracker(nil),
nodeRRDMemCache: make(map[string]rrdMemCacheEntry),
vmRRDMemCache: make(map[string]rrdMemCacheEntry),
}
defer m.alertManager.Stop()
cacheKey := guestMetadataCacheKey("pve1", "node1", 100)
m.guestMetadataCache[cacheKey] = guestMetadataCacheEntry{
ipAddresses: []string{"192.168.1.50"},
networkInterfaces: []models.GuestNetworkInterface{
{Name: "eth0", MAC: "00:11:22:33:44:55", Addresses: []string{"192.168.1.50"}},
},
osName: "Debian",
osVersion: "12",
agentVersion: "8.2.0",
fetchedAt: time.Now().Add(-time.Minute),
}
client := &mockPVEClientExtra{
vms: []proxmox.VM{
{VMID: 100, Name: "vm100", Node: "node1", Status: "running", MaxMem: 8 * 1024, Mem: 4 * 1024},
},
vmStatusErr: fmt.Errorf("API error 500: timeout"),
}
m.pollVMsWithNodes(
context.Background(),
"pve1",
"",
false,
client,
[]proxmox.Node{{Node: "node1", Status: "online"}},
map[string]string{"node1": "online"},
)
state := m.GetState()
if len(state.VMs) != 1 {
t.Fatalf("expected 1 VM, got %d", len(state.VMs))
}
vm := state.VMs[0]
if len(vm.IPAddresses) != 1 || vm.IPAddresses[0] != "192.168.1.50" {
t.Fatalf("expected cached IP addresses to be preserved, got %#v", vm.IPAddresses)
}
if len(vm.NetworkInterfaces) != 1 || vm.NetworkInterfaces[0].Name != "eth0" {
t.Fatalf("expected cached network interfaces to be preserved, got %#v", vm.NetworkInterfaces)
}
if vm.OSName != "Debian" || vm.OSVersion != "12" {
t.Fatalf("expected cached OS info to be preserved, got %q %q", vm.OSName, vm.OSVersion)
}
if vm.AgentVersion != "8.2.0" {
t.Fatalf("expected cached agent version to be preserved, got %q", vm.AgentVersion)
}
}
func TestPollVMsWithNodes_ContinuesGuestAgentQueriesAfterTransientStatusFailure(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
m := &Monitor{
state: models.NewState(),
guestAgentFSInfoTimeout: time.Second,
guestAgentRetries: 1,
guestAgentNetworkTimeout: time.Second,
guestAgentOSInfoTimeout: time.Second,
guestAgentVersionTimeout: time.Second,
guestMetadataCache: make(map[string]guestMetadataCacheEntry),
guestMetadataLimiter: make(map[string]time.Time),
rateTracker: NewRateTracker(),
metricsHistory: NewMetricsHistory(100, time.Hour),
alertManager: alerts.NewManager(),
stalenessTracker: NewStalenessTracker(nil),
nodeRRDMemCache: make(map[string]rrdMemCacheEntry),
vmRRDMemCache: make(map[string]rrdMemCacheEntry),
}
defer m.alertManager.Stop()
registry := unifiedresources.NewRegistry(nil)
guestID := makeGuestID("pve1", "node1", 100)
registry.IngestSnapshot(models.StateSnapshot{
VMs: []models.VM{
{
ID: guestID,
VMID: 100,
Name: "vm100",
Node: "node1",
Instance: "pve1",
Type: "qemu",
Status: "running",
IPAddresses: []string{"192.168.1.50"},
AgentVersion: "8.1.0",
NetworkInterfaces: []models.GuestNetworkInterface{
{Name: "eth0", MAC: "00:11:22:33:44:55", Addresses: []string{"192.168.1.50"}},
},
Disks: []models.Disk{
{Total: 100 * 1024 * 1024 * 1024, Used: 40 * 1024 * 1024 * 1024, Free: 60 * 1024 * 1024 * 1024, Usage: 40, Mountpoint: "/", Type: "ext4", Device: "/dev/vda"},
},
LastSeen: time.Now(),
},
},
})
m.resourceStore = unifiedresources.NewMonitorAdapter(registry)
prev := m.previousGuestContextForInstance("pve1")
prevVM, ok := prev.vmsByID[guestID]
if !ok || !hasRecentGuestAgentEvidence(&prevVM, time.Now()) {
t.Fatalf("expected recent guest-agent evidence for %s, got %#v", guestID, prev.vmsByID)
}
client := &mockPVEClientExtra{
vms: []proxmox.VM{
{VMID: 100, Name: "vm100", Node: "node1", Status: "running", MaxMem: 8 * 1024, Mem: 4 * 1024, MaxDisk: 100 * 1024 * 1024 * 1024},
},
vmStatusErr: fmt.Errorf("API error 500: timeout"),
fsInfo: []proxmox.VMFileSystem{
{Mountpoint: "/", Type: "ext4", TotalBytes: 100 * 1024 * 1024 * 1024, UsedBytes: 40 * 1024 * 1024 * 1024, Disk: "/dev/vda"},
},
netIfaces: []proxmox.VMNetworkInterface{
{Name: "eth0", HardwareAddr: "00:11:22:33:44:55", IPAddresses: []proxmox.VMIPAddress{{Address: "192.168.1.50", Prefix: 24}}},
},
agentInfo: map[string]interface{}{
"result": map[string]interface{}{
"name": "Debian GNU/Linux",
"version": "12",
},
},
agentVersion: "8.2.0",
}
m.pollVMsWithNodes(
context.Background(),
"pve1",
"",
false,
client,
[]proxmox.Node{{Node: "node1", Status: "online"}},
map[string]string{"node1": "online"},
)
state := m.GetState()
if len(state.VMs) != 1 {
t.Fatalf("expected 1 VM, got %d", len(state.VMs))
}
vm := state.VMs[0]
if vm.Disk.Usage != 40 {
t.Fatalf("expected guest-agent disk usage after status failure, got %.2f", vm.Disk.Usage)
}
if vm.DiskStatusReason != "" {
t.Fatalf("expected empty disk status reason, got %q", vm.DiskStatusReason)
}
if len(vm.Disks) != 1 || vm.Disks[0].Device != "/dev/vda" {
t.Fatalf("expected guest-agent disk inventory, got %#v", vm.Disks)
}
if len(vm.NetworkInterfaces) != 1 || vm.NetworkInterfaces[0].Name != "eth0" {
t.Fatalf("expected guest-agent interfaces after status failure, got %#v", vm.NetworkInterfaces)
}
if vm.AgentVersion != "8.2.0" {
t.Fatalf("expected refreshed agent version, got %q", vm.AgentVersion)
}
}
func TestBuildVMFromClusterResource_UsesLinkedHostAgentDiskFallback(t *testing.T) {
monitor := &Monitor{rateTracker: NewRateTracker()}
client := &mockPVEClientExtra{
@@ -1112,6 +1489,7 @@ func TestBuildVMFromClusterResource_UsesLinkedHostAgentDiskFallback(t *testing.T
},
},
},
nil,
)
if !ok {
t.Fatal("expected VM to be built")
+26 -14
View File
@@ -38,6 +38,7 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, clu
// Capture the previous guest context once per poll cycle so fallback behavior
// is based on a consistent pre-poll snapshot.
prevGuests := m.previousGuestContextForInstance(instanceName)
prevVMByID := prevGuests.vmsByID
vmIDToHostAgent := prevGuests.hostAgentsByVMID
log.Debug().
@@ -89,6 +90,10 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, clu
// Generate canonical guest ID: instance:node:vmid
guestID := makeGuestID(instanceName, n.Node, vm.VMID)
var prevVM *models.VM
if prev, ok := prevVMByID[guestID]; ok {
prevVM = &prev
}
guestRaw := VMMemoryRaw{
ListingMem: vm.Mem,
@@ -151,8 +156,12 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, clu
memorySource = "status-unavailable"
}
if vm.Status == "running" && vmStatus != nil {
guestIPs, guestIfaces, guestOSName, guestOSVersion, agentVersion := m.fetchGuestAgentMetadata(ctx, client, instanceName, n.Node, vm.Name, vm.VMID, vmStatus)
now := time.Now()
guestAgentAvailable := vm.Status == "running" &&
(shouldQueryGuestAgent(vmStatus, prevVM, now) ||
m.hasRecentGuestMetadataEvidence(instanceName, n.Node, vm.VMID, now))
if guestAgentAvailable {
guestIPs, guestIfaces, guestOSName, guestOSVersion, agentVersion := m.fetchGuestAgentMetadata(ctx, client, instanceName, n.Node, vm.Name, vm.VMID, vmStatus, vmStatus == nil)
if len(guestIPs) > 0 {
ipAddresses = guestIPs
}
@@ -221,21 +230,26 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, clu
// For running VMs, ALWAYS try to get filesystem info from guest agent
// The cluster/resources endpoint always returns 0 for disk usage
if vm.Status == "running" && vmStatus != nil && diskTotal > 0 {
if guestAgentAvailable && diskTotal > 0 {
agentValue := 0
if vmStatus != nil {
agentValue = vmStatus.Agent.Value
}
// Log the initial state
if logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().
Str("instance", instanceName).
Str("vm", vm.Name).
Int("vmid", vm.VMID).
Int("agent", vmStatus.Agent.Value).
Int("agent", agentValue).
Uint64("diskUsed", diskUsed).
Uint64("diskTotal", diskTotal).
Msg("VM has 0 disk usage, checking guest agent")
}
// Check if agent is enabled
if vmStatus.Agent.Value == 0 {
if vmStatus != nil && vmStatus.Agent.Value == 0 {
diskStatusReason = "agent-disabled"
if logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().
@@ -243,7 +257,7 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, clu
Str("vm", vm.Name).
Msg("Guest agent disabled in VM config")
}
} else if vmStatus.Agent.Value > 0 || diskUsed == 0 {
} else {
if logging.IsLevelEnabled(zerolog.DebugLevel) {
log.Debug().
Str("instance", instanceName).
@@ -453,17 +467,15 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, clu
Msg("Guest agent provided filesystem info but no usable filesystems found (all were special mounts)")
}
}
} else {
// No vmStatus available or agent disabled - show allocated disk
if diskTotal > 0 {
diskUsage = -1 // Show as allocated size
diskStatusReason = "no-agent"
}
}
} else if vm.Status == "running" && diskTotal > 0 {
// Running VM but no vmStatus - show allocated disk
// Running VM but no guest-agent evidence - show allocated disk
diskUsage = -1
diskStatusReason = "no-status"
if vmStatus == nil {
diskStatusReason = "no-status"
} else {
diskStatusReason = "no-agent"
}
}
if vm.Status == "running" && !diskFromAgent {
+26 -8
View File
@@ -9,6 +9,7 @@ import (
type previousGuestContext struct {
vms []models.VM
vmsByID map[string]models.VM
containers []models.Container
containerOCIByVMID map[int]bool
hostAgentsByVMID map[string]models.Host
@@ -17,6 +18,7 @@ type previousGuestContext struct {
func (m *Monitor) previousGuestContextForInstance(instanceName string) previousGuestContext {
ctx := previousGuestContext{
vms: make([]models.VM, 0),
vmsByID: make(map[string]models.VM),
containers: make([]models.Container, 0),
containerOCIByVMID: make(map[int]bool),
hostAgentsByVMID: make(map[string]models.Host),
@@ -31,7 +33,15 @@ func (m *Monitor) previousGuestContextForInstance(instanceName string) previousG
if vm == nil || vm.Instance() != instanceName {
continue
}
ctx.vms = append(ctx.vms, previousVMFromView(vm))
modelVM := previousVMFromView(vm)
ctx.vms = append(ctx.vms, modelVM)
if modelVM.ID != "" {
ctx.vmsByID[modelVM.ID] = modelVM
}
guestID := makeGuestID(modelVM.Instance, modelVM.Node, modelVM.VMID)
if guestID != "" {
ctx.vmsByID[guestID] = modelVM
}
}
for _, ct := range readState.Containers() {
@@ -84,13 +94,21 @@ func previousVMFromView(vm *unifiedresources.VMView) models.VM {
return models.VM{}
}
return models.VM{
ID: vm.ID(),
Instance: vm.Instance(),
Node: vm.Node(),
VMID: vm.VMID(),
Name: vm.Name(),
Status: string(vm.Status()),
LastSeen: vm.LastSeen(),
ID: vm.ID(),
Instance: vm.Instance(),
Node: vm.Node(),
VMID: vm.VMID(),
Name: vm.Name(),
Type: "qemu",
Status: string(vm.Status()),
IPAddresses: vm.IPAddresses(),
OSName: vm.OSName(),
OSVersion: vm.OSVersion(),
AgentVersion: vm.AgentVersion(),
NetworkInterfaces: guestNetworkInterfacesFromReadStateView(vm.NetworkInterfaces()),
Disks: guestDisksFromReadStateView(vm.Disks()),
DiskStatusReason: vm.DiskStatusReason(),
LastSeen: vm.LastSeen(),
}
}
@@ -28,6 +28,7 @@ func TestBuildVMFromClusterResource_PreservesProxmoxPool(t *testing.T) {
nil,
"",
nil,
nil,
)
if !ok {
t.Fatal("expected VM to be built")
@@ -72,7 +72,7 @@ func (m *Monitor) applyVMStatusDetails(
state.networkOutBytes = int64(status.NetOut)
// Gather guest metadata from the agent when available
guestIPs, guestIfaces, guestOSName, guestOSVersion, guestAgentVersion := m.fetchGuestAgentMetadata(ctx, client, instanceName, res.Node, res.Name, res.VMID, status)
guestIPs, guestIfaces, guestOSName, guestOSVersion, guestAgentVersion := m.fetchGuestAgentMetadata(ctx, client, instanceName, res.Node, res.Name, res.VMID, status, false)
if len(guestIPs) > 0 {
state.ipAddresses = guestIPs
}
@@ -142,6 +142,7 @@ func (m *Monitor) buildVMFromClusterResource(
client PVEClientInterface,
guestID string,
vmIDToHostAgent map[string]models.Host,
prevVM *models.VM,
) (models.VM, VMMemoryRaw, string, time.Time, bool) {
// Skip templates if configured
if res.Template == 1 {
@@ -196,6 +197,51 @@ func (m *Monitor) buildVMFromClusterResource(
Int("vmid", res.VMID).
Msg("Could not get VM status, using cluster/resources disk data")
}
now := time.Now()
guestAgentAvailable := shouldQueryGuestAgent(state.detailedStatus, prevVM, now) ||
m.hasRecentGuestMetadataEvidence(instanceName, res.Node, res.VMID, now)
if guestAgentAvailable && state.detailedStatus == nil {
guestIPs, guestIfaces, guestOSName, guestOSVersion, guestAgentVersion := m.fetchGuestAgentMetadata(
ctx,
client,
instanceName,
res.Node,
res.Name,
res.VMID,
nil,
true,
)
if len(guestIPs) > 0 {
state.ipAddresses = guestIPs
}
if len(guestIfaces) > 0 {
state.networkInterfaces = guestIfaces
}
if guestOSName != "" {
state.osName = guestOSName
}
if guestOSVersion != "" {
state.osVersion = guestOSVersion
}
if guestAgentVersion != "" {
state.agentVersion = guestAgentVersion
}
var fsDisks []models.Disk
state.diskTotal, state.diskUsed, state.diskFree, state.diskUsage, fsDisks, state.diskFromAgent = m.updateVMDisksFromGuestAgentFSInfo(
ctx,
instanceName,
res,
client,
state.diskTotal,
state.diskUsed,
state.diskUsage,
)
if len(fsDisks) > 0 {
state.individualDisks = fsDisks
}
}
}
if res.Status != "running" {
+17 -3
View File
@@ -30,7 +30,15 @@ func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceNam
// behavior is based on a consistent pre-poll snapshot.
prevGuests := m.previousGuestContextForInstance(instanceName)
allVMs, allContainers := m.collectGuestsFromClusterResources(ctx, instanceName, resources, client, prevGuests.containerOCIByVMID, prevGuests.hostAgentsByVMID)
allVMs, allContainers := m.collectGuestsFromClusterResources(
ctx,
instanceName,
resources,
client,
prevGuests.containerOCIByVMID,
prevGuests.vmsByID,
prevGuests.hostAgentsByVMID,
)
allVMs, allContainers = m.preserveGuestsForGracePeriod(instanceName, resources, prevGuests.vms, prevGuests.containers, nodeEffectiveStatus, allVMs, allContainers)
@@ -62,6 +70,7 @@ func (m *Monitor) collectGuestsFromClusterResources(
resources []proxmox.ClusterResource,
client PVEClientInterface,
prevContainerIsOCI map[int]bool,
prevVMByID map[string]models.VM,
vmIDToHostAgent map[string]models.Host,
) ([]models.VM, []models.Container) {
allVMs := make([]models.VM, 0, len(resources))
@@ -81,7 +90,11 @@ func (m *Monitor) collectGuestsFromClusterResources(
switch res.Type {
case "qemu":
vm, ok := m.handleClusterVMResource(ctx, instanceName, res, guestID, client, vmIDToHostAgent)
var prevVM *models.VM
if prev, ok := prevVMByID[guestID]; ok {
prevVM = &prev
}
vm, ok := m.handleClusterVMResource(ctx, instanceName, res, guestID, client, prevVM, vmIDToHostAgent)
if !ok {
continue
}
@@ -104,9 +117,10 @@ func (m *Monitor) handleClusterVMResource(
res proxmox.ClusterResource,
guestID string,
client PVEClientInterface,
prevVM *models.VM,
vmIDToHostAgent map[string]models.Host,
) (models.VM, bool) {
vm, guestRaw, memorySource, sampleTime, ok := m.buildVMFromClusterResource(ctx, instanceName, res, client, guestID, vmIDToHostAgent)
vm, guestRaw, memorySource, sampleTime, ok := m.buildVMFromClusterResource(ctx, instanceName, res, client, guestID, vmIDToHostAgent, prevVM)
if !ok {
return models.VM{}, false
}