Files
pulse/internal/monitoring/monitor_pve_guest_builders.go
T
rcourtman d7225a45a0 Fix Proxmox guest memory fallbacks
Also fixes Ceph pool threshold resource identity.

Refs #1341
2026-05-05 14:59:29 +01:00

710 lines
22 KiB
Go

package monitoring
import (
"context"
"fmt"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/pkg/fsfilters"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
"github.com/rs/zerolog/log"
)
type vmBuildState struct {
memTotal uint64
memUsed uint64
memorySource string
guestRaw VMMemoryRaw
diskReadBytes int64
diskWriteBytes int64
networkInBytes int64
networkOutBytes int64
diskTotal uint64
diskUsed uint64
diskFree uint64
diskUsage float64
diskFromAgent bool
diskStatusReason string
individualDisks []models.Disk
ipAddresses []string
networkInterfaces []models.GuestNetworkInterface
osName string
osVersion string
agentVersion string
detailedStatus *proxmox.VMStatus
}
func (m *Monitor) applyVMStatusDetails(
ctx context.Context,
instanceName string,
res proxmox.ClusterResource,
client PVEClientInterface,
guestID string,
vmIDToHostAgent map[string]models.Host,
status *proxmox.VMStatus,
state *vmBuildState,
) {
if status == nil || state == nil {
return
}
state.detailedStatus = status
state.memTotal, state.memUsed, state.memorySource = m.resolveGuestStatusMemory(
ctx,
client,
instanceName,
res.Name,
res.Node,
res.VMID,
guestID,
status,
vmIDToHostAgent,
state.memTotal,
state.memorySource,
&state.guestRaw,
)
// Use actual disk I/O values from detailed status
state.diskReadBytes = int64(status.DiskRead)
state.diskWriteBytes = int64(status.DiskWrite)
state.networkInBytes = int64(status.NetIn)
state.networkOutBytes = int64(status.NetOut)
// Gather guest metadata from the agent when available
guestIPs, guestIfaces, guestOSName, guestOSVersion, guestAgentVersion := m.fetchGuestAgentMetadata(ctx, client, instanceName, res.Node, res.Name, res.VMID, status, false)
if len(guestIPs) > 0 {
state.ipAddresses = guestIPs
}
if len(guestIfaces) > 0 {
state.networkInterfaces = guestIfaces
}
if guestOSName != "" {
state.osName = guestOSName
}
if guestOSVersion != "" {
state.osVersion = guestOSVersion
}
if guestAgentVersion != "" {
state.agentVersion = guestAgentVersion
}
// Always try to get filesystem info if agent is enabled
// Prefer guest agent data over cluster/resources data for accuracy
if status.Agent.IsAvailable() {
var fsDisks []models.Disk
state.diskTotal, state.diskUsed, state.diskFree, state.diskUsage, fsDisks, state.diskFromAgent, state.diskStatusReason = m.updateVMDisksFromGuestAgentFSInfo(
ctx,
instanceName,
res,
client,
state.diskTotal,
state.diskUsed,
state.diskUsage,
)
if len(fsDisks) > 0 {
state.individualDisks = fsDisks
}
} else {
// Agent disabled - show allocated disk size
if state.diskTotal > 0 {
state.diskUsage = -1 // Show as allocated size
}
if status.Agent.IsEnabled() {
state.diskStatusReason = "agent-not-running"
} else {
state.diskStatusReason = "agent-disabled"
}
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Int("agent", status.Agent.Value).
Bool("agentEnabled", status.Agent.IsEnabled()).
Msg("VM guest agent is not currently queryable")
}
}
func (m *Monitor) buildVMFromClusterResource(
ctx context.Context,
instanceName string,
res proxmox.ClusterResource,
client PVEClientInterface,
guestID string,
vmIDToHostAgent map[string]models.Host,
prevVM *models.VM,
) (models.VM, VMMemoryRaw, string, []string, time.Time, bool) {
// Skip templates if configured
if res.Template == 1 {
return models.VM{}, VMMemoryRaw{}, "", nil, time.Time{}, false
}
if guestID == "" {
guestID = makeGuestID(instanceName, res.Node, res.VMID)
}
prevSnapshot := m.previousGuestSnapshot(instanceName, "qemu", res.Node, res.VMID)
state := vmBuildState{
memTotal: res.MaxMem,
memUsed: res.Mem,
memorySource: "cluster-resources",
guestRaw: VMMemoryRaw{ListingMem: res.Mem, ListingMaxMem: res.MaxMem},
diskReadBytes: int64(res.DiskRead),
diskWriteBytes: int64(res.DiskWrite),
networkInBytes: int64(res.NetIn),
networkOutBytes: int64(res.NetOut),
diskTotal: res.MaxDisk,
diskUsed: res.Disk,
}
state.diskFree = state.diskTotal - state.diskUsed
state.diskUsage = safePercentage(float64(state.diskUsed), float64(state.diskTotal))
// If VM shows 0 disk usage but has allocated disk, it's likely guest agent issue
// Set to -1 to indicate "unknown" rather than showing misleading 0%
if res.Type == "qemu" && state.diskUsed == 0 && state.diskTotal > 0 && res.Status == "running" {
state.diskUsage = -1
}
if res.Status != "running" && state.diskTotal > 0 {
state.diskUsage = -1
state.diskStatusReason = "vm-stopped"
}
// For running VMs, always try to get filesystem info from guest agent
// The cluster/resources endpoint often returns 0 or incorrect values for disk usage
// We should prefer guest agent data when available for accurate metrics
if res.Status == "running" && res.Type == "qemu" {
// First check if agent is enabled by getting VM status
status, err := client.GetVMStatus(ctx, res.Node, res.VMID)
if err != nil {
if state.diskTotal > 0 {
state.diskStatusReason = "no-status"
}
log.Debug().
Err(err).
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Msg("Could not get VM status to check guest agent availability")
} else if status != nil {
m.applyVMStatusDetails(ctx, instanceName, res, client, guestID, vmIDToHostAgent, status, &state)
} else {
if state.diskTotal > 0 {
state.diskStatusReason = "no-status"
}
// No vmStatus available - keep cluster/resources data
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Msg("Could not get VM status, using cluster/resources disk data")
}
now := time.Now()
guestAgentAvailable := shouldQueryGuestAgent(state.detailedStatus, prevVM, now) ||
m.hasRecentGuestMetadataEvidence(instanceName, res.Node, res.VMID, now)
if guestAgentAvailable && state.detailedStatus == nil {
guestIPs, guestIfaces, guestOSName, guestOSVersion, guestAgentVersion := m.fetchGuestAgentMetadata(
ctx,
client,
instanceName,
res.Node,
res.Name,
res.VMID,
nil,
true,
)
if len(guestIPs) > 0 {
state.ipAddresses = guestIPs
}
if len(guestIfaces) > 0 {
state.networkInterfaces = guestIfaces
}
if guestOSName != "" {
state.osName = guestOSName
}
if guestOSVersion != "" {
state.osVersion = guestOSVersion
}
if guestAgentVersion != "" {
state.agentVersion = guestAgentVersion
}
var fsDisks []models.Disk
state.diskTotal, state.diskUsed, state.diskFree, state.diskUsage, fsDisks, state.diskFromAgent, state.diskStatusReason = m.updateVMDisksFromGuestAgentFSInfo(
ctx,
instanceName,
res,
client,
state.diskTotal,
state.diskUsed,
state.diskUsage,
)
if len(fsDisks) > 0 {
state.individualDisks = fsDisks
}
}
var preferred bool
state.diskTotal, state.diskUsed, state.diskFree, state.diskUsage, state.individualDisks, state.diskStatusReason, preferred = preferLinkedHostAgentDiskInventory(
guestID,
vmIDToHostAgent,
state.diskTotal,
state.diskUsed,
state.diskFree,
state.diskUsage,
state.individualDisks,
state.diskStatusReason,
)
if preferred {
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Bool("guestAgentDiskAvailable", state.diskFromAgent).
Float64("usage", state.diskUsage).
Msg("QEMU disk: preferring linked Pulse host agent disk inventory")
}
}
sampleTime := time.Now()
state.diskTotal, state.diskUsed, state.diskFree, state.diskUsage, state.individualDisks, state.diskStatusReason = stabilizeGuestLowTrustDisk(
prevVM,
res.Status,
state.diskTotal,
state.diskUsed,
state.diskFree,
state.diskUsage,
state.individualDisks,
state.diskStatusReason,
state.diskFromAgent,
sampleTime,
)
if res.Status != "running" {
state.memorySource = "powered-off"
state.memUsed = 0
}
memFree := uint64(0)
if state.memTotal >= state.memUsed {
memFree = state.memTotal - state.memUsed
}
var snapshotNotes []string
state.memUsed, state.memorySource, snapshotNotes = stabilizeGuestLowTrustMemory(
prevSnapshot,
res.Status,
state.memorySource,
state.memTotal,
state.memUsed,
sampleTime,
guestAgentSignalsHealthy(
state.detailedStatus,
state.diskFromAgent,
state.ipAddresses,
state.networkInterfaces,
state.osName,
state.osVersion,
state.agentVersion,
),
)
currentMetrics := IOMetrics{
DiskRead: state.diskReadBytes,
DiskWrite: state.diskWriteBytes,
NetworkIn: state.networkInBytes,
NetworkOut: state.networkOutBytes,
Timestamp: sampleTime,
}
diskReadRate, diskWriteRate, netInRate, netOutRate := m.rateTracker.CalculateRates(guestID, currentMetrics)
memoryUsage := safePercentage(float64(state.memUsed), float64(state.memTotal))
memory := models.Memory{
Total: int64(state.memTotal),
Used: int64(state.memUsed),
Free: int64(memFree),
Usage: memoryUsage,
}
if memory.Free < 0 {
memory.Free = 0
}
if memory.Used > memory.Total {
memory.Used = memory.Total
}
if state.detailedStatus != nil && state.detailedStatus.Balloon > 0 {
memory.Balloon = int64(state.detailedStatus.Balloon)
}
vm := models.VM{
ID: guestID,
VMID: res.VMID,
Name: res.Name,
Node: res.Node,
Pool: strings.TrimSpace(res.Pool),
Instance: instanceName,
Status: res.Status,
Type: "qemu",
CPU: safeFloat(res.CPU),
CPUs: res.MaxCPU,
Memory: memory,
Disk: models.Disk{
Total: int64(state.diskTotal),
Used: int64(state.diskUsed),
Free: int64(state.diskFree),
Usage: state.diskUsage,
},
Disks: state.individualDisks,
DiskStatusReason: state.diskStatusReason,
IPAddresses: state.ipAddresses,
OSName: state.osName,
OSVersion: state.osVersion,
AgentVersion: state.agentVersion,
NetworkInterfaces: state.networkInterfaces,
NetworkIn: max(0, int64(netInRate)),
NetworkOut: max(0, int64(netOutRate)),
DiskRead: max(0, int64(diskReadRate)),
DiskWrite: max(0, int64(diskWriteRate)),
Uptime: int64(res.Uptime),
Template: res.Template == 1,
LastSeen: sampleTime,
}
// Parse tags
if res.Tags != "" {
vm.Tags = strings.Split(res.Tags, ";")
// Log if Pulse-specific tags are detected
for _, tag := range vm.Tags {
switch tag {
case "pulse-no-alerts", "pulse-monitor-only", "pulse-relaxed":
log.Info().
Str("vm", vm.Name).
Str("node", vm.Node).
Str("tag", tag).
Msg("Pulse control tag detected on VM")
}
}
}
return vm, state.guestRaw, state.memorySource, snapshotNotes, sampleTime, true
}
type vmFSInfoSummary struct {
totalBytes uint64
usedBytes uint64
individualDisks []models.Disk
skippedFS []string
includedFS []string
}
func (m *Monitor) fetchVMFSInfo(ctx context.Context, instanceName string, res proxmox.ClusterResource, client PVEClientInterface) ([]proxmox.VMFileSystem, string, bool) {
// Use retry logic for guest agent calls to handle transient timeouts (refs #630)
fsInfoRaw, err := m.retryGuestAgentCall(ctx, m.guestAgentFSInfoTimeout, m.guestAgentRetries, func(ctx context.Context) (interface{}, error) {
return client.GetVMFSInfo(ctx, res.Node, res.VMID)
})
var fsInfo []proxmox.VMFileSystem
if err == nil {
if fs, ok := fsInfoRaw.([]proxmox.VMFileSystem); ok {
fsInfo = fs
}
}
if err != nil {
// Log more helpful error messages based on the error type
errMsg := err.Error()
if strings.Contains(errMsg, "500") || strings.Contains(errMsg, "QEMU guest agent is not running") {
log.Info().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Msg("Guest agent enabled in VM config but not running inside guest OS. Install and start qemu-guest-agent in the VM")
log.Info().
Str("instance", instanceName).
Str("vm", res.Name).
Msg("To verify: ssh into VM and run 'systemctl status qemu-guest-agent' or 'ps aux | grep qemu-ga'")
} else if strings.Contains(errMsg, "timeout") {
log.Info().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Msg("Guest agent timeout - agent may be installed but not responding")
} else if strings.Contains(errMsg, "403") || strings.Contains(errMsg, "401") || strings.Contains(errMsg, "authentication error") {
// Permission error - user/token lacks required permissions
log.Info().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Msg("VM disk monitoring permission denied. Check permissions:")
log.Info().
Str("instance", instanceName).
Str("vm", res.Name).
Msg("• Proxmox 9: Ensure token/user has VM.GuestAgent.Audit privilege (Pulse setup adds this via PulseMonitor role)")
log.Info().
Str("instance", instanceName).
Str("vm", res.Name).
Msg("• Proxmox 8: Ensure token/user has VM.Monitor privilege (Pulse setup adds this via PulseMonitor role)")
log.Info().
Str("instance", instanceName).
Str("vm", res.Name).
Msg("• All versions: Sys.Audit is recommended for Ceph metrics and applied when available")
log.Info().
Str("instance", instanceName).
Str("vm", res.Name).
Msg("• Re-run Pulse setup script if node was added before v4.7")
log.Info().
Str("instance", instanceName).
Str("vm", res.Name).
Msg("• Verify guest agent is installed and running inside the VM")
} else {
log.Debug().
Err(err).
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Msg("Failed to get filesystem info from guest agent")
}
return nil, classifyGuestAgentDiskStatusError(err), false
}
if len(fsInfo) == 0 {
log.Info().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Msg("Guest agent returned no filesystem info - agent may need restart or VM may have no mounted filesystems")
return nil, "no-filesystems", false
}
return fsInfo, "", true
}
func (m *Monitor) summarizeVMFSInfo(instanceName string, res proxmox.ClusterResource, fsInfo []proxmox.VMFileSystem) vmFSInfoSummary {
summary := vmFSInfoSummary{}
// Track seen filesystems to deduplicate mounts that share the same device:
// 1. btrfs/zfs mount multiple subvolumes from a shared pool
// 2. Kubernetes bind-mounts volumes at both pod and plugin paths
// In both cases, only count the device's capacity once.
seenFilesystems := make(map[string]bool)
// Log all filesystems received for debugging
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Int("filesystem_count", len(fsInfo)).
Msg("Processing filesystems from guest agent")
for _, fs := range fsInfo {
// Skip special filesystems and mounts
shouldSkip, reasons := fsfilters.ShouldSkipFilesystem(fs.Type, fs.Mountpoint, fs.TotalBytes, fs.UsedBytes)
if shouldSkip {
// Check if any reason is read-only for detailed logging
for _, r := range reasons {
if strings.HasPrefix(r, "read-only-") {
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Str("mountpoint", fs.Mountpoint).
Str("type", fs.Type).
Float64("total_gb", float64(fs.TotalBytes)/1073741824).
Float64("used_gb", float64(fs.UsedBytes)/1073741824).
Msg("Skipping read-only filesystem from disk aggregation")
break
}
}
summary.skippedFS = append(summary.skippedFS, fmt.Sprintf("%s(%s,%s)",
fs.Mountpoint, fs.Type, strings.Join(reasons, ",")))
continue
}
// Only count real filesystems with valid data
// Some filesystems report 0 bytes (like unformatted or system partitions)
if fs.TotalBytes > 0 {
// Deduplication: the same device can appear multiple times when:
// 1. btrfs/zfs mount multiple subvolumes from a shared pool
// 2. Kubernetes bind-mounts volumes at both pod and plugin paths
// In both cases, only count the device's capacity once.
fsTypeLower := strings.ToLower(fs.Type)
countThisFS := true
if fs.Disk != "" {
// Same device at multiple mount paths → count once
dedupeKey := fmt.Sprintf("%s:%d", fs.Disk, fs.TotalBytes)
if seenFilesystems[dedupeKey] {
countThisFS = false
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Str("mountpoint", fs.Mountpoint).
Str("type", fs.Type).
Str("device", fs.Disk).
Uint64("total", fs.TotalBytes).
Str("dedupe_key", dedupeKey).
Msg("Skipping duplicate device mount in total calculation")
} else {
seenFilesystems[dedupeKey] = true
}
} else if fsTypeLower == "btrfs" || fsTypeLower == "zfs" ||
strings.HasPrefix(fsTypeLower, "zfs") {
// No device info — fall back to type+size dedup for COW filesystems
dedupeKey := fmt.Sprintf("%s::%d", fsTypeLower, fs.TotalBytes)
if seenFilesystems[dedupeKey] {
countThisFS = false
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Str("mountpoint", fs.Mountpoint).
Str("type", fs.Type).
Uint64("total", fs.TotalBytes).
Str("dedupe_key", dedupeKey).
Msg("Skipping duplicate btrfs/zfs subvolume in total calculation")
} else {
seenFilesystems[dedupeKey] = true
}
}
if countThisFS {
summary.totalBytes += fs.TotalBytes
summary.usedBytes += fs.UsedBytes
}
summary.includedFS = append(summary.includedFS, fmt.Sprintf("%s(%s,%.1fGB)",
fs.Mountpoint, fs.Type, float64(fs.TotalBytes)/1073741824))
// Add to individual disks array (always include for display)
summary.individualDisks = append(summary.individualDisks, models.Disk{
Total: int64(fs.TotalBytes),
Used: int64(fs.UsedBytes),
Free: int64(fs.TotalBytes - fs.UsedBytes),
Usage: safePercentage(float64(fs.UsedBytes), float64(fs.TotalBytes)),
Mountpoint: fs.Mountpoint,
Type: fs.Type,
Device: fs.Disk,
})
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Str("mountpoint", fs.Mountpoint).
Str("type", fs.Type).
Uint64("total", fs.TotalBytes).
Uint64("used", fs.UsedBytes).
Float64("total_gb", float64(fs.TotalBytes)/1073741824).
Float64("used_gb", float64(fs.UsedBytes)/1073741824).
Bool("counted_in_total", countThisFS).
Msg("Including filesystem in disk usage calculation")
} else if fs.TotalBytes == 0 && len(fs.Mountpoint) > 0 {
summary.skippedFS = append(summary.skippedFS, fmt.Sprintf("%s(%s,0GB)", fs.Mountpoint, fs.Type))
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Str("mountpoint", fs.Mountpoint).
Str("type", fs.Type).
Msg("Skipping filesystem with zero total bytes")
}
}
if len(summary.skippedFS) > 0 {
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Strs("skipped", summary.skippedFS).
Msg("Skipped special filesystems")
}
if len(summary.includedFS) > 0 {
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Strs("included", summary.includedFS).
Msg("Filesystems included in disk calculation")
}
return summary
}
func (m *Monitor) updateVMDisksFromGuestAgentFSInfo(
ctx context.Context,
instanceName string,
res proxmox.ClusterResource,
client PVEClientInterface,
diskTotal uint64,
diskUsed uint64,
diskUsage float64,
) (uint64, uint64, uint64, float64, []models.Disk, bool, string) {
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Msg("Guest agent enabled, querying filesystem info for accurate disk usage")
fsInfo, diskStatusReason, ok := m.fetchVMFSInfo(ctx, instanceName, res, client)
if !ok {
return diskTotal, diskUsed, diskTotal - diskUsed, diskUsage, nil, false, diskStatusReason
}
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("filesystems", len(fsInfo)).
Msg("Got filesystem info from guest agent")
summary := m.summarizeVMFSInfo(instanceName, res, fsInfo)
// If we got valid data from guest agent, use it
if summary.totalBytes > 0 {
// Sanity check: if the reported disk is way larger than allocated disk,
// we might be getting host disk info somehow
allocatedDiskGB := float64(res.MaxDisk) / 1073741824
reportedDiskGB := float64(summary.totalBytes) / 1073741824
// If reported disk is more than 2x the allocated disk, log for debugging.
// This is expected for pass-through disks and can also indicate we're getting host disk or network shares.
if allocatedDiskGB > 0 && reportedDiskGB > allocatedDiskGB*2 {
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Float64("allocated_gb", allocatedDiskGB).
Float64("reported_gb", reportedDiskGB).
Float64("ratio", reportedDiskGB/allocatedDiskGB).
Strs("filesystems", summary.includedFS).
Msg("VM reports disk usage significantly larger than allocated disk - possible issue with filesystem detection")
}
diskTotal = summary.totalBytes
diskUsed = summary.usedBytes
diskUsage = safePercentage(float64(summary.usedBytes), float64(summary.totalBytes))
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Uint64("totalBytes", summary.totalBytes).
Uint64("usedBytes", summary.usedBytes).
Float64("total_gb", float64(summary.totalBytes)/1073741824).
Float64("used_gb", float64(summary.usedBytes)/1073741824).
Float64("allocated_gb", allocatedDiskGB).
Float64("usage", diskUsage).
Uint64("old_disk", res.Disk).
Uint64("old_maxdisk", res.MaxDisk).
Msg("Using guest agent data for accurate disk usage (replacing cluster/resources data)")
return diskTotal, diskUsed, diskTotal - diskUsed, diskUsage, summary.individualDisks, true, ""
}
// Only special filesystems found - show allocated disk size instead
if diskTotal > 0 {
diskUsage = -1 // Show as allocated size
}
log.Info().
Str("instance", instanceName).
Str("vm", res.Name).
Int("filesystems_found", len(fsInfo)).
Msg("Guest agent provided filesystem info but no usable filesystems found (all were special mounts)")
return diskTotal, diskUsed, diskTotal - diskUsed, diskUsage, nil, false, "special-filesystems-only"
}