From effa640fdaaf96ad00ea00746bb1b8a69d171354 Mon Sep 17 00:00:00 2001 From: Pulse Monitor Date: Mon, 1 Sep 2025 10:16:02 +0000 Subject: [PATCH] feat: improve VM disk stats handling and documentation - Add DiskStatusReason field to track why disk stats are unavailable - Show helpful tooltips in UI explaining specific issues: - Proxmox 9 API token limitation (401 on guest agent endpoints) - Guest agent not installed/running - Special filesystems only (Live ISOs) - Permission issues - Add comprehensive troubleshooting guide (docs/VM_DISK_STATS_TROUBLESHOOTING.md) - Document that API tokens cannot access guest agent data on PVE 9 - Tested and confirmed: only password/cookie auth works for guest agent on PVE 9 - Update README with quick reference to VM disk stats issue This addresses issues #348, #367, and #71 by clearly explaining the root cause (Proxmox API limitation) and providing actionable guidance to users. --- README.md | 6 + docs/VM_DISK_STATS_TROUBLESHOOTING.md | 187 ++++++++++++++++++ .../src/components/Dashboard/GuestRow.tsx | 38 +++- frontend-modern/src/types/api.ts | 1 + frontend-modern/vite.config.dev.ts | 31 +++ internal/models/models.go | 1 + internal/monitoring/monitor.go | 19 +- 7 files changed, 278 insertions(+), 5 deletions(-) create mode 100644 docs/VM_DISK_STATS_TROUBLESHOOTING.md create mode 100644 frontend-modern/vite.config.dev.ts diff --git a/README.md b/README.md index abfee22de..e8d149446 100644 --- a/README.md +++ b/README.md @@ -456,6 +456,12 @@ See [Reverse Proxy Configuration Guide](docs/REVERSE_PROXY.md) for nginx, Caddy, - If you provide credentials via environment variables, they take precedence - To use Quick Security Setup: Start container WITHOUT auth environment variables +### VM Disk Stats Show 0% or Dash +- **Known Proxmox 9 limitation**: API tokens cannot access guest agent data +- See [VM Disk Stats Troubleshooting Guide](docs/VM_DISK_STATS_TROUBLESHOOTING.md) for details +- Container (LXC) disk stats work normally +- This is a Proxmox API limitation, not a Pulse bug + ### Connection Issues - Check Proxmox API is accessible (port 8006/8007) - Verify credentials have PVEAuditor role minimum diff --git a/docs/VM_DISK_STATS_TROUBLESHOOTING.md b/docs/VM_DISK_STATS_TROUBLESHOOTING.md new file mode 100644 index 000000000..274dbcfeb --- /dev/null +++ b/docs/VM_DISK_STATS_TROUBLESHOOTING.md @@ -0,0 +1,187 @@ +# VM Disk Stats Troubleshooting Guide + +**Last Updated: September 1, 2025** +**Tested on: Proxmox VE 9.0.6** +**Status: Current Proxmox limitation (may be fixed in future versions)** + +## Quick Summary + +VM disk usage shows 0% or "-" in Pulse? This is a known Proxmox limitation, not a Pulse bug. + +**The Problem**: On Proxmox 9, API tokens cannot access guest agent data (returns 401 Unauthorized), even with full permissions. This prevents Pulse from retrieving actual disk usage for VMs. + +## Why VM Disk Stats Don't Work + +### 1. How Proxmox Reports VM Disk Usage + +Unlike containers (LXCs), Proxmox doesn't natively track VM disk usage. The standard API endpoints return: +- `disk: 0` - Always 0 for actual usage +- `maxdisk: ` - The allocated disk size + +To get real disk usage, you must query the QEMU Guest Agent inside the VM. + +### 2. The Proxmox 9 API Token Limitation + +**Testing Results (September 1, 2025 on PVE 9.0.6)**: + +| User | Auth Method | Guest Agent Access | +|------|-------------|-------------------| +| root@pam | API Token | ❌ 401 Unauthorized | +| root@pam | Password/Cookie | ✅ 200 OK | +| user@pam with PVEAuditor | API Token | ❌ 401 Unauthorized | +| user@pam with PVEAuditor | Password/Cookie | ✅ 200 OK | + +**Key Finding**: API tokens cannot access `/nodes/{node}/qemu/{vmid}/agent/*` endpoints on Proxmox 9, regardless of user or permissions. + +### 3. Version-Specific Behavior + +#### Proxmox 8 and earlier +- API tokens work with `VM.Monitor` permission +- Guest agent data accessible via tokens + +#### Proxmox 9+ +- `VM.Monitor` permission removed +- Replaced with `VM.GuestAgent.Audit` (part of PVEAuditor role) +- **BUT**: API tokens still cannot access guest agent endpoints (Proxmox bug/limitation) + +## What You'll See in Pulse + +### Disk Display States + +1. **Shows percentage** (e.g., "45%") + - Everything working correctly + - Guest agent installed and accessible + +2. **Shows "-" with tooltip** + - Hover to see specific reason: + - "Permission denied. On Proxmox 9, API tokens cannot access guest agent data." + - "Guest agent not running. Install qemu-guest-agent in the VM." + - "Only special filesystems detected. Normal for Live ISOs." + +3. **Shows 0%** + - Old Pulse version or edge case + - Usually means no guest agent + +## Solutions + +### Option 1: Use Password Authentication (Not Recommended) +Instead of API tokens, use username/password authentication. This works but is less secure and not recommended for production. + +### Option 2: Accept the Limitation +- Container (LXC) disk stats work fine +- VM disk stats will show "-" with explanatory tooltip +- Wait for Proxmox to fix this upstream + +### Option 3: Install Guest Agent (Partial Solution) +Even though API tokens can't access the data on PVE 9, installing guest agent helps on PVE 8 and prepares for when Proxmox fixes this: + +#### Debian/Ubuntu VMs +```bash +apt update && apt install qemu-guest-agent +systemctl enable --now qemu-guest-agent +``` + +#### RHEL/Rocky/AlmaLinux VMs +```bash +yum install qemu-guest-agent +systemctl enable --now qemu-guest-agent +``` + +#### Alpine Linux VMs +```bash +apk add qemu-guest-agent +rc-update add qemu-guest-agent +rc-service qemu-guest-agent start +``` + +#### Windows VMs +Install VirtIO drivers which include the guest agent. + +After installation, enable in Proxmox: +1. VM → Options → QEMU Guest Agent → Enable +2. Restart the VM + +## Verification Commands + +### Check if guest agent is accessible (run on Proxmox host) +```bash +# Check if agent is enabled in VM config +qm config | grep agent + +# Test agent directly (works as root on host) +qm agent ping +qm agent get-fsinfo + +# Test via API with token (will fail on PVE 9) +curl -k -H "Authorization: PVEAPIToken=user@pam!token=" \ + https://localhost:8006/api2/json/nodes//qemu//agent/get-fsinfo +``` + +## Related Issues + +- **GitHub #348**: "After 4.7 Disk usage is at 0% (Proxmox 9 related)" +- **GitHub #367**: "Strange disk usage reporting on some VMs" +- **GitHub #71**: Initial report of VM disk showing 0% +- **Proxmox Bug #1373**: Feature request for native VM disk usage in Proxmox + +## Technical Details + +### Why This Happens + +1. **Proxmox Design**: VMs are black boxes to the hypervisor. Unlike containers which share the host kernel, VMs run their own OS and Proxmox can't see inside without guest agent. + +2. **Security Model Change**: Proxmox 9 tightened security around guest agent access, but went too far and blocked API tokens entirely. + +3. **API Endpoints Affected**: + - `/nodes/{node}/qemu/{vmid}/agent/get-fsinfo` - File system information + - `/nodes/{node}/qemu/{vmid}/agent/*` - All guest agent endpoints + +### What Pulse Does + +1. Fetches VM data from `/nodes/{node}/qemu` or `/cluster/resources` +2. Sees `disk: 0` for VMs +3. Attempts to query guest agent at `/nodes/{node}/qemu/{vmid}/agent/get-fsinfo` +4. Gets 401 Unauthorized (on PVE 9 with tokens) +5. Sets disk usage to "-" with explanation in tooltip + +### Special Cases + +#### Live ISOs/Installation Media +VMs booted from ISOs show special filesystems only: +- `squashfs` - Compressed read-only filesystem +- `iso9660` - CD/DVD filesystem +- `tmpfs` - RAM-based temporary filesystem + +These are filtered out as they don't represent actual disk usage. + +#### Templates +VM templates always show 0% (they're not running). + +#### Stopped VMs +Stopped VMs show 0% (guest agent not accessible when VM is off). + +## Future Updates + +This limitation is specific to Proxmox 9.0.x as of September 2025. Check back for updates: + +- **Proxmox may fix this** in 9.1 or later versions +- **Pulse will automatically work** once Proxmox fixes the API +- **No Pulse update needed** - the fix needs to come from Proxmox + +## Workaround for Critical Monitoring + +If VM disk monitoring is critical for your environment: + +1. Consider using a dedicated monitoring solution inside VMs (Prometheus node exporter, Telegraf, etc.) +2. Use Proxmox's built-in email alerts for disk space +3. Monitor at the storage level instead of per-VM + +## Need Help? + +- Check Pulse logs: `tail -f /var/log/pulse.log | grep -i "guest agent"` +- Verify permissions: `pveum user permissions ` +- Test manually: `qm agent get-fsinfo` (as root on Proxmox host) + +--- + +*This document describes the current state as of September 2025. The situation may improve in future Proxmox releases.* \ No newline at end of file diff --git a/frontend-modern/src/components/Dashboard/GuestRow.tsx b/frontend-modern/src/components/Dashboard/GuestRow.tsx index b71775a79..625ae69df 100644 --- a/frontend-modern/src/components/Dashboard/GuestRow.tsx +++ b/frontend-modern/src/components/Dashboard/GuestRow.tsx @@ -74,7 +74,34 @@ export function GuestRow(props: GuestRowProps) { const isRunning = createMemo(() => props.guest.status === 'running'); - + // Get helpful tooltip for disk status + const getDiskStatusTooltip = () => { + if (!isVM(props.guest)) return 'Disk stats unavailable'; + + const vm = props.guest as VM; + const reason = vm.diskStatusReason; + + switch (reason) { + case 'agent-not-running': + return 'Guest agent not running. Install and start qemu-guest-agent in the VM.'; + case 'agent-timeout': + return 'Guest agent timeout. Agent may need to be restarted.'; + case 'permission-denied': + return 'Permission denied. On Proxmox 9, API tokens cannot access guest agent data. Use password authentication instead of API tokens.'; + case 'agent-disabled': + return 'Guest agent is disabled in VM configuration. Enable it in VM Options.'; + case 'no-filesystems': + return 'No filesystems found. VM may be booting or using a Live ISO.'; + case 'special-filesystems-only': + return 'Only special filesystems detected (ISO/squashfs). This is normal for Live systems.'; + case 'agent-error': + return 'Error communicating with guest agent.'; + case 'no-data': + return 'No disk data available from Proxmox API.'; + default: + return 'Disk stats unavailable. Guest agent may not be installed.'; + } + }; // Get row styling - include alert styles if present const rowClass = createMemo(() => { @@ -189,7 +216,14 @@ export function GuestRow(props: GuestRowProps) { 0 && diskPercent() !== -1} - fallback={-} + fallback={ + + - + + } > 0 && vm.Status == "running" { diskUsage = -1 + diskStatusReason = "no-data" } // For running VMs with 0 disk usage, always try guest agent (even if agent flag is 0) // The agent flag might not be reliable or the API might return 0 incorrectly if vm.Status == "running" && (vm.Agent > 0 || (diskUsed == 0 && diskTotal > 0)) { - log.Debug(). + log.Info(). Str("instance", instanceName). Str("vm", vm.Name). Int("vmid", vm.VMID). Int("agent", vm.Agent). + Uint64("diskUsed", diskUsed). + Uint64("diskTotal", diskTotal). Bool("diskIsZero", diskUsed == 0). Msg("Attempting to get filesystem info from guest agent (legacy API)") @@ -1521,12 +1525,14 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, cli // Log more helpful error messages based on the error type errMsg := err.Error() if strings.Contains(errMsg, "500") || strings.Contains(errMsg, "QEMU guest agent is not running") { + diskStatusReason = "agent-not-running" log.Info(). Str("instance", instanceName). Str("vm", vm.Name). Int("vmid", vm.VMID). Msg("Guest agent enabled in VM config but not running inside guest OS. Install and start qemu-guest-agent in the VM (legacy API)") } else if strings.Contains(errMsg, "timeout") { + diskStatusReason = "agent-timeout" log.Info(). Str("instance", instanceName). Str("vm", vm.Name). @@ -1534,6 +1540,7 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, cli Msg("Guest agent timeout - agent may be installed but not responding (legacy API)") } else if strings.Contains(errMsg, "403") || strings.Contains(errMsg, "401") || strings.Contains(errMsg, "authentication error") { // Permission error - check if it's the known PVE 9 limitation + diskStatusReason = "permission-denied" log.Info(). Str("instance", instanceName). Str("vm", vm.Name). @@ -1552,6 +1559,7 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, cli Str("vm", vm.Name). Msg("• Verify guest agent is installed and running inside the VM") } else { + diskStatusReason = "agent-error" log.Debug(). Err(err). Str("instance", instanceName). @@ -1560,13 +1568,14 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, cli Msg("Failed to get filesystem info from guest agent (legacy API)") } } else if len(fsInfo) == 0 { + diskStatusReason = "no-filesystems" log.Info(). Str("instance", instanceName). Str("vm", vm.Name). Int("vmid", vm.VMID). Msg("Guest agent returned no filesystem info - agent may need restart or VM may have no mounted filesystems (legacy API)") } else { - log.Debug(). + log.Info(). Str("instance", instanceName). Str("vm", vm.Name). Int("filesystems", len(fsInfo)). @@ -1616,6 +1625,7 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, cli diskUsed = usedBytes diskFree = totalBytes - usedBytes diskUsage = safePercentage(float64(usedBytes), float64(totalBytes)) + diskStatusReason = "" // Clear reason - we got data! log.Info(). Str("instance", instanceName). @@ -1626,6 +1636,7 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, cli Float64("usage", diskUsage). Msg("Successfully retrieved disk usage from guest agent (node API showed 0)") } else { + diskStatusReason = "special-filesystems-only" log.Info(). Str("instance", instanceName). Str("vm", vm.Name). @@ -1634,7 +1645,8 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, cli } } } else { - if vm.Agent == 0 { + if vm.Agent == 0 && diskUsed == 0 && diskTotal > 0 { + diskStatusReason = "agent-disabled" log.Debug(). Str("instance", instanceName). Str("vm", vm.Name). @@ -1666,6 +1678,7 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, cli Free: int64(diskFree), Usage: diskUsage, }, + DiskStatusReason: diskStatusReason, NetworkIn: maxInt64(0, int64(netInRate)), NetworkOut: maxInt64(0, int64(netOutRate)), DiskRead: maxInt64(0, int64(diskReadRate)),