From 6ec491bdf30434eb20419ebf9dcc89b9f16a3737 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:48:52 +0100 Subject: [PATCH] Keep VM disk diagnostics off the guest-agent channel Replace active guest probes with bounded passive host reads. Preserve errors and private configuration, distinguish setup from responsiveness, and put backup/thaw precautions ahead of troubleshooting. Exercise the actual helper and copied command with adverse and healthy controls. Contract-Neutral: Operator diagnostics and existing help only; Proxmox polling, guest-agent coordination, APIs, storage and runtime behaviour are unchanged. Change-source: pulse-maintainer --- docs/TROUBLESHOOTING.md | 19 +- docs/VM_DISK_MONITORING.md | 114 ++++++-- .../browser-tests/vm-disk-safety-help.cjs | 93 +++++++ .../public/docs/TROUBLESHOOTING.md | 19 +- .../public/docs/VM_DISK_MONITORING.md | 114 ++++++-- scripts/test-vm-disk.sh | 224 ++++++++-------- scripts/tests/test_vm_disk_diagnostics.py | 252 ++++++++++++++++++ 7 files changed, 652 insertions(+), 183 deletions(-) create mode 100644 frontend-modern/browser-tests/vm-disk-safety-help.cjs create mode 100644 scripts/tests/test_vm_disk_diagnostics.py diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 2ff1a7b0eb..b5e498ebb3 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -227,10 +227,14 @@ installed service must contain `SyslogLevelPrefix=true` and repair an older generated unit rather than adding a JSON-parsing wrapper. #### VMs show "-" for disk usage -- Install **QEMU Guest Agent** in the VM. -- Enable "QEMU Guest Agent" in Proxmox VM Options. -- Restart the VM. -- See [VM Disk Monitoring](VM_DISK_MONITORING.md). +- Read the disk value's explanation and observation time first; a dash is not + proof that the agent is missing. +- Check the guest-local service, current VM Options and the configured API + token's read permissions. Schedule any setup change or restart outside backups. +- Do not run guest-agent probes during backup freeze/thaw. An OK backup task + or an absent lock does not confirm thaw. +- See [VM Disk Monitoring](VM_DISK_MONITORING.md) for the passive host preflight + and backup safety precaution; it does not verify a fresh disk poll. #### Temperature data missing - Install `lm-sensors` on the host. @@ -615,7 +619,12 @@ For PVE 8 only, use `VM.Monitor` instead of the `VM.GuestAgent.*` privileges. Note: The built-in `PVEAuditor` role cannot be modified. Create a custom role (e.g. `PulseMonitor`) with the above privileges added, and assign it to your Pulse API token. After upgrading to PVE 9, add the `VM.GuestAgent.*` privileges and remove legacy `VM.Monitor` from the custom role. -**Rocky Linux / RHEL VMs**: The default qemu-guest-agent configuration may block file-read RPCs (`guest-file-open`, `guest-file-read`, `guest-file-close`). If memory or disk data is missing for these VMs, check `/etc/sysconfig/qemu-ga` and ensure those operations are not blocked, then restart the agent. Refer to your distro's qemu-guest-agent documentation for the exact config syntax. +**Rocky Linux / RHEL VMs**: File-read restrictions in `/etc/sysconfig/qemu-ga` +can explain missing guest memory; they do not by themselves establish why disk +usage is absent. Review the guest's policy before changing it. Schedule any +allowlist change or agent restart outside backups, following the guest OS's +documentation. See [VM Disk Monitoring](VM_DISK_MONITORING.md) for the distinct +permissions and backup safety boundary. ### Proxmox pending-update access diff --git a/docs/VM_DISK_MONITORING.md b/docs/VM_DISK_MONITORING.md index cbf6a5a328..9c0b762933 100644 --- a/docs/VM_DISK_MONITORING.md +++ b/docs/VM_DISK_MONITORING.md @@ -1,42 +1,102 @@ # 💾 VM Disk Monitoring -Monitor actual disk usage inside your VMs using the QEMU Guest Agent. +Pulse can read filesystem usage inside a Proxmox VM through the QEMU Guest +Agent. Allocated virtual-disk size is not the same as used space inside the +guest. LXC filesystem collection uses a different path. -## 🚀 Quick Start +## Backup safety -1. **Install Guest Agent**: - * **Linux**: `apt install qemu-guest-agent` (Debian/Ubuntu) or `yum install qemu-guest-agent` (RHEL). - * **Windows**: Install **virtio-win** drivers. -2. **Enable in Proxmox**: - * VM Options → **QEMU Guest Agent** → Enabled. - * Restart the VM. -3. **Verify**: - * Run `qm agent ping` on the Proxmox host. - * Check Pulse dashboard for disk usage (e.g., "5.2GB used of 32GB"). +**Do not run manual guest-agent probes during a backup, freeze or thaw.** A +guest-agent request is an active command on the same channel used by +freeze-enabled backups, even when it only reads disk or memory information. +There is a [report of a VM remaining frozen despite an OK backup task](https://github.com/rcourtman/Pulse/issues/2439). +The reported command-ID collision is not established as a reproduced cause. -## ⚙️ Requirements +If your installation is affected, stop the Pulse service before the backup +and restart it only after independently confirming the guest has thawed, using +your existing guest console or workload checks. **Pulse monitoring and alerts +are unavailable while it is stopped.** An OK backup task or an absent VM lock +does not prove thaw succeeded. This is a temporary precaution, not a claim +that the monitoring defect is fixed. -* **QEMU Guest Agent**: Must be installed and running inside the VM. -* **Proxmox Permissions**: `VM.Monitor` (Proxmox 8) or `VM.GuestAgent.Audit` + `VM.GuestAgent.FileRead` (Proxmox 9+). Note: `PVEAuditor` is a built-in read-only role that cannot be modified — create a custom role instead. +Do not test freeze/thaw commands, clear backup locks, disable backup freezing +or force-reset a guest as a disk-monitoring diagnostic. Those operations can +affect workloads or backup consistency. Preserve the existing backup task and +redacted error, rather than repeat a potentially harmful backup for a report. + +## 🚀 Setup + +Plan changes outside backup windows and follow the guest's normal maintenance +procedure; a reboot interrupts its workloads. + +1. **Install the guest agent inside the VM.** Use the guest OS's supported + package: `qemu-guest-agent` on Linux, or the QEMU Guest Agent service from + the virtio-win distribution on Windows. +2. **Enable it in Proxmox.** Review **VM Options → QEMU Guest Agent** and apply + any required VM restart in that maintenance window. +3. **Check the existing observations.** In Pulse, hover over the disk dash or + open the resource details for the collection explanation and observation + time. Check the guest-agent service from inside the guest, not by repeatedly + sending `qm agent` commands from the hypervisor. A configured agent is not + proof it responded, and a retained disk value is not proof of a fresh poll. + +## ⚙️ Permissions + +Use the dedicated account and token actually configured for that Pulse +connection, not a guessed account name. Review their effective privileges for +the affected VM; a host-root diagnostic does not test the API token's access. + +| Collection | Proxmox VE 9+ | Proxmox VE 8 | +| --- | --- | --- | +| Filesystem usage and guest information | `VM.GuestAgent.Audit` | `VM.Monitor` | +| Reading guest memory via `/proc/meminfo` | Also `VM.GuestAgent.FileRead` | `VM.Monitor` | + +Do not grant Administrator or guest execution/write permissions to diagnose a +missing disk value. The built-in `PVEAuditor` role cannot be modified; use a +custom monitoring role if the required read privileges are absent. ## 🔧 Troubleshooting -| Issue | Solution | -| :--- | :--- | -| **Disk shows "-"** | Hover over the dash for details. Common causes: Agent not running, disabled in config, or permission denied. | -| **Permission Denied** | Ensure your Proxmox token/user has `VM.GuestAgent.Audit` + `VM.GuestAgent.FileRead` (PVE 9+) or `VM.Monitor` (PVE 8). | -| **Rocky Linux / RHEL: memory or disk data missing** | The default qemu-guest-agent config may block file-read RPCs. Check `/etc/sysconfig/qemu-ga` and ensure `guest-file-open`, `guest-file-read`, and `guest-file-close` are not blocked, then restart the agent. See your distro's qemu-guest-agent docs for exact syntax. | -| **Agent Timeout** | Increase timeouts via env vars if network is slow: `GUEST_AGENT_FSINFO_TIMEOUT=10s`. | -| **Windows VMs** | Ensure the **QEMU Guest Agent** service is running in Windows Services. | +| Observation | Useful next check | +| --- | --- | +| **Disk shows “-”** | Read its explanation and observation time. Check the owning host, current VM options, guest-local service and configured API token access. | +| **Permission denied** | Review the token and account's effective read privileges on that VM, including privilege-separated token ACLs. A successful root command would not disprove this error. | +| **Timeout or backup lock** | Check the existing backup/task timeline and guest workload locally. Defer active guest-agent probes; increasing a timeout is not a contention or thaw repair. | +| **Rocky Linux / RHEL memory missing** | Review `/etc/sysconfig/qemu-ga` inside the guest. File-read restrictions can explain memory collection failure; they do not by themselves prove why filesystem usage is absent. Change the guest's allowlist or restart its agent only through your normal maintenance procedure, outside backups. | +| **Windows service stopped** | Check the QEMU Guest Agent service inside Windows. Schedule any restart outside backups. | + +### Passive host preflight + +The current `scripts/test-vm-disk.sh` helper reads only local VM status and +current configuration, with bounded timeouts. It sends **no guest-agent +commands**, reads no guest files, and changes no service, lock or ACL. Use it +on the VM's owning Proxmox host with an account permitted to read that local +configuration (normally through `sudo`). + +Use a reviewed checkout containing this passive helper. Inspect the file +before running it: older copies send `ping` and `get-fsinfo` to the guest. Do +not pipe a network download into a root shell. -### Diagnostic Script -Run this on your Proxmox host to debug specific VMs: ```bash -curl -sSL https://raw.githubusercontent.com/rcourtman/Pulse/main/scripts/test-vm-disk.sh | bash +# From the reviewed Pulse checkout; replace 100 with the affected VM ID. +sudo bash ./scripts/test-vm-disk.sh 100 ``` +The output distinguishes the running/stopped state, agent configuration and +a reported backup/other lock without dumping the full VM configuration. A +missing or unreadable lock is not permission to probe the guest: a backup can +start immediately after the read. A non-zero exit means a required read failed +or the response could not be interpreted; it is not an empty or healthy result. +**A successful preflight does not verify disk freshness, guest-agent +responsiveness or thaw.** + +Keep only the relevant time, Pulse and Proxmox versions, observation/error and +backup task sequence for a report. Redact private VM names, paths and addresses; +never include credentials, full VM configuration, service environments or a +whole host journal. See [Getting help](TROUBLESHOOTING.md#-getting-help). + ## 📝 Notes -* **Network Mounts**: NFS/SMB mounts are automatically excluded. -* **Databases**: Usage reflects filesystem usage, which may differ from database-internal metrics. -* **Containers**: LXC containers are monitored natively without the guest agent. +- **Network mounts:** NFS/SMB mounts are excluded from filesystem usage. +- **Databases:** Filesystem usage can differ from database-internal metrics. +- **Containers:** LXC collection does not use the VM's QEMU Guest Agent. diff --git a/frontend-modern/browser-tests/vm-disk-safety-help.cjs b/frontend-modern/browser-tests/vm-disk-safety-help.cjs new file mode 100644 index 0000000000..b12a380c1e --- /dev/null +++ b/frontend-modern/browser-tests/vm-disk-safety-help.cjs @@ -0,0 +1,93 @@ +// Exercise production Docs/router/Markdown. No Proxmox or guest is contacted. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { createHash } = require('node:crypto'); +const { chromium, webkit } = require('playwright'); + +(async () => { + const root = path.resolve('frontend-modern'); + const artifacts = path.join(root, 'node_modules', 'vm-disk-safety-browser'); + fs.mkdirSync(artifacts, { recursive: true }); + const guides = Object.fromEntries(['VM_DISK_MONITORING', 'TROUBLESHOOTING'].map(name => + [name, fs.readFileSync(path.join(root, 'public/docs', `${name}.md`), 'utf8')])); + const recipe = guides.VM_DISK_MONITORING.match(/```bash\n([\s\S]*?)```/)[1].trim(); + process.chdir(root); + const { createServer } = await import(path.join(root, 'node_modules/vite/dist/node/index.js')); + const server = await createServer({ + root, configFile: path.join(root, 'vite.config.ts'), + cacheDir: path.join(root, 'node_modules', '.vite-vm-disk-safety'), + server: { host: '127.0.0.1', port: 5265, strictPort: true }, + optimizeDeps: { noDiscovery: true, include: [] }, + }); + const results = []; + let browser; + try { + await server.listen(); + for (const [engine, width, tone] of [['chromium', 1280, 'light'], ['webkit', 390, 'dark']]) { + browser = await { chromium, webkit }[engine].launch(engine === 'chromium' + ? { headless: true, channel: 'chromium', args: ['--no-sandbox'] } + : { headless: true }); + const page = await browser.newPage({ viewport: { width, height: 1000 }, + ...(engine === 'webkit' ? { isMobile: true, hasTouch: true } : {}) }); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); + await page.goto('http://127.0.0.1:5265/browser-tests/docs-fragment-navigation.html', + { waitUntil: 'domcontentloaded', timeout: 60_000 }); + if (tone === 'dark') await page.evaluate(() => document.documentElement.classList.add('dark')); + await page.getByRole('link', { name: '← All documentation' }).click(); + const entry = page.locator('a[data-doc-link][href="/docs/VM_DISK_MONITORING"]').first(); + await entry.focus(); + await page.keyboard.press('Enter'); + const safety = page.getByRole('heading', { name: 'Backup safety', exact: true }); + await safety.waitFor(); + await safety.scrollIntoViewIfNeeded(); + let text = await page.locator('article').innerText(); + for (const phrase of ['Do not run manual guest-agent probes', 'monitoring and alerts', + 'does not prove thaw', 'temporary precaution', 'not established as a reproduced cause', + 'clear backup locks', 'host-root diagnostic does not test']) { + assert.ok(text.replace(/\s+/g, ' ').includes(phrase), phrase); + } + assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1)); + await page.screenshot({ path: path.join(artifacts, `${engine}-backup-safety.png`) }); + const passive = page.getByRole('heading', { name: 'Passive host preflight', exact: true }); + await passive.scrollIntoViewIfNeeded(); + const code = page.locator('pre').filter({ hasText: 'sudo bash ./scripts/test-vm-disk.sh 100' }); + assert.equal((await code.locator('code').textContent()).trim(), recipe); + const section = text.replace(/\s+/g, ' ').split('Passive host preflight')[1]; + for (const phrase of ['no guest-agent commands', 'older copies', 'bounded timeouts', + 'non-zero exit', 'does not verify disk freshness', 'before running it']) { + assert.ok(section.includes(phrase), phrase); + } + assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1)); + await page.screenshot({ path: path.join(artifacts, `${engine}-passive-preflight.png`) }); + const related = page.getByRole('link', { name: 'Getting help', exact: true }); + await related.focus(); + await page.keyboard.press('Enter'); + await page.getByRole('heading', { name: '🆘 Getting Help', exact: true }).waitFor(); + text = (await page.locator('article').innerText()).replace(/\s+/g, ' '); + for (const phrase of ['Do not run guest-agent probes during backup freeze/thaw', + 'passive host preflight', 'they do not by themselves establish why disk usage is absent']) { + assert.ok(text.includes(phrase), phrase); + } + assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1)); + assert.deepEqual(errors, []); + results.push({ engine, browserVersion: browser.version(), width, tone, + keyboardEntry: true, backupPrecautionAndOutage: true, exactCopiedShell: true, + passiveLimits: true, relatedHelpNavigation: true, noBodyOverflow: true, errors }); + await browser.close(); + browser = undefined; + } + const result = { result: 'passed', playwrightVersion: require('playwright/package.json').version, + scope: 'rendered production help and copied recipe; no native guest or backup acceptance', + docs: Object.fromEntries(Object.entries(guides).map(([name, content]) => + [`frontend-modern/public/docs/${name}.md`, createHash('sha256').update(content).digest('hex')])), + results }; + fs.writeFileSync(path.join(artifacts, 'result.json'), JSON.stringify(result, null, 2) + '\n'); + console.log(JSON.stringify(result)); + } finally { + if (browser) await browser.close(); + await server.close(); + } +})().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/frontend-modern/public/docs/TROUBLESHOOTING.md b/frontend-modern/public/docs/TROUBLESHOOTING.md index 2ff1a7b0eb..b5e498ebb3 100644 --- a/frontend-modern/public/docs/TROUBLESHOOTING.md +++ b/frontend-modern/public/docs/TROUBLESHOOTING.md @@ -227,10 +227,14 @@ installed service must contain `SyslogLevelPrefix=true` and repair an older generated unit rather than adding a JSON-parsing wrapper. #### VMs show "-" for disk usage -- Install **QEMU Guest Agent** in the VM. -- Enable "QEMU Guest Agent" in Proxmox VM Options. -- Restart the VM. -- See [VM Disk Monitoring](VM_DISK_MONITORING.md). +- Read the disk value's explanation and observation time first; a dash is not + proof that the agent is missing. +- Check the guest-local service, current VM Options and the configured API + token's read permissions. Schedule any setup change or restart outside backups. +- Do not run guest-agent probes during backup freeze/thaw. An OK backup task + or an absent lock does not confirm thaw. +- See [VM Disk Monitoring](VM_DISK_MONITORING.md) for the passive host preflight + and backup safety precaution; it does not verify a fresh disk poll. #### Temperature data missing - Install `lm-sensors` on the host. @@ -615,7 +619,12 @@ For PVE 8 only, use `VM.Monitor` instead of the `VM.GuestAgent.*` privileges. Note: The built-in `PVEAuditor` role cannot be modified. Create a custom role (e.g. `PulseMonitor`) with the above privileges added, and assign it to your Pulse API token. After upgrading to PVE 9, add the `VM.GuestAgent.*` privileges and remove legacy `VM.Monitor` from the custom role. -**Rocky Linux / RHEL VMs**: The default qemu-guest-agent configuration may block file-read RPCs (`guest-file-open`, `guest-file-read`, `guest-file-close`). If memory or disk data is missing for these VMs, check `/etc/sysconfig/qemu-ga` and ensure those operations are not blocked, then restart the agent. Refer to your distro's qemu-guest-agent documentation for the exact config syntax. +**Rocky Linux / RHEL VMs**: File-read restrictions in `/etc/sysconfig/qemu-ga` +can explain missing guest memory; they do not by themselves establish why disk +usage is absent. Review the guest's policy before changing it. Schedule any +allowlist change or agent restart outside backups, following the guest OS's +documentation. See [VM Disk Monitoring](VM_DISK_MONITORING.md) for the distinct +permissions and backup safety boundary. ### Proxmox pending-update access diff --git a/frontend-modern/public/docs/VM_DISK_MONITORING.md b/frontend-modern/public/docs/VM_DISK_MONITORING.md index cbf6a5a328..9c0b762933 100644 --- a/frontend-modern/public/docs/VM_DISK_MONITORING.md +++ b/frontend-modern/public/docs/VM_DISK_MONITORING.md @@ -1,42 +1,102 @@ # 💾 VM Disk Monitoring -Monitor actual disk usage inside your VMs using the QEMU Guest Agent. +Pulse can read filesystem usage inside a Proxmox VM through the QEMU Guest +Agent. Allocated virtual-disk size is not the same as used space inside the +guest. LXC filesystem collection uses a different path. -## 🚀 Quick Start +## Backup safety -1. **Install Guest Agent**: - * **Linux**: `apt install qemu-guest-agent` (Debian/Ubuntu) or `yum install qemu-guest-agent` (RHEL). - * **Windows**: Install **virtio-win** drivers. -2. **Enable in Proxmox**: - * VM Options → **QEMU Guest Agent** → Enabled. - * Restart the VM. -3. **Verify**: - * Run `qm agent ping` on the Proxmox host. - * Check Pulse dashboard for disk usage (e.g., "5.2GB used of 32GB"). +**Do not run manual guest-agent probes during a backup, freeze or thaw.** A +guest-agent request is an active command on the same channel used by +freeze-enabled backups, even when it only reads disk or memory information. +There is a [report of a VM remaining frozen despite an OK backup task](https://github.com/rcourtman/Pulse/issues/2439). +The reported command-ID collision is not established as a reproduced cause. -## ⚙️ Requirements +If your installation is affected, stop the Pulse service before the backup +and restart it only after independently confirming the guest has thawed, using +your existing guest console or workload checks. **Pulse monitoring and alerts +are unavailable while it is stopped.** An OK backup task or an absent VM lock +does not prove thaw succeeded. This is a temporary precaution, not a claim +that the monitoring defect is fixed. -* **QEMU Guest Agent**: Must be installed and running inside the VM. -* **Proxmox Permissions**: `VM.Monitor` (Proxmox 8) or `VM.GuestAgent.Audit` + `VM.GuestAgent.FileRead` (Proxmox 9+). Note: `PVEAuditor` is a built-in read-only role that cannot be modified — create a custom role instead. +Do not test freeze/thaw commands, clear backup locks, disable backup freezing +or force-reset a guest as a disk-monitoring diagnostic. Those operations can +affect workloads or backup consistency. Preserve the existing backup task and +redacted error, rather than repeat a potentially harmful backup for a report. + +## 🚀 Setup + +Plan changes outside backup windows and follow the guest's normal maintenance +procedure; a reboot interrupts its workloads. + +1. **Install the guest agent inside the VM.** Use the guest OS's supported + package: `qemu-guest-agent` on Linux, or the QEMU Guest Agent service from + the virtio-win distribution on Windows. +2. **Enable it in Proxmox.** Review **VM Options → QEMU Guest Agent** and apply + any required VM restart in that maintenance window. +3. **Check the existing observations.** In Pulse, hover over the disk dash or + open the resource details for the collection explanation and observation + time. Check the guest-agent service from inside the guest, not by repeatedly + sending `qm agent` commands from the hypervisor. A configured agent is not + proof it responded, and a retained disk value is not proof of a fresh poll. + +## ⚙️ Permissions + +Use the dedicated account and token actually configured for that Pulse +connection, not a guessed account name. Review their effective privileges for +the affected VM; a host-root diagnostic does not test the API token's access. + +| Collection | Proxmox VE 9+ | Proxmox VE 8 | +| --- | --- | --- | +| Filesystem usage and guest information | `VM.GuestAgent.Audit` | `VM.Monitor` | +| Reading guest memory via `/proc/meminfo` | Also `VM.GuestAgent.FileRead` | `VM.Monitor` | + +Do not grant Administrator or guest execution/write permissions to diagnose a +missing disk value. The built-in `PVEAuditor` role cannot be modified; use a +custom monitoring role if the required read privileges are absent. ## 🔧 Troubleshooting -| Issue | Solution | -| :--- | :--- | -| **Disk shows "-"** | Hover over the dash for details. Common causes: Agent not running, disabled in config, or permission denied. | -| **Permission Denied** | Ensure your Proxmox token/user has `VM.GuestAgent.Audit` + `VM.GuestAgent.FileRead` (PVE 9+) or `VM.Monitor` (PVE 8). | -| **Rocky Linux / RHEL: memory or disk data missing** | The default qemu-guest-agent config may block file-read RPCs. Check `/etc/sysconfig/qemu-ga` and ensure `guest-file-open`, `guest-file-read`, and `guest-file-close` are not blocked, then restart the agent. See your distro's qemu-guest-agent docs for exact syntax. | -| **Agent Timeout** | Increase timeouts via env vars if network is slow: `GUEST_AGENT_FSINFO_TIMEOUT=10s`. | -| **Windows VMs** | Ensure the **QEMU Guest Agent** service is running in Windows Services. | +| Observation | Useful next check | +| --- | --- | +| **Disk shows “-”** | Read its explanation and observation time. Check the owning host, current VM options, guest-local service and configured API token access. | +| **Permission denied** | Review the token and account's effective read privileges on that VM, including privilege-separated token ACLs. A successful root command would not disprove this error. | +| **Timeout or backup lock** | Check the existing backup/task timeline and guest workload locally. Defer active guest-agent probes; increasing a timeout is not a contention or thaw repair. | +| **Rocky Linux / RHEL memory missing** | Review `/etc/sysconfig/qemu-ga` inside the guest. File-read restrictions can explain memory collection failure; they do not by themselves prove why filesystem usage is absent. Change the guest's allowlist or restart its agent only through your normal maintenance procedure, outside backups. | +| **Windows service stopped** | Check the QEMU Guest Agent service inside Windows. Schedule any restart outside backups. | + +### Passive host preflight + +The current `scripts/test-vm-disk.sh` helper reads only local VM status and +current configuration, with bounded timeouts. It sends **no guest-agent +commands**, reads no guest files, and changes no service, lock or ACL. Use it +on the VM's owning Proxmox host with an account permitted to read that local +configuration (normally through `sudo`). + +Use a reviewed checkout containing this passive helper. Inspect the file +before running it: older copies send `ping` and `get-fsinfo` to the guest. Do +not pipe a network download into a root shell. -### Diagnostic Script -Run this on your Proxmox host to debug specific VMs: ```bash -curl -sSL https://raw.githubusercontent.com/rcourtman/Pulse/main/scripts/test-vm-disk.sh | bash +# From the reviewed Pulse checkout; replace 100 with the affected VM ID. +sudo bash ./scripts/test-vm-disk.sh 100 ``` +The output distinguishes the running/stopped state, agent configuration and +a reported backup/other lock without dumping the full VM configuration. A +missing or unreadable lock is not permission to probe the guest: a backup can +start immediately after the read. A non-zero exit means a required read failed +or the response could not be interpreted; it is not an empty or healthy result. +**A successful preflight does not verify disk freshness, guest-agent +responsiveness or thaw.** + +Keep only the relevant time, Pulse and Proxmox versions, observation/error and +backup task sequence for a report. Redact private VM names, paths and addresses; +never include credentials, full VM configuration, service environments or a +whole host journal. See [Getting help](TROUBLESHOOTING.md#-getting-help). + ## 📝 Notes -* **Network Mounts**: NFS/SMB mounts are automatically excluded. -* **Databases**: Usage reflects filesystem usage, which may differ from database-internal metrics. -* **Containers**: LXC containers are monitored natively without the guest agent. +- **Network mounts:** NFS/SMB mounts are excluded from filesystem usage. +- **Databases:** Filesystem usage can differ from database-internal metrics. +- **Containers:** LXC collection does not use the VM's QEMU Guest Agent. diff --git a/scripts/test-vm-disk.sh b/scripts/test-vm-disk.sh index 4e7dc187dc..b6464e7b94 100755 --- a/scripts/test-vm-disk.sh +++ b/scripts/test-vm-disk.sh @@ -1,138 +1,124 @@ #!/usr/bin/env bash -set -uo pipefail +# Passive Proxmox preflight only. A lock read is not a synchronisation gate: +# a backup can start immediately afterwards. Never send guest-agent commands. +set -euo pipefail -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -info() { - printf "%b[INFO]%b %s\n" "$BLUE" "$NC" "$1" +usage() { + printf '%s\n' 'Usage: test-vm-disk.sh VMID' \ + 'Read local VM status and current configuration on its Proxmox host.' \ + 'No guest-agent, guest filesystem, ACL or service changes are made.' } -ok() { - printf " %b✓%b %s\n" "$GREEN" "$NC" "$1" -} +if [[ $# -eq 1 && $1 == --help ]]; then + usage + exit 0 +fi +if [[ $# -ne 1 || ! $1 =~ ^[1-9][0-9]{0,8}$ ]]; then + usage >&2 + exit 2 +fi +vmid=$1 -warn() { - printf " %b⚠%b %s\n" "$YELLOW" "$NC" "$1" -} - -fail() { - printf " %b✗%b %s\n" "$RED" "$NC" "$1" -} - -require_cmd() { - if ! command -v "$1" >/dev/null 2>&1; then - printf "%b[ERROR]%b Required command '%s' not found.\n" "$RED" "$NC" "$1" >&2 +for command in qm timeout; do + if ! command -v "$command" >/dev/null 2>&1; then + printf 'Required command unavailable: %s. Run on the owning Proxmox host.\n' "$command" >&2 exit 1 fi +done + +read_failure() { + local operation=$1 code=$2 + if [[ $code -eq 124 || $code -eq 137 ]]; then + printf 'VM %s read timed out; no guest-agent commands were sent.\n' "$operation" >&2 + else + printf 'VM %s read failed (exit %s); no guest-agent commands were sent.\n' "$operation" "$code" >&2 + fi + printf '%s\n' 'Check the VM ID, owning host and local read permissions. Inspect any error privately.' >&2 + exit 1 } -# Ensure we are running on a Proxmox host -require_cmd qm -require_cmd pveversion +printf '%s\n' 'Passive VM preflight: no guest-agent commands will be sent.' +# Bound even host-local reads. Do not print raw config or command errors: +# those can contain private guest names, paths and other infrastructure data. +if status_output=$(timeout --signal=TERM --kill-after=2s 10s qm status "$vmid" 2>/dev/null); then + case "$status_output" in + 'status: running') status=running ;; + 'status: stopped') status=stopped ;; + *) printf '%s\n' 'VM status response was not understood; preflight incomplete.' >&2; exit 1 ;; + esac +else + read_failure status "$?" +fi +printf 'VM status: %s\n' "$status" -if [[ $EUID -ne 0 ]]; then - printf "%b[ERROR]%b This script must be run as root on the Proxmox host.\n" "$RED" "$NC" >&2 +if config=$(timeout --signal=TERM --kill-after=2s 10s qm config "$vmid" --current 2>/dev/null); then + if [[ -z $config ]]; then + printf '%s\n' 'VM current configuration was empty; preflight incomplete.' >&2 + exit 1 + fi +else + read_failure configuration "$?" +fi + +agent_seen=false +lock_seen=false +field_seen=false +agent_value='' +lock_value='' +while IFS= read -r line; do + if [[ $line =~ ^[A-Za-z][A-Za-z0-9_-]*: ]]; then + field_seen=true + fi + case "$line" in + agent:*) + if [[ $agent_seen == true ]]; then + printf '%s\n' 'Repeated agent configuration; preflight incomplete.' >&2 + exit 1 + fi + agent_seen=true + agent_value=${line#agent:} + agent_value=${agent_value//[[:space:]]/} + ;; + lock:*) + if [[ $lock_seen == true ]]; then + printf '%s\n' 'Repeated VM lock configuration; preflight incomplete.' >&2 + exit 1 + fi + lock_seen=true + lock_value=${line#lock:} + lock_value=${lock_value//[[:space:]]/} + ;; + esac +done <<< "$config" +if [[ $field_seen != true ]]; then + printf '%s\n' 'VM current configuration was not understood; preflight incomplete.' >&2 exit 1 fi -VMID=${1:-} -if [[ -z "${VMID}" ]]; then - read -rp "Enter the VMID to inspect: " VMID +agent=disabled +if [[ $agent_seen == true ]]; then + agent=unknown + enabled_options=0 + IFS=, read -r -a options <<< "$agent_value" + for option in "${options[@]}"; do + case "$option" in + 1|enabled=1) agent=enabled; enabled_options=$((enabled_options + 1)) ;; + 0|enabled=0) agent=disabled; enabled_options=$((enabled_options + 1)) ;; + esac + done + if [[ $enabled_options -ne 1 ]]; then + agent=unknown + fi fi - -if [[ ! $VMID =~ ^[0-9]+$ ]]; then - printf "%b[ERROR]%b VMID must be a numeric value.\n" "$RED" "$NC" >&2 - exit 1 -fi - -info "Running disk diagnostics for VMID ${VMID}" - -# --- Check VM existence and status --- -status_output=$(qm status "$VMID" 2>&1) -if [[ $? -ne 0 ]]; then - printf "%b[ERROR]%b Unable to query VM %s.\n" "$RED" "$NC" "$VMID" >&2 - printf "%s\n" "$status_output" >&2 - exit 1 -fi - -status=$(awk -F': ' '/status/ {print $2}' <<<"$status_output") -status=${status:-unknown} -ok "VM found (status: ${status})" - -running=false -if [[ $status == "running" ]]; then - running=true -else - warn "VM is not running. Start the VM before re-running disk checks for accurate results." -fi - -# --- Check guest agent configuration --- -agent_line=$(qm config "$VMID" | grep '^agent:' || true) -agent_enabled=false -if [[ -n $agent_line ]]; then - if grep -Eq '1|enabled' <<<"$agent_line"; then - agent_enabled=true - ok "QEMU guest agent is enabled in VM configuration (${agent_line})." +printf 'Guest agent configured: %s (not a responsiveness test).\n' "$agent" +if [[ $lock_seen == true ]]; then + if [[ $lock_value == backup ]]; then + printf '%s\n' 'VM lock: backup. Defer guest-agent diagnostics and configuration changes.' else - warn "QEMU guest agent is defined but not enabled (current config: ${agent_line})." + printf '%s\n' 'VM lock: present or unknown. Check the existing Proxmox task before making changes.' fi else - warn "QEMU guest agent is not enabled for this VM. Add 'agent: 1' in the VM options." + printf '%s\n' 'VM lock: not reported. This does not establish that a backup is idle or the guest is thawed.' fi - -if [[ $agent_enabled == true && $running == true ]]; then - # Guest agent ping - ping_output=$(qm agent "$VMID" ping 2>&1) - if [[ $? -eq 0 ]]; then - ok "Guest agent responded to ping." - else - warn "Guest agent did not respond to ping." - printf " %s\n" "$ping_output" - fi - - # Filesystem info - fs_output=$(qm agent "$VMID" get-fsinfo 2>&1) - if [[ $? -eq 0 ]]; then - if grep -Eq "(mount|fsname|filesystem)" <<<"$fs_output"; then - ok "Fetched filesystem information from guest." - else - warn "Guest agent returned no filesystem entries. Review the guest OS permissions." - fi - else - warn "Failed to retrieve filesystem information via guest agent." - printf " %s\n" "$fs_output" - fi -else - warn "Skipping guest agent checks because the VM is not running or the agent is disabled." -fi - -# --- Validate pulse monitoring account (optional) --- -if pveum user list 2>/dev/null | grep -q 'pulse_monitor@pam'; then - acl_output=$(pveum acl list / -user pulse_monitor@pam 2>&1) - if [[ $? -eq 0 && -n $acl_output ]]; then - ok "pulse_monitor@pam user has ACL entries configured." - else - warn "pulse_monitor@pam user exists but no ACLs were found. Ensure it has proper permissions." - fi -else - warn "pulse_monitor@pam user not found. Create it if Pulse is expected to collect agent data." -fi - -cat <<'SUMMARY' - -Next steps: - • If the guest agent is disabled, enable it in the VM Options tab (set "QEMU Guest Agent" to "Enabled"). - • Inside the guest OS, ensure the qemu-guest-agent service is installed, running, and has access to disk information. - • Verify the pulse_monitor@pam user (or your service account) has proper permissions: - - Proxmox 9: VM.GuestAgent.Audit privilege (Pulse setup adds via PulseMonitor role) - - Proxmox 8: VM.Monitor privilege (Pulse setup adds via PulseMonitor role) - - Sys.Audit is recommended for Ceph metrics and included when available - - Both API tokens and passwords work fine for guest agent access - -Diagnostics complete. -SUMMARY +printf '%s\n' 'Read-only preflight completed. Disk freshness, guest-agent responsiveness and guest thaw were not tested.' diff --git a/scripts/tests/test_vm_disk_diagnostics.py b/scripts/tests/test_vm_disk_diagnostics.py new file mode 100644 index 0000000000..3e25f290fb --- /dev/null +++ b/scripts/tests/test_vm_disk_diagnostics.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +"""Exercise the actual passive helper with synthetic Proxmox commands. + +No hypervisor, guest, backup or host service is touched. These controls prove +command selection, failure handling and private output, not native thaw. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import time +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts/test-vm-disk.sh" +DOC = ROOT / "docs/VM_DISK_MONITORING.md" +BASH = shutil.which("bash") +REAL_TIMEOUT = shutil.which("timeout") +PRIVATE = "synthetic-private-infrastructure-marker" +READS = [["status", "100"], ["config", "100", "--current"]] + +# Record every attempted operation, including prohibited ones. The timeout +# adapter checks the helper's real deadline arguments; one test uses real +# coreutils timeout around the synthetic blocked qm child. +TOOL = '''#!/usr/bin/env python3 +import json, os, pathlib, subprocess, sys, time +name = pathlib.Path(sys.argv[0]).name +args = sys.argv[1:] +root = pathlib.Path(os.environ['VM_FIXTURE']) +with (root / 'calls.jsonl').open('a') as stream: + stream.write(json.dumps({'name': name, 'args': args}) + '\\n') +settings = json.loads((root / 'settings.json').read_text()) +if name == 'timeout': + assert args[:4] == ['--signal=TERM', '--kill-after=2s', '10s', 'qm'] + if settings.get('real_timeout'): + result = subprocess.run([os.environ['REAL_TIMEOUT'], *args]) + else: + result = subprocess.run(args[3:]) + sys.exit(result.returncode) +elif name == 'qm': + if args == ['status', '100']: + operation = 'status' + default = 'status: running\\n' + elif args == ['config', '100', '--current']: + operation = 'config' + default = 'agent: 1\\nname: synthetic-private-infrastructure-marker\\n' + else: + # No fake succeeds on an unrecognised or guest-agent operation. + sys.exit(90) + if settings.get('sleep_operation') == operation: + time.sleep(20) + sys.stdout.write(settings.get(operation + '_output', default)) + sys.stderr.write('synthetic-private-infrastructure-marker\\n') + sys.exit(settings.get(operation + '_exit', 0)) +elif name == 'sudo': + assert args == ['bash', './scripts/test-vm-disk.sh', '100'] + os.execv(os.environ['REAL_BASH'], [os.environ['REAL_BASH'], *args[1:]]) +else: + sys.exit(91) +''' + + +class VMDiskDiagnosticsTest(unittest.TestCase): + def exercise(self, *, settings=None, args=None, recipe=None, source=None, + missing=None): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "settings.json").write_text(json.dumps(settings or {})) + for name in ("qm", "timeout", "sudo", "pveum", "pveversion", "curl", "systemctl"): + if name == missing: + continue + tool = root / name + tool.write_text(TOOL) + tool.chmod(0o700) + env = dict(os.environ, VM_FIXTURE=str(root), REAL_BASH=BASH, + REAL_TIMEOUT=REAL_TIMEOUT, PATH=f"{root}:{os.environ['PATH']}") + if missing: + # The adapters use an absolute interpreter, so an empty + # fallback PATH can genuinely model unavailable dependencies. + env["PATH"] = str(root) + for tool in root.iterdir(): + if tool.name in ("settings.json",): + continue + tool.write_text(TOOL.replace("/usr/bin/env python3", os.sys.executable)) + script = SCRIPT + if source is not None: + script = root / "candidate.sh" + script.write_text(source) + invocation = [BASH, str(script), *(args if args is not None else ["100"])] + if recipe: + invocation = [BASH, "-c", recipe] + result = subprocess.run(invocation, env=env, cwd=ROOT, + capture_output=True, text=True, timeout=16) + calls_file = root / "calls.jsonl" + calls = [json.loads(line) for line in calls_file.read_text().splitlines()] if calls_file.exists() else [] + self.assertNotIn(PRIVATE, result.stdout + result.stderr) + return result, calls + + def assert_passive(self, calls, *, operations=READS): + self.assertEqual([c["args"] for c in calls if c["name"] == "qm"], operations) + self.assertTrue(all(c["name"] in ("timeout", "qm", "sudo") for c in calls)) + for call in calls: + if call["name"] == "timeout": + self.assertEqual(call["args"][:4], ["--signal=TERM", "--kill-after=2s", "10s", "qm"]) + + def test_running_vm_preflight_is_passive_even_without_a_lock(self): + result, calls = self.exercise() + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_passive(calls) + self.assertIn("VM status: running", result.stdout) + self.assertIn("Guest agent configured: enabled", result.stdout) + self.assertIn("VM lock: not reported", result.stdout) + self.assertIn("does not establish", result.stdout) + self.assertIn("guest thaw were not tested", result.stdout) + + def test_stopped_guest_is_not_started_or_probed(self): + result, calls = self.exercise(settings={"status_output": "status: stopped\n"}) + self.assertEqual(result.returncode, 0) + self.assertIn("VM status: stopped", result.stdout) + self.assert_passive(calls) + + def test_backup_other_empty_and_future_locks_never_send_a_probe(self): + for lock in ("backup", "snapshot", "migrate", "", PRIVATE): + with self.subTest(lock=lock): + result, calls = self.exercise(settings={"config_output": f"agent: 1\nlock: {lock}\nname: {PRIVATE}\n"}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("VM lock: backup" if lock == "backup" else "VM lock: present or unknown", result.stdout) + self.assert_passive(calls) + + def test_agent_boolean_is_not_inferred_from_other_numeric_options(self): + cases = { + None: "disabled", "0": "disabled", "1": "enabled", + "0,freeze-fs-on-backup=1": "disabled", "enabled=0,fstrim_cloned_disks=1": "disabled", + "enabled=1,freeze-fs-on-backup=0": "enabled", "freeze-fs-on-backup=1,enabled=0": "disabled", + "fstrim_cloned_disks=1": "unknown", "enabled=2": "unknown", "": "unknown", + "0,enabled=1": "unknown", "1,enabled=1": "unknown", "disabled": "unknown", + } + for value, expected in cases.items(): + with self.subTest(value=value): + config = f"name: {PRIVATE}\n" + (f"agent: {value}\n" if value is not None else "") + result, calls = self.exercise(settings={"config_output": config}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"Guest agent configured: {expected} (not a responsiveness test)", result.stdout) + self.assert_passive(calls) + + def test_failed_status_never_reads_configuration_or_reports_completion(self): + for code in (1, 13, 124, 137, 143): + with self.subTest(code=code): + result, calls = self.exercise(settings={"status_exit": code, "status_output": PRIVATE}) + self.assertEqual(result.returncode, 1) + self.assertNotIn("preflight completed", result.stdout) + self.assertNotIn("VM status:", result.stdout) + self.assertIn("timed out" if code in (124, 137) else f"exit {code}", result.stderr) + self.assert_passive(calls, operations=READS[:1]) + + def test_config_failure_cannot_be_presented_as_an_absent_lock(self): + for code in (1, 13, 124, 137): + with self.subTest(code=code): + result, calls = self.exercise(settings={"config_exit": code, "config_output": f"agent: 1\nlock: backup\n{PRIVATE}"}) + self.assertEqual(result.returncode, 1) + self.assertNotIn("preflight completed", result.stdout) + self.assertNotIn("VM lock:", result.stdout) + self.assertIn("configuration read", result.stderr) + self.assert_passive(calls) + + def test_empty_unknown_and_repeated_responses_fail_without_dumping_config(self): + cases = [ + {"status_output": ""}, {"status_output": "status: " + PRIVATE}, + {"config_output": ""}, {"config_output": PRIVATE}, + {"config_output": f"agent: 1\nagent: 0\nname: {PRIVATE}\n"}, + {"config_output": f"lock: backup\nlock: migrate\nname: {PRIVATE}\n"}, + ] + for settings in cases: + with self.subTest(settings=settings): + result, calls = self.exercise(settings=settings) + self.assertEqual(result.returncode, 1) + self.assertIn("preflight incomplete", result.stderr) + self.assertNotIn("preflight completed", result.stdout) + self.assert_passive(calls, operations=READS[:1] if "status_output" in settings else READS) + + def test_help_and_invalid_arguments_never_contact_proxmox(self): + for args in ([], ["0"], ["-1"], ["100;touch BAD"], ["100", "--probe"], + ["9999999999"], ["--probe"], ["--help"]): + with self.subTest(args=args): + result, calls = self.exercise(args=args) + self.assertEqual(result.returncode, 0 if args == ["--help"] else 2) + self.assertEqual(calls, []) + + def test_missing_dependencies_do_not_run_a_partial_preflight(self): + for missing in ("qm", "timeout"): + with self.subTest(missing=missing): + result, calls = self.exercise(missing=missing) + self.assertEqual(result.returncode, 1) + self.assertIn(f"unavailable: {missing}", result.stderr) + self.assertEqual(calls, []) + + def test_real_deadline_stops_a_blocked_synthetic_local_read(self): + self.assertIsNotNone(REAL_TIMEOUT) + start = time.monotonic() + result, calls = self.exercise(settings={"real_timeout": True, "sleep_operation": "status"}) + elapsed = time.monotonic() - start + self.assertEqual(result.returncode, 1) + self.assertIn("status read timed out", result.stderr) + self.assertLess(elapsed, 14) + self.assert_passive(calls, operations=READS[:1]) + + def test_exact_documented_command_uses_the_passive_helper(self): + recipes = re.findall(r"```bash\n(.*?)```", DOC.read_text(), re.DOTALL) + self.assertEqual(len(recipes), 1) + result, calls = self.exercise(recipe=recipes[0]) + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_passive(calls) + self.assertEqual([c["args"] for c in calls if c["name"] == "sudo"], + [["bash", "./scripts/test-vm-disk.sh", "100"]]) + + def test_safety_oracle_rejects_reintroduced_guest_probes_and_mutations(self): + source = SCRIPT.read_text() + for unsafe in ('qm agent "$vmid" ping', 'qm agent "$vmid" get-fsinfo', + 'qm unlock "$vmid"', 'qm reset "$vmid"', + 'pveum acl modify /', 'systemctl restart pulse'): + with self.subTest(operation=unsafe): + mutated = source.replace("vmid=$1", "vmid=$1\n" + unsafe) + result, calls = self.exercise(source=mutated, settings={"config_output": "agent: 1\nlock: backup\n"}) + self.assertNotEqual(result.returncode, 0) + with self.assertRaises(AssertionError): + self.assert_passive(calls) + + +class VMDiskHelpTest(unittest.TestCase): + def test_guides_are_mirrored_and_preserve_the_safety_and_proof_boundaries(self): + for name in ("VM_DISK_MONITORING", "TROUBLESHOOTING"): + source = (ROOT / f"docs/{name}.md").read_bytes() + self.assertEqual(source, (ROOT / f"frontend-modern/public/docs/{name}.md").read_bytes()) + guide = DOC.read_text() + for phrase in ("monitoring and alerts", "not established as a reproduced cause", + "does not prove thaw", "a backup can", "older copies", "non-zero exit", + "host-root diagnostic does not test", "not a claim", "bounded timeouts"): + self.assertIn(phrase, guide) + for block in re.findall(r"```bash\n(.*?)```", guide, re.DOTALL): + self.assertNotRegex(block, r"\b(curl|wget|qm agent|pveum|systemctl)\b") + self.assertNotIn("GUEST_AGENT_FSINFO_TIMEOUT=", guide) + + +if __name__ == "__main__": + unittest.main()