diff --git a/docs/API.md b/docs/API.md index c5a7677968..46d3d3cbec 100644 --- a/docs/API.md +++ b/docs/API.md @@ -9,18 +9,65 @@ Pulse provides a comprehensive REST API for automation and integration. Most API requests require authentication via one of the following methods: ### API Token (Recommended) -Pass the token in the `X-API-Token` header. + +Create an API token in Pulse's **API Access** settings with only the scopes +needed for your task. The read-only example below requires `monitoring:read`. +Pass the token in the `X-API-Token` header, using a private header file so the +secret does not appear in shell history or process arguments. + +On the machine running curl, prepare the file and open it in an editor: + ```bash -curl -H "X-API-Token: your-token" http://localhost:7655/api/health +umask 077 +mkdir -p "$HOME/.config/pulse" +touch "$HOME/.config/pulse/api-header" +chmod 600 "$HOME/.config/pulse/api-header" +vi "$HOME/.config/pulse/api-header" ``` -### Bearer Token +In the editor, save just this line, replacing `` with the API token: + +```text +X-API-Token: +``` + +Then make a read-only request (curl 7.76 or later): + ```bash -curl -H "Authorization: Bearer your-token" http://localhost:7655/api/health +curl --fail-with-body --header "@$HOME/.config/pulse/api-header" \ + http://127.0.0.1:7655/api/state/summary +``` + +This loopback URL applies only when curl runs on the Pulse host. For remote +access, use your Pulse HTTPS URL and keep certificate verification enabled; +do not use `--insecure`. On an authentication-enabled instance, the protected +summary checks token access; `/api/health` is public and does not verify authentication. +`--fail-with-body` returns a non-zero exit on HTTP errors, including 401 and +403, while retaining the error response. + +Do not paste tokens into command lines, URLs or issue reports. Keep the header +file private and outside shared repositories and diagnostics; do not use curl +verbose/trace output when sharing a result. Share only the relevant redacted +error, not the whole infrastructure response. Revoke tokens that are no longer +needed in Pulse's API Access settings. + +### Bearer Token + +The same API token can use a Bearer header instead. Replace the header file's +line in the editor with the following, then use the same curl command above; +do not send both authentication headers: + +```text +Authorization: Bearer ``` ### Session Cookie -Standard browser session cookie (used by the UI). + +For one-off read-only diagnostics, use your signed-in Pulse browser to open +the API path on that same instance. Do not extract or paste its session cookie +into a command or report; use the scoped token-file method above for automation. +State-changing session requests also require Pulse's CSRF protection, which +the UI handles. Session endpoints: - `POST /api/login` (sets `pulse_session` + `pulse_csrf`) diff --git a/docs/PRIVACY.md b/docs/PRIVACY.md index cdebc02e3f..5de0702446 100644 --- a/docs/PRIVACY.md +++ b/docs/PRIVACY.md @@ -158,7 +158,7 @@ Every field is listed below with the reason it exists. Nothing else is included | Update available | `true`/`false` | Report whether that last check offered a newer release, without sending which release | | Service health observed | `true`/`false` | Distinguish a release that performed the bounded local UI/API self-check from an older release with no signal | | Service health healthy | `true`/`false` | Report whether Pulse's locally bound listener served a healthy API response, UI document, and every referenced local frontend asset without sending an address, URL, response, or error text | -| Service health failure category | `listener`, `startup`, `runtime`, `api_connectivity`, `api_status`, `ui_status`, `frontend_assets`, or `unknown` | Classify a failed local self-check or startup path into one fixed category without sending the listener address, request URL, HTTP body, asset name, IP address, or raw error | +| Service health failure category | `listener`, `startup`, `runtime`, `api_connectivity`, `timeout`, `api_status`, `ui_status`, `frontend_assets`, or `unknown` | Classify a failed local self-check or startup path into one fixed category without sending the listener address, request URL, HTTP body, asset name, IP address, or raw error | | Service health cohort | `first_observation`, `same_version`, or `version_change` | Mark whether this direct observation is the first schema-v13 observation, another observation of the same release, or the first observed state after a release-version change | | Service health previous version | `6.4.0` | Retain only the normalized immediately previous Pulse release identity so aggregate reporting can compare a post-upgrade before/after cohort without treating rolling counters as new-release activity | | Service health previous observed | `true`/`false` | State whether the immediately previous release recorded the bounded local service-health signal | diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index c20b71a4d5..1a1102e057 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -58,9 +58,35 @@ screen for this instance and do not paste it into support requests or issue reports. ### Port change didn't take effect -1. Check which service is running: `systemctl status pulse` (legacy installs may use `pulse-backend`). -2. Verify environment override: `systemctl show pulse --property=Environment`. -3. Docker: Ensure you updated the `-p` flag (e.g., `-p 8080:7655`). + +The web UI and API listen on `FRONTEND_PORT` (default `7655`). The deprecated +`PORT` alias applies only when `FRONTEND_PORT` is unset; changing +`frontendPort` in `system.json` has no effect. `PULSE_AGENT_INGEST_PORT` is a +separate agent listener, not the web UI port. See +[Port configuration](CONFIGURATION.md#common-overrides-environment-variables). + +- **Systemd / Proxmox LXC**: identify the active service with + `systemctl is-active pulse` (legacy installs may use `pulse-backend`). Inspect + only the port setting in its managed configuration locally. If you change a + unit or drop-in, run `sudo systemctl daemon-reload`, then restart the affected + service during a suitable maintenance window. For LXC, run these checks inside + the Pulse container, not on the Proxmox host. +- **Docker / Compose**: distinguish the published host port from the listener + inside the container. With the default listener, `8080:7655` exposes the UI on + host port `8080`; changing the host port does not require `FRONTEND_PORT`. + In the repository's Compose file, `PULSE_PORT` controls this host-side mapping. + Save the mapping in your existing Compose project and apply it with + `docker compose up -d pulse`, keeping the same image and mounted data volume. + Restarting an existing container does not apply a new port mapping. For other + container managers, use their recreate/redeploy action while preserving the + data mount; do not delete the volume. +- **Reverse proxy**: check its upstream port and the firewall separately. A + working agent connection on a split-port deployment does not prove that the + web UI is reachable. + +Do not post full service environments, `docker inspect` output or resolved +`docker compose config` output: they can include passwords and tokens. Share +only the relevant port numbers and a redacted error if help is needed. ### "Connection Refused" - Check if Pulse is running. @@ -155,11 +181,19 @@ repair an older generated unit rather than adding a JSON-parsing wrapper. - See [Temperature Monitoring](TEMPERATURE_MONITORING.md). #### Docker hosts appearing/disappearing -- **Duplicate IDs**: Cloned VMs often share `/etc/machine-id`. -- **Fix**: Run `rm /etc/machine-id && systemd-machine-id-setup` on the clone. -- **Identity note**: The displayed IP is not the durable identity. Pulse uses - the machine ID or an explicit agent ID, so two clones with the same value can - collapse into one record even when their hostnames or IP addresses differ. + +Cloned hosts can share a **saved Pulse agent ID**, not just an OS machine ID. +The agent uses an explicit ID first, then its saved `agent-id` file, and derives +one from the machine only when neither is available. Changing a hostname, IP or +`/etc/machine-id` therefore does not necessarily change its Pulse identity. + +Compare the affected hosts in **Agent Doctor** and inspect only their configured +`agent-id` files locally. Do not delete the OS machine ID, agent state or Pulse +history as a troubleshooting step. If a duplicate is confirmed, give only the +clone a stable, unique ID in its managed service or container configuration; +leave the original host unchanged. Follow +[Clone identity recovery](UNIFIED_AGENT.md#duplicate-agents) for the configuration +precedence, systemd example and checks after restart. ### Notifications @@ -309,25 +343,50 @@ check, report the Pulse version and displayed check time/status separately; API success alone does not confirm the Pulse display has recovered. ### Recovery Mode -If you are completely locked out, you can trigger a recovery token from localhost: -```bash -curl -X POST http://localhost:7655/api/security/recovery \ - -d '{"action":"generate_token","duration":30}' -``` -Use the returned token in `X-Recovery-Token` when calling `/api/security/recovery` to enable or disable local-only auth bypass (`disable_auth` / `enable_auth`). Token generation is localhost-only. -Example (enable recovery mode): -```bash -curl -X POST http://localhost:7655/api/security/recovery \ - -H "X-Recovery-Token: " \ - -d '{"action":"disable_auth"}' -``` +For a forgotten local password, follow [I forgot my password](#i-forgot-my-password) +above, using the steps for your deployment. Enter the host-only bootstrap token +in that instance's setup screen; do not paste it into a command or a report. +For OIDC, SAML or proxy login, use the identity-provider or administrator path +described there instead. + +The advanced `/api/security/recovery` API creates a **browser-bound recovery +session**, not a server-wide authentication bypass. Despite its legacy name, +`disable_auth` does not disable authentication for other clients. Recovery +sessions work only over direct loopback requests; remote and reverse-proxy +requests cannot use them. A successful curl response does not unlock a separate +browser: the session cookie belongs to the client that made the request. +`enable_auth` clears that recovery session; it does not reset a password. + +Do not transfer recovery cookies between clients or paste recovery tokens into +command arguments, URLs, screenshots or GitHub threads. The password-reset steps +above avoid that credential-handling detour. --- ## ๐Ÿ†˜ Getting Help If you're still stuck: -1. **Check Logs**: `journalctl -u pulse -n 100` or `docker logs --tail 100 pulse`. -2. **Check Version**: `curl http://localhost:7655/api/version`. -3. **Open Issue**: Report on [GitHub Issues](https://github.com/rcourtman/Pulse/issues) with your logs and version info. + +1. **Keep the original evidence**: note what you did, when it happened and the + exact error. Do not repeat an update, outage or notification storm merely to + reproduce it. A failed update banner does not prove the action left the + target unchanged; check its current state before another attempt. +2. **Identify the affected version**: give the running Pulse and relevant agent + versions, not just the version before an upgrade. For Docker, include the + running image tag or digest. If installation never started Pulse, give the + attempted release and public installer/helper source, or say "unknown". +3. **Choose relevant, safe evidence**: if Pulse is running and collection is + safe, use **Settings โ†’ Diagnostics โ†’ Export for GitHub (sanitized)** for + connection or data failures. For a visual problem, a screenshot or the exact + error may be enough. If logs are needed, inspect a bounded local excerpt + (`journalctl -u pulse -n 100 --no-pager` or `docker logs --tail 100 pulse`), + not a full configuration or data-directory upload. +4. **Review before posting**: even a sanitized export or screenshot can contain + identifying details. Remove credentials, session cookies, webhook URLs and + private host, network or personal information. Never post bootstrap/recovery + tokens, `.env` files, private keys or an unsanitized export. +5. **Use the appropriate thread**: [GitHub Issues](https://github.com/rcourtman/Pulse/issues) + for a bug, or [Discussions](https://github.com/rcourtman/Pulse/discussions) for + a setup question. Add new evidence to an existing matching report rather + than opening a duplicate. Do not refile information you have already supplied. diff --git a/docs/UNIFIED_AGENT.md b/docs/UNIFIED_AGENT.md index 7207a29560..ff7435ca9d 100644 --- a/docs/UNIFIED_AGENT.md +++ b/docs/UNIFIED_AGENT.md @@ -412,7 +412,7 @@ sudo chmod 0700 /usr/local/libexec/pulse-queue-depth | `--insecure` | `PULSE_INSECURE_SKIP_VERIFY` | Skip TLS verification | `false` | | `--allow-plaintext-http` | `PULSE_AGENT_ALLOW_PLAINTEXT_HTTP` | Allow plain HTTP to a Pulse server that does not look local (private IP, single-label, `.local`/`.lan`/`.home`/`.home.arpa`/`.internal`, or resolves to private addresses). Sends the API token in cleartext; only for networks you fully control, e.g. internal networks numbered from public IP space | `false` | | `--hostname` | `PULSE_HOSTNAME` | Override hostname | *(OS hostname)* | -| `--agent-id` | `PULSE_AGENT_ID` | Unique agent identifier | *(machine-id)* | +| `--agent-id` | `PULSE_AGENT_ID` | Unique agent identifier | *(saved ID, then machine ID)* | | `--report-ip` | `PULSE_REPORT_IP` | Override reported IP (multi-NIC) | *(auto)* | | `--disable-ceph` | `PULSE_DISABLE_CEPH` | Disable local Ceph status polling | `false` | | `--tag` | `PULSE_TAGS` | Apply tags (repeatable or CSV) | *(none)* | @@ -834,20 +834,53 @@ file and re-run the installer to pick up the rotating configuration. identity evidence. The endpoint reports repair handoffs but does not run them. ### Duplicate Agents -If cloned VMs appear as the same agent: -```bash -sudo rm /etc/machine-id && sudo systemd-machine-id-setup -``` -Or set a unique agent ID: -```bash ---agent-id my-unique-agent-id -``` +Cloning an installed agent can copy its saved identity and credentials. Different +hostnames, MAC addresses or IPs do not make those agents distinct in Pulse. +Identity selection is: -The displayed or reported IP is not the durable agent identity. Pulse normally -uses the machine ID (or an explicit `--agent-id`), so cloned systems must have -unique machine and agent IDs even when their hostnames, MAC addresses, and IPs -differ. +1. An explicit `--agent-id` argument (which overrides `PULSE_AGENT_ID`). +2. `PULSE_AGENT_ID` when no explicit argument is supplied. +3. The saved `agent-id` file, when no override is set. +4. The machine-derived ID, with hostname as a fallback, when no saved ID exists. + +First compare the affected hosts in **Agent Doctor**. Inspect the identity file +locally: the default Linux path is `/var/lib/pulse-agent/agent-id`; `--state-dir` +changes that directory and `--agent-id-file` or `PULSE_AGENT_ID_FILE` can select +another file. Do not upload `connection.env`, token files or whole service and +environment dumps; older installations can contain credentials there. + +**Do not delete `/etc/machine-id`, agent state or Pulse history to repair a +duplicate.** A saved agent ID survives an OS machine-ID change, and changing the +OS identity affects more than Pulse. Prepare OS identities when provisioning a +clone, using the distribution's instructions, not during live Pulse diagnosis. + +For a confirmed duplicate, keep the original host unchanged and choose one +stable, unique override for the clone. Persist it in the configuration that +actually starts that agent, rather than launching a second agent process: + +- **Systemd, without an existing `--agent-id` argument**: use + `sudo systemctl edit pulse-agent.service` and add the following drop-in, + replacing the example ID with the clone's unique ID: + + ```ini + [Service] + Environment="PULSE_AGENT_ID=vm-clone-02" + ``` + + Then restart only that agent with `sudo systemctl restart pulse-agent.service`. + If its managed launch command already supplies `--agent-id`, change that + existing override instead: the argument wins over the environment variable. +- **Docker Compose**: set `PULSE_AGENT_ID` for the clone's agent service, keeping + its existing volumes, collector options and token-file configuration. An + existing `--agent-id` in `command` takes precedence; update it instead if set. + +Keep the override across restarts and upgrades. After restart, verify that both +hosts report fresh, distinct IDs in Agent Doctor and that each row's metrics +belong to the correct host. Changing an ID does not split or recover historical +data already combined under the old ID. If authentication or reporting fails, +keep the bounded, redacted error for diagnosis; do not delete tokens, re-enrol +blindly or remove the original host's record. ### Permission Denied (Docker) Ensure the agent can access the Docker socket: diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 60d37911dd..8eea5b7709 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1411,6 +1411,21 @@ per-alert rate limiting; future notification-gating changes should extend that policy owner rather than burying new checks inside metric or resource-specific evaluators. +### Configured flapping thresholds remain reachable + +Every accepted positive `FlappingThreshold`, including values above ten, must +be reachable by current observations inside `FlappingWindowSeconds`. Per-key +history retains at most `max(10, FlappingThreshold)` timestamps after window +pruning; it must not grow with repeated attempts during an active cooldown. +Only the threshold-crossing attempt opens that cooldown. Later attempts keep +its original deadline even after the observation window drains, and a served +cooldown resets the episode so a later storm can arm it again. Lowering the +threshold applies the new history bound and threshold on the next check. The +read-only delivery diagnosis must explain the same threshold and suppression +actually enforced at dispatch. `internal/alerts/flapping_threshold_test.go` +pins these boundaries through normal configuration updates and notification +callbacks, alongside the existing flapping cooldown and one-shot callback tests. + ### Monitor-only delivery is terminal Monitor-only alerts remain visible, but neither a firing nor a recovery diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 0c3ed83fe3..fe54ff248f 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -4662,7 +4662,28 @@ no JavaScript. It separates the two and nothing else. A conclusive result is acted on immediately and any vulnerability at any severity still fails, even if the same response also carries a transport error, so a severity threshold must never be introduced; only an unreachable endpoint is -retried. Retrying is bounded by wall clock and not by attempt count alone, +retried. A nonempty package-finding map or any positive integer +summary count must fail immediately, even when the total is missing or +contradicts that evidence; a later response must not replace a known finding. +A clean verdict requires all six known summary counts to be integer zero, +no endpoint error and an absent or empty package-finding map. Booleans, +strings, nulls, negative counts and partial summaries are not zero-finding +evidence. Without positive evidence, malformed reports retain the existing +bounded outage policy, never a clean verdict. Summary diagnostics may print +only validated nonnegative integer counts (or `unknown`) and the package-record +count, not raw metadata strings capable of emitting workflow commands. +Finding diagnostics must use that same captured JSON response, never +a second registry request: such a request could hang outside the watchdog or +describe a different verdict. Logs retain the affected package, severity, +range, locked paths, advisory source/title/link and fix availability through +allowlisted, JSON-escaped fields, not arbitrary registry text or transport-error +details. Missing package-level detail leaves the vulnerability failure intact; +it must not trigger another request. `scripts/tests/test_npm_audit_retry.py` +executes changed-second-response, missing-detail, escaping and single-request +fixtures, including production argument forwarding and findings accompanied by +transport errors, positive package/severity evidence with missing or zero +totals, malformed zero summaries under both outage modes, and metadata +annotation injection. Retrying is bounded by wall clock and not by attempt count alone, because npm's own `fetch-timeout` defaults to five minutes and it retries internally: on 2026-09-04 three attempts against a hanging endpoint ran for 10m56s and cancelled the Frontend job at its own timeout with every test @@ -6022,3 +6043,20 @@ offline dependency snapshot for the updated lockfile, which is not present in the current assignment; a host dependency acquisition on the next launch is required before this contract's Go evidence can be produced. No passing Go suite, installed build or release acceptance is claimed here. + +## Release-body updater ownership (30 September 2026) + +Each executable `/bin/update --version` example in a published body must carry +its own nearby condition: only use it when that helper was installed by the +Pulse server installer. A warning in a distant highlight does not scope an +unqualified install, rollback or authored Before-you-upgrade command. The +existing generated sections retain their signed, version-pinned installer +fallback for community-scripts installations or an unknown helper owner. + +`validate_release_body_shape` applies `validate_server_updater_guidance` to +the complete body before accepting it; it rejects unsafe examples rather than +rewriting the author's instructions or selecting another rollback version. +`render_release_body_test.py` exercises all three unsafe published contexts, +distant-scope rejection, and valid generated and inline ownership conditions. +This changes publication validation, not installer behaviour, immutable release +contents or proof of an installed upgrade/rollback. diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 5e59ab13b5..04a4e3fdf1 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -307,6 +307,19 @@ group's rows mid-scroll. The controller samples several leading siblings and keeps the tallest, so uniform tables still measure their real row height while mixed group/content lists keep a content-scale estimate. +Shared workload, node, Docker-host, and resource-drawer history presentation +must scope retained observations to the exact resource type, resource ID and +range. An uncached target or range change clears the former points while its +read is pending, including when a PBS host link is withdrawn. A failed +replacement must never cache former-host points under the new target. Matching +cached reads and same-source background polls retain their chart without a +loading flash. Superseded, locked, unavailable and unmounted requests propagate +the query's abort signal to the Charts API; late results cannot replace current +observations. `GuestDrawerHistory.source-isolation.test.tsx` exercises the real +renderer, cache readback and cancellation, not a mocked chart. The mock-backed +PBS browser runner verifies delayed range and withdrawn-target reads at desktop +and phone widths; neither proof establishes installed collection or #1723 relief. + Shared workload, node, Docker-host, and resource-drawer history presentation keeps current readings separate from stored samples. A current metric may populate the legend while history is still being collected, but it must never diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index cd1e819a3d..453d3a5385 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -122,6 +122,25 @@ cycles, completion timestamps and unchanged failure backoff. These are synthetic runtime proofs, not native firmware timeout or reporter-resolution evidence. +### TrueNAS reporting summaries preserve raw History โ€” issue #2346 + +Every reporting time-range query requests `aggregate: true`, the TrueNAS +default, through the shared `reportingRangeQuery`: JSON-RPC host/disk-temperature +History and short temperature reads, plus legacy REST host History/live telemetry. +SCALE 25.04.2.6 rejects `aggregate: false` because its response schema requires +the min/mean/max fields omitted by that option. These summaries accompany the +raw `data` series; History still parses only raw samples, with the same +start/end window, timestamps, units and default duration. There is no new +retry, error suppression, transport selection or graph-isolation policy. + +`TestReportingHistorySupportsSCALE2504Aggregations` models the supplied +ReportingGetDataResult/EINVAL failure and preserves 3,600 raw samples through +system and disk History. `TestReportingAggregationsPreserveRPCFailures` keeps +validation, access and missing-method errors visible. The REST default-summary +regression and existing CORE graph-isolation/live-memory tests retain legacy +telemetry and canonical CPU/memory History projections. Synthetic transports +are not native appliance or containing-release acceptance. + **TrueNAS legacy-REST memory telemetry โ€” issue #2077 (20 September 2026)** A connection proven to run a recognized legacy CORE/FreeNAS release has no diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index 8df54c6569..7f8b616f6b 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -2768,6 +2768,7 @@ "internal/alerts/eventlog/reader_isolation_test.go", "internal/alerts/external_probe_test.go", "internal/alerts/filter_evaluation_test.go", + "internal/alerts/flapping_threshold_test.go", "internal/alerts/guest_snapshot_test.go", "internal/alerts/history_concurrency_test.go", "internal/alerts/history_migration_test.go", diff --git a/docs/release-control/v6/internal/subsystems/security-privacy.md b/docs/release-control/v6/internal/subsystems/security-privacy.md index 66024394cb..ad199b6b40 100644 --- a/docs/release-control/v6/internal/subsystems/security-privacy.md +++ b/docs/release-control/v6/internal/subsystems/security-privacy.md @@ -1492,9 +1492,9 @@ download URLs, command output, log lines, paths, hostnames, release asset URLs, checksums, signatures, or operator-entered values. Schema v13 adds a direct local service-health observation so a process-level telemetry heartbeat is not mistaken for proof that the installed UI and API -are being served. The runtime probes its bound listener through loopback and -reports only observed/healthy booleans, one fixed failure class (`listener`, -`startup`, `runtime`, `api_connectivity`, `api_status`, `ui_status`, +are being served. The runtime probes the address bound by its TCP listener +(loopback for wildcard binds) and reports only observed/healthy booleans, one fixed failure class (`listener`, +`startup`, `runtime`, `api_connectivity`, `timeout`, `api_status`, `ui_status`, `frontend_assets`, or `unknown`), a fixed observation cohort, and the immediately previous normalized release's observed/healthy booleans. No probe target, listener address, URL, IP address, asset path, response content, raw @@ -1502,6 +1502,19 @@ error, account, customer, or infrastructure identity may enter the payload or persisted receiver row. The previous-release fields are direct adjacent-release observations, not 30-day update counters, and are the only valid basis for a before/after release-health cohort in the adoption report. +The service-health self-check must preserve the explicit bound TCP address and +port; wildcard IPv6/unspecified-family listeners try IPv4 and IPv6 loopback, +while an IPv4-only wildcard remains IPv4-only. No external address discovery, +proxy, redirect, or remote frontend asset fetch is permitted. One bounded +settling retry may follow a failed observation in the telemetry background +runner: at most two five-second attempts, separated by one second. Each attempt +shares its deadline across API/UI/assets and reserves time for an alternate +loopback family. Once an API responds, status/body/UI/asset failures cannot be +masked by switching families. Deadline and network timeout failures use only +the fixed `timeout` bucket; other failure classes remain visible. Tests must +cover unavailable IPv6 with working IPv4, IPv6-only serving, a stalled family, +startup recovery, final timeouts, explicit addresses, HTTPS, redirect refusal, +bounded bodies/assets, closed-category serialization and background execution. That same outbound usage telemetry floor now also permits only content-free Pulse Patrol control and governed Pulse Intelligence operations adoption flags and counters inside the same rotating 30-day telemetry window: diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index e2a48b9049..1d0f8017ec 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -319,6 +319,8 @@ The backend may establish that explicit link through a running PVE guest's uniqu The operator's PBS connection label is never sufficient backend host identity. A label matching an unrelated Agent must not win over the endpoint's corroborated PVE guest-IP/Agent link or project that Agent's SMART disks under PBS. If only the label matches, the backend leaves the host link absent and the PBS service series separate; Backups presentation has its own correlation boundary. +The direct backend PBS-to-Agent path requires non-zero observations within five minutes and excludes loopback, link-local, non-unicast and recognisable Docker-local interface addresses. Private management networks, custom management bridges and explicit report-IP hints remain eligible, including correctly parsed IPv6 URL endpoints. A stale Agent at a reused endpoint must lose PBS disk membership and its presentation link; a fresh unique replacement may acquire them, while concurrent eligible Agents remain ambiguous. `pbs_pmg_registry_test.go` covers unsafe direct evidence, management-network controls and fresh/stale/replacement/ambiguity transitions through monitor registry replacement. Its three-PBS/nine-Agent fixture preserves distinct source-native host targets and disk/network/I/O telemetry, but does not establish the reporter's VirtualBox identity or installed History acceptance. + A realtime refresh can briefly omit the correlated host row while the PBS server row remains. Retain the last resolved host per PBS server across that omission instead of falling back to the service target, so the drawer's Identity rows and History series stay on the host key. Reuse the remembered host only while it is still fresh relative to the server and drop it once stale, so a removed or replaced host is not advertised indefinitely; a present but ambiguous host still declines. Verification: ProxmoxBackupServersTable.drawer.test.tsx covers standalone and diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index b19d9bc2a3..7bf7b0c4a7 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -23,6 +23,25 @@ and sort the complete canonical change table while startup and ingestion wait. ## Purpose +### Allocation-bounded snapshot and clone metadata โ€” issue #2199 + +WebSocket snapshot extraction decodes every entry's ID from the encoded array +in one batched pass, preserving standard JSON identity semantics rather than +trusting source-ID hints after checking only the first entry. Missing/duplicate +IDs still reject resource keying; infrastructure and alert arrays retain the +existing whole-field fallback. The decoder-owned raw entry buffers are retained +without a redundant copy, and must remain isolated from the original encoding +and from each other. Per-entry reference decoding, malformed/escaped/duplicate +identity fuzz cases and encoded tail-change deltas pin these boundaries. + +Resource clones still refresh canonical metadata on every read, including after +source, facet or tag edits. Platform membership uses a deduplicated small slice +with the existing canonical order; sensitivity scans normalized tags once, +preserving explicit public, restricted, sensitive and resource-type precedence. +The exhaustive source-combination test and clone scope/policy refresh test +protect page admission, policy routing and clone isolation. Component benchmarks +measure allocation/time changes, not installed fleet CPU or write-rate acceptance. + ### Bounded generation change comparison โ€” issue #2199 Accepted snapshot rebuilds compare the previous and replacement registry @@ -508,6 +527,8 @@ PBS presentation correlation accepts standalone agents and VM/system-container r When the registry uniquely corroborates a PBS connection to a host agent by PBS-reported node name, configured endpoint hostname or reported interface IP, the PBS service may expose that source-native agent ID as `pbs.linkedAgentId`. The user-editable connection name alone is not host identity: it must not attach an unrelated Agent or preempt a corroborated VM-IP/Agent link. This is a presentation link, not a replacement for the PBS service's canonical ID or metrics target. Token-auth connections may lack `nodeName`; consumers must use the explicit link in preference to display-name matching and must not infer one from an IP or name themselves. PVE-only nodes without an actual Agent facet are not host-history candidates, even if their name matches the PBS connection. A changed link invalidates any remembered correlation to an old host. +Direct corroboration also requires a source-native Agent ID and non-zero PBS/Agent observations within five minutes. A current PBS endpoint must not keep a stale Agent's link or disk membership when the address is reused. Interface evidence uses the shared provider-link address filter: loopback, link-local, non-unicast and recognisable Docker-local interfaces cannot prove machine identity; ordinary private management interfaces, custom management bridges and explicit global-unicast report-IP hints remain eligible. URL hostname extraction preserves bracketed IPv6 literals instead of truncating at their first colon. `TestPBSDirectHostLinkRejectsUnsafeObservations`, `TestPBSDirectHostLinkPreservesManagementEvidence` and `TestPBSDirectHostLinkWithdrawsAndReplacesStaleAgent` pin these boundaries. `TestPBSHostHistoryThreeServerTopology` proves source-native host targets and Agent disk/network/I/O telemetry remain distinct for two PVE guests and one independently IP-corroborated non-PVE PBS among nine Agents; it is synthetic, not the reporter's installed VirtualBox evidence. + For a PBS inside a PVE VM whose in-guest Agent omits interface IPs, the registry may also corroborate a unique Agent-to-VM state link with a unique running VM reporting the PBS endpoint's exact IP. All three observations must be within five minutes, and reused/non-unique IPs, stopped guests or competing Agent links decline to associate. This is an alternate backend evidence chain, not frontend hostname inference; it cannot identify a VirtualBox guest without equivalent observed topology. A single agent surfaced as both a VM/system-container guest and a standalone agent row is one identity, not an ambiguous pair. Collapse candidates that share an agent identity before deciding, and prefer the guest's canonical metricsTarget because it names the persisted host series; the PBS service target names the service key and now records only PBS API node-status CPU/memory history, not host-agent disk, network, or I/O. Only distinct agent identities, or candidates whose identity cannot be proven equal, decline to choose. Never use input order or a fallback that discards the PBS row id. @@ -5159,7 +5180,10 @@ to an agent host and its agent-reported SMART disk resources only when one host is uniquely corroborated by the PBS-reported node name, endpoint hostname, endpoint IP against the host report and interface addresses, or a fresh PVE guest-IP/Agent link. The user-editable connection label is not identity evidence. -Multiple matches are ambiguous and must fail closed without assigning PBS ownership. +All candidate Agent reports must be fresh relative to the PBS observation; +direct IP corroboration excludes host-local networks under the shared provider +link filter. Multiple eligible matches are ambiguous and must fail closed +without assigning PBS ownership. The same unique match may set `PBSData.LinkedAgentID` on the PBS service only after the source-native host ID resolves to a canonical resource carrying that diff --git a/frontend-modern/browser-tests/docs-fragment-navigation.tsx b/frontend-modern/browser-tests/docs-fragment-navigation.tsx index 2f929f78e7..5a9d65b9fb 100644 --- a/frontend-modern/browser-tests/docs-fragment-navigation.tsx +++ b/frontend-modern/browser-tests/docs-fragment-navigation.tsx @@ -20,6 +20,7 @@ const targets: Record = { reload: '/docs/API#resource-maintenance-and-operator-state', malformed: '/docs/API#%invalid', missing: '/docs/API#does-not-exist', + troubleshooting: '/docs/TROUBLESHOOTING#recovery-mode', }; window.history.replaceState({}, '', targets[scenario] ?? targets.plain); diff --git a/frontend-modern/browser-tests/pbs-history-source-isolation.cjs b/frontend-modern/browser-tests/pbs-history-source-isolation.cjs new file mode 100644 index 0000000000..27ba7086bb --- /dev/null +++ b/frontend-modern/browser-tests/pbs-history-source-isolation.cjs @@ -0,0 +1,231 @@ +// Mock-backed proof of the production PBS table/drawer, not installed acceptance. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { createHash } = require('node:crypto'); +const { chromium } = require('playwright'); + +(async () => { + const root = '/workspace/frontend-modern'; + process.chdir(root); + const phaseFile = path.join(root, 'node_modules/history-proof-phase.txt'); + const phase = fs.existsSync(phaseFile) ? fs.readFileSync(phaseFile, 'utf8').trim() : 'candidate'; + const artifacts = path.join(root, `node_modules/history-source-isolation-${phase}`); + fs.mkdirSync(artifacts, { recursive: true }); + const { createServer } = await import(path.join(root, 'node_modules/vite/dist/node/index.js')); + const server = await createServer({ + root, + configFile: path.join(root, 'vite.config.ts'), + server: { host: '127.0.0.1', port: 5218, strictPort: true }, + }); + let browser; + const observations = []; + try { + await server.listen(); + browser = await chromium.launch({ + headless: true, + channel: 'chromium', + args: ['--no-sandbox'], + }); + for (const width of [1365, 390]) { + const page = await browser.newPage({ + viewport: { width, height: width === 390 ? 844 : 900 }, + }); + page.setDefaultTimeout(20_000); + const requests = [], + failures = [], + errors = []; + const held = new Map(); + let holdOldRange = true, + holdService = true; + page.on('pageerror', (error) => errors.push(error.message)); + page.on('requestfailed', (request) => { + if (request.url().includes('/api/metrics-store/history')) + failures.push({ url: request.url(), failure: request.failure() }); + }); + const response = (id, range) => ({ + resourceType: 'agent', + resourceId: id, + range, + start: 1_700_000_000_000, + end: 1_700_000_060_000, + source: 'store', + metrics: Object.fromEntries( + (id === 'pbs-three' + ? ['cpu', 'memory'] + : ['cpu', 'memory', 'disk', 'netin', 'netout', 'diskread', 'diskwrite'] + ).map((metric, i) => [ + metric, + [0, 1].map((step) => ({ + timestamp: 1_700_000_000_000 + step * 60_000, + value: (id === 'pbs-three' ? 12 : range === '1h' ? 31 : 88) + i + step, + min: 10, + max: 99, + })), + ]), + ), + }); + await page.route('**/*', async (route) => { + const request = route.request(); + const url = new URL(request.url()); + if (url.origin !== 'http://127.0.0.1:5218') return route.abort(); + if (!url.pathname.startsWith('/api/')) return route.continue(); + if (url.pathname === '/api/metrics-store/history') { + const id = url.searchParams.get('resourceId'), + range = url.searchParams.get('range'); + requests.push({ method: request.method(), id, range }); + assert.ok(['agent-three', 'pbs-three'].includes(id), `unexpected target ${id}`); + if (id === 'agent-three' && range === '6h' && holdOldRange) { + holdOldRange = false; + held.set('range', route); + return; + } + if (id === 'pbs-three' && range === '1h' && holdService) { + holdService = false; + held.set('service', route); + return; + } + return route.fulfill({ json: response(id, range) }); + } + if (url.pathname === '/api/license/runtime-capabilities') + return route.fulfill({ + json: { + capabilities: [], + limits: [], + max_history_days: 7, + hosted_mode: false, + runtime: { build: 'community', label: 'Pulse Community runtime' }, + blocked_capabilities: [], + }, + }); + return route.fulfill({ json: { data: [], enabled: false } }); + }); + await page.goto('http://127.0.0.1:5218/browser-tests/pbs-identity-boundary.html', { + waitUntil: 'domcontentloaded', + timeout: 120_000, + }); + await page.getByRole('button', { name: 'Corroborate links', exact: true }).click(); + const toggle = page.getByRole('button', { + name: 'Expand details for backup-connection-three', + exact: true, + }); + if (width === 1365) { + await toggle.focus(); + await page.keyboard.press('Enter'); + } else await page.locator('td[title="backup-connection-three ยท tank"]').click(); + const detail = page.locator('[data-inline-platform-resource-detail-for="pbs-three"]'); + const plots = detail.locator('[data-testid="guest-history-plot"] path'); + await detail.getByRole('tab', { name: 'History', exact: true }).click(); + await page.waitForFunction( + () => document.querySelectorAll('[data-testid="guest-history-plot"] path').length === 7, + ); + const originalPaths = await plots.evaluateAll((paths) => + paths.map((p) => p.getAttribute('d')), + ); + const checkLayout = async () => { + const dimensions = await page.evaluate(() => ({ + scroll: document.documentElement.scrollWidth, + inner: innerWidth, + })); + assert.ok(dimensions.scroll <= dimensions.inner + 1, JSON.stringify(dimensions)); + assert.equal( + await detail + .getByRole('tab', { name: 'History', exact: true }) + .getAttribute('aria-selected'), + 'true', + ); + return dimensions; + }; + const screenshot = async (state) => + page.screenshot({ path: path.join(artifacts, `${state}-${width}.png`), fullPage: true }); + await screenshot('host-loaded'); + await detail.getByTestId('guest-history-range-control').selectOption('6h'); + await page.waitForFunction( + () => document.querySelector('[data-testid="guest-history-range-control"]').value === '6h', + ); + // Wait for the route, not an arbitrary sleep or the very assertion under test. + for (let attempt = 0; !held.has('range') && attempt < 100; attempt++) + await page.waitForTimeout(10); + assert.ok(held.has('range'), 'range request was not intercepted'); + await screenshot('range-loading'); + assert.equal(await plots.count(), 0, 'old range plotted while 6h replacement is loading'); + assert.ok((await detail.getByText('Loading history', { exact: true }).count()) > 0); + await checkLayout(); + await detail.getByTestId('guest-history-range-control').selectOption('1h'); + await page.waitForFunction( + () => document.querySelectorAll('[data-testid="guest-history-plot"] path').length === 7, + ); + const currentPaths = await plots.evaluateAll((paths) => + paths.map((p) => p.getAttribute('d')), + ); + assert.notDeepEqual(currentPaths, originalPaths); + let lateResponse; + try { + await held.get('range').fulfill({ json: response('agent-three', '6h') }); + lateResponse = 'fulfilled'; + } catch (error) { + lateResponse = String(error); + } + // A marker evaluation after the obsolete response; the unit proof also + // deliberately resolves a mock which ignores AbortSignal entirely. + await page.evaluate(() => new Promise(requestAnimationFrame)); + assert.deepEqual( + await plots.evaluateAll((paths) => paths.map((p) => p.getAttribute('d'))), + currentPaths, + ); + await page.getByRole('button', { name: 'Withdraw third link', exact: true }).click(); + for (let attempt = 0; !held.has('service') && attempt < 100; attempt++) + await page.waitForTimeout(10); + assert.ok(held.has('service'), 'withdrawal request was not intercepted'); + await screenshot('target-loading'); + assert.equal(await plots.count(), 0, 'former host plotted while revoked target is loading'); + assert.ok((await detail.getByText('Loading history', { exact: true }).count()) > 0); + await held + .get('service') + .fulfill({ status: 503, json: { error: 'Fixture replacement unavailable' } }); + await detail.getByText('Failed to load history data', { exact: true }).waitFor(); + assert.equal(await plots.count(), 0); + await screenshot('target-failed'); + await detail.getByTestId('guest-history-range-control').selectOption('6h'); + await page.waitForFunction( + () => document.querySelectorAll('[data-testid="guest-history-plot"] path').length === 2, + ); + assert.equal(await detail.locator('[data-history-group="network"] path').count(), 0); + assert.equal(await detail.locator('[data-history-group="disk-io"] path').count(), 0); + assert.equal(await detail.getByText('Loading history', { exact: true }).count(), 0); + await screenshot('service-loaded'); + const dimensions = await checkLayout(); + const beforeLocked = requests.length; + await detail.getByTestId('guest-history-range-control').selectOption('14d'); + await detail.getByText(/14 days history requires a higher license plan/).waitFor(); + assert.equal(await plots.count(), 0); + assert.equal(requests.length, beforeLocked); + assert.ok(requests.every((r) => r.method === 'GET')); + assert.deepEqual(errors, []); + observations.push({ width, requests, failures, lateResponse, dimensions, errors }); + await page.close(); + } + const file = 'src/components/Workloads/GuestDrawerHistory.tsx'; + const result = { + result: 'passed', + phase, + browser: browser.version(), + playwright: require('playwright/package.json').version, + content_sha256: { + [`frontend-modern/${file}`]: createHash('sha256') + .update(fs.readFileSync(path.join(root, file))) + .digest('hex'), + }, + observations, + }; + fs.writeFileSync(path.join(artifacts, 'result.json'), JSON.stringify(result, null, 2) + '\n'); + console.log(JSON.stringify(result)); + } finally { + if (browser) await browser.close(); + await server.close(); + for (const name of fs.readdirSync(artifacts)) fs.chmodSync(path.join(artifacts, name), 0o644); + } +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/frontend-modern/browser-tests/pbs-identity-boundary.cjs b/frontend-modern/browser-tests/pbs-identity-boundary.cjs new file mode 100644 index 0000000000..01ab2a9b0a --- /dev/null +++ b/frontend-modern/browser-tests/pbs-identity-boundary.cjs @@ -0,0 +1,201 @@ +// Offline verification of the production table, drawer and History requests. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { createHash } = require('node:crypto'); +const { chromium } = require('playwright'); + +(async () => { + const root = '/workspace/frontend-modern'; + const artifacts = path.join(root, 'node_modules/pbs-identity-boundary-proof'); + fs.mkdirSync(artifacts, { recursive: true }); + fs.chmodSync(artifacts, 0o755); + process.chdir(root); + const { createServer } = await import(path.join(root, 'node_modules/vite/dist/node/index.js')); + const server = await createServer({ + root, + configFile: path.join(root, 'vite.config.ts'), + server: { host: '127.0.0.1', port: 5211, strictPort: true }, + }); + let browser; + const results = []; + try { + await server.listen(); + browser = await chromium.launch({ + headless: true, + channel: 'chromium', + args: ['--no-sandbox'], + }); + for (const width of [1365, 390]) { + const page = await browser.newPage({ + viewport: { width, height: width === 390 ? 844 : 900 }, + }); + page.setDefaultTimeout(20_000); + const pageErrors = []; + const requests = []; + page.on('pageerror', (error) => pageErrors.push(error.message)); + await page.route('**/*', (route) => { + const url = new URL(route.request().url()); + if (url.origin !== 'http://127.0.0.1:5211') return route.abort(); + if (!url.pathname.startsWith('/api/')) return route.continue(); + if (url.pathname === '/api/metrics-store/history') { + const type = url.searchParams.get('resourceType'); + const id = url.searchParams.get('resourceId'); + requests.push({ method: route.request().method(), target: `${type}:${id}` }); + assert.ok(!id.startsWith('unrelated-'), `wrong-host History requested: ${id}`); + return route.fulfill({ + json: { + resourceType: type, + resourceId: id, + range: url.searchParams.get('range'), + start: Date.now() - 3_600_000, + end: Date.now(), + source: 'store', + // PBS API supplies only service CPU/memory, not invented host series. + metrics: Object.fromEntries( + (id.startsWith('pbs-') + ? ['cpu', 'memory'] + : ['cpu', 'memory', 'disk', 'netin', 'netout', 'diskread', 'diskwrite'] + ).map((metric) => [ + metric, + [30, 20, 10].map((minutes, index) => ({ + timestamp: Date.now() - minutes * 60_000, + value: 10 + index * 5, + min: 10 + index * 5, + max: 10 + index * 5, + })), + ]), + ), + }, + }); + } + if (url.pathname === '/api/license/runtime-capabilities') { + return route.fulfill({ + json: { + capabilities: [], + limits: [], + max_history_days: 7, + hosted_mode: false, + runtime: { build: 'community', label: 'Pulse Community runtime' }, + blocked_capabilities: [], + }, + }); + } + return route.fulfill({ json: { data: [], enabled: false } }); + }); + await page.goto('http://127.0.0.1:5211/browser-tests/pbs-identity-boundary.html', { + waitUntil: 'domcontentloaded', + timeout: 120_000, + }); + const observations = []; + const inspect = async (suffix, expected, state, open = true) => { + const detail = page.locator(`[data-inline-platform-resource-detail-for="pbs-${suffix}"]`); + if (open) { + const toggle = page.getByRole('button', { + name: `Expand details for backup-connection-${suffix}`, + exact: true, + }); + if (width === 1365) { + await toggle.focus(); + await page.keyboard.press('Enter'); + } else await page.locator(`td[title="backup-connection-${suffix} ยท tank"]`).click(); + } + await detail.waitFor(); + const overview = detail.getByRole('tab', { name: 'Overview', exact: true }); + await overview.click(); + const identity = detail + .locator('[data-testid="resource-identity-section"] tr') + .filter({ hasText: 'Metrics Target' }); + await identity.getByText(expected, { exact: true }).waitFor(); + assert.ok(!(await detail.innerText()).includes('/WRONG-HOST'), 'wrong-host disks rendered'); + await detail.getByRole('tab', { name: 'History', exact: true }).click(); + await detail.locator('[data-testid="guest-history-plot"] path').first().waitFor(); + assert.equal( + await detail + .getByRole('tab', { name: 'History', exact: true }) + .getAttribute('aria-selected'), + 'true', + ); + assert.ok( + requests.some((request) => request.target === expected), + `missing History read for ${expected}`, + ); + const targetStart = requests.length; + await detail.getByTestId('guest-history-range-control').selectOption('6h'); + // Force a fresh read even when the open drawer already has 6h selected. + await detail.getByTestId('guest-history-range-control').selectOption('1h'); + await page.waitForFunction(() => !document.body.innerText.includes('Loading history')); + const dimensions = await page.evaluate(() => ({ + scroll: document.documentElement.scrollWidth, + inner: innerWidth, + })); + assert.ok(dimensions.scroll <= dimensions.inner + 1, JSON.stringify(dimensions)); + observations.push({ + state, + suffix, + expected, + dimensions, + newTargets: requests.slice(targetStart), + }); + console.log(JSON.stringify({ width, state, expected })); + }; + for (const suffix of ['one', 'two', 'three']) + await inspect(suffix, `agent:pbs-${suffix}`, 'no-link-label-collisions'); + await page.screenshot({ + path: path.join(artifacts, `unlinked-${width}.png`), + fullPage: true, + }); + await page.getByRole('button', { name: 'Corroborate links', exact: true }).click(); + await inspect('three', 'agent:agent-three', 'explicit-link', false); + await inspect('one', 'vm:vm-one', 'same-Agent-guest-deduplication'); + await inspect('two', 'vm:vm-two', 'second-guest'); + await inspect('three', 'agent:agent-three', 'standalone-host'); + await page.screenshot({ path: path.join(artifacts, `linked-${width}.png`), fullPage: true }); + await page.getByRole('button', { name: 'Omit linked host rows', exact: true }).click(); + await inspect('three', 'agent:agent-three', 'unchanged-link-host-omission', false); + await page.getByRole('button', { name: 'Restore linked host rows', exact: true }).click(); + await inspect('three', 'agent:agent-three', 'restored-host', false); + await page.getByRole('button', { name: 'Withdraw third link', exact: true }).click(); + await inspect('three', 'agent:pbs-three', 'withdrawn-link-with-hosts-still-present', false); + assert.ok( + !( + await page.locator('[data-inline-platform-resource-detail-for="pbs-three"]').innerText() + ).includes('/real-three'), + 'withdrawn host disks remained visible', + ); + await page.screenshot({ + path: path.join(artifacts, `withdrawn-${width}.png`), + fullPage: true, + }); + await page.getByRole('button', { name: 'Report exact node hostname', exact: true }).click(); + await inspect('one', 'vm:vm-one', 'exact-reported-machine-hostname'); + await inspect('three', 'agent:pbs-three', 'remaining-uncorroborated-host'); + assert.ok(requests.length > 0); + assert.ok(requests.every((request) => request.method === 'GET')); + assert.deepEqual(pageErrors, []); + results.push({ width, observations, requests, pageErrors }); + await page.close(); + } + const file = 'src/features/proxmox/ProxmoxBackupServersTable.tsx'; + const result = { + result: 'passed', + browser: browser.version(), + playwright: '1.56.1', + content_sha256: { + [`frontend-modern/${file}`]: createHash('sha256') + .update(fs.readFileSync(path.join(root, file))) + .digest('hex'), + }, + results, + }; + fs.writeFileSync(path.join(artifacts, 'result.json'), JSON.stringify(result, null, 2) + '\n'); + for (const name of fs.readdirSync(artifacts)) fs.chmodSync(path.join(artifacts, name), 0o644); + console.log(JSON.stringify(result)); + } finally { + if (browser) await browser.close(); + await server.close(); + } +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/frontend-modern/browser-tests/pbs-identity-boundary.html b/frontend-modern/browser-tests/pbs-identity-boundary.html new file mode 100644 index 0000000000..f09fc08bf2 --- /dev/null +++ b/frontend-modern/browser-tests/pbs-identity-boundary.html @@ -0,0 +1,12 @@ + + + + + + PBS identity boundary verification + + +
+ + + diff --git a/frontend-modern/browser-tests/pbs-identity-boundary.tsx b/frontend-modern/browser-tests/pbs-identity-boundary.tsx new file mode 100644 index 0000000000..bc3881d00b --- /dev/null +++ b/frontend-modern/browser-tests/pbs-identity-boundary.tsx @@ -0,0 +1,116 @@ +// Synthetic bounded Backups/Overview resources, not installed host evidence. +import { createMemo, createSignal } from 'solid-js'; +import { render } from 'solid-js/web'; + +import { ProxmoxBackupServersTable } from '../src/features/proxmox/ProxmoxBackupServersTable'; +import type { Resource } from '../src/types/resource'; +import '../src/index.css'; + +const seen = Date.now(); +const suffixes = ['one', 'two', 'three']; +const services = suffixes.map((suffix, index): Resource => ({ + id: `pbs-${suffix}`, + type: 'pbs', + name: `backup-connection-${suffix}`, + displayName: `backup-connection-${suffix}`, + platformId: `pbs-${suffix}`, + platformType: 'proxmox-pbs', + sourceType: 'api', + sources: ['pbs'], + status: 'online', + lastSeen: seen, + cpu: { current: 10 + index }, + memory: { current: 40 }, + metricsTarget: { resourceType: 'agent', resourceId: `pbs-${suffix}` }, + canonicalIdentity: { hostname: `10.2.0.${13 + index}` }, + pbs: { + instanceId: `pbs-${suffix}`, + hostname: `10.2.0.${13 + index}`, + connectionHealth: 'healthy', + version: '3.2.1', + datastores: [{ name: 'tank', total: 1000, used: 400, available: 600, usagePercent: 40 }], + }, +})); +const hosts = suffixes.map((suffix): Resource => ({ + id: `agent-${suffix}`, + type: 'agent', + name: `Host label ${suffix}`, + platformType: 'proxmox-pbs', + sourceType: 'agent', + sources: ['agent', 'pbs'], + status: 'online', + lastSeen: seen, + agent: { + agentId: `agent-${suffix}`, + hostname: `host-${suffix}.example`, + disks: [{ mountpoint: `/real-${suffix}`, total: 1000, used: 250, free: 750, usage: 25 }], + }, + metricsTarget: { resourceType: 'agent', resourceId: `agent-${suffix}` }, +})); +const guests = hosts.slice(0, 2).map((host, index): Resource => ({ + ...host, + id: `vm-${suffixes[index]}`, + type: 'vm', + sources: ['proxmox', 'agent'], + metricsTarget: { resourceType: 'vm', resourceId: `vm-${suffixes[index]}` }, +})); +const unrelated = Array.from({ length: 6 }, (_, index): Resource => ({ + ...hosts[index % 3], + id: `unrelated-${index}`, + type: index < 3 ? 'vm' : 'agent', + name: `backup-connection-${suffixes[index % 3]}`, + displayName: `backup-connection-${suffixes[index % 3]}`, + identity: { ips: [`10.9.0.${13 + index}`] }, + agent: { + agentId: `unrelated-${index}`, + hostname: `backup-connection-${suffixes[index % 3]}`, + disks: [{ mountpoint: '/WRONG-HOST', total: 1000, used: 999, free: 1, usage: 99.9 }], + }, + metricsTarget: { resourceType: 'vm', resourceId: `unrelated-${index}` }, +})); + +const Fixture = () => { + const [mode, setMode] = createSignal('unlinked'); + const servers = createMemo(() => [ + ...services.map((server, index): Resource => ({ + ...server, + pbs: { + ...server.pbs!, + linkedAgentId: + ['linked', 'omitted', 'withdrawn'].includes(mode()) && + !(mode() === 'withdrawn' && index === 2) + ? `agent-${suffixes[index]}` + : undefined, + nodeName: mode() === 'node' && index === 0 ? 'host-one.example' : undefined, + }, + })), + ...(mode() === 'omitted' ? [] : [...hosts, ...guests]), + ...unrelated, + ]); + return ( +
+

PBS identity boundary verification

+
+ + + + + +
+ {mode()} + +
+ ); +}; + +render(() => , document.getElementById('root') as HTMLElement); diff --git a/frontend-modern/browser-verification.json b/frontend-modern/browser-verification.json index 971e11f167..46f3f9af25 100644 --- a/frontend-modern/browser-verification.json +++ b/frontend-modern/browser-verification.json @@ -1,22 +1,16 @@ { "version": 1, - "base_sha": "91d8822d76c5a7214995c0e370be637b6e6336e3", - "verified_at": "2026-09-29T22:13:56Z", + "base_sha": "21a35494aecb335755de773dd099e1364e65b3e0", + "verified_at": "2026-09-30T20:12:09Z", "result": "passed", "changed_paths": [ - "frontend-modern/src/components/shared/ContainerUpdateBadge.tsx", - "frontend-modern/src/components/shared/containerUpdateBadgeModel.ts", - "frontend-modern/src/components/shared/useContainerUpdateButtonState.ts", - "frontend-modern/src/stores/containerUpdates.ts" + "frontend-modern/src/components/Workloads/GuestDrawerHistory.tsx" ], "content_sha256": { - "frontend-modern/src/components/shared/ContainerUpdateBadge.tsx": "fd88f3e3e275926572126f43cce7296e7361b5e527868dd3c59c7eb951e9f5c9", - "frontend-modern/src/components/shared/containerUpdateBadgeModel.ts": "10bc488cb82ae84c6ae46d77ec672d34df2a1b3c28e138ae0f3b4214cccbe37e", - "frontend-modern/src/components/shared/useContainerUpdateButtonState.ts": "4b9dd4a42549b0433a1afa56f26183fe721c80626f823fb3c8489934721267c5", - "frontend-modern/src/stores/containerUpdates.ts": "b5d47132f7daf7f14bcc38462043288fc211344410af622ada7728694bcd514d" + "frontend-modern/src/components/Workloads/GuestDrawerHistory.tsx": "b2b31441b19c2e50283ceedf319cd09a1f94791750cd424c052717a4957fdc67" }, "routes": [ - "/browser-tests/update-action-receipt.html (production Docker update button with scripted action reads)" + "/browser-tests/pbs-identity-boundary.html (production PBS table and real resource History renderer; synthetic API data)" ], "viewports": [ { @@ -29,28 +23,187 @@ } ], "states": [ - "governed update receipt pending after registry updateAvailable becomes false", - "existing action re-opened by GET with no second plan or execute", - "recorded completed action after a second GET changes the displayed outcome" + "corroborated host with seven observed metric paths", + "uncached range replacement pending with no old paths", + "new range loaded before the old range response", + "withdrawn host link with replacement target pending", + "replacement target returns 503 with no former-host history", + "service-only response retains CPU/memory, not host disk/network/I/O", + "locked 14-day range issues no history read" ], "interactions": [ - "inspect pending button and absence of a Current or Completed claim", - "select Review action and inspect receipt-pending dialog", - "select Check for receipt and inspect recorded completion", - "assert both action requests are GET; no page errors or horizontal overflow" + "desktop keyboard Enter / phone row click to open PBS details", + "select 6h while its GET is held; select 1h, then fulfil the obsolete 6h request", + "withdraw third host link while service GET is held, then return 503", + "select 6h after failure and inspect the service-only response", + "select locked 14d; check all ten screenshots, retained History tab, zero horizontal overflow and no page errors" ], - "command": "pulse-worker-browser frontend-modern/browser-tests/update-action-receipt.cjs from assigned workspace root; Vite 6.4.3, Chromium 141.0.7390.37, Playwright 1.56.1", + "command": "pulse-worker-browser frontend-modern/browser-tests/pbs-history-source-isolation.cjs from assigned workspace root; Vite 6.4.3, Chromium 141.0.7390.37, Playwright 1.56.1", "artifacts": [ - "/var/lib/pulse-maintainer/worker-outputs/web-product-ahrmjhme/update-action-receipt-browser-final.log", - "/var/lib/pulse-maintainer/worker-outputs/web-product-ahrmjhme/update-action-receipt-proof-final/pending-1365.png", - "/var/lib/pulse-maintainer/worker-outputs/web-product-ahrmjhme/update-action-receipt-proof-final/review-1365.png", - "/var/lib/pulse-maintainer/worker-outputs/web-product-ahrmjhme/update-action-receipt-proof-final/completed-1365.png", - "/var/lib/pulse-maintainer/worker-outputs/web-product-ahrmjhme/update-action-receipt-proof-final/pending-390.png", - "/var/lib/pulse-maintainer/worker-outputs/web-product-ahrmjhme/update-action-receipt-proof-final/review-390.png", - "/var/lib/pulse-maintainer/worker-outputs/web-product-ahrmjhme/update-action-receipt-proof-final/completed-390.png" + "/var/lib/pulse-maintainer/worker-outputs/web-product-j4h_mbv3/receipts/history-browser-final.log", + "/var/lib/pulse-maintainer/worker-outputs/web-product-j4h_mbv3/history-browser-final/result.json", + "/var/lib/pulse-maintainer/worker-outputs/web-product-j4h_mbv3/history-browser-final/host-loaded-1365.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-j4h_mbv3/history-browser-final/host-loaded-390.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-j4h_mbv3/history-browser-final/range-loading-1365.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-j4h_mbv3/history-browser-final/range-loading-390.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-j4h_mbv3/history-browser-final/service-loaded-1365.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-j4h_mbv3/history-browser-final/service-loaded-390.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-j4h_mbv3/history-browser-final/target-failed-1365.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-j4h_mbv3/history-browser-final/target-failed-390.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-j4h_mbv3/history-browser-final/target-loading-1365.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-j4h_mbv3/history-browser-final/target-loading-390.png" ], - "notes": "Mock-backed UI proof; six screenshots inspected, including pending and completed phone states. This does not establish installed behaviour or resolve #1891.", + "notes": "Ten final screenshots inspected. The superseded GET is observed ERR_ABORTED at both widths; the unit regression also resolves mocks that ignore cancellation. Mock-backed proof establishes source/range isolation, not installed PBS collection, VirtualBox telemetry, all three #1723 drawers or release availability. Earlier unconfigured-CSS red/attempt-one results are retained separately.", "prior_verifications": [ + { + "version": 1, + "base_sha": "6ade01cbd4d02cd25b691f9ff5919206cceae764", + "verified_at": "2026-09-30T01:40:13Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx" + ], + "content_sha256": { + "frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx": "ed7207456ebbc990ada34b0c739e29565d90cfe1998415a781d87f3067d69c36" + }, + "routes": [ + "/browser-tests/pbs-identity-boundary.html (production PBS table, resource drawer and History with bounded synthetic resource props)" + ], + "viewports": [ + { + "width": 1365, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "three unlinked PBS services alongside six label/IP-colliding Agents", + "explicit backend links for two PVE guests and a standalone host, with same-Agent guest/host deduplication", + "unchanged links with host rows omitted and restored", + "withdrawn third link with real and colliding host rows still present", + "fresh exact PBS nodeName to Agent hostname match with mismatched display labels" + ], + "interactions": [ + "desktop keyboard disclosure and phone row activation for all three PBS drawers", + "inspect Metrics Target, reject wrong-host disks, select History and 6h/1h ranges", + "join, omit, restore and withdraw links while keeping the drawer open; verify only service History remains after withdrawal", + "24 target/layout observations; 76 History GETs, zero unrelated targets, page errors or horizontal overflow" + ], + "command": "pulse-worker-browser frontend-modern/browser-tests/pbs-identity-boundary.cjs; Vite 6.4.3, Playwright 1.56.1, Chromium 141.0.7390.37", + "artifacts": [ + "/var/lib/pulse-maintainer/worker-outputs/web-product-f2zda9of/pbs-browser-final.log", + "/var/lib/pulse-maintainer/worker-outputs/web-product-f2zda9of/pbs-identity-boundary.cjs", + "/var/lib/pulse-maintainer/worker-outputs/web-product-f2zda9of/pbs-browser-final-artifacts/result.json", + "/var/lib/pulse-maintainer/worker-outputs/web-product-f2zda9of/pbs-browser-final-artifacts/unlinked-1365.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-f2zda9of/pbs-browser-final-artifacts/linked-1365.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-f2zda9of/pbs-browser-final-artifacts/withdrawn-1365.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-f2zda9of/pbs-browser-final-artifacts/unlinked-390.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-f2zda9of/pbs-browser-final-artifacts/linked-390.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-f2zda9of/pbs-browser-final-artifacts/withdrawn-390.png" + ], + "notes": "All six screenshots inspected. Mock-backed UI proof, not installed topology, actual persisted host History or full #1723 relief. First browser failure retained separately; source snapshot isolation fixes that observed revocation failure." + }, + { + "version": 1, + "base_sha": "c3e8b6dd6dff36b5396ba2eb147d4e58d23001a6", + "verified_at": "2026-09-29T23:04:22Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx" + ], + "content_sha256": { + "frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx": "7172690587f8e642ee30492481abbe76ee79d277aad290c3b50310dd1c0b282b" + }, + "routes": [ + "/pbs-correlation-proof.html (production PBS backup-server table and resource drawer with scripted, source-scoped resources)" + ], + "viewports": [ + { + "width": 1365, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "three PBS services with distinct corroborated Agent links, including two PVE guests and one standalone host", + "host rows omitted in a transient refresh while the backend link is unchanged", + "third backend link withdrawn while the Agent row remains omitted" + ], + "interactions": [ + "open all three PBS drawers and inspect their Metrics Target identity", + "omit host rows and confirm targets stay on the correlated hosts", + "withdraw the third link and confirm its drawer returns to agent:pbs-three, with no page errors or horizontal overflow" + ], + "command": "pulse-worker-browser pbs-browser-proof.cjs from assigned workspace root; Vite 6.4.3, Playwright 1.56.1, Chromium desktop/phone", + "artifacts": [ + "/var/lib/pulse-maintainer/worker-outputs/web-product-sfkzldum/pbs-browser-proof-final.log", + "/var/lib/pulse-maintainer/worker-outputs/web-product-sfkzldum/pbs-browser-proof.cjs", + "/var/lib/pulse-maintainer/worker-outputs/web-product-sfkzldum/pbs-browser-proof-artifacts/pbs-1365-linked.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-sfkzldum/pbs-browser-proof-artifacts/pbs-1365-withdrawn.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-sfkzldum/pbs-browser-proof-artifacts/pbs-390-linked.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-sfkzldum/pbs-browser-proof-artifacts/pbs-390-withdrawn.png" + ], + "notes": "Mock-backed proof of the real PBS table and drawer; all four screenshots inspected. It does not establish installed #1723 relief or observed Agent History series." + }, + { + "version": 1, + "base_sha": "91d8822d76c5a7214995c0e370be637b6e6336e3", + "verified_at": "2026-09-29T22:13:56Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/shared/ContainerUpdateBadge.tsx", + "frontend-modern/src/components/shared/containerUpdateBadgeModel.ts", + "frontend-modern/src/components/shared/useContainerUpdateButtonState.ts", + "frontend-modern/src/stores/containerUpdates.ts" + ], + "content_sha256": { + "frontend-modern/src/components/shared/ContainerUpdateBadge.tsx": "fd88f3e3e275926572126f43cce7296e7361b5e527868dd3c59c7eb951e9f5c9", + "frontend-modern/src/components/shared/containerUpdateBadgeModel.ts": "10bc488cb82ae84c6ae46d77ec672d34df2a1b3c28e138ae0f3b4214cccbe37e", + "frontend-modern/src/components/shared/useContainerUpdateButtonState.ts": "4b9dd4a42549b0433a1afa56f26183fe721c80626f823fb3c8489934721267c5", + "frontend-modern/src/stores/containerUpdates.ts": "b5d47132f7daf7f14bcc38462043288fc211344410af622ada7728694bcd514d" + }, + "routes": [ + "/browser-tests/update-action-receipt.html (production Docker update button with scripted action reads)" + ], + "viewports": [ + { + "width": 1365, + "height": 900 + }, + { + "width": 390, + "height": 844 + } + ], + "states": [ + "governed update receipt pending after registry updateAvailable becomes false", + "existing action re-opened by GET with no second plan or execute", + "recorded completed action after a second GET changes the displayed outcome" + ], + "interactions": [ + "inspect pending button and absence of a Current or Completed claim", + "select Review action and inspect receipt-pending dialog", + "select Check for receipt and inspect recorded completion", + "assert both action requests are GET; no page errors or horizontal overflow" + ], + "command": "pulse-worker-browser frontend-modern/browser-tests/update-action-receipt.cjs from assigned workspace root; Vite 6.4.3, Chromium 141.0.7390.37, Playwright 1.56.1", + "artifacts": [ + "/var/lib/pulse-maintainer/worker-outputs/web-product-ahrmjhme/update-action-receipt-browser-final.log", + "/var/lib/pulse-maintainer/worker-outputs/web-product-ahrmjhme/update-action-receipt-proof-final/pending-1365.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-ahrmjhme/update-action-receipt-proof-final/review-1365.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-ahrmjhme/update-action-receipt-proof-final/completed-1365.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-ahrmjhme/update-action-receipt-proof-final/pending-390.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-ahrmjhme/update-action-receipt-proof-final/review-390.png", + "/var/lib/pulse-maintainer/worker-outputs/web-product-ahrmjhme/update-action-receipt-proof-final/completed-390.png" + ], + "notes": "Mock-backed UI proof; six screenshots inspected, including pending and completed phone states. This does not establish installed behaviour or resolve #1891." + }, { "version": 1, "base_sha": "fbf17b271abe70ad3e12042506b52f85b6cbc484", diff --git a/frontend-modern/public/docs/API.md b/frontend-modern/public/docs/API.md index c5a7677968..46d3d3cbec 100644 --- a/frontend-modern/public/docs/API.md +++ b/frontend-modern/public/docs/API.md @@ -9,18 +9,65 @@ Pulse provides a comprehensive REST API for automation and integration. Most API requests require authentication via one of the following methods: ### API Token (Recommended) -Pass the token in the `X-API-Token` header. + +Create an API token in Pulse's **API Access** settings with only the scopes +needed for your task. The read-only example below requires `monitoring:read`. +Pass the token in the `X-API-Token` header, using a private header file so the +secret does not appear in shell history or process arguments. + +On the machine running curl, prepare the file and open it in an editor: + ```bash -curl -H "X-API-Token: your-token" http://localhost:7655/api/health +umask 077 +mkdir -p "$HOME/.config/pulse" +touch "$HOME/.config/pulse/api-header" +chmod 600 "$HOME/.config/pulse/api-header" +vi "$HOME/.config/pulse/api-header" ``` -### Bearer Token +In the editor, save just this line, replacing `` with the API token: + +```text +X-API-Token: +``` + +Then make a read-only request (curl 7.76 or later): + ```bash -curl -H "Authorization: Bearer your-token" http://localhost:7655/api/health +curl --fail-with-body --header "@$HOME/.config/pulse/api-header" \ + http://127.0.0.1:7655/api/state/summary +``` + +This loopback URL applies only when curl runs on the Pulse host. For remote +access, use your Pulse HTTPS URL and keep certificate verification enabled; +do not use `--insecure`. On an authentication-enabled instance, the protected +summary checks token access; `/api/health` is public and does not verify authentication. +`--fail-with-body` returns a non-zero exit on HTTP errors, including 401 and +403, while retaining the error response. + +Do not paste tokens into command lines, URLs or issue reports. Keep the header +file private and outside shared repositories and diagnostics; do not use curl +verbose/trace output when sharing a result. Share only the relevant redacted +error, not the whole infrastructure response. Revoke tokens that are no longer +needed in Pulse's API Access settings. + +### Bearer Token + +The same API token can use a Bearer header instead. Replace the header file's +line in the editor with the following, then use the same curl command above; +do not send both authentication headers: + +```text +Authorization: Bearer ``` ### Session Cookie -Standard browser session cookie (used by the UI). + +For one-off read-only diagnostics, use your signed-in Pulse browser to open +the API path on that same instance. Do not extract or paste its session cookie +into a command or report; use the scoped token-file method above for automation. +State-changing session requests also require Pulse's CSRF protection, which +the UI handles. Session endpoints: - `POST /api/login` (sets `pulse_session` + `pulse_csrf`) diff --git a/frontend-modern/public/docs/PRIVACY.md b/frontend-modern/public/docs/PRIVACY.md index cdebc02e3f..5de0702446 100644 --- a/frontend-modern/public/docs/PRIVACY.md +++ b/frontend-modern/public/docs/PRIVACY.md @@ -158,7 +158,7 @@ Every field is listed below with the reason it exists. Nothing else is included | Update available | `true`/`false` | Report whether that last check offered a newer release, without sending which release | | Service health observed | `true`/`false` | Distinguish a release that performed the bounded local UI/API self-check from an older release with no signal | | Service health healthy | `true`/`false` | Report whether Pulse's locally bound listener served a healthy API response, UI document, and every referenced local frontend asset without sending an address, URL, response, or error text | -| Service health failure category | `listener`, `startup`, `runtime`, `api_connectivity`, `api_status`, `ui_status`, `frontend_assets`, or `unknown` | Classify a failed local self-check or startup path into one fixed category without sending the listener address, request URL, HTTP body, asset name, IP address, or raw error | +| Service health failure category | `listener`, `startup`, `runtime`, `api_connectivity`, `timeout`, `api_status`, `ui_status`, `frontend_assets`, or `unknown` | Classify a failed local self-check or startup path into one fixed category without sending the listener address, request URL, HTTP body, asset name, IP address, or raw error | | Service health cohort | `first_observation`, `same_version`, or `version_change` | Mark whether this direct observation is the first schema-v13 observation, another observation of the same release, or the first observed state after a release-version change | | Service health previous version | `6.4.0` | Retain only the normalized immediately previous Pulse release identity so aggregate reporting can compare a post-upgrade before/after cohort without treating rolling counters as new-release activity | | Service health previous observed | `true`/`false` | State whether the immediately previous release recorded the bounded local service-health signal | diff --git a/frontend-modern/public/docs/TROUBLESHOOTING.md b/frontend-modern/public/docs/TROUBLESHOOTING.md index c20b71a4d5..1a1102e057 100644 --- a/frontend-modern/public/docs/TROUBLESHOOTING.md +++ b/frontend-modern/public/docs/TROUBLESHOOTING.md @@ -58,9 +58,35 @@ screen for this instance and do not paste it into support requests or issue reports. ### Port change didn't take effect -1. Check which service is running: `systemctl status pulse` (legacy installs may use `pulse-backend`). -2. Verify environment override: `systemctl show pulse --property=Environment`. -3. Docker: Ensure you updated the `-p` flag (e.g., `-p 8080:7655`). + +The web UI and API listen on `FRONTEND_PORT` (default `7655`). The deprecated +`PORT` alias applies only when `FRONTEND_PORT` is unset; changing +`frontendPort` in `system.json` has no effect. `PULSE_AGENT_INGEST_PORT` is a +separate agent listener, not the web UI port. See +[Port configuration](CONFIGURATION.md#common-overrides-environment-variables). + +- **Systemd / Proxmox LXC**: identify the active service with + `systemctl is-active pulse` (legacy installs may use `pulse-backend`). Inspect + only the port setting in its managed configuration locally. If you change a + unit or drop-in, run `sudo systemctl daemon-reload`, then restart the affected + service during a suitable maintenance window. For LXC, run these checks inside + the Pulse container, not on the Proxmox host. +- **Docker / Compose**: distinguish the published host port from the listener + inside the container. With the default listener, `8080:7655` exposes the UI on + host port `8080`; changing the host port does not require `FRONTEND_PORT`. + In the repository's Compose file, `PULSE_PORT` controls this host-side mapping. + Save the mapping in your existing Compose project and apply it with + `docker compose up -d pulse`, keeping the same image and mounted data volume. + Restarting an existing container does not apply a new port mapping. For other + container managers, use their recreate/redeploy action while preserving the + data mount; do not delete the volume. +- **Reverse proxy**: check its upstream port and the firewall separately. A + working agent connection on a split-port deployment does not prove that the + web UI is reachable. + +Do not post full service environments, `docker inspect` output or resolved +`docker compose config` output: they can include passwords and tokens. Share +only the relevant port numbers and a redacted error if help is needed. ### "Connection Refused" - Check if Pulse is running. @@ -155,11 +181,19 @@ repair an older generated unit rather than adding a JSON-parsing wrapper. - See [Temperature Monitoring](TEMPERATURE_MONITORING.md). #### Docker hosts appearing/disappearing -- **Duplicate IDs**: Cloned VMs often share `/etc/machine-id`. -- **Fix**: Run `rm /etc/machine-id && systemd-machine-id-setup` on the clone. -- **Identity note**: The displayed IP is not the durable identity. Pulse uses - the machine ID or an explicit agent ID, so two clones with the same value can - collapse into one record even when their hostnames or IP addresses differ. + +Cloned hosts can share a **saved Pulse agent ID**, not just an OS machine ID. +The agent uses an explicit ID first, then its saved `agent-id` file, and derives +one from the machine only when neither is available. Changing a hostname, IP or +`/etc/machine-id` therefore does not necessarily change its Pulse identity. + +Compare the affected hosts in **Agent Doctor** and inspect only their configured +`agent-id` files locally. Do not delete the OS machine ID, agent state or Pulse +history as a troubleshooting step. If a duplicate is confirmed, give only the +clone a stable, unique ID in its managed service or container configuration; +leave the original host unchanged. Follow +[Clone identity recovery](UNIFIED_AGENT.md#duplicate-agents) for the configuration +precedence, systemd example and checks after restart. ### Notifications @@ -309,25 +343,50 @@ check, report the Pulse version and displayed check time/status separately; API success alone does not confirm the Pulse display has recovered. ### Recovery Mode -If you are completely locked out, you can trigger a recovery token from localhost: -```bash -curl -X POST http://localhost:7655/api/security/recovery \ - -d '{"action":"generate_token","duration":30}' -``` -Use the returned token in `X-Recovery-Token` when calling `/api/security/recovery` to enable or disable local-only auth bypass (`disable_auth` / `enable_auth`). Token generation is localhost-only. -Example (enable recovery mode): -```bash -curl -X POST http://localhost:7655/api/security/recovery \ - -H "X-Recovery-Token: " \ - -d '{"action":"disable_auth"}' -``` +For a forgotten local password, follow [I forgot my password](#i-forgot-my-password) +above, using the steps for your deployment. Enter the host-only bootstrap token +in that instance's setup screen; do not paste it into a command or a report. +For OIDC, SAML or proxy login, use the identity-provider or administrator path +described there instead. + +The advanced `/api/security/recovery` API creates a **browser-bound recovery +session**, not a server-wide authentication bypass. Despite its legacy name, +`disable_auth` does not disable authentication for other clients. Recovery +sessions work only over direct loopback requests; remote and reverse-proxy +requests cannot use them. A successful curl response does not unlock a separate +browser: the session cookie belongs to the client that made the request. +`enable_auth` clears that recovery session; it does not reset a password. + +Do not transfer recovery cookies between clients or paste recovery tokens into +command arguments, URLs, screenshots or GitHub threads. The password-reset steps +above avoid that credential-handling detour. --- ## ๐Ÿ†˜ Getting Help If you're still stuck: -1. **Check Logs**: `journalctl -u pulse -n 100` or `docker logs --tail 100 pulse`. -2. **Check Version**: `curl http://localhost:7655/api/version`. -3. **Open Issue**: Report on [GitHub Issues](https://github.com/rcourtman/Pulse/issues) with your logs and version info. + +1. **Keep the original evidence**: note what you did, when it happened and the + exact error. Do not repeat an update, outage or notification storm merely to + reproduce it. A failed update banner does not prove the action left the + target unchanged; check its current state before another attempt. +2. **Identify the affected version**: give the running Pulse and relevant agent + versions, not just the version before an upgrade. For Docker, include the + running image tag or digest. If installation never started Pulse, give the + attempted release and public installer/helper source, or say "unknown". +3. **Choose relevant, safe evidence**: if Pulse is running and collection is + safe, use **Settings โ†’ Diagnostics โ†’ Export for GitHub (sanitized)** for + connection or data failures. For a visual problem, a screenshot or the exact + error may be enough. If logs are needed, inspect a bounded local excerpt + (`journalctl -u pulse -n 100 --no-pager` or `docker logs --tail 100 pulse`), + not a full configuration or data-directory upload. +4. **Review before posting**: even a sanitized export or screenshot can contain + identifying details. Remove credentials, session cookies, webhook URLs and + private host, network or personal information. Never post bootstrap/recovery + tokens, `.env` files, private keys or an unsanitized export. +5. **Use the appropriate thread**: [GitHub Issues](https://github.com/rcourtman/Pulse/issues) + for a bug, or [Discussions](https://github.com/rcourtman/Pulse/discussions) for + a setup question. Add new evidence to an existing matching report rather + than opening a duplicate. Do not refile information you have already supplied. diff --git a/frontend-modern/public/docs/UNIFIED_AGENT.md b/frontend-modern/public/docs/UNIFIED_AGENT.md index 7207a29560..ff7435ca9d 100644 --- a/frontend-modern/public/docs/UNIFIED_AGENT.md +++ b/frontend-modern/public/docs/UNIFIED_AGENT.md @@ -412,7 +412,7 @@ sudo chmod 0700 /usr/local/libexec/pulse-queue-depth | `--insecure` | `PULSE_INSECURE_SKIP_VERIFY` | Skip TLS verification | `false` | | `--allow-plaintext-http` | `PULSE_AGENT_ALLOW_PLAINTEXT_HTTP` | Allow plain HTTP to a Pulse server that does not look local (private IP, single-label, `.local`/`.lan`/`.home`/`.home.arpa`/`.internal`, or resolves to private addresses). Sends the API token in cleartext; only for networks you fully control, e.g. internal networks numbered from public IP space | `false` | | `--hostname` | `PULSE_HOSTNAME` | Override hostname | *(OS hostname)* | -| `--agent-id` | `PULSE_AGENT_ID` | Unique agent identifier | *(machine-id)* | +| `--agent-id` | `PULSE_AGENT_ID` | Unique agent identifier | *(saved ID, then machine ID)* | | `--report-ip` | `PULSE_REPORT_IP` | Override reported IP (multi-NIC) | *(auto)* | | `--disable-ceph` | `PULSE_DISABLE_CEPH` | Disable local Ceph status polling | `false` | | `--tag` | `PULSE_TAGS` | Apply tags (repeatable or CSV) | *(none)* | @@ -834,20 +834,53 @@ file and re-run the installer to pick up the rotating configuration. identity evidence. The endpoint reports repair handoffs but does not run them. ### Duplicate Agents -If cloned VMs appear as the same agent: -```bash -sudo rm /etc/machine-id && sudo systemd-machine-id-setup -``` -Or set a unique agent ID: -```bash ---agent-id my-unique-agent-id -``` +Cloning an installed agent can copy its saved identity and credentials. Different +hostnames, MAC addresses or IPs do not make those agents distinct in Pulse. +Identity selection is: -The displayed or reported IP is not the durable agent identity. Pulse normally -uses the machine ID (or an explicit `--agent-id`), so cloned systems must have -unique machine and agent IDs even when their hostnames, MAC addresses, and IPs -differ. +1. An explicit `--agent-id` argument (which overrides `PULSE_AGENT_ID`). +2. `PULSE_AGENT_ID` when no explicit argument is supplied. +3. The saved `agent-id` file, when no override is set. +4. The machine-derived ID, with hostname as a fallback, when no saved ID exists. + +First compare the affected hosts in **Agent Doctor**. Inspect the identity file +locally: the default Linux path is `/var/lib/pulse-agent/agent-id`; `--state-dir` +changes that directory and `--agent-id-file` or `PULSE_AGENT_ID_FILE` can select +another file. Do not upload `connection.env`, token files or whole service and +environment dumps; older installations can contain credentials there. + +**Do not delete `/etc/machine-id`, agent state or Pulse history to repair a +duplicate.** A saved agent ID survives an OS machine-ID change, and changing the +OS identity affects more than Pulse. Prepare OS identities when provisioning a +clone, using the distribution's instructions, not during live Pulse diagnosis. + +For a confirmed duplicate, keep the original host unchanged and choose one +stable, unique override for the clone. Persist it in the configuration that +actually starts that agent, rather than launching a second agent process: + +- **Systemd, without an existing `--agent-id` argument**: use + `sudo systemctl edit pulse-agent.service` and add the following drop-in, + replacing the example ID with the clone's unique ID: + + ```ini + [Service] + Environment="PULSE_AGENT_ID=vm-clone-02" + ``` + + Then restart only that agent with `sudo systemctl restart pulse-agent.service`. + If its managed launch command already supplies `--agent-id`, change that + existing override instead: the argument wins over the environment variable. +- **Docker Compose**: set `PULSE_AGENT_ID` for the clone's agent service, keeping + its existing volumes, collector options and token-file configuration. An + existing `--agent-id` in `command` takes precedence; update it instead if set. + +Keep the override across restarts and upgrades. After restart, verify that both +hosts report fresh, distinct IDs in Agent Doctor and that each row's metrics +belong to the correct host. Changing an ID does not split or recover historical +data already combined under the old ID. If authentication or reporting fails, +keep the bounded, redacted error for diagnosis; do not delete tokens, re-enrol +blindly or remove the original host's record. ### Permission Denied (Docker) Ensure the agent can access the Docker socket: diff --git a/frontend-modern/src/components/Workloads/GuestDrawerHistory.tsx b/frontend-modern/src/components/Workloads/GuestDrawerHistory.tsx index 42de4f2f20..369646de81 100644 --- a/frontend-modern/src/components/Workloads/GuestDrawerHistory.tsx +++ b/frontend-modern/src/components/Workloads/GuestDrawerHistory.tsx @@ -427,17 +427,21 @@ export const GuestDrawerHistory: Component = (props) => range: props.range, }; }, - fetcher: async (key) => + fetcher: async (key, signal) => normalizeHistoryResponse( await ChartsAPI.getMetricsHistory({ resourceType: key.resourceType, resourceId: key.resourceId, range: key.range, maxPoints: GUEST_DRAWER_HISTORY_MAX_POINTS, + signal, }), ), initialValue: EMPTY_HISTORY_RESPONSE, cacheKey: (key) => `guest-drawer-history:${key.resourceType}:${key.resourceId}:${key.range}`, + // Former-host or former-range observations are not evidence for this + // target. Matching cached reads and same-source polling remain retained. + retainPreviousValueOnSourceChange: false, pollMs: GUEST_DRAWER_HISTORY_POLL_MS, }); diff --git a/frontend-modern/src/components/Workloads/__tests__/GuestDrawerHistory.source-isolation.test.tsx b/frontend-modern/src/components/Workloads/__tests__/GuestDrawerHistory.source-isolation.test.tsx new file mode 100644 index 0000000000..e5c16ba3d6 --- /dev/null +++ b/frontend-modern/src/components/Workloads/__tests__/GuestDrawerHistory.source-isolation.test.tsx @@ -0,0 +1,237 @@ +import { cleanup, render, screen, waitFor } from '@solidjs/testing-library'; +import { createSignal } from 'solid-js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { ChartsAPI, type AllMetricsHistoryResponse, type HistoryTimeRange } from '@/api/charts'; +import { + createNonSuspendingQuery, + resetCreateNonSuspendingQueryCacheForTest, +} from '@/hooks/createNonSuspendingQuery'; +import { GuestDrawerHistory } from '../GuestDrawerHistory'; +import type { GuestDrawerHistoryTarget } from '../guestDrawerModel'; + +vi.mock('@/stores/license', () => ({ + loadRuntimeCapabilities: vi.fn(async () => undefined), + maxHistoryDays: () => 7, + isRangeLocked: (range: HistoryTimeRange) => ['14d', '30d', '90d'].includes(range), +})); + +afterEach(() => { + cleanup(); + resetCreateNonSuspendingQueryCacheForTest(); + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +const response = ( + resourceId: string, + range = '24h', + value = 88, + resourceType = 'agent', +): AllMetricsHistoryResponse => ({ + resourceType, + resourceId, + range, + start: 1_700_000_000_000, + end: 1_700_000_060_000, + source: 'store', + metrics: { + cpu: [ + { timestamp: 1_700_000_000_000, value: value - 1, min: value - 1, max: value - 1 }, + { timestamp: 1_700_000_060_000, value, min: value, max: value }, + ], + }, +}); + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + return { promise, resolve, reject }; +} + +const paths = (container: HTMLElement) => + container.querySelectorAll('[data-testid="guest-history-plot"] path'); + +describe('GuestDrawerHistory source isolation', () => { + it.each(['resource', 'resource type', 'range'] as const)( + 'clears old points while an uncached %s replacement is loading', + async (change) => { + const pending = deferred(); + const fetch = vi + .spyOn(ChartsAPI, 'getMetricsHistory') + .mockResolvedValueOnce(response('host-old')) + .mockReturnValueOnce(pending.promise); + const [target, setTarget] = createSignal({ + resourceType: 'agent', + resourceId: 'host-old', + }); + const [range, setRange] = createSignal('24h'); + const { container } = render(() => ); + await waitFor(() => expect(paths(container)).toHaveLength(1)); + + if (change === 'resource') setTarget({ resourceType: 'agent', resourceId: 'pbs-service' }); + if (change === 'resource type') setTarget({ resourceType: 'vm', resourceId: 'host-old' }); + if (change === 'range') setRange('6h'); + await waitFor(() => expect(fetch).toHaveBeenCalledTimes(2)); + + expect(paths(container)).toHaveLength(0); + expect(screen.getAllByText('Loading history').length).toBeGreaterThan(0); + pending.resolve(response(target().resourceId, range(), 12, target().resourceType)); + await waitFor(() => expect(paths(container)).toHaveLength(1)); + expect(screen.queryByText('Loading history')).not.toBeInTheDocument(); + }, + ); + + it('does not cache former-host points under a failed replacement target', async () => { + const pending = deferred(); + const fetch = vi + .spyOn(ChartsAPI, 'getMetricsHistory') + .mockResolvedValueOnce(response('host-old')) + .mockReturnValueOnce(pending.promise) + .mockImplementation(() => new Promise(() => {})); + const [target, setTarget] = createSignal({ + resourceType: 'agent', + resourceId: 'host-old', + }); + const first = render(() => ); + await waitFor(() => expect(paths(first.container)).toHaveLength(1)); + setTarget({ resourceType: 'agent', resourceId: 'pbs-service' }); + await waitFor(() => expect(fetch).toHaveBeenCalledTimes(2)); + pending.reject(new Error('Replacement unavailable')); + await screen.findByText('Failed to load history data'); + first.unmount(); + + // Read the same public query/cache API as a remount. A cached failure may + // be retained, but it must not contain former-host observations. + const CacheProbe = () => { + const query = createNonSuspendingQuery({ + source: () => 'pbs-service', + cacheKey: (id) => `guest-drawer-history:agent:${id}:24h`, + fetcher: () => new Promise(() => {}), + initialValue: { ...response(''), metrics: {} }, + }); + return {query.value().resourceId}; + }; + render(() => ); + expect(screen.getByTestId('cached-resource')).toBeEmptyDOMElement(); + }); + + it('aborts a superseded read and ignores late success and failure from earlier targets', async () => { + const old = deferred(); + const middle = deferred(); + const fetch = vi + .spyOn(ChartsAPI, 'getMetricsHistory') + .mockReturnValueOnce(old.promise) + .mockReturnValueOnce(middle.promise) + .mockResolvedValueOnce(response('current-host', '24h', 12)); + const [target, setTarget] = createSignal({ + resourceType: 'agent', + resourceId: 'old-host', + }); + const { container } = render(() => ); + await waitFor(() => expect(fetch).toHaveBeenCalledTimes(1)); + const firstSignal = fetch.mock.calls[0][0].signal; + expect(firstSignal).toBeInstanceOf(AbortSignal); + setTarget({ resourceType: 'agent', resourceId: 'middle-host' }); + await waitFor(() => expect(fetch).toHaveBeenCalledTimes(2)); + expect(firstSignal!.aborted).toBe(true); + setTarget({ resourceType: 'agent', resourceId: 'current-host' }); + await waitFor(() => expect(paths(container)).toHaveLength(1)); + const currentPath = paths(container)[0].getAttribute('d'); + old.resolve(response('old-host')); + middle.reject(new Error('Late failure')); + await old.promise; + await middle.promise.catch(() => undefined); + await Promise.resolve(); + + expect(paths(container)[0]).toHaveAttribute('d', currentPath); + expect(screen.queryByText('Failed to load history data')).not.toBeInTheDocument(); + expect(fetch.mock.calls[1][0].signal!.aborted).toBe(true); + }); + + it('keeps only matching cached history when revisiting a resource and range', async () => { + const fetch = vi + .spyOn(ChartsAPI, 'getMetricsHistory') + .mockResolvedValueOnce(response('host-old')) + .mockResolvedValueOnce(response('host-new', '24h', 12)) + .mockImplementation(() => new Promise(() => {})); + const [target, setTarget] = createSignal({ + resourceType: 'agent', + resourceId: 'host-old', + }); + const { container } = render(() => ); + await waitFor(() => expect(paths(container)).toHaveLength(1)); + const oldPath = paths(container)[0].getAttribute('d'); + setTarget({ resourceType: 'agent', resourceId: 'host-new' }); + await waitFor(() => expect(paths(container)[0]).not.toHaveAttribute('d', oldPath)); + setTarget({ resourceType: 'agent', resourceId: 'host-old' }); + await waitFor(() => expect(fetch).toHaveBeenCalledTimes(3)); + + expect(paths(container)[0]).toHaveAttribute('d', oldPath); + expect(screen.queryByText('Loading history')).not.toBeInTheDocument(); + }); + + it('retains same-source points during background polling without a loading flash', async () => { + vi.useFakeTimers(); + const pending = deferred(); + const fetch = vi + .spyOn(ChartsAPI, 'getMetricsHistory') + .mockResolvedValueOnce(response('host-old')) + .mockReturnValueOnce(pending.promise); + const target: GuestDrawerHistoryTarget = { resourceType: 'agent', resourceId: 'host-old' }; + const { container } = render(() => ); + await vi.advanceTimersByTimeAsync(0); + expect(paths(container)).toHaveLength(1); + const originalPath = paths(container)[0].getAttribute('d'); + await vi.advanceTimersByTimeAsync(30_000); + expect(fetch).toHaveBeenCalledTimes(2); + expect(paths(container)[0]).toHaveAttribute('d', originalPath); + expect(screen.queryByText('Loading history')).not.toBeInTheDocument(); + pending.resolve(response('host-old', '24h', 12)); + await vi.advanceTimersByTimeAsync(0); + expect(paths(container)[0]).not.toHaveAttribute('d', originalPath); + }); + + it('aborts and rejects late points when the history target becomes unavailable', async () => { + const pending = deferred(); + const fetch = vi.spyOn(ChartsAPI, 'getMetricsHistory').mockReturnValue(pending.promise); + const [target, setTarget] = createSignal({ + resourceType: 'agent', + resourceId: 'host-old', + }); + const { container } = render(() => ); + await waitFor(() => expect(fetch).toHaveBeenCalledTimes(1)); + setTarget(null); + await screen.findByText('History unavailable'); + expect(fetch.mock.calls[0][0].signal!.aborted).toBe(true); + pending.resolve(response('host-old')); + await pending.promise; + await Promise.resolve(); + expect(paths(container)).toHaveLength(0); + expect(screen.getByText('History unavailable')).toBeInTheDocument(); + }); + + it('aborts a pending read when its range is locked or its drawer unmounts', async () => { + const fetch = vi + .spyOn(ChartsAPI, 'getMetricsHistory') + .mockImplementation(() => new Promise(() => {})); + const [range, setRange] = createSignal('24h'); + const target: GuestDrawerHistoryTarget = { resourceType: 'agent', resourceId: 'host-old' }; + const view = render(() => ); + await waitFor(() => expect(fetch).toHaveBeenCalledTimes(1)); + setRange('14d'); + await screen.findByText(/14 days history requires a higher license plan/); + expect(fetch.mock.calls[0][0].signal).toBeInstanceOf(AbortSignal); + expect(fetch.mock.calls[0][0].signal!.aborted).toBe(true); + expect(fetch).toHaveBeenCalledTimes(1); + setRange('24h'); + await waitFor(() => expect(fetch).toHaveBeenCalledTimes(2)); + const active = fetch.mock.calls[1][0].signal; + expect(active!.aborted).toBe(false); + view.unmount(); + expect(active!.aborted).toBe(true); + }); +}); diff --git a/frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx b/frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx index 6a29b01ea5..313270eca0 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx @@ -1,4 +1,5 @@ import { For, Show, createMemo, type Accessor, type JSX } from 'solid-js'; +import { unwrap } from 'solid-js/store'; import { StatusDot } from '@/components/shared/StatusDot'; import { TableCell, TableHead, TableRow } from '@/components/shared/Table'; @@ -23,7 +24,7 @@ import { } from '@/features/platformPage/PlatformResourceDetailTableRow'; import type { PBSBackup } from '@/types/api'; import type { Resource, ResourcePBSDatastore } from '@/types/resource'; -import { getNormalizedIdentityLookupVariants } from '@/utils/resourceIdentity'; +import { getPlatformAgentRecord } from '@/utils/agentResources'; import type { StatusIndicatorVariant } from '@/utils/status'; import { useObservedElementWidth } from '@/hooks/useObservedElementWidth'; @@ -103,27 +104,19 @@ function usageToneClass(pct: number | undefined): string { return 'text-base-content'; } -const identityValues = (resource: Resource): Array => [ - resource.canonicalIdentity?.hostname, - resource.canonicalIdentity?.platformId, - resource.identity?.hostname, - ...(resource.identity?.ips ?? []), - resource.agent?.hostname, - resource.pbs?.hostname, - // The node hostname PBS reports about itself is machine identity. The - // connection may be configured by an IP or DNS alias the agent never - // reports, so without this the host correlation can drop out between - // snapshots and the drawer falls back to the service history target - // (#1723). - resource.pbs?.nodeName, - resource.pbs?.instanceId, - resource.platformId, - resource.name, - resource.displayName, -]; +// Only the hostname PBS reports about itself can corroborate a host without +// a backend link. Connection labels, instance IDs, canonical presentation +// fields and configured endpoints are not machine identity. In particular, +// the shared dotted-token helper would make unrelated IPs/FQDNs collide. +const machineHostname = (value: unknown): string => + typeof value === 'string' ? value.trim().toLowerCase().replace(/\.$/, '') : ''; -const identityTokens = (resource: Resource): Set => - new Set(identityValues(resource).flatMap((value) => getNormalizedIdentityLookupVariants(value))); +const correlationEvidenceKey = (server: Resource): string | undefined => { + const linkedAgentId = server.pbs?.linkedAgentId?.trim(); + if (linkedAgentId) return `agent:${linkedAgentId}`; + const nodeName = machineHostname(server.pbs?.nodeName); + return nodeName ? `node:${nodeName}` : undefined; +}; const stringValues = (...candidates: unknown[]): string[] => candidates.flatMap((candidate) => @@ -156,14 +149,32 @@ const isHostWithAgent = (candidate: Resource): boolean => (candidate.type === 'agent' || isGuestWithAgent(candidate)) && Boolean(candidate.agent ?? candidate.platformData?.agent); -// Host telemetry can be merged into a PVE guest rather than a standalone -// agent. Keep the unique-identity check and require an actual agent facet. -const isCorrelationCandidate = (serverTokens: Set, candidate: Resource): boolean => { - if (!isHostWithAgent(candidate)) return false; - for (const token of identityTokens(candidate)) { - if (serverTokens.has(token)) return true; +const matchingAgentHosts = (server: Resource, candidates: readonly Resource[]): Resource[] => { + // Endpoint/interface and provider guest links belong to the backend. An + // explicit link also vetoes same-name Agents with a different identity. + const linkedAgentId = server.pbs?.linkedAgentId?.trim(); + if (linkedAgentId) { + return candidates.filter( + (candidate) => isHostWithAgent(candidate) && correlatedAgentKey(candidate) === linkedAgentId, + ); } - return false; + const nodeName = machineHostname(server.pbs?.nodeName); + if (!nodeName) return []; + return candidates.filter((candidate) => { + if (!isHostWithAgent(candidate) || !correlatedAgentKey(candidate)) return false; + const agent = candidate.agent ?? getPlatformAgentRecord(candidate); + // Do not let stale host observations undo a withdrawn backend link. + const reportSeen = + typeof agent?.lastReportAt === 'string' ? Date.parse(agent.lastReportAt) : candidate.lastSeen; + const delta = Math.abs(server.lastSeen - reportSeen); + return ( + agent?.stale !== true && + server.lastSeen > 0 && + reportSeen > 0 && + delta <= PBS_CORRELATION_RETENTION_MAX_STALENESS_MS && + machineHostname(agent?.hostname) === nodeName + ); + }); }; const preferredCorrelatedHost = (matches: readonly Resource[]): Resource | undefined => @@ -175,22 +186,7 @@ const uniquelyCorrelatedAgent = ( server: Resource, candidates: readonly Resource[], ): Resource | undefined => { - // The backend can corroborate a PBS connection with a unique agent from - // its reported interface IP even when the PBS token cannot read nodeName - // and the configured connection name differs from the PVE/agent hostname. - // An explicit link also vetoes same-name agents with a different ID. - const linkedAgentId = server.pbs?.linkedAgentId?.trim(); - if (linkedAgentId) { - return preferredCorrelatedHost( - candidates.filter( - (candidate) => - isHostWithAgent(candidate) && correlatedAgentKey(candidate) === linkedAgentId, - ), - ); - } - const serverTokens = identityTokens(server); - if (serverTokens.size === 0) return undefined; - const matches = candidates.filter((candidate) => isCorrelationCandidate(serverTokens, candidate)); + const matches = matchingAgentHosts(server, candidates); if (matches.length === 0) return undefined; if (matches.length === 1) return matches[0]; @@ -221,11 +217,8 @@ const uniquelyCorrelatedAgent = ( // distinction matters for correlation retention: a snapshot that simply omits // the host row is a transient refresh gap, while an ambiguous snapshot is a // deliberate decline that must not be papered over with a remembered guess. -const hasCorrelationCandidate = (server: Resource, candidates: readonly Resource[]): boolean => { - const serverTokens = identityTokens(server); - if (serverTokens.size === 0) return false; - return candidates.some((candidate) => isCorrelationCandidate(serverTokens, candidate)); -}; +const hasCorrelationCandidate = (server: Resource, candidates: readonly Resource[]): boolean => + matchingAgentHosts(server, candidates).length > 0; // A live refresh can briefly omit the correlated host row (for example while a // realtime snapshot replaces the merged estate), which used to flip the Backups @@ -234,7 +227,12 @@ const hasCorrelationCandidate = (server: Resource, candidates: readonly Resource // only across such an omission, and only while the remembered host is still // plausibly current, so a genuinely removed or replaced host is not advertised // indefinitely. A host that is present but ambiguous still declines. -export type PbsCorrelationRetention = Map; +interface RetainedPbsCorrelation { + agent: Resource; + evidenceKey?: string; +} + +export type PbsCorrelationRetention = Map; export const createPbsCorrelationRetention = (): PbsCorrelationRetention => new Map(); @@ -299,24 +297,29 @@ export function buildBackupServerRows( .map((server) => { const agent = uniquelyCorrelatedAgent(server, servers); if (agent) { - retention?.set(server.id, agent); + retention?.set(server.id, { + agent, + evidenceKey: correlationEvidenceKey(server), + }); return mergePBSAgentPresentation(server, agent); } const retained = retention?.get(server.id); if (retained) { - // A changed backend link is evidence of host replacement; an old - // correlation must not survive merely because its row is absent. - const linkedAgentId = server.pbs?.linkedAgentId?.trim(); - if (linkedAgentId && correlatedAgentKey(retained) !== linkedAgentId) { + // A changed or withdrawn backend link is evidence that the old host + // correlation is no longer trusted. A transiently missing row is not + // enough to keep advertising that host's History after revocation. + if (correlationEvidenceKey(server) !== retained.evidenceKey) { retention?.delete(server.id); return server; } const fresh = - server.lastSeen - retained.lastSeen <= PBS_CORRELATION_RETENTION_MAX_STALENESS_MS; + server.lastSeen - retained.agent.lastSeen <= PBS_CORRELATION_RETENTION_MAX_STALENESS_MS; if (fresh && !hasCorrelationCandidate(server, servers)) { - return mergePBSAgentPresentation(server, retained); + return mergePBSAgentPresentation(server, retained.agent); } - if (!fresh) retention?.delete(server.id); + // An ambiguous match revokes the remembered choice too; a later + // omission must not resurrect an identity we already declined. + retention?.delete(server.id); } return server; }) @@ -334,7 +337,6 @@ export function buildBackupServerRows( .sort((left, right) => left.name.localeCompare(right.name)); const memoryTotal = server.memory?.total ?? 0; const host = { - resource: server, serverName: server.name, online: serverIsOnline(server), connectionLabel: connectionLabel(server), @@ -350,14 +352,20 @@ export function buildBackupServerRows( memoryTotal: memoryTotal > 0 ? memoryTotal : undefined, uptimeSeconds: server.uptime ?? server.pbs?.uptimeSeconds, }; + // Each keyed row has a reconciled Solid store. Give it its own JSON DTO + // snapshot: otherwise joining an Agent mutates the source's service + // metricsTarget (or another datastore row), defeating later revocation. + // Unwrap reactive source DTOs before cloning; never clone their proxies. + const resourceSnapshot = () => structuredClone(unwrap(server)); if (datastores.length === 0) { - rows.push({ key: server.id, ...host, backupCount: 0 }); + rows.push({ key: server.id, ...host, resource: resourceSnapshot(), backupCount: 0 }); continue; } for (const datastore of datastores) { rows.push({ key: `${server.id}:${datastore.name}`, ...host, + resource: resourceSnapshot(), // Row stores reconcile independently from the nested PBS snapshot. Do // not alias its datastore object: a reordered snapshot reconciles that // array by position and would overwrite this row's datastore identity. diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupServersTable.drawer.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupServersTable.drawer.test.tsx index 52a65298c5..05854d4787 100644 --- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupServersTable.drawer.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupServersTable.drawer.test.tsx @@ -46,6 +46,7 @@ const makePbsResource = (): Resource => memory: { current: 40, total: 8_000, used: 3_200, free: 4_800 }, pbs: { instanceId: 'pbs-main', + nodeName: 'pbs-main.local', version: '3.2.1', connectionHealth: 'healthy', datastores: [{ name: 'tank', total: 1_000, used: 400, available: 600, usagePercent: 40 }], @@ -187,6 +188,70 @@ describe('ProxmoxBackupServersTable details', () => { ); }); + it('updates an open drawer to the service target when the link is withdrawn beside a colliding guest', async () => { + const pbs = makePbsResource(); + pbs.agent = undefined; + pbs.platformData = { sources: ['pbs'] }; + pbs.pbs = { ...pbs.pbs!, nodeName: undefined, linkedAgentId: 'agent-pbs-1' }; + pbs.metricsTarget = { resourceType: 'agent', resourceId: 'pbs-main' }; + const host = { + ...pbs, + id: 'agent-pbs-1', + type: 'agent', + pbs: undefined, + agent: { agentId: 'agent-pbs-1', hostname: 'real-host' }, + metricsTarget: { resourceType: 'agent', resourceId: 'agent-pbs-1' }, + } as Resource; + const unrelated = { + ...host, + id: 'vm-other', + type: 'vm', + name: pbs.name, + agent: { agentId: 'agent-other', hostname: pbs.name }, + metricsTarget: { resourceType: 'vm', resourceId: 'vm-other' }, + } as Resource; + const [servers, setServers] = createSignal([pbs, host, unrelated]); + render(() => ); + fireEvent.click(screen.getByRole('button', { name: 'Expand details for pbs-main' })); + const detail = screen.getByTestId('pbs-resource-detail'); + expect(detail).toHaveAttribute('data-metrics-resource-id', 'agent-pbs-1'); + + setServers([{ ...pbs, pbs: { ...pbs.pbs!, linkedAgentId: undefined } }, host, unrelated]); + await waitFor(() => expect(detail).toHaveAttribute('data-metrics-resource-id', 'pbs-main')); + expect(detail).not.toHaveAttribute('data-agent-id'); + expect(resourceDetailDrawerMount).toHaveBeenCalledTimes(1); + }); + + it('does not mutate reusable service/Agent snapshots while joining and withdrawing host History', async () => { + const pbs = makePbsResource(); + pbs.pbs = { ...pbs.pbs!, nodeName: undefined }; + pbs.platformData = { sources: ['pbs'] }; + pbs.metricsTarget = { resourceType: 'agent', resourceId: 'pbs-main' }; + const host = { + ...pbs, + id: 'agent-pbs-1', + type: 'agent', + pbs: undefined, + agent: { agentId: 'agent-pbs-1', hostname: 'real-host' }, + metricsTarget: { resourceType: 'agent', resourceId: 'agent-pbs-1' }, + } as Resource; + const original = JSON.stringify([pbs, host]); + const [servers, setServers] = createSignal([pbs, host]); + render(() => ); + fireEvent.click(screen.getByRole('button', { name: 'Expand details for pbs-main' })); + const detail = screen.getByTestId('pbs-resource-detail'); + expect(detail).toHaveAttribute('data-metrics-resource-id', 'pbs-main'); + + setServers([{ ...pbs, pbs: { ...pbs.pbs!, linkedAgentId: 'agent-pbs-1' } }, host]); + await waitFor(() => expect(detail).toHaveAttribute('data-metrics-resource-id', 'agent-pbs-1')); + expect(JSON.stringify([pbs, host])).toBe(original); + setServers([pbs, host]); + await waitFor(() => expect(detail).toHaveAttribute('data-metrics-resource-id', 'pbs-main')); + expect(detail).not.toHaveAttribute('data-agent-id'); + expect(JSON.stringify([pbs, host])).toBe(original); + expect(resourceDetailDrawerMount).toHaveBeenCalledTimes(1); + }); + it.each(['agent', 'vm'] as const)( 'does not guess when an agent and %s share the PBS hostname', (type) => { @@ -283,11 +348,22 @@ describe('ProxmoxBackupServersTable details', () => { pbs.name = 'proxback'; pbs.displayName = 'proxback'; pbs.platformId = 'pbs-1'; - pbs.pbs = { ...pbs.pbs!, instanceId: 'proxback', hostname: 'proxback-vm' }; + // The reported node, not the configured endpoint, corroborates this host. + pbs.pbs = { + ...pbs.pbs!, + instanceId: 'proxback', + hostname: 'proxback-vm', + nodeName: 'proxback-vm', + }; pbs.metricsTarget = { resourceType: 'agent', resourceId: 'pbs-1' }; pbs.platformData = { sources: ['pbs'], - pbs: { instanceId: 'proxback', hostname: 'proxback-vm', datastoreCount: 1 }, + pbs: { + instanceId: 'proxback', + hostname: 'proxback-vm', + nodeName: 'proxback-vm', + datastoreCount: 1, + }, }; const sharedAgent = { agentId: 'agent-proxback', hostname: 'proxback-vm' }; const guest = { @@ -347,6 +423,7 @@ describe('ProxmoxBackupServersTable details', () => { pbs.pbs = { instanceId: 'proxback', hostname: 'proxback-vm', + linkedAgentId: 'agent-proxback', version: '3.2.1', connectionHealth: 'healthy', datastores: [{ name: 'tank', total: 1000, used: 400, available: 600, usagePercent: 40 }], diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupServersTable.test.ts b/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupServersTable.test.ts index 46d366a2e3..6dfa3b16c3 100644 --- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupServersTable.test.ts +++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupServersTable.test.ts @@ -121,6 +121,7 @@ const makeCorrelatablePbs = (lastSeen = 1_700_000_000_000): Resource => pbs: { instanceId: 'proxback', hostname: 'proxback-vm', + nodeName: 'proxback-vm', connectionHealth: 'healthy', datastores: [{ name: 'tank', total: 1_000, used: 400, available: 600 }], }, @@ -140,12 +141,264 @@ const makeCorrelatedAgent = (overrides: Partial = {}): Resource => sources: ['agent', 'pbs'], status: 'online', lastSeen: 1_700_000_000_000, - agent: { agentId: 'agent-proxback', hostname: 'proxback' }, + agent: { agentId: 'agent-proxback', hostname: 'proxback-vm' }, metricsTarget: { resourceType: 'agent', resourceId: 'agent-proxback' }, ...overrides, }) as Resource; +describe('buildBackupServerRows PBS machine-identity boundary', () => { + it.each(['name', 'displayName', 'platformId', 'instanceId', 'canonicalIdentity', 'identity'])( + 'does not import an unrelated guest History or host disks from a PBS %s collision', + (field) => { + const pbs = makeCorrelatablePbs(); + pbs.name = 'backup-connection'; + pbs.displayName = 'Backup connection'; + pbs.platformId = 'pbs-service'; + pbs.pbs = { + ...pbs.pbs!, + instanceId: 'pbs-service', + hostname: '10.2.0.13', + nodeName: undefined, + }; + if (field === 'instanceId') pbs.pbs.instanceId = 'other-host'; + else if (field === 'canonicalIdentity') { + pbs.canonicalIdentity = { hostname: 'other-host', platformId: 'other-host' }; + } else if (field === 'identity') + pbs.identity = { hostname: 'other-host', ips: ['10.9.0.13'] }; + else pbs[field as 'name' | 'displayName' | 'platformId'] = 'other-host'; + const unrelated = makeCorrelatedAgent({ + id: 'vm-other', + type: 'vm', + name: 'other-host', + agent: { + agentId: 'agent-other', + hostname: 'other-host', + disks: [{ mountpoint: '/wrong-host' }], + }, + disk: { current: 99 }, + metricsTarget: { resourceType: 'vm', resourceId: 'vm-other' }, + }); + + const row = buildBackupServerRows([pbs, unrelated])[0]; + expect(row.resource.metricsTarget).toEqual(pbs.metricsTarget); + expect(row.resource.agent).toBeUndefined(); + expect(row.resource.disk).toBeUndefined(); + expect(row.resource.sourceType).toBe('api'); + }, + ); + + it('does not correlate different endpoint IPs through their first dotted token', () => { + const pbs = makeCorrelatablePbs(); + pbs.name = 'backup-connection'; + pbs.pbs = { ...pbs.pbs!, hostname: '10.2.0.13', nodeName: undefined }; + const unrelated = makeCorrelatedAgent({ + name: 'other-host', + agent: { agentId: 'agent-other', hostname: 'other-host' }, + identity: { ips: ['10.9.0.13'] }, + }); + expect(buildBackupServerRows([pbs, unrelated])[0].resource.metricsTarget).toEqual( + pbs.metricsTarget, + ); + }); + + it('does not use a guessed endpoint-to-Agent match without backend corroboration', () => { + const pbs = makeCorrelatablePbs(); + pbs.pbs = { ...pbs.pbs!, hostname: '10.2.0.13', nodeName: undefined }; + const unrelated = makeCorrelatedAgent({ identity: { ips: ['10.2.0.13'] } }); + expect(buildBackupServerRows([pbs, unrelated])[0].resource.metricsTarget).toEqual( + pbs.metricsTarget, + ); + }); + + it('does not let a present label-colliding guest undo backend link withdrawal', () => { + const retention = createPbsCorrelationRetention(); + const pbs = makeCorrelatablePbs(); + pbs.pbs = { ...pbs.pbs!, hostname: '10.2.0.13', linkedAgentId: 'agent-proxback' }; + buildBackupServerRows([pbs, makeCorrelatedAgent()], [], retention); + const withdrawn = { ...pbs, pbs: { ...pbs.pbs!, linkedAgentId: undefined } }; + const unrelated = makeCorrelatedAgent({ + id: 'vm-other', + type: 'vm', + agent: { agentId: 'agent-other', hostname: pbs.name }, + metricsTarget: { resourceType: 'vm', resourceId: 'vm-other' }, + disk: { current: 99 }, + }); + for (let refresh = 0; refresh < 2; refresh++) { + const row = buildBackupServerRows([withdrawn, unrelated], [], retention); + expect(row[0].resource.metricsTarget).toEqual(pbs.metricsTarget); + expect(row[0].resource.agent).toBeUndefined(); + expect(row[0].resource.disk).toBeUndefined(); + expect(retention.size).toBe(0); + } + }); + + it('does not match a reported PBS node to a guest display name instead of its Agent hostname', () => { + const pbs = makeCorrelatablePbs(); + pbs.pbs = { ...pbs.pbs!, nodeName: 'real-pbs.example' }; + const unrelated = makeCorrelatedAgent({ + name: 'real-pbs.example', + agent: { agentId: 'agent-other', hostname: 'other-host.example' }, + }); + expect(buildBackupServerRows([pbs, unrelated])[0].resource.metricsTarget).toEqual( + pbs.metricsTarget, + ); + }); + + it('does not treat different qualified machine names as the same host', () => { + const pbs = makeCorrelatablePbs(); + pbs.pbs = { ...pbs.pbs!, nodeName: 'backup.production.example' }; + const unrelated = makeCorrelatedAgent({ + agent: { agentId: 'agent-other', hostname: 'backup.test.example' }, + }); + expect(buildBackupServerRows([pbs, unrelated])[0].resource.metricsTarget).toEqual( + pbs.metricsTarget, + ); + }); + + it.each(['direct', 'platform'])( + 'preserves a fresh reported node-to-%s-Agent hostname match, ignoring display labels', + (facet) => { + const pbs = makeCorrelatablePbs(); + pbs.pbs = { ...pbs.pbs!, nodeName: ' PROXBACK-VM.EXAMPLE. ' }; + const agent = makeCorrelatedAgent({ + name: 'Custom host label', + displayName: 'Custom display label', + agent: + facet === 'direct' + ? { agentId: 'agent-proxback', hostname: 'proxback-vm.example' } + : undefined, + platformData: + facet === 'platform' + ? { + agent: { agentId: 'agent-proxback', hostname: 'proxback-vm.example' }, + } + : undefined, + }); + expect(buildBackupServerRows([pbs, agent])[0].resource.metricsTarget).toEqual( + agent.metricsTarget, + ); + }, + ); + + it('deduplicates a genuinely matching node/Agent surfaced as both guest and host', () => { + const pbs = makeCorrelatablePbs(); + const host = makeCorrelatedAgent(); + const guest = { + ...host, + id: 'vm-proxback', + type: 'vm', + metricsTarget: { + resourceType: 'vm', + resourceId: 'vm-proxback', + }, + } as Resource; + expect(buildBackupServerRows([pbs, host, guest])[0].resource.metricsTarget).toEqual( + guest.metricsTarget, + ); + }); + + it.each([-6, 6])('declines a node-name match with a %s-minute observation gap', (minutes) => { + const pbs = makeCorrelatablePbs(); + const agent = makeCorrelatedAgent({ lastSeen: pbs.lastSeen + minutes * 60_000 }); + expect(buildBackupServerRows([pbs, agent])[0].resource.metricsTarget).toEqual( + pbs.metricsTarget, + ); + }); + + it.each(['stale', 'old-report', 'invalid-report'])( + 'does not let a recent PVE row hide %s Agent evidence', + (state) => { + const pbs = makeCorrelatablePbs(); + const agent = makeCorrelatedAgent({ + type: 'vm', + agent: { + agentId: 'agent-proxback', + hostname: 'proxback-vm', + stale: state === 'stale', + lastReportAt: + state === 'old-report' + ? new Date(pbs.lastSeen - 6 * 60_000).toISOString() + : state === 'invalid-report' + ? 'invalid' + : undefined, + }, + }); + expect(buildBackupServerRows([pbs, agent])[0].resource.metricsTarget).toEqual( + pbs.metricsTarget, + ); + }, + ); + + it('declines an observed hostname without a concrete Agent identity', () => { + const pbs = makeCorrelatablePbs(); + const agent = makeCorrelatedAgent({ agent: { hostname: 'proxback-vm' } }); + expect(buildBackupServerRows([pbs, agent])[0].resource.metricsTarget).toEqual( + pbs.metricsTarget, + ); + }); + + it('revokes a retained node-name match when the reported machine identity disappears', () => { + const retention = createPbsCorrelationRetention(); + const pbs = makeCorrelatablePbs(); + buildBackupServerRows([pbs, makeCorrelatedAgent()], [], retention); + expect(retention.size).toBe(1); + const uncorroborated = { ...pbs, pbs: { ...pbs.pbs!, nodeName: undefined } }; + expect( + buildBackupServerRows([uncorroborated], [], retention)[0].resource.metricsTarget, + ).toEqual(pbs.metricsTarget); + expect(retention.size).toBe(0); + }); +}); + describe('buildBackupServerRows PBS host correlation retention', () => { + it('keeps three PBS guests on their own History targets across PVE and VirtualBox hosts', () => { + const pbsServers = ['one', 'two', 'three'].map((suffix) => { + const pbs = makeCorrelatablePbs(); + pbs.id = `pbs-${suffix}`; + pbs.name = `backup-connection-${suffix}`; + pbs.pbs = { + ...pbs.pbs!, + instanceId: `pbs-${suffix}`, + hostname: `10.2.0.${suffix === 'one' ? 13 : suffix === 'two' ? 23 : 33}`, + nodeName: undefined, + linkedAgentId: `agent-${suffix}`, + }; + pbs.metricsTarget = { resourceType: 'agent', resourceId: `pbs-${suffix}` }; + return pbs; + }); + const agents = ['one', 'two', 'three'].map((suffix) => + makeCorrelatedAgent({ + id: `agent-${suffix}`, + name: `guest-${suffix}`, + agent: { agentId: `agent-${suffix}`, hostname: `guest-${suffix}` }, + metricsTarget: { resourceType: 'agent', resourceId: `agent-${suffix}` }, + }), + ); + const pveGuests = agents.slice(0, 2).map((agent): Resource => ({ + ...agent, + id: `vm-${agent.agent!.agentId}`, + type: 'vm', + sources: ['proxmox', 'agent'], + metricsTarget: { resourceType: 'vm', resourceId: `vm-${agent.agent!.agentId}` }, + })); + const otherAgents = Array.from({ length: 6 }, (_, index) => + makeCorrelatedAgent({ + id: `agent-unrelated-${index}`, + name: `backup-connection-${index % 3 === 0 ? 'one' : index % 3 === 1 ? 'two' : 'three'}`, + agent: { agentId: `agent-unrelated-${index}` }, + }), + ); + + const rows = buildBackupServerRows([...pbsServers, ...agents, ...pveGuests, ...otherAgents]); + expect( + rows.map((row) => [row.resource.id, row.resource.agent?.agentId, row.resource.metricsTarget]), + ).toEqual([ + ['pbs-one', 'agent-one', { resourceType: 'vm', resourceId: 'vm-agent-one' }], + ['pbs-three', 'agent-three', { resourceType: 'agent', resourceId: 'agent-three' }], + ['pbs-two', 'agent-two', { resourceType: 'vm', resourceId: 'vm-agent-two' }], + ]); + }); + it('keeps the agent UUID through PBS-only to side-by-side PVE/PBS refreshes without nodeName', () => { const retention = createPbsCorrelationRetention(); const pbs = makeCorrelatablePbs(); @@ -245,6 +498,22 @@ describe('buildBackupServerRows PBS host correlation retention', () => { expect(retention.size).toBe(0); }); + it('drops retained host History when a backend link is withdrawn', () => { + const retention = createPbsCorrelationRetention(); + const pbs = makeCorrelatablePbs(); + pbs.pbs = { ...pbs.pbs!, linkedAgentId: 'agent-proxback' }; + expect( + buildBackupServerRows([pbs, makeCorrelatedAgent()], [], retention)[0].resource.metricsTarget + ?.resourceId, + ).toBe('agent-proxback'); + + const withdrawn = { ...pbs, pbs: { ...pbs.pbs!, linkedAgentId: undefined } } as Resource; + expect( + buildBackupServerRows([withdrawn], [], retention)[0].resource.metricsTarget?.resourceId, + ).toBe('proxback'); + expect(retention.size).toBe(0); + }); + it('retains the resolved host target while a refresh snapshot omits the host row', () => { const retention = createPbsCorrelationRetention(); const pbs = makeCorrelatablePbs(); @@ -271,12 +540,16 @@ describe('buildBackupServerRows PBS host correlation retention', () => { const otherAgent = makeCorrelatedAgent({ id: 'agent-other', platformId: 'agent-other', - agent: { agentId: 'agent-other', hostname: 'proxback' }, + agent: { agentId: 'agent-other', hostname: 'proxback-vm' }, metricsTarget: { resourceType: 'agent', resourceId: 'agent-other' }, }); const ambiguous = buildBackupServerRows([pbs, agent, otherAgent], [], retention); expect(ambiguous[0].resource.metricsTarget?.resourceId).toBe('proxback'); + expect(retention.size).toBe(0); + expect(buildBackupServerRows([pbs], [], retention)[0].resource.metricsTarget?.resourceId).toBe( + 'proxback', + ); }); it('drops a remembered host once it is stale relative to the server', () => { diff --git a/frontend-modern/src/utils/__tests__/docsLinks.test.ts b/frontend-modern/src/utils/__tests__/docsLinks.test.ts index 86f70f2d23..172e0345b4 100644 --- a/frontend-modern/src/utils/__tests__/docsLinks.test.ts +++ b/frontend-modern/src/utils/__tests__/docsLinks.test.ts @@ -131,6 +131,71 @@ describe('docsLinks', () => { } }); + it('keeps lockout and incident reporting guidance safe for the affected client', () => { + const troubleshooting = readFileSync(path.join(repoRoot, 'docs', 'TROUBLESHOOTING.md'), 'utf8'); + const recovery = troubleshooting.split('### Recovery Mode')[1].split('\n---')[0]; + expect(recovery).toContain('[I forgot my password](#i-forgot-my-password)'); + expect(recovery).toContain('browser-bound recovery'); + expect(recovery).toContain('A successful curl response does not unlock a separate'); + expect(recovery).toContain('remote and reverse-proxy'); + expect(recovery).not.toContain('generate_token'); + expect(recovery).not.toMatch(/```(?:bash|sh)[\s\S]*?X-Recovery-Token/); + + const help = troubleshooting.split('## ๐Ÿ†˜ Getting Help')[1]; + expect(help).toContain('Do not repeat an update, outage or notification storm'); + expect(help).toContain('Export for GitHub (sanitized)'); + expect(help).toContain('Review before posting'); + expect(help).toContain('session cookies, webhook URLs'); + expect(help).toContain('If installation never started Pulse'); + }); + + it('keeps port troubleshooting specific without exposing deployment credentials', () => { + const troubleshooting = readFileSync(path.join(repoRoot, 'docs', 'TROUBLESHOOTING.md'), 'utf8'); + const ports = troubleshooting.split("### Port change didn't take effect")[1].split('\n### ')[0]; + + expect(ports).toContain('CONFIGURATION.md#common-overrides-environment-variables'); + expect(ports).toContain('`PORT` alias applies only when `FRONTEND_PORT` is unset'); + expect(ports).toContain('`frontendPort` in `system.json` has no effect'); + expect(ports).toContain('separate agent listener, not the web UI port'); + expect(ports).toContain('`systemctl is-active pulse`'); + expect(ports).toContain('`sudo systemctl daemon-reload`'); + expect(ports).toContain('inside\n the Pulse container, not on the Proxmox host'); + expect(ports).toContain('`docker compose up -d pulse`'); + expect(ports).toContain('same image and mounted data volume'); + expect(ports).toContain('Restarting an existing container does not apply a new port mapping'); + expect(ports).toContain('do not delete the volume'); + expect(ports).toContain('Do not post full service environments'); + expect(ports).not.toContain('--property=Environment'); + expect(ports).not.toMatch( + /`(?:sudo )?(?:systemctl (?:status|cat)|docker inspect|docker compose config)[^`]*`(?! output)/, + ); + }); + + it('keeps clone identity recovery separate from destructive OS or credential resets', () => { + const troubleshooting = readFileSync(path.join(repoRoot, 'docs', 'TROUBLESHOOTING.md'), 'utf8'); + const agentGuide = readFileSync(path.join(repoRoot, 'docs', 'UNIFIED_AGENT.md'), 'utf8'); + const shortGuide = troubleshooting + .split('#### Docker hosts appearing/disappearing')[1] + .split('\n### ')[0]; + const cloneGuide = agentGuide.split('### Duplicate Agents')[1].split('\n### ')[0]; + + expect(shortGuide).toContain('UNIFIED_AGENT.md#duplicate-agents'); + for (const section of [shortGuide, cloneGuide]) { + expect(section).toContain('saved'); + expect(section).toContain('Agent Doctor'); + expect(section).toContain('original host unchanged'); + expect(section).not.toContain('systemd-machine-id-setup'); + expect(section).not.toMatch(/(?:sudo\s+)?rm\s+(?:-\S+\s+)*\/etc\/machine-id/); + } + expect(cloneGuide).toContain('argument wins over the environment variable'); + expect(cloneGuide).toContain('PULSE_AGENT_ID_FILE'); + expect(cloneGuide).toContain('Environment="PULSE_AGENT_ID=vm-clone-02"'); + expect(cloneGuide).toContain('fresh, distinct IDs'); + expect(cloneGuide).toContain('does not split or recover historical'); + expect(cloneGuide).toContain('Do not upload `connection.env`, token files'); + expect(cloneGuide).toContain('do not delete tokens, re-enrol'); + }); + it('ships a bounded maintenance example separately from incident snoozing', () => { const apiReference = readFileSync(path.join(repoRoot, 'docs', 'API.md'), 'utf8'); const maintenance = apiReference diff --git a/internal/alerts/flapping_threshold_test.go b/internal/alerts/flapping_threshold_test.go new file mode 100644 index 0000000000..25872dca0e --- /dev/null +++ b/internal/alerts/flapping_threshold_test.go @@ -0,0 +1,154 @@ +package alerts + +import ( + "fmt" + "testing" + "time" +) + +// A positive threshold survives the normal configuration update. Its history +// bound must not silently make that accepted storm-control setting unreachable. +func TestFlappingConfiguredThresholdGatesDispatch(t *testing.T) { + for _, threshold := range []int{1, 5, 10, 11, 25, 100} { + t.Run(fmt.Sprintf("threshold_%d", threshold), func(t *testing.T) { + m := NewManagerWithDataDir(t.TempDir()) + t.Cleanup(m.Stop) + cfg := m.GetConfig() + cfg.Enabled = true + cfg.ActivationState = ActivationActive + cfg.FlappingEnabled = true + cfg.FlappingThreshold = threshold + cfg.FlappingWindowSeconds = 300 + cfg.FlappingCooldownMinutes = 15 + m.UpdateConfig(cfg) + if got := m.GetConfig().FlappingThreshold; got != threshold { + t.Fatalf("accepted threshold = %d, want %d", got, threshold) + } + + _, alert := testNewCanonicalAlert("vm-100", "vm-100-cpu", "vm", "cpu") + alert.Level = AlertLevelWarning + alert.StartTime = time.Now() + alert.LastSeen = alert.StartTime + m.mu.Lock() + m.setActiveAlertNoLock(alert.ID, alert) + m.mu.Unlock() + + delivered := 0 + m.SetAlertCallback(func(*Alert) { delivered++ }) + for attempt := 1; attempt <= threshold; attempt++ { + m.mu.Lock() + dispatched := m.dispatchAlert(alert, false) + m.mu.Unlock() + if want := attempt < threshold; dispatched != want { + t.Fatalf("dispatch %d = %v, want %v at threshold %d", attempt, dispatched, want, threshold) + } + } + if delivered != threshold-1 { + t.Fatalf("notifications admitted = %d, want %d", delivered, threshold-1) + } + + // Repeated attempts during the same cooldown must neither escape to a + // destination nor grow the retained history or extend the deadline. + key := canonicalTrackingKeyForAlert(alert) + m.mu.Lock() + deadline := m.suppressedUntil[key] + lastNotified := alert.LastNotified + for attempt := 0; attempt < 500; attempt++ { + if m.dispatchAlert(alert, false) { + m.mu.Unlock() + t.Fatal("notification escaped the flapping cooldown") + } + } + historySize := len(m.flappingHistory[key]) + unchanged := m.suppressedUntil[key].Equal(deadline) && alert.LastNotified == lastNotified + m.mu.Unlock() + if delivered != threshold-1 || historySize != threshold || !unchanged { + t.Fatalf("cooldown: delivered=%d history=%d unchanged=%v", delivered, historySize, unchanged) + } + + diagnosis, found := m.DiagnoseAlertDelivery(alert.ID) + if !found || diagnosis.Reason != AlertDeliveryReasonFlapping || !diagnosis.FlappingActive || + diagnosis.FlappingThreshold != threshold || diagnosis.FlappingHistoryInWindow != threshold || + diagnosis.SuppressedUntil == nil || !diagnosis.SuppressedUntil.Equal(deadline) { + t.Fatalf("delivery diagnosis does not explain the enforced setting: %+v", diagnosis) + } + }) + } +} + +func TestFlappingHighThresholdPrunesExpiresAndRearms(t *testing.T) { + m := NewManagerWithDataDir(t.TempDir()) + t.Cleanup(m.Stop) + cfg := m.GetConfig() + cfg.FlappingEnabled = true + cfg.FlappingThreshold = 25 + cfg.FlappingWindowSeconds = 60 + cfg.FlappingCooldownMinutes = 15 + m.UpdateConfig(cfg) + + const key = "high-threshold-window" + m.mu.Lock() + defer m.mu.Unlock() + now := time.Now() + for i := 0; i < 100; i++ { + m.flappingHistory[key] = append(m.flappingHistory[key], now.Add(-2*time.Minute)) + } + for i := 0; i < 23; i++ { + m.flappingHistory[key] = append(m.flappingHistory[key], now.Add(-time.Second)) + } + suppressed, transitioned := m.checkFlappingLocked(key) + if suppressed || transitioned || len(m.flappingHistory[key]) != 24 { + t.Fatalf("old changes counted or current changes lost: suppressed=%v transitioned=%v history=%d", suppressed, transitioned, len(m.flappingHistory[key])) + } + suppressed, transitioned = m.checkFlappingLocked(key) + if !suppressed || !transitioned || len(m.flappingHistory[key]) != 25 { + t.Fatalf("25th current change did not arm cooldown: suppressed=%v transitioned=%v history=%d", suppressed, transitioned, len(m.flappingHistory[key])) + } + + deadline := m.suppressedUntil[key] + m.flappingHistory[key] = nil // The observation window drains before cooldown ends. + suppressed, transitioned = m.checkFlappingLocked(key) + if !suppressed || transitioned || !m.suppressedUntil[key].Equal(deadline) { + t.Fatal("cooldown did not hold its original deadline after the window drained") + } + m.suppressedUntil[key] = now.Add(-time.Second) + suppressed, transitioned = m.checkFlappingLocked(key) + if suppressed || transitioned || m.flappingActive[key] || len(m.flappingHistory[key]) != 1 { + t.Fatal("served cooldown did not release and reset the episode") + } + for change := 2; change <= 25; change++ { + suppressed, transitioned = m.checkFlappingLocked(key) + if want := change == 25; suppressed != want || transitioned != want { + t.Fatalf("new episode change %d: suppressed=%v transitioned=%v", change, suppressed, transitioned) + } + } +} + +func TestFlappingLoweredThresholdBoundsRetainedHistory(t *testing.T) { + m := NewManagerWithDataDir(t.TempDir()) + t.Cleanup(m.Stop) + cfg := m.GetConfig() + cfg.FlappingEnabled = true + cfg.FlappingThreshold = 100 + m.UpdateConfig(cfg) + + const key = "threshold-edited" + m.mu.Lock() + for change := 0; change < 30; change++ { + if suppressed, _ := m.checkFlappingLocked(key); suppressed { + m.mu.Unlock() + t.Fatal("notification suppressed below the original threshold") + } + } + m.mu.Unlock() + cfg.FlappingThreshold = 12 + m.UpdateConfig(cfg) + + m.mu.Lock() + suppressed, transitioned := m.checkFlappingLocked(key) + historySize := len(m.flappingHistory[key]) + m.mu.Unlock() + if !suppressed || !transitioned || historySize != 12 { + t.Fatalf("lowered threshold not enforced with bounded history: suppressed=%v transitioned=%v history=%d", suppressed, transitioned, historySize) + } +} diff --git a/internal/alerts/notification_policy.go b/internal/alerts/notification_policy.go index ca0218adb8..24a3b49510 100644 --- a/internal/alerts/notification_policy.go +++ b/internal/alerts/notification_policy.go @@ -122,8 +122,12 @@ func (m *Manager) checkFlappingLocked(trackingKey string) (suppress bool, justTr validHistory = append(validHistory, t) } } - // Limit to max 10 entries to prevent unbounded growth - const maxFlappingHistory = 10 + // Keep enough observations to reach the configured threshold. A fixed + // ten-entry cap silently disabled flapping suppression for accepted + // thresholds above ten. Retain the old diagnostic allowance for smaller + // thresholds, while still bounding history by configuration rather than + // the number of dispatch attempts. + maxFlappingHistory := max(10, m.config.FlappingThreshold) if len(validHistory) > maxFlappingHistory { validHistory = validHistory[len(validHistory)-maxFlappingHistory:] } diff --git a/internal/telemetry/service_health.go b/internal/telemetry/service_health.go index 0bee33b7ff..5ab2f11935 100644 --- a/internal/telemetry/service_health.go +++ b/internal/telemetry/service_health.go @@ -85,6 +85,7 @@ func canonicalServiceHealthFailureCategory(value string) string { ServiceHealthFailureStartup, ServiceHealthFailureRuntime, ServiceHealthFailureAPIConnectivity, + ServiceHealthFailureTimeout, ServiceHealthFailureAPIStatus, ServiceHealthFailureUIStatus, ServiceHealthFailureFrontendAssets: diff --git a/internal/telemetry/service_health_test.go b/internal/telemetry/service_health_test.go index 62e3909eeb..9de4ee77fc 100644 --- a/internal/telemetry/service_health_test.go +++ b/internal/telemetry/service_health_test.go @@ -165,3 +165,82 @@ func TestServiceHealthStateRejectsTamperedPreviousVersion(t *testing.T) { t.Fatalf("tampered previous release escaped local boundary: %#v", ping) } } + +// Closed categories survive both the outbound JSON payload and the persisted +// adjacent-version cohort. No transport error text is part of either shape. +func TestServiceHealthFailureCategoriesSerializeAndPersist(t *testing.T) { + categories := []string{ + ServiceHealthFailureListener, ServiceHealthFailureStartup, + ServiceHealthFailureRuntime, ServiceHealthFailureAPIConnectivity, + ServiceHealthFailureTimeout, ServiceHealthFailureAPIStatus, + ServiceHealthFailureUIStatus, ServiceHealthFailureFrontendAssets, + ServiceHealthFailureUnknown, + } + for _, category := range categories { + t.Run(category, func(t *testing.T) { + dir := t.TempDir() + cfg := Config{Version: "6.4.6", DataDir: dir, GetServiceHealth: func() ServiceHealthObservation { + return ServiceHealthObservation{Observed: true, FailureCategory: " " + strings.ToUpper(category) + " "} + }} + ping, err := buildPingAt(cfg, "startup", time.Now().UTC()) + if err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(ping) + if err != nil { + t.Fatal(err) + } + var fields map[string]interface{} + if err := json.Unmarshal(encoded, &fields); err != nil { + t.Fatal(err) + } + if fields["service_health_failure_category"] != category || fields["service_health_observed"] != true || fields["service_health_healthy"] != false { + t.Fatalf("serialized service-health fields = %#v", fields) + } + record := readServiceHealthRecord(dir) + if record.CurrentFailureCategory != category || !record.CurrentObserved || record.CurrentHealthy { + t.Fatalf("persisted observation = %#v", record) + } + cfg.Version = "6.4.7" + cfg.GetServiceHealth = func() ServiceHealthObservation { return ServiceHealthObservation{Observed: true, Healthy: true} } + recovered, err := buildPingAt(cfg, "startup", time.Now().UTC()) + if err != nil { + t.Fatal(err) + } + record = readServiceHealthRecord(dir) + if recovered.ServiceHealthFailureCategory != "" || !recovered.ServiceHealthHealthy || record.PreviousFailureCategory != category || record.PreviousHealthy { + t.Fatalf("recovery lost the previous failure: %#v / %#v", recovered, record) + } + }) + } + for _, category := range []string{"api_timeout: GET http://192.0.2.1/private", "timeout private.js", "timeout\nAuthorization"} { + if got := canonicalServiceHealthFailureCategory(category); got != ServiceHealthFailureUnknown { + t.Fatalf("free-form category accepted: %q -> %q", category, got) + } + } +} + +func TestTelemetryStartDoesNotWaitForServiceHealth(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + entered, release, started := make(chan struct{}), make(chan struct{}), make(chan struct{}) + dir := t.TempDir() + t.Cleanup(func() { close(release); cancel(); Stop() }) + go func() { + startWithDelay(ctx, Config{Enabled: true, Version: "6.4.6", DataDir: dir, GetServiceHealth: func() ServiceHealthObservation { + close(entered) + <-release + return ServiceHealthObservation{Observed: true, Healthy: true} + }}, 0) + close(started) + }() + select { + case <-entered: + case <-time.After(2 * time.Second): + t.Fatal("background probe did not begin") + } + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("telemetry startup blocked on its service-health probe") + } +} diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index 5e47bbd3bf..3501e340e3 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -254,6 +254,7 @@ const ( ServiceHealthFailureStartup = "startup" ServiceHealthFailureRuntime = "runtime" ServiceHealthFailureAPIConnectivity = "api_connectivity" + ServiceHealthFailureTimeout = "timeout" ServiceHealthFailureAPIStatus = "api_status" ServiceHealthFailureUIStatus = "ui_status" ServiceHealthFailureFrontendAssets = "frontend_assets" @@ -1124,6 +1125,12 @@ var ( // // This is a no-op when outbound usage telemetry is disabled. func Start(ctx context.Context, cfg Config) { + startWithDelay(ctx, cfg, startupDelay) +} + +// Keep the production settling delay constant while allowing deterministic +// tests to exercise the same background runner without a two-minute sleep. +func startWithDelay(ctx context.Context, cfg Config, initialDelay time.Duration) { if !cfg.Enabled { log.Info().Msg("Outbound usage telemetry is disabled (enable via PULSE_TELEMETRY=true or Settings โ†’ System)") return @@ -1153,7 +1160,7 @@ func Start(ctx context.Context, cfg Config) { defer r.wg.Done() // Wait for the monitor to connect and populate state before the first ping. - startTimer := time.NewTimer(startupDelay) + startTimer := time.NewTimer(initialDelay) select { case <-ctx.Done(): startTimer.Stop() diff --git a/internal/truenas/client.go b/internal/truenas/client.go index 7d19143468..6d51ff1c0b 100644 --- a/internal/truenas/client.go +++ b/internal/truenas/client.go @@ -283,11 +283,7 @@ func (c *Client) getSystemTelemetryREST(ctx context.Context) (*SystemInfo, error if start <= 0 { start = end } - response, err := c.getLegacySystemReportingData(ctx, map[string]any{ - "aggregate": false, - "start": start, - "end": end, - }) + response, err := c.getLegacySystemReportingData(ctx, reportingRangeQuery(start, end)) if err != nil { return nil, err } @@ -361,6 +357,18 @@ func reportingGraph(name, identifier string) map[string]any { return graph } +// reportingRangeQuery requests the default min/mean/max summaries alongside +// the unchanged raw data series. SCALE 25.04 validates those summaries even +// when aggregate=false would omit them (#2346). Parsers use data, not summaries; +// enabling aggregation does not change the requested window or sample timestamps. +func reportingRangeQuery(start, end int64) map[string]any { + return map[string]any{ + "aggregate": true, + "start": start, + "end": end, + } +} + // legacyRESTReportingGraphs is the reporting.get_data graph set used to // reconstruct live telemetry and history over the legacy REST transport. The // identifier is omitted for device-independent graphs so the request matches @@ -463,11 +471,7 @@ func (c *Client) getSystemMetricHistoryREST(ctx context.Context, duration time.D if start <= 0 { start = end } - response, err := c.getLegacySystemReportingData(ctx, map[string]any{ - "aggregate": false, - "start": start, - "end": end, - }) + response, err := c.getLegacySystemReportingData(ctx, reportingRangeQuery(start, end)) if err != nil { return nil, err } @@ -2868,11 +2872,7 @@ func (c *trueNASRPCClient) getSystemMetricHistory(ctx context.Context, duration {"name": "arcsize", "identifier": nil}, {"name": "interface", "identifier": nil}, {"name": "disk", "identifier": nil}, - }, map[string]any{ - "aggregate": false, - "start": start, - "end": end, - }) + }, reportingRangeQuery(start, end)) if err != nil { return nil, err } @@ -2965,11 +2965,7 @@ func (c *trueNASRPCClient) getDiskTemperatureHistory(ctx context.Context, identi }) } - response, err := c.getReportingDataWithQuery(ctx, graphs, map[string]any{ - "aggregate": false, - "start": start, - "end": end, - }) + response, err := c.getReportingDataWithQuery(ctx, graphs, reportingRangeQuery(start, end)) if err != nil { return nil, err } @@ -2990,11 +2986,7 @@ func (c *trueNASRPCClient) getReportingData(ctx context.Context, graphs []map[st start = end } - return c.getReportingDataWithQuery(ctx, graphs, map[string]any{ - "aggregate": true, - "start": start, - "end": end, - }) + return c.getReportingDataWithQuery(ctx, graphs, reportingRangeQuery(start, end)) } func (c *trueNASRPCClient) getReportingDataWithQuery(ctx context.Context, graphs []map[string]any, query map[string]any) ([]trueNASReportingGetDataResponse, error) { diff --git a/internal/truenas/client_test.go b/internal/truenas/client_test.go index 0e633283d6..f5818d337f 100644 --- a/internal/truenas/client_test.go +++ b/internal/truenas/client_test.go @@ -1088,8 +1088,8 @@ func TestGetSystemMetricHistoryUsesReportingRPC(t *testing.T) { t.Fatalf("unexpected history graphs: %#v", params[0]) } query, ok := params[1].(map[string]any) - if !ok || query["aggregate"] != false { - t.Fatalf("expected aggregate=false history query, got %#v", params[1]) + if !ok || query["aggregate"] != true { + t.Fatalf("expected aggregate=true history query, got %#v", params[1]) } now := time.Now().UTC().Truncate(time.Second) @@ -1529,8 +1529,8 @@ func TestGetDiskTemperatureHistoryUsesReportingRPC(t *testing.T) { if !ok { t.Fatalf("unexpected history query: %#v", params[1]) } - if aggregate := query["aggregate"]; aggregate != false { - t.Fatalf("expected aggregate=false for history query, got %#v", aggregate) + if aggregate := query["aggregate"]; aggregate != true { + t.Fatalf("expected aggregate=true for history query, got %#v", aggregate) } now := time.Now().UTC().Truncate(time.Second) @@ -2139,7 +2139,7 @@ func (s *isolatedReportingTransport) RoundTrip(r *http.Request) (*http.Response, return nil, err } status, body := s.status, `[]` - if r.Method != http.MethodPost || len(request.Graphs) == 0 || request.Query["aggregate"] != false || request.Query["end"].(float64) <= request.Query["start"].(float64) { + if r.Method != http.MethodPost || len(request.Graphs) == 0 || request.Query["aggregate"] != true || request.Query["end"].(float64) <= request.Query["start"].(float64) { return nil, fmt.Errorf("invalid reporting request") } if status == 0 { diff --git a/internal/truenas/reporting_aggregation_test.go b/internal/truenas/reporting_aggregation_test.go new file mode 100644 index 0000000000..b819a33192 --- /dev/null +++ b/internal/truenas/reporting_aggregation_test.go @@ -0,0 +1,276 @@ +package truenas + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/websocket" +) + +// This models the response-validation failure supplied in #2346, not a capture +// of the appliance response. Aggregation summaries accompany, not replace, data. +const scale2504ReportingValidationError = "6 validation errors for ReportingGetDataResult\n" + + "result.0.aggregations.min\n Field required [type=missing, input_value={}, input_type=dict]\n" + + "result.0.aggregations.mean\n Field required [type=missing, input_value={}, input_type=dict]\n" + + "result.0.aggregations.max\n Field required [type=missing, input_value={}, input_type=dict]\n" + + "result.1.aggregations.min\n Field required [type=missing, input_value={}, input_type=dict]\n" + + "result.1.aggregations.mean\n Field required [type=missing, input_value={}, input_type=dict]\n" + + "result.1.aggregations.max\n Field required [type=missing, input_value={}, input_type=dict]" + +func writeReportingRPCFailure(t *testing.T, conn *websocket.Conn, id int64, code int, name, message string) { + t.Helper() + data, err := json.Marshal(map[string]any{"errname": name}) + if err != nil { + t.Fatal(err) + } + if err := conn.WriteJSON(trueNASRPCResponse{ + JSONRPC: "2.0", ID: id, + Error: &trueNASRPCError{Code: code, Message: message, Data: data}, + }); err != nil { + t.Fatal(err) + } +} + +func aggregatedReportingFixture(name string, end int64, count int) map[string]any { + legend := map[string][]string{ + "cpu": {"usage"}, "memory": {"used", "total"}, "arcsize": {"size"}, + "interface": {"received", "sent"}, "disk": {"read", "write"}, + "disktemp": {"temperature"}, + }[name] + data := make([]any, count) + for i := range data { + row := []any{end - int64(count-1) + int64(i)} + switch name { + case "cpu": + row = append(row, 10+i%20) + case "memory": + row = append(row, int64(4+i%2)<<30, int64(16)<<30) + case "arcsize": + row = append(row, int64(1)<<30) + case "interface": + row = append(row, 4096, 2048) + case "disk": + row = append(row, 8192, 4096) + case "disktemp": + row = append(row, 30+i%5) + } + data[i] = row + } + return map[string]any{ + "name": name, "legend": legend, "data": data, + "start": end - int64(count-1), "end": end, + // Deliberately different from all sample values: summaries must not be + // substituted for raw samples in History or latest live telemetry. + "aggregations": map[string]any{ + "min": []int{-999}, "mean": []int{999}, "max": []int{9999}, + }, + } +} + +func assertReportingRange(t *testing.T, query map[string]any, seconds int64) int64 { + t.Helper() + start, startOK := query["start"].(float64) + end, endOK := query["end"].(float64) + if !startOK || !endOK || len(query) != 3 || int64(end-start) != seconds { + t.Fatalf("reporting query window changed: %#v, want %ds", query, seconds) + } + if delta := time.Now().Unix() - int64(end); delta < 0 || delta > 5 { + t.Fatalf("reporting end is not current: %#v", query) + } + return int64(end) +} + +func assertRawReportingSeries(t *testing.T, points []TimeSeriesPoint, end int64, value func(int) float64) { + t.Helper() + if len(points) != 3600 { + t.Fatalf("raw series has %d samples, want 3600", len(points)) + } + for i, point := range points { + if !point.Timestamp.Equal(time.Unix(end-3599+int64(i), 0)) || point.Value != value(i) { + t.Fatalf("raw sample %d changed: %+v", i, point) + } + } +} + +func TestReportingHistorySupportsSCALE2504Aggregations(t *testing.T) { + for _, test := range []struct { + name string + disk bool + duration time.Duration + seconds int64 + }{ + {"system", false, time.Hour, 3600}, + {"system default", false, -time.Second, 86400}, + {"disk", true, time.Hour, 3600}, + {"disk default", true, 0, 86400}, + } { + t.Run(test.name, func(t *testing.T) { + var responseEnd atomic.Int64 + server := newMockServerWithRPC(t, defaultAPIResponses(), nil, func(t *testing.T, conn *websocket.Conn) { + auth := readRPCRequest(t, conn) + writeRPCResult(t, conn, auth.ID, true) + request := readRPCRequest(t, conn) + if request.Method != "reporting.get_data" { + t.Fatalf("unexpected method: %s", request.Method) + } + params := request.Params.([]any) + query := params[1].(map[string]any) + if query["aggregate"] != true { + writeReportingRPCFailure(t, conn, request.ID, -32001, "EINVAL", scale2504ReportingValidationError) + return + } + end := assertReportingRange(t, query, test.seconds) + responseEnd.Store(end) + graphs := params[0].([]any) + wantGraphs := 5 + if test.disk { + wantGraphs = 1 + } + if len(graphs) != wantGraphs { + t.Fatalf("graphs = %#v, want %d", graphs, wantGraphs) + } + var result []map[string]any + for _, raw := range graphs { + graph := raw.(map[string]any) + entry := aggregatedReportingFixture(graph["name"].(string), end, 3600) + entry["identifier"] = graph["identifier"] + result = append(result, entry) + } + writeRPCResult(t, conn, request.ID, result) + }) + t.Cleanup(server.Close) + client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "fixture-key"}) + if test.disk { + history, err := client.GetDiskTemperatureHistory(context.Background(), []string{"sda"}, test.duration) + if err != nil { + t.Fatal(err) + } + assertRawReportingSeries(t, history["sda"], responseEnd.Load(), func(i int) float64 { return float64(30 + i%5) }) + return + } + // Use the real API history fetcher and canonical provider projection; + // cached inventory is synthetic and is not an appliance identity proof. + fixtures := DefaultFixtures() + fixtures.System.MemoryTotalBytes = 16 << 30 + provider := NewProvider(fixtures) + provider.fetcher = &APIFetcher{Client: client} + resourceID, history, err := provider.SystemMetricHistory(context.Background(), test.duration) + if err != nil { + t.Fatal(err) + } + if resourceID != "truenas-main" { + t.Fatalf("canonical metrics target changed: %q", resourceID) + } + end := responseEnd.Load() + assertRawReportingSeries(t, history["cpu"], end, func(i int) float64 { return float64(10 + i%20) }) + assertRawReportingSeries(t, history["memory"], end, func(i int) float64 { return float64(3+i%2) / 16 * 100 }) + for key, value := range map[string]float64{"netin": 4096, "netout": 2048, "diskread": 8192, "diskwrite": 4096} { + assertRawReportingSeries(t, history[key], end, func(int) float64 { return value }) + } + }) + } +} + +func TestReportingAggregationsPreserveRPCFailures(t *testing.T) { + for _, test := range []struct { + name, errname, message string + code int + }{ + {"validation", "EINVAL", scale2504ReportingValidationError, -32001}, + {"access", "EACCES", "permission denied", -32001}, + {"unrelated validation", "EINVAL", "unknown graph", -32001}, + {"missing method", "ENOSYS", "method not found", -32601}, + } { + t.Run(test.name, func(t *testing.T) { + var calls atomic.Int32 + server := newMockServerWithRPC(t, defaultAPIResponses(), nil, func(t *testing.T, conn *websocket.Conn) { + auth := readRPCRequest(t, conn) + writeRPCResult(t, conn, auth.ID, true) + request := readRPCRequest(t, conn) + calls.Add(1) + writeReportingRPCFailure(t, conn, request.ID, test.code, test.errname, test.message) + }) + t.Cleanup(server.Close) + client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "fixture-key"}) + history, err := client.GetSystemMetricHistory(context.Background(), time.Hour) + var rpcErr *RPCError + if history != nil || !errors.As(err, &rpcErr) || rpcErr.Code != test.code || rpcErr.Errname != test.errname || rpcErr.Message != test.message { + t.Fatalf("reporting error lost: history=%+v, err=%v", history, err) + } + if calls.Load() != 1 { + t.Fatalf("unexpected reporting retry: %d calls", calls.Load()) + } + }) + } +} + +func TestRESTReportingDefaultAggregationsPreserveSamples(t *testing.T) { + for _, live := range []bool{false, true} { + name := "history" + seconds := int64(3600) + if live { + name, seconds = "live telemetry", legacyRESTTelemetryWindowSeconds + } + t.Run(name, func(t *testing.T) { + var end int64 + var calls int + client := newLegacyRESTReportingClient(t, nil) + client.httpClient.Transport = reportingAggregationRESTFunc(func(r *http.Request) (*http.Response, error) { + calls++ + var request struct { + Graphs []map[string]any `json:"graphs"` + Query map[string]any `json:"query"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + return nil, err + } + if r.Method != http.MethodPost || request.Query["aggregate"] != true { + t.Fatalf("unexpected reporting request: %s %#v", r.Method, request.Query) + } + end = assertReportingRange(t, request.Query, seconds) + var result []map[string]any + for _, graph := range request.Graphs { + if _, present := graph["identifier"]; present { + t.Fatalf("legacy graph shape changed: %#v", graph) + } + result = append(result, aggregatedReportingFixture(graph["name"].(string), end, int(seconds))) + } + body, err := json.Marshal(result) + if err != nil { + return nil, err + } + return (alertArgsTransport{r.URL.Path: apiResponse{body: string(body)}}).RoundTrip(r) + }) + if live { + system, err := client.GetSystemTelemetry(context.Background()) + if err != nil { + t.Fatal(err) + } + if system.CPUPercent != 29 || system.MemoryTotalBytes != 16<<30 || system.ARCSizeBytes != 1<<30 || system.NetInRate != 4096 || system.DiskWriteRate != 4096 { + t.Fatalf("latest raw telemetry changed: %+v", system) + } + } else { + history, err := client.GetSystemMetricHistory(context.Background(), time.Hour) + if err != nil { + t.Fatal(err) + } + assertRawReportingSeries(t, history.CPUPercent, end, func(i int) float64 { return float64(10 + i%20) }) + assertRawReportingSeries(t, systemMemoryPercentHistory(history, 16<<30), end, func(i int) float64 { return float64(3+i%2) / 16 * 100 }) + } + if calls != 1 { + t.Fatalf("successful REST batch issued %d requests, want 1", calls) + } + }) + } +} + +type reportingAggregationRESTFunc func(*http.Request) (*http.Response, error) + +func (f reportingAggregationRESTFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} diff --git a/internal/unifiedresources/adapters.go b/internal/unifiedresources/adapters.go index 7e4b9e79a5..b3e9096cc6 100644 --- a/internal/unifiedresources/adapters.go +++ b/internal/unifiedresources/adapters.go @@ -3908,11 +3908,7 @@ func extractHostname(raw string) string { } parsed, err := url.Parse(raw) if err == nil && parsed.Host != "" { - host := parsed.Host - if strings.Contains(host, ":") { - host = strings.Split(host, ":")[0] - } - return host + return parsed.Hostname() } if strings.Contains(raw, "/") { diff --git a/internal/unifiedresources/adapters_test.go b/internal/unifiedresources/adapters_test.go index 4768e5a8a8..3f9c2e4630 100644 --- a/internal/unifiedresources/adapters_test.go +++ b/internal/unifiedresources/adapters_test.go @@ -1980,3 +1980,16 @@ func TestUnraidArrayDiskCountAdapters(t *testing.T) { } } } + +func TestProviderURLHostnamePreservesIPv6(t *testing.T) { + for _, test := range []struct{ endpoint, want string }{ + {"https://[fd00::13]:8007", "fd00::13"}, + {"https://[::1]:8007", "::1"}, + {"https://backup-host.local:8007", "backup-host.local"}, + {"https://10.2.0.13:8007", "10.2.0.13"}, + } { + if got := extractHostname(test.endpoint); got != test.want { + t.Errorf("extractHostname(%q) = %q, want %q", test.endpoint, got, test.want) + } + } +} diff --git a/internal/unifiedresources/clone_test.go b/internal/unifiedresources/clone_test.go index 5a0dd1ddd9..6f6b837be0 100644 --- a/internal/unifiedresources/clone_test.go +++ b/internal/unifiedresources/clone_test.go @@ -1,6 +1,7 @@ package unifiedresources import ( + "reflect" "testing" "time" @@ -9,6 +10,55 @@ import ( "github.com/rcourtman/pulse-go-rewrite/pkg/tlsutil" ) +func TestCloneResourceRefreshesScopeAndPolicyMetadata(t *testing.T) { + // Registry mutation paths can leave derived metadata stale until the next + // read. Keep refresh on every clone, including after source/tag edits. + original := &Resource{ + ID: "app-container:lab", Type: ResourceTypeAppContainer, + Sources: []DataSource{SourceDocker, SourceAgent, SourceDocker}, + PlatformScopes: []string{"stale"}, + Agent: &AgentData{Hostname: "lab"}, + Docker: &DockerData{HostSourceID: "proxmox-lxc-docker:lab:node:100"}, + Policy: &ResourcePolicy{Sensitivity: ResourceSensitivityRestricted}, + AISafeSummary: "stale", + } + for _, test := range []struct { + name string + tags []string + sensitivity ResourceSensitivity + routing ResourceRoutingScope + }{ + {"public-last", []string{"backup", "PII", " PUBLIC "}, ResourceSensitivityPublic, ResourceRoutingScopeCloudSummary}, + {"public-first", []string{"public", "restricted", "database"}, ResourceSensitivityPublic, ResourceRoutingScopeCloudSummary}, + {"restricted-last", []string{"backup", " Customer_Data "}, ResourceSensitivityRestricted, ResourceRoutingScopeLocalOnly}, + {"restricted-first", []string{" SECRETS ", "storage"}, ResourceSensitivityRestricted, ResourceRoutingScopeLocalOnly}, + {"sensitive", []string{"", "unrecognised", " DaTaBaSe ", "database"}, ResourceSensitivitySensitive, ResourceRoutingScopeLocalFirst}, + {"internal", []string{"prod", ""}, ResourceSensitivityInternal, ResourceRoutingScopeCloudSummary}, + } { + t.Run(test.name, func(t *testing.T) { + original.Tags = test.tags + cloned := cloneResource(original) + assertStringSliceEqual(t, cloned.PlatformScopes, []string{"agent", "proxmox-pve", "docker"}) + if cloned.Policy.Sensitivity != test.sensitivity || cloned.Policy.Routing.Scope != test.routing || cloned.AISafeSummary == "stale" { + t.Fatalf("clone retained stale policy: %#v, summary=%q", cloned.Policy, cloned.AISafeSummary) + } + cloned.PlatformScopes[0] = "mutated" + cloned.Policy.Sensitivity = ResourceSensitivityPublic + if !reflect.DeepEqual(original.PlatformScopes, []string{"stale"}) || original.Policy.Sensitivity != ResourceSensitivityRestricted || original.AISafeSummary != "stale" { + t.Fatal("clone modified the original derived metadata") + } + }) + } + original.Sources = []DataSource{SourceTrueNAS, SourceAgent} + original.TrueNAS = &TrueNASData{Hostname: "lab"} + original.Docker = nil + cloned := cloneResource(original) + assertStringSliceEqual(t, cloned.PlatformScopes, []string{"agent", "truenas"}) + if cloned.Policy.Sensitivity != ResourceSensitivitySensitive { + t.Fatalf("edited provider facet lost its default sensitivity: %#v", cloned.Policy) + } +} + // --- cloneResource: top-level isolation --- func TestCloneResource_Nil(t *testing.T) { diff --git a/internal/unifiedresources/metadata_bench_test.go b/internal/unifiedresources/metadata_bench_test.go new file mode 100644 index 0000000000..5a5bc958d5 --- /dev/null +++ b/internal/unifiedresources/metadata_bench_test.go @@ -0,0 +1,30 @@ +package unifiedresources + +import "testing" + +var benchmarkSensitivity ResourceSensitivity +var benchmarkScopes []string + +// Component measurements of the per-read metadata path, not installed fleet +// CPU/write attribution or release qualification thresholds. +func BenchmarkRefreshPlatformScopes(b *testing.B) { + resource := Resource{ + Type: ResourceTypeAppContainer, Sources: []DataSource{SourceDocker, SourceAgent}, + Agent: &AgentData{}, Docker: &DockerData{HostSourceID: "proxmox-lxc-docker:lab:node:100"}, + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + RefreshPlatformScopes(&resource) + } + benchmarkScopes = resource.PlatformScopes +} + +func BenchmarkClassifyResourceSensitivity(b *testing.B) { + resource := Resource{Type: ResourceTypeVM, Tags: []string{"prod", "web", "database", "customer-data"}} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + benchmarkSensitivity = classifyResourceSensitivity(resource) + } +} diff --git a/internal/unifiedresources/pbs_host_disks.go b/internal/unifiedresources/pbs_host_disks.go index f795501647..f6ff16c9f3 100644 --- a/internal/unifiedresources/pbs_host_disks.go +++ b/internal/unifiedresources/pbs_host_disks.go @@ -94,7 +94,9 @@ func (rr *ResourceRegistry) associatePBSHostAgentResources( func uniquePBSHostAgent(instance models.PBSInstance, hosts []models.Host, vms []models.VM) *models.Host { var match *models.Host for index := range hosts { - if !pbsInstanceCorroboratesHost(instance, hosts[index]) { + if strings.TrimSpace(hosts[index].ID) == "" || + !pbsHostLinkObservationFresh(instance.LastSeen, hosts[index].LastSeen) || + !pbsInstanceCorroboratesHost(instance, hosts[index]) { continue } if match != nil { @@ -119,7 +121,7 @@ func uniquePBSHostAgent(instance models.PBSInstance, hosts []models.Host, vms [] guestID := "" for _, vm := range vms { if !strings.EqualFold(strings.TrimSpace(vm.Status), "running") || - !pbsGuestLinkObservationFresh(instance.LastSeen, vm.LastSeen) { + !pbsHostLinkObservationFresh(instance.LastSeen, vm.LastSeen) { continue } for _, address := range vm.IPAddresses { @@ -141,7 +143,8 @@ func uniquePBSHostAgent(instance models.PBSInstance, hosts []models.Host, vms [] if strings.TrimSpace(hosts[index].LinkedVMID) != guestID { continue } - if !pbsGuestLinkObservationFresh(instance.LastSeen, hosts[index].LastSeen) { + if strings.TrimSpace(hosts[index].ID) == "" || + !pbsHostLinkObservationFresh(instance.LastSeen, hosts[index].LastSeen) { continue } if match != nil { @@ -152,7 +155,10 @@ func uniquePBSHostAgent(instance models.PBSInstance, hosts []models.Host, vms [] return match } -func pbsGuestLinkObservationFresh(pbsSeen, peerSeen time.Time) bool { +// Endpoint addresses and machine names can be reused after replacement. A +// current PBS observation must not borrow identity from an old Agent report, +// whether corroboration is direct or passes through a PVE guest. +func pbsHostLinkObservationFresh(pbsSeen, peerSeen time.Time) bool { if pbsSeen.IsZero() || peerSeen.IsZero() { return false } @@ -162,9 +168,6 @@ func pbsGuestLinkObservationFresh(pbsSeen, peerSeen time.Time) bool { func pbsInstanceCorroboratesHost(instance models.PBSInstance, host models.Host) bool { hostName := NormalizeHostname(host.Hostname) - if hostName == "" { - return false - } // The node hostname the PBS API reports about itself is machine identity. // It is the strongest evidence when the connection is configured by IP or // a DNS alias the agent never reports, which the connection label and the @@ -180,18 +183,14 @@ func pbsInstanceCorroboratesHost(instance models.PBSInstance, host models.Host) return false } if ip := NormalizeIP(endpoint); ip != "" { - if NormalizeIP(host.ReportIP) == ip { + if providerLinkIP(host.ReportIP) == ip { return true } - for _, iface := range host.NetworkInterfaces { - for _, address := range iface.Addresses { - if NormalizeIP(address) == ip { - return true - } - } + if _, ok := providerLinkNetworkIPs(host.NetworkInterfaces)[ip]; ok { + return true } return false } - return NormalizeHostname(endpoint) == hostName + return hostName != "" && NormalizeHostname(endpoint) == hostName } diff --git a/internal/unifiedresources/pbs_pmg_registry_test.go b/internal/unifiedresources/pbs_pmg_registry_test.go index bf31164d0e..0146f353fa 100644 --- a/internal/unifiedresources/pbs_pmg_registry_test.go +++ b/internal/unifiedresources/pbs_pmg_registry_test.go @@ -1,6 +1,7 @@ package unifiedresources import ( + "fmt" "testing" "time" @@ -601,3 +602,244 @@ func containsPlatformScope(values []string, expected string) bool { } return false } + +// Direct PBS/Agent correlation must use the same usable machine addresses as +// provider-link inference, not host-local addresses repeated on every server. +func TestPBSDirectHostLinkRejectsUnsafeObservations(t *testing.T) { + now := time.Now().UTC() + for _, test := range []struct { + name string + endpoint string + mutate func(*models.Host) + }{ + {"loopback-report", "127.0.0.1", func(h *models.Host) { h.ReportIP = "127.0.0.1" }}, + {"ipv6-loopback", "[::1]", func(h *models.Host) { h.ReportIP = "::1" }}, + {"unspecified-report", "0.0.0.0", func(h *models.Host) { h.ReportIP = "0.0.0.0" }}, + {"link-local-report", "169.254.2.3", func(h *models.Host) { h.ReportIP = "169.254.2.3" }}, + {"multicast-report", "224.0.0.2", func(h *models.Host) { h.ReportIP = "224.0.0.2" }}, + {"loopback-interface", "127.0.0.1", func(h *models.Host) { + h.NetworkInterfaces = []models.HostNetworkInterface{{Name: "lo", Addresses: []string{"127.0.0.1/8"}}} + }}, + {"docker-interface", "172.17.0.1", func(h *models.Host) { + h.NetworkInterfaces = []models.HostNetworkInterface{{Name: "docker0", Addresses: []string{"172.17.0.1/16"}}} + }}, + {"docker-generated-bridge", "172.22.0.1", func(h *models.Host) { + h.NetworkInterfaces = []models.HostNetworkInterface{{Name: "br-abcdef123456", Addresses: []string{"172.22.0.1/16"}}} + }}, + {"stale-report", "10.2.0.13", func(h *models.Host) { h.LastSeen = now.Add(-10 * time.Minute) }}, + {"missing-report-time", "10.2.0.13", func(h *models.Host) { h.LastSeen = time.Time{} }}, + {"missing-agent-id", "10.2.0.13", func(h *models.Host) { h.ID = "" }}, + } { + t.Run(test.name, func(t *testing.T) { + host := models.Host{ + ID: "unrelated-agent", Hostname: "unrelated-host", ReportIP: "10.2.0.13", + Status: "online", LastSeen: now, + Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{Device: "/dev/sda", Serial: "UNRELATED-DISK"}}}, + } + test.mutate(&host) + registry := NewRegistry(nil) + registry.IngestSnapshot(models.StateSnapshot{ + Hosts: []models.Host{host}, + PBSInstances: []models.PBSInstance{{ + ID: "pbs-service", Name: "backup", Host: "https://" + test.endpoint + ":8007", + Status: "online", LastSeen: now, + }}, + }) + for _, resource := range registry.ListForPresentation() { + if resource.PBS != nil && resource.PBS.LinkedAgentID != "" { + t.Errorf("unsafe observation linked PBS to %q", resource.PBS.LinkedAgentID) + } + if (resource.Agent != nil || resource.PhysicalDisk != nil) && containsDataSource(resource.Sources, SourcePBS) { + t.Errorf("unsafe observation lent PBS ownership to %s (%s)", resource.ID, resource.Type) + } + } + }) + } +} + +// Corroboration stays possible for ordinary private management networks, +// explicit multi-NIC report-IP hints, and PBS-reported machine hostnames. +func TestPBSDirectHostLinkPreservesManagementEvidence(t *testing.T) { + now := time.Now().UTC() + for _, test := range []struct { + name string + endpoint string + nodeName string + host models.Host + }{ + {"private-report", "10.2.0.13", "", models.Host{Hostname: "different", ReportIP: "10.2.0.13"}}, + {"interface-without-hostname", "10.2.0.13", "", models.Host{NetworkInterfaces: []models.HostNetworkInterface{{Name: "eth0", Addresses: []string{"10.2.0.13/24"}}}}}, + {"172-management-report", "172.22.0.1", "", models.Host{Hostname: "different", ReportIP: "172.22.0.1"}}, + {"management-bridge", "172.22.0.1", "", models.Host{Hostname: "different", NetworkInterfaces: []models.HostNetworkInterface{{Name: "vmbr0", Addresses: []string{"172.22.0.1/24"}}}}}, + {"custom-management-bridge", "172.22.0.1", "", models.Host{Hostname: "different", NetworkInterfaces: []models.HostNetworkInterface{{Name: "br-management", Addresses: []string{"172.22.0.1/24"}}}}}, + {"ipv6-management", "[fd00::13]", "", models.Host{Hostname: "different", NetworkInterfaces: []models.HostNetworkInterface{{Name: "eth0", Addresses: []string{"fd00::13/64"}}}}}, + {"reported-node-name", "10.2.0.13", "backup-host", models.Host{Hostname: "backup-host.local"}}, + {"endpoint-hostname", "backup-host.local", "", models.Host{Hostname: "backup-host"}}, + } { + t.Run(test.name, func(t *testing.T) { + host := test.host + host.ID, host.LastSeen = "host-agent", now + instance := models.PBSInstance{Host: "https://" + test.endpoint + ":8007", NodeName: test.nodeName, LastSeen: now} + if got := uniquePBSHostAgent(instance, []models.Host{host}, nil); got == nil || got.ID != host.ID { + t.Fatalf("usable management evidence lost PBS Agent link: %+v", got) + } + }) + } +} + +// A previously observed address may be reused. Registry replacement must +// withdraw the old host link and PBS disk membership, then safely select a +// fresh unique replacement without changing the service's History target. +func TestPBSDirectHostLinkWithdrawsAndReplacesStaleAgent(t *testing.T) { + now := time.Now().UTC() + host := models.Host{ + ID: "old-agent", Hostname: "backup-host", ReportIP: "10.2.0.13", LastSeen: now, + Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{Device: "/dev/sda", Serial: "OLD-DISK"}}}, + } + snapshot := models.StateSnapshot{ + Hosts: []models.Host{host}, + PBSInstances: []models.PBSInstance{{ID: "pbs-service", Host: "https://10.2.0.13:8007", LastSeen: now}}, + } + adapter := NewMonitorAdapter(NewRegistry(nil)) + for _, phase := range []string{"fresh", "stale", "replaced", "ambiguous"} { + switch phase { + case "stale": + snapshot.PBSInstances[0].LastSeen = now.Add(10 * time.Minute) + case "replaced": + replacement := host + replacement.ID, replacement.Hostname = "new-agent", "replacement-host" + replacement.LastSeen = snapshot.PBSInstances[0].LastSeen + replacement.Sensors = models.HostSensorSummary{SMART: []models.HostDiskSMART{{Device: "/dev/sda", Serial: "NEW-DISK"}}} + snapshot.Hosts = append(snapshot.Hosts, replacement) + case "ambiguous": + snapshot.Hosts[0].LastSeen = snapshot.PBSInstances[0].LastSeen + } + adapter.replaceRegistry(snapshot, nil) + registry := adapter.currentRegistry() + want := "" + if phase == "fresh" { + want = "old-agent" + } else if phase == "replaced" { + want = "new-agent" + } + for _, resource := range registry.List() { + if resource.PBS != nil { + if resource.PBS.LinkedAgentID != want { + t.Errorf("%s: PBS link = %q, want %q", phase, resource.PBS.LinkedAgentID, want) + } + if got := registry.MetricsTarget(resource.ID); got == nil || *got != (MetricsTarget{ResourceType: "agent", ResourceID: "pbs-service"}) { + t.Errorf("%s: PBS service target changed: %+v", phase, got) + } + } + if resource.Agent != nil { + if got := containsDataSource(resource.Sources, SourcePBS); got != (resource.Agent.AgentID == want) { + t.Errorf("%s: Agent %s PBS membership = %t", phase, resource.Agent.AgentID, got) + } + } + if resource.PhysicalDisk != nil { + wantMembership := phase == "fresh" && resource.PhysicalDisk.Serial == "OLD-DISK" || + phase == "replaced" && resource.PhysicalDisk.Serial == "NEW-DISK" + if got := containsDataSource(resource.Sources, SourcePBS); got != wantMembership { + t.Errorf("%s: disk %s PBS membership = %t", phase, resource.PhysicalDisk.Serial, got) + } + } + } + } +} + +// Mirrors #1723's two PVE-hosted PBS machines and one non-PVE machine among +// nine Agents. The non-PVE case supplies observed interface-IP evidence: the +// test is not a claim that the reporter's VirtualBox Agent supplies that IP. +func TestPBSHostHistoryThreeServerTopology(t *testing.T) { + now := time.Now().UTC() + snapshot := models.StateSnapshot{} + for index := 1; index <= 9; index++ { + host := models.Host{ + ID: fmt.Sprintf("agent-%d", index), Hostname: fmt.Sprintf("machine-%d", index), LastSeen: now, + CPUUsage: float64(index * 10), Memory: models.Memory{Total: 1000, Used: int64(index * 100), Usage: float64(index * 10)}, + Disks: []models.Disk{{Device: "/dev/sda", Total: 1000, Used: int64(index * 100), Usage: float64(index * 10)}}, + NetworkInterfaces: []models.HostNetworkInterface{{Name: "eth0", RXBytes: uint64(index * 11), TXBytes: uint64(index * 12)}}, + NetInRate: float64(index * 11), NetOutRate: float64(index * 12), + DiskReadRate: float64(index * 13), DiskWriteRate: float64(index * 14), + DiskIO: []models.DiskIO{{Device: "sda", ReadBytes: uint64(index * 13), WriteBytes: uint64(index * 14)}}, + Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{Device: "/dev/sda", Serial: fmt.Sprintf("DISK-%d", index)}}}, + } + if index <= 2 { + host.LinkedVMID = fmt.Sprintf("pve:node:%d", index+100) + snapshot.VMs = append(snapshot.VMs, models.VM{ + ID: host.LinkedVMID, VMID: index + 100, Instance: "pve", Node: "node", Name: host.Hostname, + Status: "running", IPAddresses: []string{fmt.Sprintf("10.2.0.%d", index)}, LastSeen: now, + }) + } else if index == 3 { + host.NetworkInterfaces[0].Addresses = []string{"10.2.0.3/24"} + } + snapshot.Hosts = append(snapshot.Hosts, host) + if index <= 3 { + snapshot.PBSInstances = append(snapshot.PBSInstances, models.PBSInstance{ + ID: fmt.Sprintf("pbs-%d", index), Name: fmt.Sprintf("backup-%d", index), Host: fmt.Sprintf("https://10.2.0.%d:8007", index), + Status: "online", LastSeen: now, CPU: float64(index), Memory: float64(index * 2), + }) + } + } + registry := NewRegistry(nil) + registry.IngestSnapshot(snapshot) + services, linkedHosts, linkedDisks := 0, 0, 0 + for _, resource := range registry.ListForPresentation() { + if resource.PBS != nil { + services++ + var index int + // Registry order is not identity. Extract the fixture's stable ID. + if _, err := fmt.Sscanf(resource.PBS.InstanceID, "pbs-%d", &index); err != nil { + t.Fatal(err) + } + if resource.PBS.LinkedAgentID != fmt.Sprintf("agent-%d", index) { + t.Errorf("%s linked to %q", resource.PBS.InstanceID, resource.PBS.LinkedAgentID) + } + if got := registry.MetricsTarget(resource.ID); got == nil || *got != (MetricsTarget{ResourceType: "agent", ResourceID: resource.PBS.InstanceID}) { + t.Errorf("service target replaced with a host target: %+v", got) + } + } + if resource.Agent != nil && containsDataSource(resource.Sources, SourcePBS) { + linkedHosts++ + var index int + if _, err := fmt.Sscanf(resource.Agent.AgentID, "agent-%d", &index); err != nil || index < 1 || index > 3 { + t.Fatalf("unrelated Agent gained PBS membership: %s", resource.Agent.AgentID) + } + if got := registry.MetricsTarget(resource.ID); got == nil || *got != (MetricsTarget{ResourceType: "agent", ResourceID: resource.Agent.AgentID}) { + t.Errorf("host target lost source-native Agent identity: %+v", got) + } + if resource.Metrics == nil { + t.Fatalf("%s lost all Agent metrics", resource.Agent.AgentID) + } + for name, metric := range map[string]*MetricValue{"memory": resource.Metrics.Memory, "disk": resource.Metrics.Disk} { + if metric == nil || metric.Source != SourceAgent || metric.Used == nil || *metric.Used != int64(index*100) { + t.Errorf("%s %s lost distinct Agent usage: %+v", resource.Agent.AgentID, name, metric) + } + } + for _, observation := range []struct { + name string + metric *MetricValue + want float64 + }{ + {"CPU", resource.Metrics.CPU, float64(index * 10)}, + {"network in", resource.Metrics.NetIn, float64(index * 11)}, + {"network out", resource.Metrics.NetOut, float64(index * 12)}, + {"disk read", resource.Metrics.DiskRead, float64(index * 13)}, + {"disk write", resource.Metrics.DiskWrite, float64(index * 14)}, + } { + if metric := observation.metric; metric == nil || metric.Source != SourceAgent || metric.Value != observation.want { + t.Errorf("%s %s lost distinct Agent telemetry (want %g): %+v", resource.Agent.AgentID, observation.name, observation.want, metric) + } + } + } + if resource.PhysicalDisk != nil && containsDataSource(resource.Sources, SourcePBS) { + linkedDisks++ + if resource.PhysicalDisk.Serial != "DISK-1" && resource.PhysicalDisk.Serial != "DISK-2" && resource.PhysicalDisk.Serial != "DISK-3" { + t.Errorf("unrelated disk gained PBS membership: %s", resource.PhysicalDisk.Serial) + } + } + } + if services != 3 || linkedHosts != 3 || linkedDisks != 3 { + t.Fatalf("three-server topology: services=%d linked hosts=%d linked disks=%d", services, linkedHosts, linkedDisks) + } +} diff --git a/internal/unifiedresources/platform_scopes.go b/internal/unifiedresources/platform_scopes.go index b14abb8d09..774e1c44a6 100644 --- a/internal/unifiedresources/platform_scopes.go +++ b/internal/unifiedresources/platform_scopes.go @@ -28,21 +28,35 @@ func RefreshPlatformScopes(resource *Resource) { return } - scopes := make(map[string]struct{}, len(resource.Sources)+2) + // Clones refresh scopes on registry reads as well as writes. These small + // membership sets do not need a map on every resource read. + scopes := make([]string, 0, 4) + add := func(scope string) { + scope = strings.ToLower(strings.TrimSpace(scope)) + if scope == "" { + return + } + for _, existing := range scopes { + if existing == scope { + return + } + } + scopes = append(scopes, scope) + } for _, source := range resource.Sources { - addPlatformScope(scopes, platformScopeForSource(source)) + add(platformScopeForSource(source)) } - addPlatformScopesForFacets(scopes, *resource) + addPlatformScopesForFacets(add, *resource) if shouldAddDockerPlatformScope(*resource) { - addPlatformScope(scopes, "docker") + add("docker") } if resource.Docker != nil { hostSourceID := strings.TrimSpace(resource.Docker.HostSourceID) if strings.HasPrefix(hostSourceID, proxmoxLXCDockerHostSourcePrefix) { - addPlatformScope(scopes, "proxmox-pve") - addPlatformScope(scopes, "docker") + add("proxmox-pve") + add("docker") } } @@ -74,30 +88,30 @@ func platformScopeForSource(source DataSource) string { } } -func addPlatformScopesForFacets(scopes map[string]struct{}, resource Resource) { +func addPlatformScopesForFacets(add func(string), resource Resource) { if resource.Agent != nil { - addPlatformScope(scopes, "agent") + add("agent") } if resource.TrueNAS != nil { - addPlatformScope(scopes, "truenas") + add("truenas") } if resource.Proxmox != nil { - addPlatformScope(scopes, "proxmox-pve") + add("proxmox-pve") } if resource.PBS != nil { - addPlatformScope(scopes, "proxmox-pbs") + add("proxmox-pbs") } if resource.PMG != nil { - addPlatformScope(scopes, "proxmox-pmg") + add("proxmox-pmg") } if resource.Kubernetes != nil { - addPlatformScope(scopes, "kubernetes") + add("kubernetes") } if resource.VMware != nil { - addPlatformScope(scopes, "vmware-vsphere") + add("vmware-vsphere") } if len(AvailabilityChecksForResource(resource)) > 0 || CanonicalResourceType(resource.Type) == ResourceTypeNetworkEndpoint { - addPlatformScope(scopes, "availability") + add("availability") } } @@ -111,29 +125,22 @@ func shouldAddDockerPlatformScope(resource Resource) bool { return true } -func addPlatformScope(scopes map[string]struct{}, scope string) { - scope = strings.ToLower(strings.TrimSpace(scope)) - if scope == "" { - return - } - scopes[scope] = struct{}{} -} - -func orderPlatformScopes(scopes map[string]struct{}) []string { +// orderPlatformScopes consumes the deduplicated slice, placing known scopes in +// canonical order and sorting any remaining scopes alphabetically in place. +func orderPlatformScopes(scopes []string) []string { if len(scopes) == 0 { return nil } - out := make([]string, 0, len(scopes)) - for _, scope := range canonicalPlatformScopeOrder { - if _, ok := scopes[scope]; ok { - out = append(out, scope) - delete(scopes, scope) + knownCount := 0 + for _, canonical := range canonicalPlatformScopeOrder { + for i := knownCount; i < len(scopes); i++ { + if scopes[i] == canonical { + scopes[knownCount], scopes[i] = scopes[i], scopes[knownCount] + knownCount++ + break + } } } - unknownStart := len(out) - for scope := range scopes { - out = append(out, scope) - } - sort.Strings(out[unknownStart:]) - return out + sort.Strings(scopes[knownCount:]) + return scopes } diff --git a/internal/unifiedresources/platform_scopes_test.go b/internal/unifiedresources/platform_scopes_test.go index ca03c2cbfc..c60eb012e5 100644 --- a/internal/unifiedresources/platform_scopes_test.go +++ b/internal/unifiedresources/platform_scopes_test.go @@ -54,6 +54,32 @@ func TestRefreshPlatformScopes_TrueNASAppContainerKeepsOwningPlatform(t *testing assertStringSliceEqual(t, resource.PlatformScopes, []string{"truenas"}) } +func TestRefreshPlatformScopesAllSourceCombinations(t *testing.T) { + // Include every membership combination, reverse input ordering, duplicates + // and unknown sources: changing the membership representation must not + // change page admission or canonical ordering. + sources := []DataSource{SourceAgent, SourceTrueNAS, SourceProxmox, SourcePBS, SourcePMG, SourceDocker, SourceK8s, SourceVMware, SourceAvailability} + for mask := 0; mask < 1<= 0; i-- { + if mask&(1< restricted > sensitive precedence independently + // of tag order, and normalising each tag only once. + tagSensitivity := ResourceSensitivityInternal + for _, tag := range resource.Tags { + switch strings.ToLower(strings.TrimSpace(tag)) { + case "public": + return ResourceSensitivityPublic + case "restricted", "customer-data", "customer_data", "pii", "phi", "pci", "regulated", "secret", "secrets": + tagSensitivity = ResourceSensitivityRestricted + case "sensitive", "backup", "mail", "storage", "database", "dataset": + if tagSensitivity != ResourceSensitivityRestricted { + tagSensitivity = ResourceSensitivitySensitive + } + } + } + if tagSensitivity != ResourceSensitivityInternal { + return tagSensitivity } if resource.PMG != nil { @@ -367,18 +373,6 @@ func resourceSummaryType(resource Resource) string { } } -func normalizedTagSet(tags []string) map[string]bool { - out := make(map[string]bool, len(tags)) - for _, tag := range tags { - trimmed := strings.ToLower(strings.TrimSpace(tag)) - if trimmed == "" { - continue - } - out[trimmed] = true - } - return out -} - func resourceHasHostname(resource Resource) bool { if canonicalHostname(resource) != "" { return true diff --git a/internal/unifiedresources/policy_metadata_test.go b/internal/unifiedresources/policy_metadata_test.go index 030a3d5326..459002e787 100644 --- a/internal/unifiedresources/policy_metadata_test.go +++ b/internal/unifiedresources/policy_metadata_test.go @@ -5,6 +5,33 @@ import ( "testing" ) +func TestClassifyResourceSensitivityTagAliasesAndPrecedence(t *testing.T) { + for _, group := range []struct { + tags []string + want ResourceSensitivity + }{ + {[]string{"restricted", "customer-data", "customer_data", "pii", "phi", "pci", "regulated", "secret", "secrets"}, ResourceSensitivityRestricted}, + {[]string{"sensitive", "backup", "mail", "storage", "database", "dataset"}, ResourceSensitivitySensitive}, + } { + for _, tag := range group.tags { + t.Run(tag, func(t *testing.T) { + normalised := " " + strings.ToUpper(tag) + " " + for _, tags := range [][]string{{normalised, "unknown", normalised}, {"unknown", normalised}} { + if got := classifyResourceSensitivity(Resource{Type: ResourceTypeVM, Tags: tags}); got != group.want { + t.Fatalf("tags %v sensitivity = %q, want %q", tags, got, group.want) + } + } + for _, tags := range [][]string{{" PUBLIC ", normalised}, {normalised, "public"}} { + // Explicit public wins even over secret-type defaults, as before. + if got := classifyResourceSensitivity(Resource{Type: ResourceTypeK8sSecret, Tags: tags}); got != ResourceSensitivityPublic { + t.Fatalf("public precedence changed for %v: %q", tags, got) + } + } + }) + } + } +} + func TestRefreshPolicyMetadata_ClassifiesRestrictedResources(t *testing.T) { resource := Resource{ ID: "vm-100", diff --git a/internal/websocket/state_delta.go b/internal/websocket/state_delta.go index 4cc37294dc..d62937d894 100644 --- a/internal/websocket/state_delta.go +++ b/internal/websocket/state_delta.go @@ -56,23 +56,31 @@ func extractKeyedEntries( if err := json.Unmarshal(encoded, &entries); err != nil { return nil, nil, fmt.Errorf("decode state %s: %w", field, err) } - byID := make(map[string]json.RawMessage) + // Decode all identities in one pass rather than allocating a decoder and + // throwaway struct for every entry. Keys must come from the encoded entries, + // not hints from the source value: checking one hinted ID cannot establish + // the identity of the rest, especially if a marshaler changes their order. + identities := make([]struct { + ID string `json:"id"` + }, len(entries)) + if err := json.Unmarshal(encoded, &identities); err != nil { + return nil, nil, fmt.Errorf("decode state %s identity: %w", field, err) + } + + byID := make(map[string]json.RawMessage, len(entries)) order := make([]string, 0, len(entries)) - for _, entry := range entries { - var identity struct { - ID string `json:"id"` - } - if err := json.Unmarshal(entry, &identity); err != nil { - return nil, nil, fmt.Errorf("decode state %s identity: %w", field, err) - } - if identity.ID == "" { + for i, entry := range entries { + id := identities[i].ID + if id == "" { return nil, nil, fmt.Errorf("state %s entry is missing id", field) } - if _, exists := byID[identity.ID]; exists { - return nil, nil, fmt.Errorf("state %s id %q is duplicated", field, identity.ID) + if _, exists := byID[id]; exists { + return nil, nil, fmt.Errorf("state %s id %q is duplicated", field, id) } - byID[identity.ID] = append(json.RawMessage(nil), entry...) - order = append(order, identity.ID) + // RawMessage.UnmarshalJSON already copied each entry into its own buffer. + // Retaining that buffer is safe; copying it a second time is redundant. + byID[id] = entry + order = append(order, id) } return byID, order, nil } diff --git a/internal/websocket/state_delta_test.go b/internal/websocket/state_delta_test.go index 33dddcf112..29d30408b8 100644 --- a/internal/websocket/state_delta_test.go +++ b/internal/websocket/state_delta_test.go @@ -1,13 +1,153 @@ package websocket import ( + "bytes" "encoding/json" + "fmt" + "reflect" "strings" "testing" "github.com/rcourtman/pulse-go-rewrite/internal/models" ) +// Keep the previous per-entry decoder as an independent compatibility oracle. +// This includes encoding/json's escaped, case-insensitive and duplicate-key +// handling instead of assuming the first JSON field is the authoritative ID. +func extractKeyedEntriesIndividually(encoded json.RawMessage, field string) (map[string]json.RawMessage, []string, error) { + var entries []json.RawMessage + if err := json.Unmarshal(encoded, &entries); err != nil { + return nil, nil, err + } + byID := make(map[string]json.RawMessage) + order := make([]string, 0, len(entries)) + for _, entry := range entries { + var identity struct { + ID string `json:"id"` + } + if err := json.Unmarshal(entry, &identity); err != nil { + return nil, nil, err + } + if identity.ID == "" { + return nil, nil, fmt.Errorf("state %s entry is missing id", field) + } + if _, exists := byID[identity.ID]; exists { + return nil, nil, fmt.Errorf("state %s id %q is duplicated", field, identity.ID) + } + byID[identity.ID] = append(json.RawMessage(nil), entry...) + order = append(order, identity.ID) + } + return byID, order, nil +} + +func assertKeyedEntriesMatchPerEntryDecoding(t *testing.T, encoded []byte) { + t.Helper() + wantEntries, wantOrder, wantErr := extractKeyedEntriesIndividually(encoded, "resource") + gotEntries, gotOrder, gotErr := extractKeyedEntries(encoded, "resource") + if (wantErr == nil) != (gotErr == nil) { + t.Fatalf("error changed: individual=%v batch=%v; JSON=%q", wantErr, gotErr, encoded) + } + if wantErr == nil && (!reflect.DeepEqual(gotEntries, wantEntries) || !reflect.DeepEqual(gotOrder, wantOrder)) { + t.Fatalf("identity/order/payload changed: individual=%v/%v batch=%v/%v; JSON=%q", wantEntries, wantOrder, gotEntries, gotOrder, encoded) + } +} + +func FuzzExtractKeyedEntriesMatchesPerEntryDecoding(f *testing.F) { + for _, encoded := range []string{ + `null`, `[]`, `[{"id":"r-1"},{"id":"r-2"}]`, + `[{"name":"before-id","id":"ๅฐพ\\\"\n\u0026","nested":{"id":"not-the-key"}}]`, + `[{"id":"r-1"},{"id":"old","id":"r-3"}]`, + `[{"ID":"case-insensitive"}]`, `[{"i\u0064":"escaped-key"}]`, + `[{"id":"r-1"},{"id":"r-1"}]`, `[{"id":"r-1"},{}]`, + `[null]`, `[{"id":null}]`, `[{"id":42}]`, `[42]`, `{}`, `[{`, + } { + f.Add([]byte(encoded)) + } + f.Fuzz(assertKeyedEntriesMatchPerEntryDecoding) +} + +func TestExtractKeyedEntriesOwnsEncodedBuffers(t *testing.T) { + encoded := []byte(`[{"id":"r-1","name":"alpha"},{"id":"r-2","name":"beta"}]`) + entries, order, err := extractKeyedEntries(encoded, "resource") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(order, []string{"r-1", "r-2"}) { + t.Fatalf("order = %v", order) + } + for i := range encoded { + encoded[i] = 'x' + } + if !bytes.Equal(entries["r-1"], []byte(`{"id":"r-1","name":"alpha"}`)) || + !bytes.Equal(entries["r-2"], []byte(`{"id":"r-2","name":"beta"}`)) { + t.Fatal("retained entries alias the original encoded buffer") + } + entries["r-1"][0] = 'x' + if entries["r-2"][0] != '{' { + t.Fatal("separate keyed entries alias each other") + } +} + +// A custom marshaler may preserve the first ID while replacing/reordering the +// tail. The encoded IDs, not a sampled check of source IDs, own delta identity. +type encodedStateFixture struct{ encoded json.RawMessage } + +func (s encodedStateFixture) MarshalJSON() ([]byte, error) { return s.encoded, nil } + +func TestSnapshotDeltaUsesEveryEncodedTailIdentity(t *testing.T) { + previous, err := buildClientStateSnapshot(encodedStateFixture{json.RawMessage(`{ + "resources":[{"id":"r-1"},{"id":"r-2"}], + "connectedInfrastructure":[{"id":"i-1"},{"id":"i-2"}], + "activeAlerts":[{"id":"a-1"},{"id":"a-2"}] + }`)}) + if err != nil { + t.Fatal(err) + } + current, err := buildClientStateSnapshot(encodedStateFixture{json.RawMessage(`{ + "resources":[{"id":"r-1"},{"id":"r-3"}], + "connectedInfrastructure":[{"id":"i-1"},{"id":"i-3"}], + "activeAlerts":[{"id":"a-1"},{"id":"a-3"}] + }`)}) + if err != nil { + t.Fatal(err) + } + delta, err := buildClientStateDelta(previous, current) + if err != nil { + t.Fatal(err) + } + for field, prefix := range map[string]string{ + resourceDeltaField: "r", infrastructureDeltaField: "i", activeAlertsDeltaField: "a", + } { + payload, ok := delta[field].(resourceDeltaPayload) + if !ok || !reflect.DeepEqual(payload.Removed, []string{prefix + "-2"}) || + !reflect.DeepEqual(payload.Order, []string{prefix + "-1", prefix + "-3"}) || + len(payload.Upserts) != 1 || string(payload.Upserts[0]) != `{"id":"`+prefix+`-3"}` { + t.Fatalf("%s: wrong encoded tail identity: %#v", field, delta[field]) + } + } +} + +func TestFrontendSnapshotValuePointerAndGenericEncodingAgree(t *testing.T) { + state := benchmarkFrontendState(100) + state.Resources[99].ID = "ๅฐพ<&\"\\\n" + state.ConnectedInfrastructure = []models.ConnectedInfrastructureItemFrontend{{ID: "i-1"}, {ID: "i-2"}} + state.ActiveAlerts = []models.Alert{{ID: "a-1"}, {ID: "a-2"}} + encoded, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + want, err := buildClientStateSnapshot(encodedStateFixture{encoded}) + if err != nil { + t.Fatal(err) + } + for _, value := range []interface{}{state, &state} { + got, err := buildClientStateSnapshot(value) + if err != nil || !reflect.DeepEqual(got, want) { + t.Fatalf("%T snapshot differs from its encoding: %v", value, err) + } + } +} + func TestClientStateDeltaOmitsUnchangedResourcePayload(t *testing.T) { previousState := models.EmptyStateFrontend() previousState.LastUpdate = 100 diff --git a/pkg/server/service_health.go b/pkg/server/service_health.go index 46869c21e6..b0518f7754 100644 --- a/pkg/server/service_health.go +++ b/pkg/server/service_health.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "encoding/json" + "errors" "fmt" "io" "net" @@ -18,6 +19,7 @@ import ( const ( serviceHealthProbeTimeout = 5 * time.Second + serviceHealthRetryDelay = time.Second serviceHealthBodyLimit = 2 << 20 serviceHealthAssetLimit = 32 ) @@ -25,21 +27,18 @@ const ( var frontendAssetReferencePattern = regexp.MustCompile(`(?i)(?:src|href)\s*=\s*["'](/assets/[^"'#?]+(?:\?[^"'#]*)?)["']`) func newServiceHealthProbe(listener net.Listener, tlsEnabled bool) func() telemetry.ServiceHealthObservation { - baseURL, ok := localServiceHealthBaseURL(listener, tlsEnabled) - if !ok { + baseURLs := localServiceHealthBaseURLs(listener, tlsEnabled) + if len(baseURLs) == 0 { return func() telemetry.ServiceHealthObservation { - return telemetry.ServiceHealthObservation{ - Observed: true, - FailureCategory: telemetry.ServiceHealthFailureAPIConnectivity, - } + return unhealthyServiceObservation(telemetry.ServiceHealthFailureAPIConnectivity) } } transport := &http.Transport{} if tlsEnabled { - // The probe stays inside this process and connects only to the address - // already bound by listener. Certificate trust is a client-facing concern, - // while this probe verifies that Pulse can serve its own HTTPS handler. + // The probe connects only to the address already bound by listener + // (or loopback for a wildcard). Certificate trust is a client-facing + // concern; this verifies Pulse can serve its own HTTPS handler. transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec G402 } client := &http.Client{ @@ -50,66 +49,144 @@ func newServiceHealthProbe(listener net.Listener, tlsEnabled bool) func() teleme }, } + return serviceHealthProbe(baseURLs, client, serviceHealthProbeTimeout, serviceHealthRetryDelay) +} + +// Each observation gets at most two five-second attempts with one second for +// startup to settle between them. Telemetry invokes this in its background +// runner, never on the monitoring or HTTP serving path. +func serviceHealthProbe(baseURLs []string, client *http.Client, timeout, retryDelay time.Duration) func() telemetry.ServiceHealthObservation { return func() telemetry.ServiceHealthObservation { - ctx, cancel := context.WithTimeout(context.Background(), serviceHealthProbeTimeout) - defer cancel() - - apiBody, status, err := serviceHealthGET(ctx, client, baseURL+"/api/health") - if err != nil { - return unhealthyServiceObservation(telemetry.ServiceHealthFailureAPIConnectivity) - } - if status < http.StatusOK || status >= http.StatusMultipleChoices { - return unhealthyServiceObservation(telemetry.ServiceHealthFailureAPIStatus) - } - var health struct { - Status string `json:"status"` - } - if json.Unmarshal(apiBody, &health) != nil || health.Status != "healthy" { - return unhealthyServiceObservation(telemetry.ServiceHealthFailureAPIStatus) - } - - indexBody, status, err := serviceHealthGET(ctx, client, baseURL+"/") - if err != nil || status < http.StatusOK || status >= http.StatusMultipleChoices || - !strings.Contains(strings.ToLower(string(indexBody)), "= http.StatusMultipleChoices || len(body) == 0 { - return unhealthyServiceObservation(telemetry.ServiceHealthFailureFrontendAssets) + defer client.CloseIdleConnections() + var observation telemetry.ServiceHealthObservation + for attempt := 0; attempt < 2; attempt++ { + if attempt > 0 { + timer := time.NewTimer(retryDelay) + <-timer.C + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + observation = observeServiceHealth(ctx, client, baseURLs) + cancel() + if observation.Healthy { + return observation } } - - return telemetry.ServiceHealthObservation{Observed: true, Healthy: true} + return observation } } -func localServiceHealthBaseURL(listener net.Listener, tlsEnabled bool) (string, bool) { +func observeServiceHealth(ctx context.Context, client *http.Client, baseURLs []string) telemetry.ServiceHealthObservation { + baseURL, apiBody, status, err := serviceHealthAPI(ctx, client, baseURLs) + if err != nil { + category := telemetry.ServiceHealthFailureAPIConnectivity + if status != 0 { + category = telemetry.ServiceHealthFailureAPIStatus + } + return unhealthyServiceObservation(serviceHealthErrorCategory(err, category)) + } + if status < http.StatusOK || status >= http.StatusMultipleChoices { + return unhealthyServiceObservation(telemetry.ServiceHealthFailureAPIStatus) + } + var health struct { + Status string `json:"status"` + } + if json.Unmarshal(apiBody, &health) != nil || health.Status != "healthy" { + return unhealthyServiceObservation(telemetry.ServiceHealthFailureAPIStatus) + } + + indexBody, status, err := serviceHealthGET(ctx, client, baseURL+"/") + if err != nil { + return unhealthyServiceObservation(serviceHealthErrorCategory(err, telemetry.ServiceHealthFailureUIStatus)) + } + if status < http.StatusOK || status >= http.StatusMultipleChoices || + !strings.Contains(strings.ToLower(string(indexBody)), "= http.StatusMultipleChoices || len(body) == 0 { + return unhealthyServiceObservation(telemetry.ServiceHealthFailureFrontendAssets) + } + } + + return telemetry.ServiceHealthObservation{Observed: true, Healthy: true} +} + +// Only a transport failure tries the other loopback family. A response pins +// the entire API/UI/asset observation to that address, even when it is unhealthy. +// Divide the remaining API budget so a stalled family cannot starve the other. +func serviceHealthAPI(ctx context.Context, client *http.Client, baseURLs []string) (string, []byte, int, error) { + var lastErr, timeoutErr error + for i, baseURL := range baseURLs { + deadline, _ := ctx.Deadline() + candidateCtx, cancel := context.WithTimeout(ctx, time.Until(deadline)/time.Duration(len(baseURLs)-i)) + body, status, err := serviceHealthGET(candidateCtx, client, baseURL+"/api/health") + cancel() + if err == nil { + return baseURL, body, status, nil + } + if status != 0 { + // A response whose body fails or exceeds the limit is still an + // observation of this server, not an unavailable address family. + return baseURL, nil, status, err + } + lastErr = err + if serviceHealthErrorCategory(err, "") == telemetry.ServiceHealthFailureTimeout { + timeoutErr = err + } + } + if timeoutErr != nil { + return "", nil, 0, timeoutErr + } + return "", nil, 0, lastErr +} + +func serviceHealthErrorCategory(err error, fallback string) string { + var netErr net.Error + if errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &netErr) && netErr.Timeout()) { + return telemetry.ServiceHealthFailureTimeout + } + return fallback +} + +func localServiceHealthBaseURLs(listener net.Listener, tlsEnabled bool) []string { if listener == nil { - return "", false + return nil } tcpAddr, ok := listener.Addr().(*net.TCPAddr) - if !ok || tcpAddr.Port <= 0 { - return "", false + if !ok || tcpAddr.Port <= 0 || tcpAddr.Port > 65535 { + return nil } - ip := tcpAddr.IP - if ip == nil || ip.IsUnspecified() { - if ip != nil && ip.To4() == nil { - ip = net.IPv6loopback - } else { - ip = net.IPv4(127, 0, 0, 1) + hosts := []string{tcpAddr.IP.String()} + if tcpAddr.IP == nil || tcpAddr.IP.IsUnspecified() { + hosts = []string{"127.0.0.1"} + // An IPv6 wildcard can be dual-stack or IPv6-only. IPv4-first also + // works when IPv6 loopback is disabled. An IPv4-only wildcard must + // not inspect a different IPv6 listener that happens to share its port. + if tcpAddr.IP == nil || tcpAddr.IP.To4() == nil { + hosts = append(hosts, "::1") } + } else if tcpAddr.Zone != "" && tcpAddr.IP.To4() == nil { + hosts[0] += "%" + tcpAddr.Zone } scheme := "http" if tlsEnabled { scheme = "https" } - return fmt.Sprintf("%s://%s", scheme, net.JoinHostPort(ip.String(), fmt.Sprintf("%d", tcpAddr.Port))), true + baseURLs := make([]string, 0, len(hosts)) + for _, host := range hosts { + baseURL := url.URL{Scheme: scheme, Host: net.JoinHostPort(host, fmt.Sprintf("%d", tcpAddr.Port))} + baseURLs = append(baseURLs, baseURL.String()) + } + return baseURLs } func frontendAssetPaths(index []byte) []string { diff --git a/pkg/server/service_health_regression_test.go b/pkg/server/service_health_regression_test.go new file mode 100644 index 0000000000..1cf04937a2 --- /dev/null +++ b/pkg/server/service_health_regression_test.go @@ -0,0 +1,310 @@ +package server + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/telemetry" +) + +// Override only Addr, retaining the real listener for serving. This models an +// IPv6 wildcard reported on an install where IPv6 loopback is unavailable. +type serviceHealthAddrListener struct { + net.Listener + addr net.Addr +} + +func (l serviceHealthAddrListener) Addr() net.Addr { return l.addr } + +type serviceHealthRoundTripper func(*http.Request) (*http.Response, error) + +func (f serviceHealthRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func healthyServiceHealthResponse(r *http.Request) (*http.Response, error) { + body := "app()" + switch r.URL.Path { + case "/api/health": + body = `{"status":"healthy"}` + case "/": + body = `` + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header), Request: r}, nil +} + +func TestServiceHealthWildcardFamiliesAndExplicitAddresses(t *testing.T) { + for _, tc := range []struct { + name string + addr net.Addr + tls bool + want []string + }{ + {"IPv6 wildcard", &net.TCPAddr{IP: net.IPv6unspecified, Port: 7655}, false, []string{"http://127.0.0.1:7655", "http://[::1]:7655"}}, + {"unspecified family", &net.TCPAddr{Port: 7655}, false, []string{"http://127.0.0.1:7655", "http://[::1]:7655"}}, + {"IPv4 wildcard", &net.TCPAddr{IP: net.IPv4zero, Port: 7655}, false, []string{"http://127.0.0.1:7655"}}, + {"explicit IPv4", &net.TCPAddr{IP: net.ParseIP("192.0.2.1"), Port: 7655}, false, []string{"http://192.0.2.1:7655"}}, + {"explicit IPv6", &net.TCPAddr{IP: net.IPv6loopback, Port: 7655}, true, []string{"https://[::1]:7655"}}, + {"explicit scoped IPv6", &net.TCPAddr{IP: net.ParseIP("fe80::1"), Zone: "eth0", Port: 7655}, true, []string{"https://[fe80::1%25eth0]:7655"}}, + {"not TCP", &net.UnixAddr{Name: "/tmp/unused", Net: "unix"}, false, nil}, + {"no port", &net.TCPAddr{IP: net.IPv4zero}, false, nil}, + {"invalid port", &net.TCPAddr{IP: net.IPv4zero, Port: 65536}, false, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + got := localServiceHealthBaseURLs(serviceHealthAddrListener{addr: tc.addr}, tc.tls) + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("targets = %#v, want %#v", got, tc.want) + } + }) + } + if got := localServiceHealthBaseURLs(nil, false); got != nil { + t.Fatalf("nil listener targets = %#v", got) + } +} + +func TestServiceHealthIPv4SurvivesUnavailableIPv6(t *testing.T) { + for _, failure := range []error{errors.New("IPv6 disabled"), errors.New("IPv6 unreachable"), context.DeadlineExceeded} { + client := &http.Client{Transport: serviceHealthRoundTripper(func(r *http.Request) (*http.Response, error) { + if r.URL.Hostname() == "::1" { + return nil, failure + } + if r.URL.Hostname() != "127.0.0.1" { + t.Fatalf("non-loopback request: %s", r.URL) + } + return healthyServiceHealthResponse(r) + })} + // Red control: the former IPv6-only target cannot observe this healthy install. + old := serviceHealthProbe([]string{"http://[::1]:7655"}, client, time.Second, 0)() + if old.Healthy || !old.Observed { + t.Fatalf("IPv6-only control = %#v", old) + } + targets := localServiceHealthBaseURLs(serviceHealthAddrListener{addr: &net.TCPAddr{IP: net.IPv6unspecified, Port: 7655}}, false) + got := serviceHealthProbe(targets, client, time.Second, 0)() + if !got.Observed || !got.Healthy || got.FailureCategory != "" { + t.Fatalf("wildcard with available IPv4 = %#v", got) + } + } +} + +func TestServiceHealthWildcardOnRealIPv4AndIPv6OnlyListeners(t *testing.T) { + for _, network := range []string{"tcp4", "tcp6"} { + t.Run(network, func(t *testing.T) { + address := "127.0.0.1:0" + if network == "tcp6" { + address = "[::1]:0" + } + listener, err := net.Listen(network, address) + if err != nil { + t.Fatalf("required %s loopback listen: %v", network, err) + } + server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + response, _ := healthyServiceHealthResponse(r) + defer response.Body.Close() + _, _ = io.Copy(w, response.Body) + }), ReadHeaderTimeout: time.Second} + done := make(chan struct{}) + go func() { defer close(done); _ = server.Serve(listener) }() + t.Cleanup(func() { _ = server.Close(); <-done }) + bound := listener.Addr().(*net.TCPAddr) + wildcard := serviceHealthAddrListener{Listener: listener, addr: &net.TCPAddr{IP: net.IPv6unspecified, Port: bound.Port}} + got := newServiceHealthProbe(wildcard, false)() + if !got.Observed || !got.Healthy || got.FailureCategory != "" { + t.Fatalf("real %s server through wildcard = %#v", network, got) + } + }) + } +} + +func TestServiceHealthStalledFamilyCannotStarveIPv6(t *testing.T) { + var hosts []string + client := &http.Client{Transport: serviceHealthRoundTripper(func(r *http.Request) (*http.Response, error) { + hosts = append(hosts, r.URL.Hostname()) + if r.URL.Hostname() == "127.0.0.1" { + <-r.Context().Done() + return nil, r.Context().Err() + } + if r.Context().Err() != nil { + t.Fatalf("IPv6 has no reserved budget: %v", r.Context().Err()) + } + return healthyServiceHealthResponse(r) + })} + got := serviceHealthProbe([]string{"http://127.0.0.1:7655", "http://[::1]:7655"}, client, 200*time.Millisecond, 0)() + if !got.Healthy || !reflect.DeepEqual(hosts, []string{"127.0.0.1", "::1", "::1", "::1"}) { + t.Fatalf("observation = %#v, requested hosts = %#v", got, hosts) + } +} + +func TestServiceHealthSlowStartupRetriesOnceAndRecovers(t *testing.T) { + var apiCalls int + var firstDeadline time.Time + client := &http.Client{Transport: serviceHealthRoundTripper(func(r *http.Request) (*http.Response, error) { + if r.URL.Path == "/api/health" { + apiCalls++ + deadline, ok := r.Context().Deadline() + if !ok { + t.Fatal("unbounded probe") + } + if apiCalls == 1 { + firstDeadline = deadline + <-r.Context().Done() + return nil, r.Context().Err() + } + if !deadline.After(firstDeadline) || r.Context().Err() != nil { + t.Fatal("retry reused exhausted startup budget") + } + } + return healthyServiceHealthResponse(r) + })} + got := serviceHealthProbe([]string{"http://127.0.0.1:7655"}, client, 50*time.Millisecond, time.Millisecond)() + if !got.Healthy || apiCalls != 2 { + t.Fatalf("observation = %#v, API calls = %d", got, apiCalls) + } +} + +func TestServiceHealthFinalTimeoutAtEveryStageIsBounded(t *testing.T) { + for _, stage := range []string{"/api/health", "/", "/assets/app.js"} { + t.Run(stage, func(t *testing.T) { + apiCalls, timeouts := 0, 0 + client := &http.Client{Transport: serviceHealthRoundTripper(func(r *http.Request) (*http.Response, error) { + if r.URL.Path == "/api/health" { + apiCalls++ + } + if r.URL.Path == stage { + timeouts++ + if _, ok := r.Context().Deadline(); !ok { + t.Fatal("unbounded request") + } + <-r.Context().Done() + return nil, r.Context().Err() + } + return healthyServiceHealthResponse(r) + })} + got := serviceHealthProbe([]string{"http://127.0.0.1:7655"}, client, 20*time.Millisecond, 0)() + if !got.Observed || got.Healthy || got.FailureCategory != telemetry.ServiceHealthFailureTimeout || apiCalls != 2 || timeouts != 2 { + t.Fatalf("observation = %#v, API calls/timeouts = %d/%d", got, apiCalls, timeouts) + } + }) + } +} + +func TestServiceHealthDoesNotHideGenuineFailuresOnAnotherFamily(t *testing.T) { + for _, tc := range []struct { + name, path, body, want string + status int + err error + }{ + {name: "connectivity", path: "/api/health", err: errors.New("refused"), want: telemetry.ServiceHealthFailureAPIConnectivity}, + {name: "API status", path: "/api/health", status: 503, body: `{"status":"unhealthy"}`, want: telemetry.ServiceHealthFailureAPIStatus}, + {name: "API invalid JSON", path: "/api/health", status: 200, body: "invalid", want: telemetry.ServiceHealthFailureAPIStatus}, + {name: "API unhealthy body", path: "/api/health", status: 200, body: `{"status":"unhealthy"}`, want: telemetry.ServiceHealthFailureAPIStatus}, + {name: "API oversized body", path: "/api/health", status: 200, body: strings.Repeat("x", serviceHealthBodyLimit+1), want: telemetry.ServiceHealthFailureAPIStatus}, + {name: "UI status", path: "/", status: 404, want: telemetry.ServiceHealthFailureUIStatus}, + {name: "UI invalid HTML", path: "/", status: 200, body: "not HTML", want: telemetry.ServiceHealthFailureUIStatus}, + {name: "UI oversized body", path: "/", status: 200, body: strings.Repeat("x", serviceHealthBodyLimit+1), want: telemetry.ServiceHealthFailureUIStatus}, + {name: "missing asset references", path: "/", status: 200, body: "", want: telemetry.ServiceHealthFailureFrontendAssets}, + {name: "asset status", path: "/assets/app.js", status: 404, want: telemetry.ServiceHealthFailureFrontendAssets}, + {name: "asset empty", path: "/assets/app.js", status: 200, want: telemetry.ServiceHealthFailureFrontendAssets}, + {name: "asset oversized", path: "/assets/app.js", status: 200, body: strings.Repeat("x", serviceHealthBodyLimit+1), want: telemetry.ServiceHealthFailureFrontendAssets}, + } { + t.Run(tc.name, func(t *testing.T) { + apiCalls, ipv6Calls := 0, 0 + client := &http.Client{Transport: serviceHealthRoundTripper(func(r *http.Request) (*http.Response, error) { + if r.URL.Path == "/api/health" { + apiCalls++ + } + if r.URL.Hostname() == "::1" { + ipv6Calls++ + if tc.err == nil { + t.Fatal("response failure switched listener family") + } + } + if r.URL.Path == tc.path { + if tc.err != nil { + return nil, tc.err + } + return &http.Response{StatusCode: tc.status, Body: io.NopCloser(strings.NewReader(tc.body)), Header: make(http.Header)}, nil + } + return healthyServiceHealthResponse(r) + })} + got := serviceHealthProbe([]string{"http://127.0.0.1:7655", "http://[::1]:7655"}, client, time.Second, 0)() + wantAPICalls := 2 + if tc.err != nil { + wantAPICalls = 4 + } + if !got.Observed || got.Healthy || got.FailureCategory != tc.want || apiCalls != wantAPICalls { + t.Fatalf("observation = %#v, API calls = %d, IPv6 calls = %d", got, apiCalls, ipv6Calls) + } + }) + } +} + +func TestServiceHealthHTTPSAndRedirectRefusal(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + response, _ := healthyServiceHealthResponse(r) + defer response.Body.Close() + _, _ = io.Copy(w, response.Body) + })) + t.Cleanup(server.Close) + if got := newServiceHealthProbe(server.Listener, true)(); !got.Healthy { + t.Fatalf("HTTPS service = %#v", got) + } + + var redirected atomic.Int32 + external := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { redirected.Add(1) })) + t.Cleanup(external.Close) + for _, stage := range []string{"/api/health", "/", "/assets/app.js"} { + t.Run(stage, func(t *testing.T) { + local := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == stage { + http.Redirect(w, r, external.URL, http.StatusFound) + return + } + response, _ := healthyServiceHealthResponse(r) + defer response.Body.Close() + _, _ = io.Copy(w, response.Body) + })) + t.Cleanup(local.Close) + got := newServiceHealthProbe(local.Listener, false)() + if got.Healthy || redirected.Load() != 0 { + t.Fatalf("redirect followed/accepted: %#v, external requests = %d", got, redirected.Load()) + } + }) + } +} + +func TestServiceHealthConcurrentObservations(t *testing.T) { + probe := serviceHealthProbe([]string{"http://127.0.0.1:7655"}, &http.Client{Transport: serviceHealthRoundTripper(healthyServiceHealthResponse)}, time.Second, 0) + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if got := probe(); !got.Healthy { + t.Errorf("concurrent observation = %#v", got) + } + }() + } + wg.Wait() +} + +func TestFrontendAssetPathsEnforceExistingBounds(t *testing.T) { + var index strings.Builder + for i := 0; i < serviceHealthAssetLimit+10; i++ { + fmt.Fprintf(&index, ``, i) + } + if paths := frontendAssetPaths([]byte(index.String())); len(paths) != serviceHealthAssetLimit { + t.Fatalf("asset count = %d, want %d", len(paths), serviceHealthAssetLimit) + } + if paths := frontendAssetPaths([]byte(``)); !reflect.DeepEqual(paths, []string{"/assets/a.js"}) { + t.Fatalf("duplicate/external paths escaped: %#v", paths) + } +} diff --git a/pkg/server/service_health_test.go b/pkg/server/service_health_test.go index 7716ca13bd..80841a857b 100644 --- a/pkg/server/service_health_test.go +++ b/pkg/server/service_health_test.go @@ -105,9 +105,15 @@ func TestServiceHealthProbeCoversAPIUIAndFrontendAssets(t *testing.T) { } func TestServiceHealthProbeReportsConnectivityWithoutListener(t *testing.T) { - got := newServiceHealthProbe(nil, false)() - if !got.Observed || got.Healthy || got.FailureCategory != telemetry.ServiceHealthFailureAPIConnectivity { - t.Fatalf("service-health observation = %#v", got) + for _, listener := range []net.Listener{ + nil, + serviceHealthAddrListener{addr: &net.UnixAddr{Name: "/tmp/unused", Net: "unix"}}, + serviceHealthAddrListener{addr: &net.TCPAddr{IP: net.IPv4zero}}, + } { + got := newServiceHealthProbe(listener, false)() + if !got.Observed || got.Healthy || got.FailureCategory != telemetry.ServiceHealthFailureAPIConnectivity { + t.Fatalf("invalid listener service-health observation = %#v", got) + } } } diff --git a/scripts/check-clone-identity-guidance.cjs b/scripts/check-clone-identity-guidance.cjs new file mode 100644 index 0000000000..d99f4539cc --- /dev/null +++ b/scripts/check-clone-identity-guidance.cjs @@ -0,0 +1,107 @@ +// Real shipped documents, production Docs renderer/router; no live host changes. +// Run from the assigned workspace root: +// pulse-worker-browser scripts/check-clone-identity-guidance.cjs +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { chromium } = require('playwright'); + +const root = process.cwd(); +const frontend = path.join(root, 'frontend-modern'); +const output = path.join(root, 'tmp', 'clone-identity-guidance'); +const port = 5198; +const observations = []; +const errors = []; + +function check(condition, message) { + observations.push({ passed: condition, message }); + if (!condition) throw new Error(message); +} + +async function sectionText(page, id) { + return page.locator(`#${id}`).evaluate((heading) => { + const parts = [heading.textContent]; + for (let node = heading.nextElementSibling; node; node = node.nextElementSibling) { + if (/^H[1-3]$/.test(node.tagName)) break; + parts.push(node.textContent); + } + return parts.join('\n').replace(/\s+/g, ' '); + }); +} + +(async () => { + fs.mkdirSync(output, { recursive: true }); + process.chdir(frontend); + const { createServer } = await import( + path.join(frontend, 'node_modules', 'vite', 'dist', 'node', 'index.js') + ); + const server = await createServer({ + root: frontend, + configFile: path.join(frontend, 'vite.config.ts'), + server: { host: '127.0.0.1', port, strictPort: true }, + }); + let browser; + let result; + try { + await server.listen(); + browser = await chromium.launch({ + headless: true, channel: 'chromium', args: ['--no-sandbox'], + }); + const page = await browser.newPage(); + page.on('pageerror', (error) => errors.push(error.message)); + page.on('response', (response) => { + if (response.status() >= 400) errors.push(`${response.status()} ${response.url()}`); + }); + // Authentication probing only; both Markdown documents are real assets. + await page.route(new RegExp(`^http://127\\.0\\.0\\.1:${port}/api/`), (route) => + route.fulfill({ + status: 200, contentType: 'application/json', + body: JSON.stringify({ hasAuthentication: false }), + }), + ); + + for (const [size, width, height] of [['desktop', 1440, 900], ['phone', 390, 844]]) { + await page.setViewportSize({ width, height }); + await page.goto(`http://127.0.0.1:${port}/browser-tests/docs-fragment-navigation.html?scenario=troubleshooting`); + await page.waitForFunction(() => document.activeElement?.id === 'recovery-mode'); + await page.locator('#docker-hosts-appearingdisappearing').scrollIntoViewIfNeeded(); + const shortGuide = await sectionText(page, 'docker-hosts-appearingdisappearing'); + check(shortGuide.includes('saved Pulse agent ID'), `${size}: saved clone identity is identified`); + check(shortGuide.includes('Do not delete the OS machine ID'), `${size}: no destructive OS reset is prescribed`); + const link = page.getByRole('link', { name: 'Clone identity recovery', exact: true }); + check((await link.getAttribute('href')) === '/docs/UNIFIED_AGENT#duplicate-agents', `${size}: link targets the shipped agent guide`); + check(!(await page.evaluate(() => document.documentElement.scrollWidth > document.documentElement.clientWidth)), `${size}: troubleshooting does not overflow`); + await page.screenshot({ path: path.join(output, `${size}-troubleshooting.png`) }); + + await link.focus(); + await page.keyboard.press('Enter'); + await page.waitForFunction(() => document.activeElement?.id === 'duplicate-agents'); + const cloneGuide = await sectionText(page, 'duplicate-agents'); + check(cloneGuide.includes('argument wins over the environment variable'), `${size}: argument precedence is explicit`); + check(cloneGuide.includes('PULSE_AGENT_ID_FILE'), `${size}: custom identity files are covered`); + check(cloneGuide.includes('Environment="PULSE_AGENT_ID=vm-clone-02"'), `${size}: persistent systemd configuration is rendered`); + check(cloneGuide.includes('fresh, distinct IDs') && cloneGuide.includes('does not split or recover historical'), `${size}: verification and history limitations are rendered`); + check(cloneGuide.includes('Do not upload connection.env, token files'), `${size}: credential-bearing evidence is excluded`); + check(!(await page.evaluate(() => document.documentElement.scrollWidth > document.documentElement.clientWidth)), `${size}: agent guide does not overflow`); + await page.locator('article pre').filter({ hasText: 'PULSE_AGENT_ID=vm-clone-02' }).scrollIntoViewIfNeeded(); + await page.screenshot({ path: path.join(output, `${size}-agent-configuration.png`) }); + } + check(errors.length === 0, 'no page errors or failed HTTP responses'); + result = { + result: 'passed', observations, errors, chromium: browser.version(), + playwright: require('playwright/package.json').version, + sourceAssets: ['TROUBLESHOOTING.md', 'UNIFIED_AGENT.md'].map((name) => ({ + path: `frontend-modern/public/docs/${name}`, + sha256: crypto.createHash('sha256').update(fs.readFileSync(path.join(frontend, 'public', 'docs', name))).digest('hex'), + })), + }; + } catch (error) { + result = { result: 'failed', observations, errors, error: error.message }; + } finally { + if (browser) await browser.close(); + await server.close(); + } + fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(result, null, 2)); + console.log(JSON.stringify(result, null, 2)); + process.exitCode = result.result === 'passed' ? 0 : 1; +})(); diff --git a/scripts/check-community-troubleshooting.cjs b/scripts/check-community-troubleshooting.cjs new file mode 100644 index 0000000000..447e8efac0 --- /dev/null +++ b/scripts/check-community-troubleshooting.cjs @@ -0,0 +1,88 @@ +// Render the real shipped troubleshooting guide with the production Docs page. +// Run from the assigned workspace root: +// pulse-worker-browser scripts/check-community-troubleshooting.cjs +const fs = require('node:fs'); +const path = require('node:path'); +const { chromium } = require('playwright'); + +const root = process.cwd(); +const frontend = path.join(root, 'frontend-modern'); +const output = path.join(root, 'tmp', 'community-troubleshooting'); +const port = 5198; +const observations = []; +const errors = []; + +function check(condition, message) { + observations.push({ passed: condition, message }); + if (!condition) throw new Error(message); +} + +(async () => { + fs.mkdirSync(output, { recursive: true }); + // Tailwind resolves its config/content globs from the frontend working dir. + process.chdir(frontend); + const { createServer } = await import( + path.join(frontend, 'node_modules', 'vite', 'dist', 'node', 'index.js') + ); + const server = await createServer({ + root: frontend, + configFile: path.join(frontend, 'vite.config.ts'), + server: { host: '127.0.0.1', port, strictPort: true }, + }); + let browser; + let result; + try { + await server.listen(); + browser = await chromium.launch({ + headless: true, channel: 'chromium', args: ['--no-sandbox'], + }); + const page = await browser.newPage(); + page.on('pageerror', (error) => errors.push(error.message)); + page.on('response', (response) => { + if (response.status() >= 400) errors.push(`${response.status()} ${response.url()}`); + }); + // Only app-runtime auth probes are mocked; the Markdown asset is real. + await page.route(new RegExp(`^http://127\\.0\\.0\\.1:${port}/api/`), (route) => + route.fulfill({ + status: 200, contentType: 'application/json', + body: JSON.stringify({ hasAuthentication: false }), + }), + ); + + for (const [size, width, height] of [['desktop', 1440, 900], ['phone', 390, 844]]) { + await page.setViewportSize({ width, height }); + await page.goto(`http://127.0.0.1:${port}/browser-tests/docs-fragment-navigation.html?scenario=troubleshooting`); + await page.waitForFunction(() => document.activeElement?.id === 'recovery-mode'); + const article = page.locator('article'); + const text = await article.innerText(); + check(text.includes('browser-bound recovery session'), `${size}: browser-bound recovery is explained`); + check(text.includes('A successful curl response does not unlock a separate browser'), `${size}: curl is not presented as a browser unlock`); + check(!(await article.locator('pre').allTextContents()).some((code) => code.includes('X-Recovery-Token') || code.includes('generate_token')), `${size}: no recovery credential command is offered`); + check(text.includes('Export for GitHub (sanitized)') && text.includes('Review before posting'), `${size}: safe reporting evidence is visible`); + check(text.includes('Do not repeat an update, outage or notification storm'), `${size}: unsafe reproduction is not required`); + check(!(await page.evaluate(() => document.documentElement.scrollWidth > document.documentElement.clientWidth)), `${size}: no horizontal page overflow`); + await page.screenshot({ path: path.join(output, `${size}-recovery.png`) }); + + // The supported reset link must work with the real router and heading ID. + await article.getByRole('link', { name: 'I forgot my password', exact: true }).click(); + await page.waitForFunction(() => document.activeElement?.id === 'i-forgot-my-password'); + check((await page.locator('#i-forgot-my-password').innerText()) === 'I forgot my password', `${size}: reset link reaches the deployment-specific steps`); + await article.getByText(size === 'phone' ? 'Review before posting' : 'Keep the original evidence', { exact: true }).scrollIntoViewIfNeeded(); + await page.screenshot({ path: path.join(output, `${size}-help.png`) }); + } + check(errors.length === 0, 'no page errors or failed HTTP responses'); + result = { + result: 'passed', observations, errors, chromium: browser.version(), + playwright: require('playwright/package.json').version, + sourceAsset: 'frontend-modern/public/docs/TROUBLESHOOTING.md', + }; + } catch (error) { + result = { result: 'failed', observations, errors, error: error.message }; + } finally { + if (browser) await browser.close(); + await server.close(); + } + fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(result, null, 2)); + console.log(JSON.stringify(result, null, 2)); + process.exitCode = result.result === 'passed' ? 0 : 1; +})(); diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index 1f8fca5a6a..86b842960f 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -4118,7 +4118,8 @@ func TestFrontendDependencySecurityAuditsAreRequired(t *testing.T) { ) // The gate itself must stay strict. An unreachable endpoint may be // retried, but no severity threshold may be introduced that lets a real - // advisory through, and any vulnerability total must still fail. + // advisory through. Any positive package, severity or total evidence + // must fail, even when the summary is missing or contradictory. runnerPath := repoFile("scripts", "npm-audit-retry.sh") runner, err := os.ReadFile(runnerPath) if err != nil { @@ -4130,8 +4131,13 @@ func TestFrontendDependencySecurityAuditsAreRequired(t *testing.T) { assertFileContainsAll(t, runnerPath, `NPM_AUDIT_REQUIRE_RESULT:-true`, `AUDIT_ARGS=("$@")`, - `if isinstance(vulns, dict) and "total" in vulns:`, - `print("vulnerable" if total else "clean")`, + `if type(value) is int and value >= 0:`, + `has_findings = isinstance(findings, dict) and bool(findings)`, + `if has_findings or any(count > 0 for count in counts.values()):`, + `print("vulnerable")`, + `isinstance(report, dict) and not report.get("error")`, + `len(counts) == len(count_names) and all(count == 0 for count in counts.values())`, + `and ("vulnerabilities" not in report or isinstance(findings, dict))`, ) // Retrying must be bounded by wall clock, not by attempt count alone. // npm's own fetch-timeout defaults to five minutes and it retries diff --git a/scripts/npm-audit-retry.sh b/scripts/npm-audit-retry.sh index 894fef3da6..95ddc1d2d9 100755 --- a/scripts/npm-audit-retry.sh +++ b/scripts/npm-audit-retry.sh @@ -111,7 +111,8 @@ run_audit() { return 0 } -# Classify one audit run. Prints a verdict word on stdout: +# Classify one audit run. Prints a verdict word, summary and (for a finding) +# allowlisted, JSON-escaped package/advisory details from that same response: # clean โ€” audit completed, no vulnerabilities # vulnerable โ€” audit completed, vulnerabilities present # unreachable โ€” npm could not get an answer from the advisory endpoint @@ -131,29 +132,75 @@ except ValueError: meta = report.get("metadata") if isinstance(report, dict) else None vulns = meta.get("vulnerabilities") if isinstance(meta, dict) else None -if isinstance(vulns, dict) and "total" in vulns: - # A usable advisory verdict takes precedence even if npm also includes a - # transport error. Never turn a real finding into a retryable outage. - total = vulns.get("total", 0) +findings = report.get("vulnerabilities") if isinstance(report, dict) else None +count_names = ("total", "critical", "high", "moderate", "low", "info") +# Counts are numbers, not arbitrary registry strings. In particular, Python +# bools are ints too, but neither false nor null is evidence of zero findings. +counts = {} +if isinstance(vulns, dict): + for name in count_names: + value = vulns.get(name) + if type(value) is int and value >= 0: + counts[name] = value +has_findings = isinstance(findings, dict) and bool(findings) +if has_findings or any(count > 0 for count in counts.values()): + # Any positive package or severity evidence wins, even if the summary is + # missing/inconsistent or npm also reports a transport error. Retrying must + # never replace an already observed finding with a later clean response. detail = " ".join( - f"{name}={vulns.get(name, 0)}" - for name in ("critical", "high", "moderate", "low", "info") + "{}={}".format(name, counts.get(name, "unknown")) for name in count_names ) - print("vulnerable" if total else "clean") - print(f"total={total} {detail}") + if has_findings: + detail += f" package_records={len(findings)}" + print("vulnerable") + print(detail) + # Do not query npm again for human-readable detail. A second request + # can hang outside the watchdog or return different advisory evidence. + # JSON encoding keeps registry text from becoming terminal escapes or + # GitHub workflow commands; transport errors and unknown fields stay out. + emitted = False + if isinstance(findings, dict): + for name, finding in sorted(findings.items()): + if not isinstance(finding, dict): + continue + projected = {"name": name} + for key in ("name", "severity", "isDirect", "range", "nodes"): + if key in finding: + projected[key] = finding[key] + via = finding.get("via") + if isinstance(via, list): + projected["via"] = [ + {key: advisory[key] for key in + ("source", "name", "dependency", "title", "url", "severity", "range") + if key in advisory} + if isinstance(advisory, dict) else advisory + for advisory in via if isinstance(advisory, (dict, str)) + ] + fix = finding.get("fixAvailable") + if isinstance(fix, bool): + projected["fixAvailable"] = fix + elif isinstance(fix, dict): + projected["fixAvailable"] = { + key: fix[key] for key in ("name", "version", "isSemVerMajor") + if key in fix + } + print("audit finding " + json.dumps(projected, ensure_ascii=True, sort_keys=True)) + emitted = True + if not emitted: + print("npm audit: package-level detail unavailable in captured verdict") sys.exit(0) -if isinstance(report, dict) and report.get("error"): - # npm reports an unusable endpoint as an error object, ENOAUDIT being the - # code it uses for 5xx, timeouts and offline runs alike. +if ( + isinstance(report, dict) and not report.get("error") + and len(counts) == len(count_names) and all(count == 0 for count in counts.values()) + and ("vulnerabilities" not in report or isinstance(findings, dict)) +): + print("clean") + print(" ".join(f"{name}=0" for name in count_names)) +else: + # A malformed/partial zero summary or endpoint error is not a clean verdict. + # With no positive finding, retain the existing bounded outage policy. print("unreachable") - sys.exit(0) - -if not isinstance(vulns, dict) or "total" not in vulns: - # No usable verdict in the payload: treat as unreachable rather than - # silently passing on a shape we do not understand. - print("unreachable") - sys.exit(0) ' } @@ -194,9 +241,9 @@ while [ "${attempt}" -le "${ATTEMPTS}" ]; do vulnerable) echo "npm audit (${SCOPE}): vulnerabilities present (${summary})" echo "::error::npm audit (${SCOPE}) found vulnerabilities: ${summary}" - # Re-run without --json so the log carries the human-readable advisory - # detail a maintainer needs to act on. - "${NPM_BIN}" audit "${AUDIT_ARGS[@]}" "${SCOPE_ARGS[@]}" || true + # Keep diagnostics bound to the conclusive response, with no extra + # registry request outside the attempt/total wall-clock limits. + printf '%s\n' "${verdict_output}" | tail -n +3 exit 1 ;; *) diff --git a/scripts/release_control/render_release_body.py b/scripts/release_control/render_release_body.py index ce091c9e32..40e63e72c1 100644 --- a/scripts/release_control/render_release_body.py +++ b/scripts/release_control/render_release_body.py @@ -360,6 +360,33 @@ def strip_validation_status_block(text: str) -> str: return _canonical_body(stripped.lstrip("\n")) +def validate_server_updater_guidance(text: str) -> None: + """Every executable updater example needs its own nearby ownership scope. + + A warning in a distant highlight does not make an unqualified install or + rollback command safe. This covers authored bullets as well as the generated + sections, without rewriting the author's instructions or rollback target. + """ + command = re.compile(r"(? 1 else "" + if not ownership.search(context): + raise ReleaseBodyIntegrityError( + "each /bin/update --version command must say nearby that it is " + "only for the helper installed by the Pulse server installer" + ) + + def validate_release_body_shape( body: str, version: str, @@ -412,6 +439,7 @@ def validate_release_body_shape( ) authored_notes = authored_prefix.split(visual_heading, 1)[0] validate_release_notes_shape(authored_notes, version) + validate_server_updater_guidance(clean_body) if expected_body is not None: expected_clean = strip_validation_status_block(expected_body) diff --git a/scripts/release_control/render_release_body_test.py b/scripts/release_control/render_release_body_test.py index 481302bacc..23b092e25a 100644 --- a/scripts/release_control/render_release_body_test.py +++ b/scripts/release_control/render_release_body_test.py @@ -608,6 +608,34 @@ Controls remain readable without horizontal scrolling. body, ) + def test_updater_ownership_scope_cannot_be_borrowed_from_a_distant_warning(self) -> None: + args = type("Args", (), {"version": "6.3.2", "rollback_target": "v6.3.1", + "rollback_command": "sudo /bin/update --version v6.3.1"})() + authored = "# Pulse v6.3.2 Release Notes\n\nSummary.\n\n## Before you upgrade\n\n" + install = render_release_body.build_installation_section("6.3.2") + rollback = render_release_body.build_rollback_section(args) + safe = authored + "Keep a backup.\n\n" + install + "\n\n" + rollback + render_release_body.validate_release_body_shape(safe, "6.3.2") + unsafe = ( + safe.replace("Use this CLI command only when `/bin/update` was installed by the Pulse server installer:", + "For systemd and Proxmox LXC installs, use:"), + safe.replace("For systemd and Proxmox LXC servers, use this command only when " + "`/bin/update` was installed by the Pulse server installer:", "Run:"), + safe.replace("Keep a backup.", "On systemd and Proxmox LXC installs, use " + "`sudo /bin/update --version v6.3.1`."), + safe.replace("Keep a backup.", "Use this CLI command only when `/bin/update` was installed " + "by the Pulse server installer.\n\nA separate unrelated bullet.\n\n" + "```bash\nsudo /bin/update --version v6.3.1\n```"), + ) + for body in unsafe: + with self.subTest(body=body), self.assertRaisesRegex( + render_release_body.ReleaseBodyIntegrityError, "each /bin/update"): + render_release_body.validate_release_body_shape(body, "6.3.2") + render_release_body.validate_server_updater_guidance( + "Use this command only when `/bin/update` was installed by\n" + "the Pulse server installer: `sudo /bin/update --version v6.3.1`." + ) + def test_flattened_release_notes_fail_closed(self) -> None: flattened = ( "# Pulse v6.1.0-rc.2 Release Notes" diff --git a/scripts/tests/test_api_auth_docs.py b/scripts/tests/test_api_auth_docs.py new file mode 100644 index 0000000000..0b9ab7adf7 --- /dev/null +++ b/scripts/tests/test_api_auth_docs.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Exercise the API guide's credential-file commands with synthetic secrets. + +The HTTP tests need guest-local loopback; run via pulse-worker-source-proof. +No real Pulse instance, credentials or external network is used. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, HTTPServer +import json +import os +from pathlib import Path +import re +import shutil +import stat +import subprocess +import tempfile +import threading +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +DOC = ROOT / "docs/API.md" +TEST_TOKEN = "synthetic-doc-test-token" + + +def auth_section() -> str: + return DOC.read_text(encoding="utf-8").split("## ๐Ÿ” Authentication\n", 1)[1].split("\n## ", 1)[0] + + +def commands() -> tuple[str, str]: + blocks = re.findall(r"```bash\n(.*?)```", auth_section(), re.DOTALL) + preparation = next(block for block in blocks if "umask 077" in block) + request = next(block for block in blocks if block.startswith("curl ")) + return preparation, request + + +@contextmanager +def recording_server(status: int = 200): + requests = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + requests.append((self.path, dict(self.headers))) + self.send_response(status) + self.end_headers() + self.wfile.write(b'{"fixture":true}\n') + + def log_message(self, *_args): + pass + + server = HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server.server_address[1], requests + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +class APIAuthDocsTest(unittest.TestCase): + def test_credentials_are_not_documented_as_command_arguments(self): + section = auth_section() + bash = "\n".join(re.findall(r"```bash\n(.*?)```", section, re.DOTALL)) + self.assertNotRegex(bash, r"(?:X-API-Token:|Authorization:|Bearer\s)") + self.assertIn('--header "@$HOME/.config/pulse/api-header"', bash) + self.assertIn("HTTPS", section) + self.assertIn("does not verify authentication", section) + self.assertIn("monitoring:read", section) + self.assertIn("Do not paste tokens", section) + + def test_shipped_copy_is_identical(self): + self.assertEqual(DOC.read_bytes(), (ROOT / "frontend-modern/public/docs/API.md").read_bytes()) + + def test_preparation_protects_new_and_existing_header_files(self): + preparation, _ = commands() + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + tools = home / "tools" + tools.mkdir() + # The editor must be interactive in the real recipe; this fixture + # verifies its filename without putting a synthetic token in argv. + editor = tools / "vi" + editor.write_text('#!/bin/sh\n[ "$#" = 1 ] && [ "$1" = "$HOME/.config/pulse/api-header" ]\n') + editor.chmod(0o700) + env = dict(os.environ, HOME=str(home), PATH=f"{tools}:{os.environ['PATH']}") + header = home / ".config/pulse/api-header" + for existing in (False, True): + if existing: + header.write_text(f"X-API-Token: {TEST_TOKEN}\n") + header.chmod(0o644) + subprocess.run(["bash", "-eu", "-c", preparation], env=env, check=True) + self.assertEqual(stat.S_IMODE(header.stat().st_mode), 0o600) + self.assertEqual(header.read_text(), f"X-API-Token: {TEST_TOKEN}\n" if existing else "") + + def run_documented_request(self, home: Path, header_text: str, port: int): + _, request = commands() + header = home / ".config/pulse/api-header" + header.parent.mkdir(parents=True, exist_ok=True) + header.write_text(header_text + "\n") + header.chmod(0o600) + real_curl = shutil.which("curl") + self.assertIsNotNone(real_curl, "curl is required to exercise the documented command") + tools = home / "tools" + tools.mkdir(exist_ok=True) + recorder = tools / "curl" + recorder.write_text( + "#!/usr/bin/env python3\nimport json, os, sys\n" + "from pathlib import Path\n" + "Path(os.environ['ARGV_RECEIPT']).write_text(json.dumps(sys.argv[1:]))\n" + "os.execv(os.environ['REAL_CURL'], [os.environ['REAL_CURL'], *sys.argv[1:]])\n" + ) + recorder.chmod(0o700) + receipt = home / "argv.json" + env = dict(os.environ, HOME=str(home), PATH=f"{tools}:{os.environ['PATH']}", + REAL_CURL=real_curl, ARGV_RECEIPT=str(receipt)) + request = request.replace("http://127.0.0.1:7655", f"http://127.0.0.1:{port}") + result = subprocess.run(["bash", "-eu", "-c", request], env=env, capture_output=True, timeout=10) + argv = json.loads(receipt.read_text()) + self.assertNotIn(TEST_TOKEN, " ".join(argv)) + self.assertIn("@" + str(header), argv) + self.assertNotIn(b"synthetic-doc-test-token", result.stdout + result.stderr) + return result + + def test_header_file_sends_each_supported_header_without_exposing_argv(self): + with tempfile.TemporaryDirectory() as temporary, recording_server() as (port, requests): + for key, value in (("X-API-Token", TEST_TOKEN), ("Authorization", "Bearer " + TEST_TOKEN)): + with self.subTest(header=key): + result = self.run_documented_request(Path(temporary), f"{key}: {value}", port) + self.assertEqual(result.returncode, 0, result.stderr.decode()) + self.assertEqual(requests[-1][0], "/api/state/summary") + self.assertEqual(requests[-1][1][key], value) + + def test_http_auth_errors_fail_instead_of_looking_successful(self): + for status in (401, 403): + with self.subTest(status=status), tempfile.TemporaryDirectory() as temporary: + with recording_server(status) as (port, _): + result = self.run_documented_request(Path(temporary), "X-API-Token: " + TEST_TOKEN, port) + self.assertEqual(result.returncode, 22, result.stderr.decode()) + self.assertIn(b'{"fixture":true}', result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_npm_audit_retry.py b/scripts/tests/test_npm_audit_retry.py index b9f5b05c28..667a4754ff 100755 --- a/scripts/tests/test_npm_audit_retry.py +++ b/scripts/tests/test_npm_audit_retry.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os from pathlib import Path import subprocess @@ -17,11 +18,15 @@ SCRIPT = ROOT / "scripts" / "npm-audit-retry.sh" class NpmAuditRetryTest(unittest.TestCase): - def run_check(self, mode: str, *arguments: str, require: str = "true"): + def run_check( + self, mode: str, *arguments: str, require: str = "true", report=None + ): with tempfile.TemporaryDirectory() as directory: fake_bin = Path(directory) count = fake_bin / "count" calls = fake_bin / "calls" + report_path = fake_bin / "report.json" + report_path.write_text(json.dumps(report), encoding="utf-8") count.write_text("0\n", encoding="utf-8") fake_npm = fake_bin / "npm" fake_npm.write_text( @@ -61,6 +66,20 @@ class NpmAuditRetryTest(unittest.TestCase): printf '%s\n' "$vulnerable_with_error" exit 1 ;; + captured-vulnerability) + if [ "$count" -eq 1 ]; then + cat "$FAKE_NPM_REPORT" + else + # A diagnostic re-query could return a different verdict. + printf '%s\n' "$clean" + fi + exit 1 + ;; + captured-report) + # Malformed reports must remain the same on retry. + cat "$FAKE_NPM_REPORT" + exit 1 + ;; garbage) echo 'not json' exit 1 @@ -78,6 +97,7 @@ class NpmAuditRetryTest(unittest.TestCase): "FAKE_NPM_CALLS": str(calls), "FAKE_NPM_COUNT": str(count), "FAKE_NPM_MODE": mode, + "FAKE_NPM_REPORT": str(report_path), "NPM_AUDIT_ATTEMPTS": "3", "NPM_AUDIT_CMD": str(fake_npm), "NPM_AUDIT_REQUIRE_RESULT": require, @@ -123,10 +143,7 @@ class NpmAuditRetryTest(unittest.TestCase): self.assertEqual(result.returncode, 1) self.assertEqual( calls, - [ - "audit --json", - "audit", - ], + ["audit --json"], ) self.assertIn("vulnerabilities present", result.stdout) self.assertNotIn("retrying", result.stdout) @@ -134,10 +151,194 @@ class NpmAuditRetryTest(unittest.TestCase): def test_vulnerability_verdict_precedes_a_transport_error(self) -> None: result, calls = self.run_check("vulnerability-with-error", "all") self.assertEqual(result.returncode, 1) - self.assertEqual(len(calls), 2) + self.assertEqual(calls, ["audit --json"]) self.assertIn("vulnerabilities present", result.stdout) self.assertNotIn("retrying", result.stdout) + def test_reports_the_original_advisory_without_an_unbounded_requery(self) -> None: + nodes = [ + "node_modules/@eslint/config-array/node_modules/brace-expansion", + "node_modules/@eslint/eslintrc/node_modules/brace-expansion", + "node_modules/brace-expansion", + "node_modules/eslint/node_modules/brace-expansion", + ] + report = { + "metadata": {"vulnerabilities": {"high": 1, "total": 1}}, + "vulnerabilities": { + "brace-expansion": { + "name": "brace-expansion", + "severity": "high", + "isDirect": False, + "range": "<=1.1.20 || 4.0.0 - 5.0.11", + "nodes": nodes, + "fixAvailable": True, + "via": [ + { + "source": 123456, + "name": "brace-expansion", + "title": "Quadratic-time expansion causes CPU denial of service", + "url": "https://github.com/advisories/GHSA-q2hr-2g5m-vwhr", + "severity": "high", + "range": "<=1.1.20", + "unrecognised": "must not be printed", + }, + "transitive-dependency", + ], + } + }, + "error": {"detail": "must not be printed"}, + } + result, calls = self.run_check( + "captured-vulnerability", "all", require="false", report=report + ) + self.assertEqual(result.returncode, 1) + self.assertEqual(calls, ["audit --json"]) + finding_lines = [ + line.removeprefix("audit finding ") + for line in result.stdout.splitlines() + if line.startswith("audit finding ") + ] + self.assertEqual(len(finding_lines), 1, result.stdout) + finding = json.loads(finding_lines[0]) + self.assertEqual(finding["nodes"], nodes) + self.assertIs(finding["fixAvailable"], True) + self.assertEqual(finding["name"], "brace-expansion") + self.assertEqual(finding["range"], "<=1.1.20 || 4.0.0 - 5.0.11") + expected_advisory = report["vulnerabilities"]["brace-expansion"]["via"][0] + self.assertEqual( + finding["via"][0], + {key: value for key, value in expected_advisory.items() if key != "unrecognised"}, + ) + self.assertEqual(finding["via"][1], "transitive-dependency") + self.assertNotIn("must not be printed", result.stdout) + self.assertNotIn("no vulnerabilities", result.stdout) + + def test_missing_advisory_detail_cannot_trigger_a_requery_or_clear_the_failure(self) -> None: + result, calls = self.run_check("vulnerability", "all") + self.assertEqual(result.returncode, 1) + self.assertEqual(calls, ["audit --json"]) + self.assertIn("package-level detail unavailable", result.stdout) + + def test_escapes_advisory_text_instead_of_emitting_workflow_commands(self) -> None: + title = "unsafe title\n::error::injected annotation\x1b[31m" + report = { + "metadata": {"vulnerabilities": {"high": 1, "total": 1}}, + "vulnerabilities": { + "dependency": {"name": "dependency", "via": [{"title": title}]} + }, + } + result, calls = self.run_check( + "captured-vulnerability", "production", "--package-lock-only", report=report + ) + self.assertEqual(result.returncode, 1) + self.assertEqual(calls, ["audit --json --package-lock-only --omit=dev"]) + finding_line = next( + (line for line in result.stdout.splitlines() if line.startswith("audit finding ")), + "audit finding {}", + ) + finding = json.loads(finding_line.removeprefix("audit finding ")) + self.assertEqual(finding.get("via"), [{"title": title}], result.stdout) + self.assertNotIn("\n::error::injected", result.stdout) + self.assertNotIn("\x1b", result.stdout) + + def test_package_findings_fail_even_when_summary_is_missing_or_zero(self) -> None: + for metadata in [None, {}, {"vulnerabilities": {"total": 0}}]: + for require in ["true", "false"]: + with self.subTest(metadata=metadata, require=require): + report = { + "vulnerabilities": { + "dependency": {"name": "dependency", "severity": "high"} + } + } + if metadata is not None: + report["metadata"] = metadata + result, calls = self.run_check( + "captured-vulnerability", "all", require=require, report=report + ) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertEqual(calls, ["audit --json"], result.stdout) + self.assertIn("vulnerabilities present", result.stdout) + self.assertIn("package_records=1", result.stdout) + self.assertNotIn("no vulnerabilities", result.stdout) + + def test_positive_severity_counts_fail_without_a_consistent_total(self) -> None: + for severity in ["info", "low", "moderate", "high", "critical"]: + for total in [None, 0]: + with self.subTest(severity=severity, total=total): + counts = {severity: 1} + if total is not None: + counts["total"] = total + report = { + "metadata": {"vulnerabilities": counts}, + "error": {"code": "ETIMEDOUT"}, + } + result, calls = self.run_check( + "captured-vulnerability", "production", + require="false", report=report, + ) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertEqual(calls, ["audit --json --omit=dev"], result.stdout) + self.assertIn("vulnerabilities present", result.stdout) + + def test_malformed_zero_verdicts_cannot_pass_as_clean(self) -> None: + zero_counts = dict.fromkeys( + ["info", "low", "moderate", "high", "critical", "total"], 0 + ) + reports = [ + {"metadata": {"vulnerabilities": {**zero_counts, "total": value}}} + for value in [None, False, -1, 0.0, "0", [], {}] + ] + reports.extend([ + {"metadata": {"vulnerabilities": {**zero_counts, "high": "0"}}}, + {"metadata": {"vulnerabilities": {**zero_counts, "high": None}}}, + {"metadata": {"vulnerabilities": {"total": 0}}}, + {"metadata": {"vulnerabilities": zero_counts}, "vulnerabilities": []}, + {"metadata": {"vulnerabilities": zero_counts}, "vulnerabilities": None}, + { + "metadata": {"vulnerabilities": zero_counts}, + "error": {"code": "ENOAUDIT"}, + }, + ]) + for report in reports: + for require in ["true", "false"]: + with self.subTest(report=report, require=require): + result, calls = self.run_check( + "captured-report", "all", require=require, report=report + ) + self.assertEqual( + result.returncode, 1 if require == "true" else 0, result.stdout + ) + self.assertEqual(calls, ["audit --json"] * 3, result.stdout) + self.assertNotIn("no vulnerabilities", result.stdout) + self.assertIn("could not reach", result.stdout) + if require == "false": + self.assertIn("::warning::", result.stdout) + + def test_summary_cannot_emit_untrusted_metadata_as_workflow_commands(self) -> None: + unsafe = "0\n::error::injected metadata\x1b[31m" + report = {"metadata": {"vulnerabilities": {"total": 1, "high": unsafe}}} + result, calls = self.run_check("captured-vulnerability", "all", report=report) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertEqual(calls, ["audit --json"]) + self.assertIn("vulnerabilities present", result.stdout) + self.assertIn("high=unknown", result.stdout) + self.assertNotIn("injected metadata", result.stdout) + self.assertNotIn("\x1b", result.stdout) + + def test_passes_a_complete_zero_summary_and_empty_package_map(self) -> None: + report = { + "metadata": { + "vulnerabilities": dict.fromkeys( + ["info", "low", "moderate", "high", "critical", "total"], 0 + ) + }, + "vulnerabilities": {}, + } + result, calls = self.run_check("captured-report", "all", report=report) + self.assertEqual(result.returncode, 0, result.stdout) + self.assertEqual(calls, ["audit --json"]) + self.assertIn("no vulnerabilities", result.stdout) + def test_persistent_outage_fails_when_a_result_is_required(self) -> None: result, calls = self.run_check("transient-failure", "all") self.assertEqual(result.returncode, 1) diff --git a/scripts/tests/test_telemetry_adoption_report.py b/scripts/tests/test_telemetry_adoption_report.py index 135465295d..66511111fa 100644 --- a/scripts/tests/test_telemetry_adoption_report.py +++ b/scripts/tests/test_telemetry_adoption_report.py @@ -595,6 +595,27 @@ class TelemetryAdoptionReportTest(unittest.TestCase): {"healthy": 1, "frontend_assets": 1}, ) + def test_target_release_service_health_keeps_timeouts_distinct_from_connectivity(self) -> None: + now = datetime(2026, 9, 30, 20, tzinfo=timezone.utc) + rows = { + category: { + "received_at": "2026-09-30 19:00:00", + "version": "6.4.6", + "service_health_observed": 1, + "service_health_healthy": 0, + "service_health_failure_category": category, + } + for category in ("timeout", "api_connectivity") + } + summary = report.summarize_target_release_service_health( + rows, {"6.4.6"}, "6.4.6", now=now + ) + self.assertEqual(summary["unhealthy_installs"], 2) + self.assertEqual( + {entry["category"]: entry["installs"] for entry in summary["failure_categories"]}, + {"timeout": 1, "api_connectivity": 1}, + ) + def test_target_release_followup_excludes_first_heartbeat_baselines_and_flags_rollbacks(self) -> None: now = datetime(2026, 8, 19, 12, tzinfo=timezone.utc) diff --git a/tests/integration/tests/63-pbs-history-targets.spec.ts b/tests/integration/tests/63-pbs-history-targets.spec.ts index 2c85e80166..2abc4c08c8 100644 --- a/tests/integration/tests/63-pbs-history-targets.spec.ts +++ b/tests/integration/tests/63-pbs-history-targets.spec.ts @@ -95,7 +95,8 @@ for (const width of [1280, 390]) { id: `pbs-${type}`, type: 'pbs', name: `backup-${type}`, displayName: `backup-${type}`, platformId: `backup-${type}`, platformType: 'proxmox-pbs', sources: ['pbs'], metricsTarget: { resourceType: 'agent', resourceId: `wrong-${type}` }, - pbs: { instanceId: `backup-${type}`, hostname: `backup-${type}`, datastores: [] }, + // Correlate with the machine name PBS reports, never its presentation label. + pbs: { instanceId: `backup-${type}`, nodeName: `backup-${type}`, datastores: [] }, })); const hosts = ['vm', 'agent'].map(type => ({ ...common, id: `host-${type}`, type, name: `backup-${type}`, displayName: `backup-${type}`, platformId: `host-${type}`,