@@ -314,7 +322,7 @@ export const AvailabilityFleetView: Component<{
setSelectedResource(undefined)}
+ onClose={() => setSelectedResourceId(undefined)}
/>
)}
diff --git a/frontend-modern/src/features/standalone/__tests__/AvailabilityFleetView.test.tsx b/frontend-modern/src/features/standalone/__tests__/AvailabilityFleetView.test.tsx
index b4f185890b..f31851a32f 100644
--- a/frontend-modern/src/features/standalone/__tests__/AvailabilityFleetView.test.tsx
+++ b/frontend-modern/src/features/standalone/__tests__/AvailabilityFleetView.test.tsx
@@ -1,12 +1,17 @@
import { cleanup, fireEvent, render, screen } from '@solidjs/testing-library';
import { afterEach, describe, expect, it, vi } from 'vitest';
+import { createSignal } from 'solid-js';
import type { AvailabilityHistoryTarget } from '@/api/availabilityHistory';
import type { Resource } from '@/types/resource';
import { AvailabilityFleetView } from '../AvailabilityFleetView';
vi.mock('@/components/Infrastructure/ResourceDetailDrawer', () => ({
ResourceDetailDrawer: (props: { resource: Resource; onClose?: () => void }) => (
-
+
@@ -92,6 +97,28 @@ const history = (index: number): AvailabilityHistoryTarget => ({
afterEach(cleanup);
describe('AvailabilityFleetView', () => {
+ it('keeps open details bound to the selected ID in the current fleet, not the clicked snapshot', () => {
+ const original = resource(1);
+ const [resources, setResources] = createSignal([original]);
+ render(() => (
+
+ ));
+ fireEvent.click(screen.getByRole('button', { name: 'Open details for Service 1' }));
+ setResources([{ ...original, status: 'offline' }]);
+ expect(screen.getByTestId('resource-detail-drawer')).toHaveAttribute(
+ 'data-resource-status',
+ 'offline',
+ );
+ setResources([]);
+ expect(screen.queryByTestId('resource-detail-drawer')).not.toBeInTheDocument();
+ setResources([original]);
+ expect(screen.queryByTestId('resource-detail-drawer')).not.toBeInTheDocument();
+ });
+
it('renders fifty keyboard-accessible attention tiles with non-color history labels', () => {
const resources = Array.from({ length: 50 }, (_, index) => resource(index));
const historyByTarget = new Map(
From 286c8a1f510134002c90ac919ce98273fb4f3e9e Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 00:45:25 +0100
Subject: [PATCH 02/86] Make write and CPU troubleshooting bounded and
evidence-preserving
Give existing help readers scoped process/container samples, honest units and unchanged-workload baselines. Exercise the exact copied recipes, including stopped services, missing counters and failed or empty container reads, without touching a database or service.
Contract-Neutral: Existing-guide documentation and recipe tests only; no runtime, API, persistence, access or retention change.
Change-source: pulse-maintainer
---
docs/TROUBLESHOOTING.md | 93 ++++++++
.../public/docs/TROUBLESHOOTING.md | 93 ++++++++
.../test_performance_troubleshooting_docs.py | 209 ++++++++++++++++++
3 files changed, 395 insertions(+)
create mode 100644 scripts/tests/test_performance_troubleshooting_docs.py
diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md
index 032d808db2..9b11c341cd 100644
--- a/docs/TROUBLESHOOTING.md
+++ b/docs/TROUBLESHOOTING.md
@@ -216,6 +216,99 @@ leave the original host unchanged. Follow
[Clone identity recovery](UNIFIED_AGENT.md#duplicate-agents) for the configuration
precedence, systemd example and checks after restart.
+#### Excessive CPU, writes or database growth
+
+First distinguish **Pulse server activity**, agent activity and total host or
+storage-device activity. A database's size is retained space, not the number of
+bytes written: repeated updates can produce high writes without growing the file.
+Keep the original running version, uptime, measurement time and workload before
+changing anything. A different binary can migrate persistent data; switching
+back to an older version does not make that data a clean older-version baseline.
+
+- For CPU, record the measuring tool, allocated CPUs or container quota and
+ whether its percentage represents one core or the whole allocation. A `ps`
+ `%CPU` value is an average since process start, not a recent sampling window.
+- Note fleet size, polling interval and whether Pulse dashboards are open. If
+ safe, close all Pulse tabs and compare another equal-length window, without
+ restarting the server or changing polling. A decrease narrows the workload;
+ it does not establish the cause or prove monitoring is healthy.
+- If the host is unresponsive or storage is nearly full, do not prolong the
+ failure to collect a benchmark. Keep existing observations and report the
+ interruption instead. Do not enable Debug or repeatedly export diagnostics
+ merely to measure performance; those actions add work and can change the result.
+
+For a responsive **Linux systemd / Proxmox LXC** install, the following reads
+two process-I/O samples, waiting 60 seconds between them. Run it inside the
+Pulse container for LXC, not on the Proxmox host. Substitute the actual service
+name (`pulse-backend` on some older installs). Use an account authorised to
+read the process counters; no service restart or database access is needed.
+
+```bash
+# systemd / Proxmox LXC: bounded process-write samples
+(
+ set -e
+ for sample in 1 2; do
+ date -u +'%Y-%m-%dT%H:%M:%SZ'
+ pid=$(systemctl show pulse --property=MainPID --value)
+ case "$pid" in
+ ''|0|*[!0-9]*) printf 'No running Pulse PID; sample unavailable.\n' >&2; exit 1 ;;
+ esac
+ TZ=UTC ps -p "$pid" -o pid=,lstart=
+ sudo awk '
+ $1 == "write_bytes:" || $1 == "cancelled_write_bytes:" { print; fields++ }
+ END { if (fields != 2) exit 1 }
+ ' "/proc/$pid/io"
+ if [ "$sample" -eq 1 ]; then sleep 60; fi
+ done
+)
+```
+
+Compare `write_bytes` only when both samples have the same PID and process
+start time, no restart occurred, and the counter did not decrease. Divide the
+byte difference by the **actual elapsed seconds**. This is storage-accounted
+process I/O, not filesystem growth or physical SSD wear; cancelled writes and
+background writeback can differ from device measurements. Keep
+`cancelled_write_bytes` alongside it, not as proof of bytes reaching the drive.
+An extrapolated GB/day rate is a projection of that short window, not a measured
+day's total. Preserve the window and units with the result.
+
+For **Docker / Compose**, run this on the Docker host, replacing `pulse` with
+the running container name. It reads only the start time and selected statistics,
+not the container environment or configuration.
+
+```bash
+# Docker: bounded container statistics
+(
+ set -e
+ for sample in 1 2; do
+ date -u +'%Y-%m-%dT%H:%M:%SZ'
+ docker inspect --format 'Started={{.State.StartedAt}}' pulse
+ stats=$(docker stats --no-stream --format \
+ 'CPU={{.CPUPerc}} Memory={{.MemUsage}} BlockIO={{.BlockIO}}' pulse)
+ if [ -z "$stats" ]; then
+ printf 'Container statistics unavailable; no zero inferred.\n' >&2
+ exit 1
+ fi
+ printf '%s\n' "$stats"
+ if [ "$sample" -eq 1 ]; then sleep 60; fi
+ done
+)
+```
+
+Docker **BlockIO** is cumulative read / write activity, not bytes per second;
+compare its write side only across the same uninterrupted container run. Its
+displayed units are rounded. Container, process and whole-device counters have
+different scopes and must not be added together or compared as interchangeable
+measurements. A failed, denied or empty read is unavailable, not zero activity.
+
+Keep persistent data on durable storage. Do not delete or truncate history,
+incident, queue or audit files, remove database indexes, or move the data
+directory to tmpfs as a diagnostic workaround. These actions can lose evidence
+or protections without fixing the writer. Do not post databases, profiles, full
+`/proc` dumps, container configuration or raw environments. Share only the bounded
+measurements, workload, versions and a relevant manually redacted error; follow
+[Getting Help](#-getting-help) for any additional evidence.
+
### Notifications
#### No alert when Pulse, power or internet goes down
diff --git a/frontend-modern/public/docs/TROUBLESHOOTING.md b/frontend-modern/public/docs/TROUBLESHOOTING.md
index 032d808db2..9b11c341cd 100644
--- a/frontend-modern/public/docs/TROUBLESHOOTING.md
+++ b/frontend-modern/public/docs/TROUBLESHOOTING.md
@@ -216,6 +216,99 @@ leave the original host unchanged. Follow
[Clone identity recovery](UNIFIED_AGENT.md#duplicate-agents) for the configuration
precedence, systemd example and checks after restart.
+#### Excessive CPU, writes or database growth
+
+First distinguish **Pulse server activity**, agent activity and total host or
+storage-device activity. A database's size is retained space, not the number of
+bytes written: repeated updates can produce high writes without growing the file.
+Keep the original running version, uptime, measurement time and workload before
+changing anything. A different binary can migrate persistent data; switching
+back to an older version does not make that data a clean older-version baseline.
+
+- For CPU, record the measuring tool, allocated CPUs or container quota and
+ whether its percentage represents one core or the whole allocation. A `ps`
+ `%CPU` value is an average since process start, not a recent sampling window.
+- Note fleet size, polling interval and whether Pulse dashboards are open. If
+ safe, close all Pulse tabs and compare another equal-length window, without
+ restarting the server or changing polling. A decrease narrows the workload;
+ it does not establish the cause or prove monitoring is healthy.
+- If the host is unresponsive or storage is nearly full, do not prolong the
+ failure to collect a benchmark. Keep existing observations and report the
+ interruption instead. Do not enable Debug or repeatedly export diagnostics
+ merely to measure performance; those actions add work and can change the result.
+
+For a responsive **Linux systemd / Proxmox LXC** install, the following reads
+two process-I/O samples, waiting 60 seconds between them. Run it inside the
+Pulse container for LXC, not on the Proxmox host. Substitute the actual service
+name (`pulse-backend` on some older installs). Use an account authorised to
+read the process counters; no service restart or database access is needed.
+
+```bash
+# systemd / Proxmox LXC: bounded process-write samples
+(
+ set -e
+ for sample in 1 2; do
+ date -u +'%Y-%m-%dT%H:%M:%SZ'
+ pid=$(systemctl show pulse --property=MainPID --value)
+ case "$pid" in
+ ''|0|*[!0-9]*) printf 'No running Pulse PID; sample unavailable.\n' >&2; exit 1 ;;
+ esac
+ TZ=UTC ps -p "$pid" -o pid=,lstart=
+ sudo awk '
+ $1 == "write_bytes:" || $1 == "cancelled_write_bytes:" { print; fields++ }
+ END { if (fields != 2) exit 1 }
+ ' "/proc/$pid/io"
+ if [ "$sample" -eq 1 ]; then sleep 60; fi
+ done
+)
+```
+
+Compare `write_bytes` only when both samples have the same PID and process
+start time, no restart occurred, and the counter did not decrease. Divide the
+byte difference by the **actual elapsed seconds**. This is storage-accounted
+process I/O, not filesystem growth or physical SSD wear; cancelled writes and
+background writeback can differ from device measurements. Keep
+`cancelled_write_bytes` alongside it, not as proof of bytes reaching the drive.
+An extrapolated GB/day rate is a projection of that short window, not a measured
+day's total. Preserve the window and units with the result.
+
+For **Docker / Compose**, run this on the Docker host, replacing `pulse` with
+the running container name. It reads only the start time and selected statistics,
+not the container environment or configuration.
+
+```bash
+# Docker: bounded container statistics
+(
+ set -e
+ for sample in 1 2; do
+ date -u +'%Y-%m-%dT%H:%M:%SZ'
+ docker inspect --format 'Started={{.State.StartedAt}}' pulse
+ stats=$(docker stats --no-stream --format \
+ 'CPU={{.CPUPerc}} Memory={{.MemUsage}} BlockIO={{.BlockIO}}' pulse)
+ if [ -z "$stats" ]; then
+ printf 'Container statistics unavailable; no zero inferred.\n' >&2
+ exit 1
+ fi
+ printf '%s\n' "$stats"
+ if [ "$sample" -eq 1 ]; then sleep 60; fi
+ done
+)
+```
+
+Docker **BlockIO** is cumulative read / write activity, not bytes per second;
+compare its write side only across the same uninterrupted container run. Its
+displayed units are rounded. Container, process and whole-device counters have
+different scopes and must not be added together or compared as interchangeable
+measurements. A failed, denied or empty read is unavailable, not zero activity.
+
+Keep persistent data on durable storage. Do not delete or truncate history,
+incident, queue or audit files, remove database indexes, or move the data
+directory to tmpfs as a diagnostic workaround. These actions can lose evidence
+or protections without fixing the writer. Do not post databases, profiles, full
+`/proc` dumps, container configuration or raw environments. Share only the bounded
+measurements, workload, versions and a relevant manually redacted error; follow
+[Getting Help](#-getting-help) for any additional evidence.
+
### Notifications
#### No alert when Pulse, power or internet goes down
diff --git a/scripts/tests/test_performance_troubleshooting_docs.py b/scripts/tests/test_performance_troubleshooting_docs.py
new file mode 100644
index 0000000000..329b5a2c77
--- /dev/null
+++ b/scripts/tests/test_performance_troubleshooting_docs.py
@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""Exercise the guide's bounded I/O recipes with synthetic OS/container readers.
+
+The real awk filters run against fixtures. No Pulse process, database, Docker
+daemon or host service is inspected. These checks establish recipe behaviour,
+not native write rates, CPU relief or an operational repair.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import tempfile
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+DOC = ROOT / "docs/TROUBLESHOOTING.md"
+
+
+def section() -> str:
+ return DOC.read_text(encoding="utf-8").split(
+ "#### Excessive CPU, writes or database growth\n", 1
+ )[1].split("\n### Notifications\n", 1)[0]
+
+
+def recipes() -> dict[str, str]:
+ blocks = re.findall(r"```bash\n(.*?)```", section(), re.DOTALL)
+ if len(blocks) != 2:
+ raise AssertionError("expected two bounded deployment recipes")
+ return {"docker" if "# Docker:" in block else "systemd": block for block in blocks}
+
+
+# All external commands record exact arguments. sudo is a test-only adapter:
+# it replaces only the documented /proc/PID/io input and executes real awk.
+READER = '''#!/usr/bin/env python3
+import json, os, pathlib, subprocess, sys
+name = pathlib.Path(sys.argv[0]).name
+args = sys.argv[1:]
+root = pathlib.Path(os.environ['RECIPE_FIXTURE'])
+log = root / 'calls.jsonl'
+previous = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else []
+count = sum(call['name'] == name for call in previous)
+with log.open('a') as stream:
+ stream.write(json.dumps({'name': name, 'args': args}) + '\\n')
+if name == 'date':
+ print('2026-10-03T00:00:00Z' if count == 0 else '2026-10-03T00:01:00Z')
+elif name == 'sleep':
+ assert args == ['60']
+elif name == 'systemctl':
+ print(os.environ.get('FIXTURE_PID', '4321'))
+ sys.exit(int(os.environ.get('SYSTEMCTL_EXIT', '0')))
+elif name == 'ps':
+ assert os.environ['TZ'] == 'UTC'
+ print('4321 Sat Oct 3 00:00:00 2026')
+ sys.exit(int(os.environ.get('PS_EXIT', '0')))
+elif name == 'sudo':
+ assert args[0] == 'awk'
+ assert args[-1] == '/proc/4321/io'
+ target = root / ('missing' if os.environ.get('IO_MISSING') else f'io-{count}')
+ result = subprocess.run([os.environ['REAL_AWK'], *args[1:-1], str(target)])
+ sys.exit(result.returncode)
+elif name == 'docker':
+ if args[0] == 'inspect':
+ print('Started=2026-10-02T20:00:00Z')
+ elif args[0] == 'stats':
+ text = os.environ.get('DOCKER_STATS', 'CPU=12.50% Memory=100MiB / 1GiB BlockIO=1MB / 12MB')
+ if text:
+ print(text)
+ else:
+ raise AssertionError('unsupported docker command')
+ setting = 'DOCKER_INSPECT_EXIT' if args[0] == 'inspect' else 'DOCKER_STATS_EXIT'
+ sys.exit(int(os.environ.get(setting, '0')))
+else:
+ raise AssertionError('unexpected command')
+'''
+
+
+class PerformanceTroubleshootingDocsTest(unittest.TestCase):
+ def exercise(self, deployment: str, *, counters: tuple[str, str] | None = None,
+ **settings: str):
+ import shutil
+
+ with tempfile.TemporaryDirectory() as temporary:
+ directory = Path(temporary)
+ for name in ("date", "sleep", "systemctl", "ps", "sudo", "docker"):
+ command = directory / name
+ command.write_text(READER, encoding="utf-8")
+ command.chmod(0o700)
+ counters = counters or (
+ "rchar: 987654\nwchar: 999999\nwrite_bytes: 1000\ncancelled_write_bytes: 20\n",
+ "rchar: 999999\nwchar: 999999\nwrite_bytes: 13000\ncancelled_write_bytes: 30\n",
+ )
+ for index, text in enumerate(counters):
+ (directory / f"io-{index}").write_text(text, encoding="utf-8")
+ env = dict(os.environ, PATH=f"{directory}:{os.environ['PATH']}",
+ RECIPE_FIXTURE=str(directory), REAL_AWK=shutil.which("awk"), **settings)
+ result = subprocess.run(["bash", "-c", recipes()[deployment]], env=env,
+ text=True, capture_output=True, timeout=10)
+ calls = [json.loads(line) for line in (directory / "calls.jsonl").read_text().splitlines()]
+ return result, calls
+
+ def test_shipped_document_matches_the_exercised_source(self):
+ self.assertEqual(DOC.read_bytes(),
+ (ROOT / "frontend-modern/public/docs/TROUBLESHOOTING.md").read_bytes())
+
+ def test_systemd_samples_only_write_counters_and_process_identity(self):
+ result, calls = self.exercise("systemd")
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual([call["name"] for call in calls],
+ ["date", "systemctl", "ps", "sudo", "sleep",
+ "date", "systemctl", "ps", "sudo"])
+ for call in calls:
+ if call["name"] == "systemctl":
+ self.assertEqual(call["args"], ["show", "pulse", "--property=MainPID", "--value"])
+ elif call["name"] == "ps":
+ self.assertEqual(call["args"], ["-p", "4321", "-o", "pid=,lstart="])
+ self.assertIn("write_bytes: 1000", result.stdout)
+ self.assertIn("write_bytes: 13000", result.stdout)
+ self.assertIn("cancelled_write_bytes: 20", result.stdout)
+ self.assertNotIn("rchar", result.stdout)
+ self.assertNotIn("wchar", result.stdout)
+
+ def test_unavailable_pid_never_reads_a_process_or_sleeps(self):
+ for pid in ("", "0", "not-a-pid", "4321 extra"):
+ with self.subTest(pid=pid):
+ result, calls = self.exercise("systemd", FIXTURE_PID=pid)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("sample unavailable", result.stderr)
+ self.assertEqual([c["name"] for c in calls], ["date", "systemctl"])
+
+ def test_service_and_process_read_failures_stop_the_collection(self):
+ for settings, expected in (({"SYSTEMCTL_EXIT": "3"}, ["date", "systemctl"]),
+ ({"PS_EXIT": "4"}, ["date", "systemctl", "ps"])):
+ with self.subTest(settings=settings):
+ result, calls = self.exercise("systemd", **settings)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual([c["name"] for c in calls], expected)
+
+ def test_missing_process_io_is_failure_not_zero_writes(self):
+ result, calls = self.exercise("systemd", IO_MISSING="1")
+ self.assertNotEqual(result.returncode, 0)
+ self.assertNotIn("write_bytes:", result.stdout)
+ self.assertNotIn("sleep", [c["name"] for c in calls])
+
+ def test_incomplete_process_counters_do_not_become_a_success(self):
+ for text in ("", "rchar: 999\n", "write_bytes: 1000\n"):
+ with self.subTest(text=text):
+ result, calls = self.exercise("systemd", counters=(text, text))
+ self.assertNotEqual(result.returncode, 0)
+ self.assertNotIn("sleep", [c["name"] for c in calls])
+
+ def test_decreasing_counters_are_retained_not_turned_into_a_rate(self):
+ result, _ = self.exercise("systemd", counters=(
+ "write_bytes: 13000\ncancelled_write_bytes: 30\n",
+ "write_bytes: 1000\ncancelled_write_bytes: 20\n",
+ ))
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertIn("write_bytes: 13000", result.stdout)
+ self.assertIn("write_bytes: 1000", result.stdout)
+ self.assertNotRegex(result.stdout, r"GB/day|bytes/s|bytes: -[0-9]+")
+
+ def test_docker_reads_only_start_time_and_selected_statistics_twice(self):
+ result, calls = self.exercise("docker")
+ self.assertEqual(result.returncode, 0, result.stderr)
+ expected = [
+ ["inspect", "--format", "Started={{.State.StartedAt}}", "pulse"],
+ ["stats", "--no-stream", "--format",
+ "CPU={{.CPUPerc}} Memory={{.MemUsage}} BlockIO={{.BlockIO}}", "pulse"],
+ ]
+ self.assertEqual([c["args"] for c in calls if c["name"] == "docker"], expected * 2)
+ self.assertEqual([c["name"] for c in calls],
+ ["date", "docker", "docker", "sleep", "date", "docker", "docker"])
+ self.assertEqual(result.stdout.count("BlockIO=1MB / 12MB"), 2)
+
+ def test_empty_docker_statistics_are_unavailable_not_zero(self):
+ result, calls = self.exercise("docker", DOCKER_STATS="")
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("no zero inferred", result.stderr)
+ self.assertNotIn("sleep", [c["name"] for c in calls])
+
+ def test_docker_reader_failure_stops_before_another_sample(self):
+ for settings, expected in (
+ ({"DOCKER_INSPECT_EXIT": "2"}, ["date", "docker"]),
+ ({"DOCKER_STATS_EXIT": "3"}, ["date", "docker", "docker"]),
+ ):
+ with self.subTest(settings=settings):
+ result, calls = self.exercise("docker", **settings)
+ self.assertEqual(result.returncode, int(next(iter(settings.values()))))
+ self.assertEqual([c["name"] for c in calls], expected)
+
+ def test_guidance_keeps_scope_units_baselines_and_data_safety(self):
+ guide = " ".join(section().split())
+ for phrase in ("same PID and process start time", "actual elapsed seconds",
+ "not a measured day's total", "not bytes per second",
+ "not a recent sampling window", "different scopes",
+ "inside the Pulse container", "does not make that data a clean",
+ "Do not enable Debug", "remove database indexes", "tmpfs",
+ "Do not post databases", "manually redacted error"):
+ self.assertIn(phrase, guide)
+ for recipe in recipes().values():
+ self.assertNotRegex(recipe, r"\b(?:restart|kill|rm|truncate|sqlite3|curl|printenv)\b|--follow|--token")
+
+
+if __name__ == "__main__":
+ unittest.main()
From f56e28e6afd2c4586fbaf3db761a488c9615ca6b Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 01:05:16 +0100
Subject: [PATCH 03/86] Own demo update and recovery through sustained host
observation
Compose the existing signed stable demo paths with the Relay host lock, durable systemd ownership, full persistent estate capture and five-minute forward and recovery watches. Preserve uncertain attempts and failed rollback, refuse destructive headroom pruning, and keep native installed acceptance separate from connected source controls.
Change-source: pulse-maintainer
---
.github/scripts/demo-runtime-transaction.py | 504 ++++++++++++++++++
.github/scripts/dispatch-demo-runtime.py | 113 ++++
.github/scripts/recover-demo-runtime.sh | 395 +-------------
.../tests/test_demo_runtime_transaction.py | 443 +++++++++++++++
.github/workflows/recover-demo-server.yml | 34 +-
.github/workflows/update-demo-server.yml | 326 +----------
.../v6/internal/DEMO_RUNTIME_TRANSACTION.md | 60 +++
.../subsystems/deployment-installability.md | 32 ++
.../v6/internal/subsystems/registry.json | 28 +-
.../installtests/build_release_assets_test.go | 43 +-
scripts/installtests/demo_transaction_test.go | 56 ++
11 files changed, 1278 insertions(+), 756 deletions(-)
create mode 100644 .github/scripts/demo-runtime-transaction.py
create mode 100644 .github/scripts/dispatch-demo-runtime.py
create mode 100644 .github/scripts/tests/test_demo_runtime_transaction.py
create mode 100644 docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md
create mode 100644 scripts/installtests/demo_transaction_test.go
diff --git a/.github/scripts/demo-runtime-transaction.py b/.github/scripts/demo-runtime-transaction.py
new file mode 100644
index 0000000000..60193e378e
--- /dev/null
+++ b/.github/scripts/demo-runtime-transaction.py
@@ -0,0 +1,504 @@
+#!/usr/bin/env python3
+"""Demo-only host transaction. Called by the reviewed CI dispatcher, not a worker tool."""
+from __future__ import annotations
+
+import fcntl
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import signal
+import socket
+import stat
+import subprocess
+import sys
+import time
+from urllib.parse import urlsplit
+
+WINDOW = 300
+INTERVAL = 5
+READY_TIMEOUT = 60
+SERVICES = ("pulse", "pulse-relay", "caddy")
+ROOT = Path("/var/lib/pulse-deploy/demo")
+LOCK = Path("/var/lib/pulse-deploy/relay/deploy.lock")
+PATHS = {"binary": Path("/opt/pulse/bin/pulse"),
+ "unit": Path("/etc/systemd/system/pulse.service"),
+ "dropins": Path("/etc/systemd/system/pulse.service.d"),
+ "data": Path("/etc/pulse")}
+RELAY_HEALTH = "https://relay.pulserelay.pro/health"
+COUNT_KEYS = ("nodes", "vms_per_node", "lxcs_per_node", "docker_hosts",
+ "docker_containers", "generic_hosts", "k8s_clusters", "k8s_nodes",
+ "k8s_pods", "k8s_deployments")
+DURATION_KEYS = ("seed_duration", "sample_interval", "update_interval")
+# Only generated demo operational memory is reset on the already-unhealthy
+# recovery route. Its complete original estate remains in the private snapshot.
+DEMO_HISTORY = ("ai_incidents.json", "ai_incidents.json.tmp",
+ "alerts/alert-history.json", "alerts/alert-history.backup.json",
+ "alerts/alert-history.json.imported", "alerts/alert-history.backup.json.imported",
+ "alerts/events.db", "alerts/events.db-shm", "alerts/events.db-wal")
+CRASH = re.compile(r"\bpanic\b|\bfatal error\b|segmentation fault|core dumped|watchdog timeout", re.I)
+TERMINAL = {"committed", "healthy_noop", "rolled_back", "refused"}
+
+
+class Failure(Exception):
+ """Only fixed stage names, never command output or private response bodies."""
+
+
+def atomic_json(path, value):
+ tmp = path.with_suffix(".tmp")
+ with open(tmp, "w", encoding="utf-8") as stream:
+ os.chmod(tmp, 0o600)
+ json.dump(value, stream, sort_keys=True)
+ stream.flush()
+ os.fsync(stream.fileno())
+ os.replace(tmp, path)
+ fd = os.open(path.parent, os.O_DIRECTORY)
+ try:
+ os.fsync(fd)
+ finally:
+ os.close(fd)
+
+
+def validate(request):
+ if set(request) != {"mode", "hostname", "local_url", "public_url", "version", "profile", "control_sha", "run_id", "run_attempt", "installer_sha256"}:
+ raise Failure("request-shape")
+ if request["mode"] not in {"update", "recover"}:
+ raise Failure("request-mode")
+ for field, pattern in (("version", r"v[0-9]+\.[0-9]+\.[0-9]+"),
+ ("hostname", r"[a-zA-Z0-9][a-zA-Z0-9.-]{0,252}"),
+ ("control_sha", r"[0-9a-f]{40}"),
+ ("run_id", r"[1-9][0-9]*"), ("run_attempt", r"[1-9][0-9]*")):
+ if not isinstance(request[field], str) or not re.fullmatch(pattern, request[field]):
+ raise Failure("request-identity")
+ for field, public in (("local_url", False), ("public_url", True)):
+ try:
+ url = urlsplit(request[field])
+ valid = (url.scheme == ("https" if public else "http")
+ and url.hostname and not url.username and not url.password
+ and not url.query and not url.fragment
+ and (url.path == "/api/health" if public else url.path in {"", "/"})
+ and (public or url.hostname in {"localhost", "127.0.0.1", "::1"})
+ and 0 < (url.port or (443 if public else 80)) < 65536)
+ except (TypeError, ValueError):
+ valid = False
+ if not valid:
+ raise Failure("request-url")
+ profile = request["profile"]
+ if not isinstance(profile, dict) or set(profile) != set(COUNT_KEYS + DURATION_KEYS):
+ raise Failure("request-profile")
+ for name, value in profile.items():
+ pattern = r"[1-9][0-9]{0,3}" if name in COUNT_KEYS else r"[1-9][0-9]{0,3}[smhd]"
+ if not isinstance(value, str) or not re.fullmatch(pattern, value):
+ raise Failure("request-profile")
+ digest = request["installer_sha256"]
+ if (request["mode"] == "update" and not re.fullmatch(r"[0-9a-f]{64}", digest)) or (request["mode"] == "recover" and digest != ""):
+ raise Failure("request-installer")
+
+
+def copy_path(src, dst):
+ """Copy without following symlinks, retaining ownership as well as modes."""
+ info = src.lstat()
+ if stat.S_ISLNK(info.st_mode):
+ dst.symlink_to(os.readlink(src))
+ elif stat.S_ISDIR(info.st_mode):
+ dst.mkdir(mode=0o700)
+ for child in src.iterdir():
+ copy_path(child, dst / child.name)
+ shutil.copystat(src, dst, follow_symlinks=False)
+ elif stat.S_ISREG(info.st_mode):
+ shutil.copy2(src, dst, follow_symlinks=False)
+ else:
+ raise Failure("snapshot-special-file")
+ os.chown(dst, info.st_uid, info.st_gid, follow_symlinks=False)
+ if not stat.S_ISLNK(info.st_mode):
+ fd = os.open(dst, os.O_RDONLY | (os.O_DIRECTORY if stat.S_ISDIR(info.st_mode) else 0))
+ try:
+ os.fsync(fd)
+ finally:
+ os.close(fd)
+
+
+def remove_path(path):
+ if path.is_symlink() or path.is_file():
+ path.unlink()
+ elif path.is_dir():
+ shutil.rmtree(path)
+
+
+def estate_hash(paths):
+ rows = []
+ for label, root in sorted(paths.items()):
+ exists = root.exists() or root.is_symlink()
+ if not exists:
+ rows.append((label, "absent"))
+ continue
+ for path in [root] + (sorted(root.rglob("*")) if root.is_dir() and not root.is_symlink() else []):
+ info = path.lstat()
+ kind = stat.S_IFMT(info.st_mode)
+ content = os.readlink(path) if stat.S_ISLNK(info.st_mode) else ""
+ if stat.S_ISREG(info.st_mode):
+ digest = hashlib.sha256()
+ with open(path, "rb") as stream:
+ for block in iter(lambda: stream.read(1024 * 1024), b""):
+ digest.update(block)
+ content = digest.hexdigest()
+ rows.append((label, str(path.relative_to(root)), kind, stat.S_IMODE(info.st_mode), info.st_uid, info.st_gid, content))
+ return hashlib.sha256(json.dumps(rows, sort_keys=True).encode()).hexdigest()
+
+
+class Host:
+ now = staticmethod(time.monotonic)
+ sleep = staticmethod(time.sleep)
+
+ def command(self, argv, *, timeout=20, env=None):
+ try:
+ result = subprocess.run(argv, capture_output=True, timeout=timeout, env=env, check=False)
+ except (OSError, subprocess.TimeoutExpired) as exc:
+ raise Failure("command-unavailable") from exc
+ if result.returncode or len(result.stdout) > 1024 * 1024:
+ raise Failure("command-failed")
+ return result.stdout.decode("utf-8", errors="strict")
+
+ def identity(self, hostname):
+ if socket.gethostname() != hostname or os.geteuid() != 0:
+ raise Failure("host-identity")
+ unit = self.command(["systemctl", "show", "pulse", "--property=FragmentPath,User,Group,ExecStart,DropInPaths", "--no-pager"])
+ fields = dict(line.split("=", 1) for line in unit.splitlines() if "=" in line)
+ if (fields.get("FragmentPath") != str(PATHS["unit"])
+ or fields.get("User") != "pulse" or fields.get("Group") != "pulse"
+ or str(PATHS["binary"]) not in fields.get("ExecStart", "")):
+ raise Failure("service-identity")
+ for name in ("binary", "unit", "data"):
+ if PATHS[name].is_symlink() or not PATHS[name].exists():
+ raise Failure("estate-identity")
+ if PATHS["dropins"].is_symlink():
+ raise Failure("dropin-identity")
+ for path in fields.get("DropInPaths", "").split():
+ if Path(path).parent != PATHS["dropins"] or Path(path).is_symlink():
+ raise Failure("unsupported-dropin")
+ if self.command(["systemctl", "is-enabled", "pulse"]).strip() != "enabled":
+ raise Failure("service-disabled")
+
+ def state(self):
+ states = {}
+ for service in SERVICES:
+ text = self.command(["systemctl", "show", service, "--property=ActiveState,SubState,MainPID,NRestarts", "--no-pager"])
+ fields = dict(line.split("=", 1) for line in text.splitlines() if "=" in line)
+ if (set(fields) != {"ActiveState", "SubState", "MainPID", "NRestarts"}
+ or not fields["MainPID"].isdigit() or not fields["NRestarts"].isdigit()):
+ raise Failure("service-observation")
+ states[service] = fields
+ return states
+
+ def status(self, url):
+ # No redirects, response bodies, credential arguments or ambient curl config.
+ try:
+ result = self.command(["curl", "--disable", "--silent", "--connect-timeout", "3", "--max-time", "8", "--output", "/dev/null", "--write-out", "%{http_code}", "--", url], timeout=10)
+ except Failure:
+ return "000"
+ return result if re.fullmatch(r"[1-5][0-9]{2}", result) else "000"
+
+ def version(self, url):
+ try:
+ body = self.command(["curl", "--disable", "--silent", "--fail", "--connect-timeout", "3", "--max-time", "8", "--", url.rstrip("/") + "/api/version"], timeout=10)
+ if len(body) > 8192:
+ raise Failure("version-observation")
+ value = json.loads(body)["version"]
+ if not isinstance(value, str) or not re.fullmatch(r"v?[0-9]+\.[0-9]+\.[0-9]+", value):
+ raise Failure("version-observation")
+ return value.lstrip("v")
+ except (ValueError, KeyError, TypeError) as exc:
+ raise Failure("version-observation") from exc
+
+ def cursors(self):
+ cursors = {}
+ for service in SERVICES:
+ text = self.command(["journalctl", "-u", service, "-n", "1", "--output=json", "--show-cursor", "--no-pager"])
+ matches = re.findall(r"^-- cursor: ([^\s]+)$", text, re.M)
+ if len(matches) != 1:
+ raise Failure("journal-cursor")
+ cursors[service] = matches[0]
+ return cursors
+
+ def check_journal(self, cursors):
+ for service, cursor in cursors.items():
+ text = self.command(["journalctl", "-u", service, "--after-cursor", cursor, "--output=json", "--no-pager"])
+ for line in text.splitlines():
+ try:
+ entry = json.loads(line)
+ if not isinstance(entry.get("MESSAGE"), str) or not isinstance(entry.get("__CURSOR"), str):
+ raise Failure("journal-observation")
+ except (ValueError, TypeError, AttributeError) as exc:
+ raise Failure("journal-observation") from exc
+ if CRASH.search(entry["MESSAGE"]):
+ raise Failure("new-service-crash")
+ cursors[service] = entry["__CURSOR"]
+
+ def stop(self):
+ self.command(["systemctl", "stop", "pulse"], timeout=90)
+
+ def start(self):
+ self.command(["systemctl", "daemon-reload"])
+ self.command(["systemctl", "start", "pulse"], timeout=90)
+
+ def install(self, attempt, request):
+ installer = attempt / "installer.sh"
+ if hashlib.sha256(installer.read_bytes()).hexdigest() != request["installer_sha256"]:
+ raise Failure("installer-identity")
+ self.command(["bash", str(installer), "--version", request["version"]], timeout=900,
+ env={"PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "HOME": "/root", "PULSE_SERVICE_NAME": "pulse"})
+
+ def profile_values(self, request):
+ values = {"DEMO_MODE": "true", "PULSE_MOCK_MODE": "true", "PULSE_MOCK_RANDOM_METRICS": "true",
+ "PULSE_MOCK_STOPPED_PERCENT": "6", "PULSE_MOCK_SEED_METRICS_STORE": "false"}
+ for key, value in request["profile"].items():
+ key = {"seed_duration": "trends_seed_duration", "sample_interval": "trends_sample_interval"}.get(key, key)
+ values["PULSE_MOCK_" + key.upper()] = value
+ return values
+
+ def profile_matches(self, request):
+ text = (PATHS["data"] / ".env").read_text().splitlines()
+ values = self.profile_values(request)
+ return (all(text.count(key + "=" + value) == 1 for key, value in values.items())
+ and "demo_fixtures" in json.loads((PATHS["data"] / "billing.json").read_text()).get("capabilities", []))
+
+ def profile(self, request, unhealthy):
+ env_file = PATHS["data"] / ".env"
+ billing_file = PATHS["data"] / "billing.json"
+ if env_file.is_symlink() or billing_file.is_symlink():
+ raise Failure("configuration-symlink")
+ values = self.profile_values(request)
+ text = env_file.read_text()
+ for key, value in values.items():
+ pattern = re.compile(r"^[ \t]*" + re.escape(key) + r"=.*$", re.M)
+ text = pattern.sub(key + "=" + value, text) if pattern.search(text) else text.rstrip("\n") + "\n" + key + "=" + value + "\n"
+ env_file.write_text(text)
+ os.chmod(env_file, 0o600)
+ billing = json.loads(billing_file.read_text())
+ if not isinstance(billing, dict) or not isinstance(billing.get("capabilities", []), list):
+ raise Failure("billing-shape")
+ billing["capabilities"] = sorted(set(billing.get("capabilities", []) + ["demo_fixtures"]))
+ billing.pop("integrity", None)
+ billing_file.write_text(json.dumps(billing))
+ os.chmod(billing_file, 0o600)
+ if unhealthy:
+ for relative in DEMO_HISTORY:
+ path = PATHS["data"] / relative
+ # Never follow a writable parent symlink to a path outside the estate.
+ if any(p.is_symlink() for p in path.parents if p != PATHS["data"].parent):
+ raise Failure("history-symlink")
+ if path.is_symlink():
+ raise Failure("history-symlink")
+ if path.is_file():
+ path.unlink()
+
+
+def active(state):
+ return state["ActiveState"] == "active" and state["SubState"] == "running" and int(state["MainPID"]) > 0
+
+
+class Transaction:
+ def __init__(self, host, request, attempt):
+ self.host, self.request, self.attempt = host, request, attempt
+ self.receipt = {"schema_version": 2, "status": "intent", "mutated": False,
+ "control_sha": request["control_sha"], "run_id": request["run_id"],
+ "run_attempt": request["run_attempt"], "mode": request["mode"],
+ "expected_version": request["version"], "installer_sha256": request["installer_sha256"],
+ "forward": {}, "recovery": {}, "recovery_required": False}
+
+ def save(self, status):
+ self.receipt["status"] = status
+ atomic_json(self.attempt / "receipt.json", self.receipt)
+
+ def urls(self):
+ return (self.request["local_url"].rstrip("/") + "/api/health", self.request["public_url"], RELAY_HEALTH)
+
+ def healthy(self, states=None):
+ return all(active(s) for s in (states or self.host.state()).values()) and all(self.host.status(url) == "200" for url in self.urls())
+
+ def ready(self, cursors):
+ deadline = self.host.now() + READY_TIMEOUT
+ while True:
+ self.host.check_journal(cursors)
+ if self.healthy():
+ return
+ if self.host.now() >= deadline:
+ raise Failure("listener-readiness")
+ self.host.sleep(INTERVAL)
+
+ def watch(self, phase, expected_version, cursors):
+ baseline = self.host.state()
+ started = self.host.now()
+ evidence = self.receipt[phase]
+ evidence.update({"required_seconds": WINDOW, "samples": 0, "elapsed_seconds": 0})
+ while True:
+ self.host.check_journal(cursors)
+ state = self.host.state()
+ if state != baseline or not self.healthy(state):
+ raise Failure("sustained-health")
+ if self.host.version(self.request["local_url"]) != expected_version:
+ raise Failure("runtime-version")
+ evidence["samples"] += 1
+ elapsed = self.host.now() - started
+ evidence["elapsed_seconds"] = elapsed
+ if elapsed >= WINDOW:
+ return
+ self.host.sleep(min(INTERVAL, WINDOW - elapsed))
+
+ def capture(self):
+ snapshot = self.attempt / "snapshot"
+ size = sum(path.lstat().st_size for root in PATHS.values() if root.exists()
+ for path in ([root] + (list(root.rglob("*")) if root.is_dir() else []))
+ if path.is_file() and not path.is_symlink())
+ # Both our independent estate and the signed installer's own backup need
+ # space. Refuse shortage; never delete backups or databases for headroom.
+ if any(shutil.disk_usage(parent).free < 2 * size + 64 * 1024 * 1024
+ for parent in (self.attempt, PATHS["data"].parent)):
+ raise Failure("snapshot-headroom")
+ snapshot.mkdir(mode=0o700)
+ manifest = {}
+ for label, path in PATHS.items():
+ manifest[label] = path.exists() or path.is_symlink()
+ if manifest[label]:
+ copy_path(path, snapshot / label)
+ atomic_json(snapshot / "paths.json", manifest)
+ digest = estate_hash(PATHS)
+ self.receipt["snapshot_sha256"] = digest
+ self.receipt["snapshot_retained"] = True
+ return digest
+
+ def restore(self):
+ snapshot = self.attempt / "snapshot"
+ for label, exists in json.loads((snapshot / "paths.json").read_text()).items():
+ remove_path(PATHS[label])
+ if exists:
+ copy_path(snapshot / label, PATHS[label])
+ if estate_hash(PATHS) != self.receipt["snapshot_sha256"]:
+ raise Failure("restored-estate-identity")
+
+ def run(self):
+ old_handlers = {}
+ healthy_baseline = False
+ stopped = False
+ original_version = ""
+ states = {}
+
+ def cancel(_signum, _frame):
+ raise Failure("cancelled-forward")
+
+ try:
+ validate(self.request)
+ self.host.identity(self.request["hostname"])
+ for entry in ROOT.glob("attempts/*"):
+ if entry == self.attempt:
+ continue
+ receipt = entry / "receipt.json"
+ if not receipt.exists() or json.loads(receipt.read_text()).get("status") not in TERMINAL:
+ raise Failure("prior-recovery-required")
+ states = self.host.state()
+ if not all(active(states[name]) for name in ("pulse-relay", "caddy")) or self.host.status(RELAY_HEALTH) != "200":
+ raise Failure("cohost-unhealthy")
+ healthy_baseline = self.healthy(states)
+ if healthy_baseline:
+ original_version = self.host.version(self.request["local_url"])
+ if self.request["mode"] == "recover" and original_version != self.request["version"].lstrip("v"):
+ raise Failure("recovery-source-mismatch")
+ elif self.request["mode"] == "update":
+ raise Failure("update-baseline-unhealthy")
+ self.receipt["healthy_baseline"] = healthy_baseline
+ # The no-op is observed too; an immediate healthy sample is not success.
+ if self.request["mode"] == "recover" and healthy_baseline and self.host.profile_matches(self.request):
+ self.watch("forward", original_version, self.host.cursors())
+ self.save("healthy_noop")
+ return 0
+ old_handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP)}
+ for sig in old_handlers:
+ signal.signal(sig, cancel)
+ self.save("capturing")
+ stopped = True
+ self.host.stop()
+ self.capture()
+ self.save("applying")
+ self.receipt["mutated"] = True
+ cursors = self.host.cursors()
+ if self.request["mode"] == "update":
+ self.host.install(self.attempt, self.request)
+ # The installer may start the unit; finish profile work quiescent.
+ self.host.stop()
+ self.host.profile(self.request, not healthy_baseline)
+ self.host.start()
+ self.ready(cursors)
+ self.watch("forward", self.request["version"].lstrip("v"), cursors)
+ # No unrelated Relay/Caddy restart is owned by this transaction.
+ final = self.host.state()
+ for service in ("pulse-relay", "caddy"):
+ if final[service] != states[service]:
+ raise Failure("cohost-identity-changed")
+ self.save("committed")
+ return 0
+ except Exception as exc:
+ self.receipt["failure"] = str(exc) if isinstance(exc, Failure) else "transaction-observation"
+ # Owned restoration, observation and the terminal receipt survive cancellation.
+ for sig in old_handlers:
+ signal.signal(sig, signal.SIG_IGN)
+ if stopped:
+ try:
+ self.save("recovering")
+ self.host.stop()
+ if "snapshot_sha256" in self.receipt:
+ self.restore()
+ if not healthy_baseline:
+ self.receipt["recovery_required"] = True
+ self.receipt["rollback"] = "unavailable-unhealthy-baseline"
+ self.save("recovery_required")
+ else:
+ cursors = self.host.cursors()
+ self.host.start()
+ self.ready(cursors)
+ self.watch("recovery", original_version, cursors)
+ final = self.host.state()
+ if any(final[name] != states[name] for name in ("pulse-relay", "caddy")):
+ raise Failure("cohost-identity-changed")
+ self.receipt["rollback"] = "verified"
+ self.save("rolled_back")
+ except Exception as recovery:
+ self.receipt["rollback"] = "failed"
+ self.receipt["recovery_failure"] = str(recovery) if isinstance(recovery, Failure) else "recovery-observation"
+ self.receipt["recovery_required"] = True
+ self.save("rollback_failed")
+ else:
+ self.save("refused")
+ return 1
+ finally:
+ for sig, handler in old_handlers.items():
+ signal.signal(sig, handler)
+
+
+def main():
+ # Attempt paths are generated by the fixed CI bootstrap, not CLI overrides.
+ if len(sys.argv) != 2:
+ return 2
+ attempt = Path(sys.argv[1])
+ if attempt.parent != ROOT / "attempts" or not re.fullmatch(r"[0-9a-f]{64}", attempt.name) or attempt.is_symlink():
+ return 2
+ request = json.loads((attempt / "request.json").read_text())
+ transaction = Transaction(Host(), request, attempt)
+ fd = None
+ try:
+ LOCK.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+ fd = os.open(LOCK, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ return transaction.run()
+ except (OSError, ValueError):
+ transaction.receipt["failure"] = "host-lock-unavailable"
+ transaction.save("refused")
+ return 1
+ finally:
+ if fd is not None:
+ os.close(fd)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/scripts/dispatch-demo-runtime.py b/.github/scripts/dispatch-demo-runtime.py
new file mode 100644
index 0000000000..499602fd17
--- /dev/null
+++ b/.github/scripts/dispatch-demo-runtime.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""The two existing demo CI paths share one retained, independently owned child."""
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import os
+from pathlib import Path
+import shlex
+import subprocess
+import sys
+
+BOOTSTRAP = r'''
+import base64, fcntl, hashlib, json, os, pathlib, subprocess, sys, time
+root = pathlib.Path('/var/lib/pulse-deploy/demo')
+payload = json.loads(sys.stdin.read(512 * 1024))
+assert os.geteuid() == 0 and set(payload) == {'request', 'source', 'installer'}
+source = base64.b64decode(payload['source'], validate=True)
+installer = base64.b64decode(payload['installer'], validate=True)
+assert len(source) < 128 * 1024 and len(installer) < 256 * 1024
+assert (hashlib.sha256(installer).hexdigest() == payload['request']['installer_sha256']) if installer else (payload['request']['installer_sha256'] == '')
+identity = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
+root.mkdir(mode=0o700, parents=True, exist_ok=True)
+assert not root.is_symlink() and root.stat().st_uid == 0
+os.chmod(root, 0o700)
+(root / 'attempts').mkdir(mode=0o700, exist_ok=True)
+assert not (root / 'attempts').is_symlink()
+fd = os.open(root / 'submit.lock', os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
+fcntl.flock(fd, fcntl.LOCK_EX)
+attempt = root / 'attempts' / identity
+if not attempt.exists():
+ attempt.mkdir(mode=0o700)
+ for name, data in [('transaction.py', source), ('installer.sh', installer),
+ ('request.json', json.dumps(payload['request']).encode()),
+ ('intent.json', json.dumps({'schema_version': 1, 'identity': identity, 'state': 'intent'}).encode())]:
+ with open(attempt / name, 'xb') as stream:
+ os.chmod(attempt / name, 0o600)
+ stream.write(data); stream.flush(); os.fsync(stream.fileno())
+ # Intent is durable before starting the unit. An uncertain launch is never replayed.
+ dfd = os.open(attempt, os.O_DIRECTORY); os.fsync(dfd); os.close(dfd)
+ try:
+ child = subprocess.run(['systemd-run', '--quiet', '--collect',
+ '--unit=pulse-demo-' + identity[:32],
+ '--property=RuntimeMaxSec=45min',
+ '--property=TimeoutStopSec=20min', '--property=KillMode=mixed',
+ '/usr/bin/python3', str(attempt / 'transaction.py'), str(attempt)],
+ capture_output=True, timeout=30)
+ launch = {'schema_version': 1, 'state': 'submitted' if child.returncode == 0 else 'uncertain',
+ 'unit': 'pulse-demo-' + identity[:32]}
+ except (OSError, subprocess.TimeoutExpired):
+ launch = {'schema_version': 1, 'state': 'uncertain'}
+ with open(attempt / 'launch.json', 'x') as stream:
+ os.chmod(attempt / 'launch.json', 0o600)
+ json.dump(launch, stream); stream.flush(); os.fsync(stream.fileno())
+os.close(fd)
+# Closing or losing this observer never stops the independently owned unit.
+deadline = time.monotonic() + 4000
+terminal = {'committed', 'healthy_noop', 'rolled_back', 'rollback_failed', 'recovery_required', 'refused'}
+while time.monotonic() < deadline:
+ try:
+ receipt = json.loads((attempt / 'receipt.json').read_text())
+ except (FileNotFoundError, ValueError):
+ receipt = None
+ if receipt and receipt.get('status') in terminal:
+ receipt['request_identity'] = identity
+ print(json.dumps(receipt))
+ sys.exit(0 if receipt['status'] in {'committed', 'healthy_noop'} else 1)
+ launch = attempt / 'launch.json'
+ if launch.exists() and json.loads(launch.read_text()).get('state') == 'uncertain':
+ break
+ time.sleep(5)
+print(json.dumps({'schema_version': 2, 'status': 'uncertain', 'request_identity': identity,
+ 'failure': 'terminal-receipt-unavailable', 'recovery_required': True}))
+sys.exit(1)
+'''
+
+
+def payload(mode, environ=os.environ):
+ profile = {key: environ['MOCK_' + key.upper()] for key in
+ ('nodes', 'vms_per_node', 'lxcs_per_node', 'docker_hosts', 'docker_containers',
+ 'generic_hosts', 'k8s_clusters', 'k8s_nodes', 'k8s_pods', 'k8s_deployments',
+ 'seed_duration', 'sample_interval', 'update_interval')}
+ installer = Path('/tmp/pulse-install.sh').read_bytes() if mode == 'update' else b''
+ request = {'mode': mode, 'hostname': environ['DEMO_EXPECTED_HOSTNAME'],
+ 'local_url': environ['DEMO_LOCAL_BASE_URL'], 'public_url': environ['DEMO_PUBLIC_HEALTH_URL'],
+ 'version': environ['EXPECTED_VERSION'], 'profile': profile,
+ 'control_sha': environ['GITHUB_SHA'], 'run_id': environ['GITHUB_RUN_ID'],
+ 'run_attempt': environ['GITHUB_RUN_ATTEMPT'],
+ 'installer_sha256': hashlib.sha256(installer).hexdigest() if installer else ''}
+ return {'request': request, 'source': base64.b64encode(Path(__file__).with_name('demo-runtime-transaction.py').read_bytes()).decode(),
+ 'installer': base64.b64encode(installer).decode()}
+
+
+def main():
+ if len(sys.argv) != 2 or sys.argv[1] not in {'update', 'recover'}:
+ return 2
+ # Same project CI SSH identity and pinned host verification as before. This
+ # is not an installed worker dispatcher or a new project credential.
+ command = ['ssh', '-i', str(Path.home() / '.ssh/id_ed25519'), '-o', 'IdentitiesOnly=yes',
+ '-o', 'StrictHostKeyChecking=yes', '-o', 'UserKnownHostsFile=' + str(Path.home() / '.ssh/known_hosts'),
+ os.environ['DEMO_SERVER_USER'] + '@' + os.environ['DEMO_SERVER_HOST'],
+ 'sudo python3 -c ' + shlex.quote(BOOTSTRAP)]
+ try:
+ result = subprocess.run(command, input=json.dumps(payload(sys.argv[1])).encode(), timeout=4100, check=False)
+ except (OSError, subprocess.TimeoutExpired):
+ print(json.dumps({'schema_version': 2, 'status': 'uncertain', 'failure': 'observer-unavailable', 'recovery_required': True}))
+ return 1
+ return result.returncode
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/.github/scripts/recover-demo-runtime.sh b/.github/scripts/recover-demo-runtime.sh
index 132c5cf48f..81612502a7 100755
--- a/.github/scripts/recover-demo-runtime.sh
+++ b/.github/scripts/recover-demo-runtime.sh
@@ -1,395 +1,4 @@
#!/usr/bin/env bash
set -euo pipefail
-
-if [ "$#" -ne 16 ]; then
- echo "usage: recover-demo-runtime.sh EXPECTED_HOSTNAME LOCAL_BASE_URL EXPECTED_VERSION MOCK_NODES MOCK_VMS_PER_NODE MOCK_LXCS_PER_NODE MOCK_DOCKER_HOSTS MOCK_DOCKER_CONTAINERS MOCK_GENERIC_HOSTS MOCK_K8S_CLUSTERS MOCK_K8S_NODES MOCK_K8S_PODS MOCK_K8S_DEPLOYMENTS MOCK_SEED_DURATION MOCK_SAMPLE_INTERVAL MOCK_UPDATE_INTERVAL" >&2
- exit 2
-fi
-
-EXPECTED_HOSTNAME="$1"
-LOCAL_BASE_URL="${2%/}"
-EXPECTED_VERSION="${3#v}"
-MOCK_NODES="$4"
-MOCK_VMS_PER_NODE="$5"
-MOCK_LXCS_PER_NODE="$6"
-MOCK_DOCKER_HOSTS="$7"
-MOCK_DOCKER_CONTAINERS="$8"
-MOCK_GENERIC_HOSTS="$9"
-MOCK_K8S_CLUSTERS="${10}"
-MOCK_K8S_NODES="${11}"
-MOCK_K8S_PODS="${12}"
-MOCK_K8S_DEPLOYMENTS="${13}"
-MOCK_SEED_DURATION="${14}"
-MOCK_SAMPLE_INTERVAL="${15}"
-MOCK_UPDATE_INTERVAL="${16}"
-SERVICE_NAME="pulse"
-RELAY_SERVICE_NAME="pulse-relay"
-EXPECTED_BINARY="/opt/pulse/bin/pulse"
-EXPECTED_UNIT="/etc/systemd/system/pulse.service"
-MUTATED=false
-BACKUP_DIR=""
-HISTORY_BACKUP_ARCHIVE=""
-DEMO_HISTORY_PATHS=(
- ai_incidents.json
- ai_incidents.json.tmp
- alerts/alert-history.json
- alerts/alert-history.backup.json
- alerts/alert-history.json.imported
- alerts/alert-history.backup.json.imported
- alerts/events.db
- alerts/events.db-shm
- alerts/events.db-wal
-)
-
-for fixture_count in \
- "$MOCK_NODES" \
- "$MOCK_VMS_PER_NODE" \
- "$MOCK_LXCS_PER_NODE" \
- "$MOCK_DOCKER_HOSTS" \
- "$MOCK_DOCKER_CONTAINERS" \
- "$MOCK_GENERIC_HOSTS" \
- "$MOCK_K8S_CLUSTERS" \
- "$MOCK_K8S_NODES" \
- "$MOCK_K8S_PODS" \
- "$MOCK_K8S_DEPLOYMENTS"; do
- [[ "$fixture_count" =~ ^[1-9][0-9]*$ ]]
-done
-[[ "$MOCK_SEED_DURATION" =~ ^[1-9][0-9]*[smhd]$ ]]
-[[ "$MOCK_SAMPLE_INTERVAL" =~ ^[1-9][0-9]*[smh]$ ]]
-[[ "$MOCK_UPDATE_INTERVAL" =~ ^[1-9][0-9]*[smh]$ ]]
-
-log() {
- printf '[demo-recovery] %s\n' "$*" >&2
-}
-
-unit_hash() {
- sudo sha256sum "$EXPECTED_UNIT" | awk '{print $1}'
-}
-
-binary_hash() {
- sudo sha256sum "$EXPECTED_BINARY" | awk '{print $1}'
-}
-
-dropin_manifest_hash() {
- local paths path
- paths="$(systemctl show "$SERVICE_NAME" --property=DropInPaths --value)"
- if [ -z "$paths" ]; then
- printf '%s' 'none' | sha256sum | awk '{print $1}'
- return
- fi
- for path in $paths; do
- sudo test -f "$path"
- printf '%s\t' "$path"
- sudo sha256sum "$path" | awk '{print $1}'
- done | sort | sha256sum | awk '{print $1}'
-}
-
-runtime_config_hash() {
- local path
- for path in /etc/pulse/.env /etc/pulse/billing.json; do
- sudo test -f "$path"
- printf '%s\t' "$path"
- sudo sha256sum "$path" | awk '{print $1}'
- done | sha256sum | awk '{print $1}'
-}
-
-set_env_value() {
- local key="$1"
- local value="$2"
- local env_file="/etc/pulse/.env"
- if sudo grep -Eq "^[[:space:]]*${key}=" "$env_file"; then
- sudo sed -i "s|^[[:space:]]*${key}=.*|${key}=${value}|" "$env_file"
- else
- printf '\n%s=%s\n' "$key" "$value" | sudo tee -a "$env_file" >/dev/null
- fi
-}
-
-runtime_profile_matches() {
- sudo grep -Fxq "PULSE_MOCK_NODES=${MOCK_NODES}" /etc/pulse/.env &&
- sudo grep -Fxq "PULSE_MOCK_VMS_PER_NODE=${MOCK_VMS_PER_NODE}" /etc/pulse/.env &&
- sudo grep -Fxq "PULSE_MOCK_LXCS_PER_NODE=${MOCK_LXCS_PER_NODE}" /etc/pulse/.env &&
- sudo grep -Fxq "PULSE_MOCK_DOCKER_HOSTS=${MOCK_DOCKER_HOSTS}" /etc/pulse/.env &&
- sudo grep -Fxq "PULSE_MOCK_DOCKER_CONTAINERS=${MOCK_DOCKER_CONTAINERS}" /etc/pulse/.env &&
- sudo grep -Fxq "PULSE_MOCK_GENERIC_HOSTS=${MOCK_GENERIC_HOSTS}" /etc/pulse/.env &&
- sudo grep -Fxq "PULSE_MOCK_K8S_CLUSTERS=${MOCK_K8S_CLUSTERS}" /etc/pulse/.env &&
- sudo grep -Fxq "PULSE_MOCK_K8S_NODES=${MOCK_K8S_NODES}" /etc/pulse/.env &&
- sudo grep -Fxq "PULSE_MOCK_K8S_PODS=${MOCK_K8S_PODS}" /etc/pulse/.env &&
- sudo grep -Fxq "PULSE_MOCK_K8S_DEPLOYMENTS=${MOCK_K8S_DEPLOYMENTS}" /etc/pulse/.env &&
- sudo grep -Fxq "PULSE_MOCK_SEED_METRICS_STORE=false" /etc/pulse/.env &&
- sudo grep -Fxq "PULSE_MOCK_TRENDS_SEED_DURATION=${MOCK_SEED_DURATION}" /etc/pulse/.env &&
- sudo grep -Fxq "PULSE_MOCK_TRENDS_SAMPLE_INTERVAL=${MOCK_SAMPLE_INTERVAL}" /etc/pulse/.env &&
- sudo grep -Fxq "PULSE_MOCK_UPDATE_INTERVAL=${MOCK_UPDATE_INTERVAL}" /etc/pulse/.env
-}
-
-cleanup_config_backup() {
- [ -n "$BACKUP_DIR" ] || return 0
- sudo unlink "$BACKUP_DIR/runtime.env" 2>/dev/null || true
- sudo unlink "$BACKUP_DIR/billing.json" 2>/dev/null || true
- if [ -n "$HISTORY_BACKUP_ARCHIVE" ]; then
- sudo unlink "$HISTORY_BACKUP_ARCHIVE" 2>/dev/null || true
- HISTORY_BACKUP_ARCHIVE=""
- fi
- rmdir "$BACKUP_DIR" 2>/dev/null || true
- BACKUP_DIR=""
-}
-
-clear_demo_operational_history() {
- local rel_path
- local -a existing=()
- for rel_path in "${DEMO_HISTORY_PATHS[@]}"; do
- if sudo test -f "/etc/pulse/${rel_path}"; then
- existing+=("$rel_path")
- fi
- done
- ((${#existing[@]} > 0)) || return 0
-
- HISTORY_BACKUP_ARCHIVE="$BACKUP_DIR/demo-operational-history.tar"
- sudo tar --create --preserve-permissions \
- --file "$HISTORY_BACKUP_ARCHIVE" --directory /etc/pulse -- "${existing[@]}"
- for rel_path in "${existing[@]}"; do
- sudo unlink "/etc/pulse/${rel_path}"
- done
-}
-
-restore_demo_operational_history() {
- local rel_path
- [ -n "$HISTORY_BACKUP_ARCHIVE" ] || return 0
- sudo test -f "$HISTORY_BACKUP_ARCHIVE" || return 0
- for rel_path in "${DEMO_HISTORY_PATHS[@]}"; do
- sudo unlink "/etc/pulse/${rel_path}" 2>/dev/null || true
- done
- sudo tar --extract --preserve-permissions \
- --file "$HISTORY_BACKUP_ARCHIVE" --directory /etc/pulse
-}
-
-restore_runtime_config() {
- [ -n "$BACKUP_DIR" ] || return 0
- sudo cp --archive "$BACKUP_DIR/runtime.env" /etc/pulse/.env
- sudo cp --archive "$BACKUP_DIR/billing.json" /etc/pulse/billing.json
-}
-
-local_version() {
- curl --connect-timeout 3 --max-time 8 -fsS \
- "${LOCAL_BASE_URL}/api/version" | jq -er '.version'
-}
-
-local_healthy() {
- curl --connect-timeout 3 --max-time 8 -fsS \
- "${LOCAL_BASE_URL}/api/health" >/dev/null && local_version >/dev/null
-}
-
-emit_evidence() {
- local status="$1"
- local before_pid="$2"
- local after_pid="$3"
- local before_relay_pid="$4"
- local after_relay_pid="$5"
- local before_binary_sha="$6"
- local after_binary_sha="$7"
- local before_unit_sha="$8"
- local after_unit_sha="$9"
- local before_dropins_sha="${10}"
- local after_dropins_sha="${11}"
- local version="${12}"
- local before_config_sha="${13}"
- local after_config_sha="${14}"
- jq -n \
- --arg status "$status" \
- --argjson mutated "$MUTATED" \
- --arg hostname "$(hostname)" \
- --arg service "$SERVICE_NAME" \
- --arg before_pid "$before_pid" \
- --arg after_pid "$after_pid" \
- --arg relay_service "$RELAY_SERVICE_NAME" \
- --arg before_relay_pid "$before_relay_pid" \
- --arg after_relay_pid "$after_relay_pid" \
- --arg binary "$EXPECTED_BINARY" \
- --arg before_binary_sha256 "$before_binary_sha" \
- --arg after_binary_sha256 "$after_binary_sha" \
- --arg unit "$EXPECTED_UNIT" \
- --arg before_unit_sha256 "$before_unit_sha" \
- --arg after_unit_sha256 "$after_unit_sha" \
- --arg before_dropins_sha256 "$before_dropins_sha" \
- --arg after_dropins_sha256 "$after_dropins_sha" \
- --arg before_runtime_config_sha256 "$before_config_sha" \
- --arg after_runtime_config_sha256 "$after_config_sha" \
- --arg version "$version" \
- --arg expected_version "$EXPECTED_VERSION" \
- --arg mock_nodes "$MOCK_NODES" \
- --arg mock_vms_per_node "$MOCK_VMS_PER_NODE" \
- --arg mock_lxcs_per_node "$MOCK_LXCS_PER_NODE" \
- --arg mock_docker_hosts "$MOCK_DOCKER_HOSTS" \
- --arg mock_docker_containers "$MOCK_DOCKER_CONTAINERS" \
- --arg mock_generic_hosts "$MOCK_GENERIC_HOSTS" \
- --arg mock_k8s_clusters "$MOCK_K8S_CLUSTERS" \
- --arg mock_k8s_nodes "$MOCK_K8S_NODES" \
- --arg mock_k8s_pods "$MOCK_K8S_PODS" \
- --arg mock_k8s_deployments "$MOCK_K8S_DEPLOYMENTS" \
- --arg mock_seed_duration "$MOCK_SEED_DURATION" \
- --arg mock_sample_interval "$MOCK_SAMPLE_INTERVAL" \
- --arg mock_update_interval "$MOCK_UPDATE_INTERVAL" \
- '{schema_version: 1, status: $status, mutated: $mutated,
- hostname: $hostname, service: $service,
- before_pid: $before_pid, after_pid: $after_pid,
- relay_service: $relay_service,
- before_relay_pid: $before_relay_pid,
- after_relay_pid: $after_relay_pid,
- binary: $binary,
- before_binary_sha256: $before_binary_sha256,
- after_binary_sha256: $after_binary_sha256,
- unit: $unit,
- before_unit_sha256: $before_unit_sha256,
- after_unit_sha256: $after_unit_sha256,
- before_dropins_sha256: $before_dropins_sha256,
- after_dropins_sha256: $after_dropins_sha256,
- before_runtime_config_sha256: $before_runtime_config_sha256,
- after_runtime_config_sha256: $after_runtime_config_sha256,
- version: $version, expected_version: $expected_version,
- runtime_profile: {mock_nodes: $mock_nodes,
- mock_vms_per_node: $mock_vms_per_node,
- mock_lxcs_per_node: $mock_lxcs_per_node,
- mock_docker_hosts: $mock_docker_hosts,
- mock_docker_containers: $mock_docker_containers,
- mock_generic_hosts: $mock_generic_hosts,
- mock_k8s_clusters: $mock_k8s_clusters,
- mock_k8s_nodes: $mock_k8s_nodes,
- mock_k8s_pods: $mock_k8s_pods,
- mock_k8s_deployments: $mock_k8s_deployments,
- mock_seed_duration: $mock_seed_duration,
- mock_sample_interval: $mock_sample_interval,
- mock_update_interval: $mock_update_interval}}'
-}
-
-[ "$(hostname)" = "$EXPECTED_HOSTNAME" ] || {
- log "target hostname does not match the governed demo environment"
- exit 1
-}
-[ "$(systemctl show "$SERVICE_NAME" --property=FragmentPath --value)" = "$EXPECTED_UNIT" ]
-[ "$(systemctl show "$SERVICE_NAME" --property=User --value)" = "pulse" ]
-[ "$(systemctl show "$SERVICE_NAME" --property=Group --value)" = "pulse" ]
-[ "$(systemctl is-enabled "$SERVICE_NAME")" = "enabled" ]
-[ "$(systemctl is-active "$RELAY_SERVICE_NAME")" = "active" ]
-systemctl show "$SERVICE_NAME" --property=ExecStart --value | grep -Fq "$EXPECTED_BINARY"
-sudo test -x "$EXPECTED_BINARY"
-sudo test -f "$EXPECTED_UNIT"
-
-BEFORE_PID="$(systemctl show "$SERVICE_NAME" --property=MainPID --value)"
-BEFORE_RELAY_PID="$(systemctl show "$RELAY_SERVICE_NAME" --property=MainPID --value)"
-BEFORE_BINARY_SHA="$(binary_hash)"
-BEFORE_UNIT_SHA="$(unit_hash)"
-BEFORE_DROPINS_SHA="$(dropin_manifest_hash)"
-BEFORE_CONFIG_SHA="$(runtime_config_hash)"
-
-if local_healthy && runtime_profile_matches; then
- VERSION="$(local_version)"
- [ "${VERSION#v}" = "$EXPECTED_VERSION" ]
- emit_evidence healthy_noop "$BEFORE_PID" "$BEFORE_PID" \
- "$BEFORE_RELAY_PID" "$BEFORE_RELAY_PID" \
- "$BEFORE_BINARY_SHA" "$BEFORE_BINARY_SHA" \
- "$BEFORE_UNIT_SHA" "$BEFORE_UNIT_SHA" \
- "$BEFORE_DROPINS_SHA" "$BEFORE_DROPINS_SHA" "$VERSION" \
- "$BEFORE_CONFIG_SHA" "$BEFORE_CONFIG_SHA"
- exit 0
-fi
-
-rollback_on_error() {
- local rc=$?
- local failed_pid
- trap - ERR
- if [ "$MUTATED" = true ]; then
- failed_pid="$(systemctl show "$SERVICE_NAME" --property=MainPID --value || true)"
- if [[ "$failed_pid" =~ ^[1-9][0-9]*$ ]] && sudo kill -0 "$failed_pid" 2>/dev/null; then
- # The HTTP listener never opened, so pprof is unavailable. SIGQUIT gives
- # one bounded Go goroutine dump in the service journal before the already
- # failed process is stopped; it does not widen recovery to any other unit.
- log "capturing failed Pulse startup goroutines before compensation"
- sudo kill -QUIT "$failed_pid" || true
- sleep 2
- fi
- log "recovery validation failed; compensating by stopping only pulse.service"
- sudo systemctl stop "$SERVICE_NAME" || true
- restore_runtime_config || true
- restore_demo_operational_history || true
- fi
- emit_evidence compensated_unavailable "$BEFORE_PID" \
- "$(systemctl show "$SERVICE_NAME" --property=MainPID --value || true)" \
- "$BEFORE_RELAY_PID" \
- "$(systemctl show "$RELAY_SERVICE_NAME" --property=MainPID --value || true)" \
- "$BEFORE_BINARY_SHA" "$(binary_hash || true)" \
- "$BEFORE_UNIT_SHA" "$(unit_hash || true)" \
- "$BEFORE_DROPINS_SHA" "$(dropin_manifest_hash || true)" "" \
- "$BEFORE_CONFIG_SHA" "$(runtime_config_hash || true)"
- cleanup_config_backup
- exit "$rc"
-}
-BACKUP_DIR="$(mktemp -d)"
-sudo cp --archive /etc/pulse/.env "$BACKUP_DIR/runtime.env"
-sudo cp --archive /etc/pulse/billing.json "$BACKUP_DIR/billing.json"
-trap rollback_on_error ERR
-
-log "applying the target-compatible demo runtime profile and restarting only pulse.service"
-MUTATED=true
-set_env_value PULSE_MOCK_NODES "$MOCK_NODES"
-set_env_value PULSE_MOCK_VMS_PER_NODE "$MOCK_VMS_PER_NODE"
-set_env_value PULSE_MOCK_LXCS_PER_NODE "$MOCK_LXCS_PER_NODE"
-set_env_value PULSE_MOCK_DOCKER_HOSTS "$MOCK_DOCKER_HOSTS"
-set_env_value PULSE_MOCK_DOCKER_CONTAINERS "$MOCK_DOCKER_CONTAINERS"
-set_env_value PULSE_MOCK_GENERIC_HOSTS "$MOCK_GENERIC_HOSTS"
-set_env_value PULSE_MOCK_K8S_CLUSTERS "$MOCK_K8S_CLUSTERS"
-set_env_value PULSE_MOCK_K8S_NODES "$MOCK_K8S_NODES"
-set_env_value PULSE_MOCK_K8S_PODS "$MOCK_K8S_PODS"
-set_env_value PULSE_MOCK_K8S_DEPLOYMENTS "$MOCK_K8S_DEPLOYMENTS"
-# Persistent mock-store backfill is reserved for the isolated local-dev data
-# directory. The governed demo already has persistent runtime data, so a stale
-# opt-in here can turn startup into an unbounded SQLite backfill.
-set_env_value PULSE_MOCK_SEED_METRICS_STORE false
-set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"
-set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL"
-set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"
-sudo chown pulse:pulse /etc/pulse/.env
-sudo chmod 600 /etc/pulse/.env
-# Alert lifecycle and AI incident history on this host are generated demo
-# operational memory. A failed 50-node bootstrap can leave thousands of
-# lifecycle events whose v6.4 replay queues one serialized incident save per
-# event before HTTP opens. Back up and clear only those fixed demo-history
-# files, including the retired JSON sources that repopulate a cleared event
-# log; compensation restores them if the bounded profile still fails.
-clear_demo_operational_history
-sudo systemctl restart "$SERVICE_NAME"
-
-for _ in $(seq 1 12); do
- if [ "$(systemctl is-active "$SERVICE_NAME" || true)" = "active" ] && local_healthy; then
- break
- fi
- sleep 5
-done
-
-[ "$(systemctl is-active "$SERVICE_NAME")" = "active" ]
-[ "$(systemctl show "$SERVICE_NAME" --property=SubState --value)" = "running" ]
-local_healthy
-runtime_profile_matches
-
-AFTER_PID="$(systemctl show "$SERVICE_NAME" --property=MainPID --value)"
-AFTER_RELAY_PID="$(systemctl show "$RELAY_SERVICE_NAME" --property=MainPID --value)"
-AFTER_BINARY_SHA="$(binary_hash)"
-AFTER_UNIT_SHA="$(unit_hash)"
-AFTER_DROPINS_SHA="$(dropin_manifest_hash)"
-AFTER_CONFIG_SHA="$(runtime_config_hash)"
-VERSION="$(local_version)"
-
-[ "$AFTER_PID" -gt 0 ]
-[ "$AFTER_PID" != "$BEFORE_PID" ]
-[ "$AFTER_RELAY_PID" = "$BEFORE_RELAY_PID" ]
-[ "$AFTER_BINARY_SHA" = "$BEFORE_BINARY_SHA" ]
-[ "$AFTER_UNIT_SHA" = "$BEFORE_UNIT_SHA" ]
-[ "$AFTER_DROPINS_SHA" = "$BEFORE_DROPINS_SHA" ]
-[ "${VERSION#v}" = "$EXPECTED_VERSION" ]
-
-trap - ERR
-cleanup_config_backup
-emit_evidence recovered "$BEFORE_PID" "$AFTER_PID" \
- "$BEFORE_RELAY_PID" "$AFTER_RELAY_PID" \
- "$BEFORE_BINARY_SHA" "$AFTER_BINARY_SHA" \
- "$BEFORE_UNIT_SHA" "$AFTER_UNIT_SHA" \
- "$BEFORE_DROPINS_SHA" "$AFTER_DROPINS_SHA" "$VERSION" \
- "$BEFORE_CONFIG_SHA" "$AFTER_CONFIG_SHA"
+# CI observer only; remote systemd owns mutation, restoration and both watches.
+exec python3 .github/scripts/dispatch-demo-runtime.py recover
diff --git a/.github/scripts/tests/test_demo_runtime_transaction.py b/.github/scripts/tests/test_demo_runtime_transaction.py
new file mode 100644
index 0000000000..36bb810f44
--- /dev/null
+++ b/.github/scripts/tests/test_demo_runtime_transaction.py
@@ -0,0 +1,443 @@
+import copy
+import hashlib
+import importlib.util
+import json
+import os
+from pathlib import Path
+import signal
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+
+
+SCRIPTS = Path(__file__).resolve().parents[1]
+
+
+def load(name, filename):
+ spec = importlib.util.spec_from_file_location(name, SCRIPTS / filename)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+engine = load("demo_transaction", "demo-runtime-transaction.py")
+dispatcher = load("demo_dispatcher", "dispatch-demo-runtime.py")
+
+
+def request(mode="recover"):
+ return {"mode": mode, "hostname": "fixture", "local_url": "http://127.0.0.1:7655",
+ "public_url": "https://demo.example/api/health", "version": "v6.4.5",
+ "control_sha": "a" * 40, "run_id": "123", "run_attempt": "1",
+ "installer_sha256": hashlib.sha256(b"signed-installer-fixture").hexdigest() if mode == "update" else "",
+ "profile": {**{name: "2" for name in engine.COUNT_KEYS},
+ "seed_duration": "2h", "sample_interval": "5m", "update_interval": "15s"}}
+
+
+class FixtureHost(engine.Host):
+ """Real filesystem estate with a virtual *unchanged* 300-second clock.
+
+ Commands/services/journal/HTTP are adapters, not native or installed proof.
+ """
+ def __init__(self):
+ self.clock = 0
+ self.pid = 100
+ self.running = True
+ self.runtime_version = "6.4.5"
+ self.starts = 0
+ self.stops = 0
+ self.install_count = 0
+ self.fail_at = None
+ self.failed = False
+ self.rollback_failure = False
+ self.unhealthy = False
+ self.journal_failure = False
+ self.ready_until = 0
+ self.cancel_at = None
+ self.cancel_in_recovery = False
+ self.co_host_change = False
+ self.co_pid = 200
+ self.transaction = None
+
+ def now(self):
+ return self.clock
+
+ def sleep(self, seconds):
+ self.clock += seconds
+
+ def identity(self, hostname):
+ if hostname != "fixture":
+ raise engine.Failure("host-identity")
+
+ def state(self):
+ state = {"ActiveState": "active", "SubState": "running", "MainPID": str(self.pid), "NRestarts": "0"}
+ result = {"pulse": state, "pulse-relay": {**state, "MainPID": "200"}, "caddy": {**state, "MainPID": str(self.co_pid)}}
+ if not self.running:
+ result["pulse"] = {**state, "ActiveState": "inactive", "SubState": "dead", "MainPID": "0"}
+ return result
+
+ def status(self, url):
+ if url == engine.RELAY_HEALTH:
+ return "200"
+ if self.clock < self.ready_until or self.unhealthy:
+ return "503"
+ if self.failed and self.rollback_failure:
+ return "503"
+ return "200" if self.running else "000"
+
+ def version(self, url):
+ return self.runtime_version
+
+ def cursors(self):
+ return {name: str(self.clock) for name in engine.SERVICES}
+
+ def check_journal(self, cursors):
+ phase = self.transaction.receipt["status"] if self.transaction else ""
+ if self.journal_failure:
+ raise engine.Failure("journal-observation")
+ if self.co_host_change and self.clock >= 55:
+ self.co_pid = 201
+ if self.cancel_at is not None and self.clock >= self.cancel_at and not self.failed:
+ self.failed = True
+ handler = signal.getsignal(signal.SIGTERM)
+ handler(signal.SIGTERM, None)
+ if self.cancel_in_recovery and phase == "recovering" and self.clock >= 350:
+ # Exercise the actual installed handler decision; do not signal the
+ # source-proof executor or kill another process to prove it.
+ assert signal.getsignal(signal.SIGTERM) == signal.SIG_IGN
+ self.cancel_in_recovery = False
+ if self.fail_at is not None and self.clock >= self.fail_at and not self.failed:
+ self.failed = True
+ raise engine.Failure("new-service-crash")
+
+ def stop(self):
+ self.stops += 1
+ self.running = False
+
+ def start(self):
+ self.starts += 1
+ self.pid += 1
+ self.running = True
+ if self.transaction.receipt["status"] == "recovering":
+ self.runtime_version = "6.4.5"
+ self.unhealthy = False
+
+ def install(self, attempt, req):
+ if hashlib.sha256((attempt / "installer.sh").read_bytes()).hexdigest() != req["installer_sha256"]:
+ raise engine.Failure("installer-identity")
+ self.install_count += 1
+ self.runtime_version = req["version"].lstrip("v")
+ engine.PATHS["binary"].write_bytes(b"new executable")
+ engine.PATHS["unit"].write_bytes(b"new unit")
+ (engine.PATHS["data"] / "persisted-history").write_bytes(b"forward history")
+
+ def profile(self, req, unhealthy):
+ super().profile(req, unhealthy)
+ if self.unhealthy:
+ self.unhealthy = False
+
+
+class TransactionTest(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.tmp.cleanup)
+ root = Path(self.tmp.name)
+ self.paths = {name: root / "estate" / name for name in engine.PATHS}
+ for name, path in self.paths.items():
+ path.parent.mkdir(parents=True, exist_ok=True)
+ if name in {"dropins", "data"}:
+ path.mkdir()
+ else:
+ path.write_bytes(b"old " + name.encode())
+ path.chmod(0o755 if name == "binary" else 0o644)
+ (self.paths["dropins"] / "limits.conf").write_bytes(b"old limits")
+ (self.paths["data"] / ".env").write_text("PRIVATE_TOKEN=not-logged\n")
+ (self.paths["data"] / "billing.json").write_text('{"capabilities":[],"integrity":"private"}')
+ (self.paths["data"] / "persisted-history").write_bytes(b"original history")
+ (self.paths["data"] / "alerts").mkdir()
+ (self.paths["data"] / "alerts/events.db").write_bytes(b"synthetic opaque persistent file")
+ self.root = root / "control"
+ self.attempt = self.root / "attempts" / ("a" * 64)
+ self.attempt.mkdir(parents=True)
+ (self.attempt / "installer.sh").write_bytes(b"signed-installer-fixture")
+ self.addCleanup(patch.stopall)
+ patch.object(engine, "ROOT", self.root).start()
+ patch.object(engine, "PATHS", self.paths).start()
+ self.before = engine.estate_hash(self.paths)
+ self.host = FixtureHost()
+
+ def run_transaction(self, mode="recover", version=None):
+ req = request(mode)
+ if version:
+ req["version"] = version
+ tx = engine.Transaction(self.host, req, self.attempt)
+ self.host.transaction = tx
+ result = tx.run()
+ self.assertEqual(tx.receipt, json.loads((self.attempt / "receipt.json").read_text()))
+ return result, tx.receipt
+
+ def test_healthy_profile_change_keeps_data_and_watches_complete_window(self):
+ code, receipt = self.run_transaction()
+ self.assertEqual(code, 0)
+ self.assertEqual(receipt["status"], "committed")
+ self.assertEqual(receipt["forward"]["elapsed_seconds"], 300)
+ self.assertEqual(receipt["forward"]["samples"], 61)
+ self.assertEqual((self.paths["data"] / "alerts/events.db").read_bytes(), b"synthetic opaque persistent file")
+ self.assertTrue((self.attempt / "snapshot/data/persisted-history").exists())
+ self.assertNotIn("not-logged", json.dumps(receipt))
+
+ def test_healthy_noop_requires_profile_and_full_watch_without_restart(self):
+ self.host.profile(request(), False)
+ code, receipt = self.run_transaction()
+ self.assertEqual(code, 0)
+ self.assertEqual(receipt["status"], "healthy_noop")
+ self.assertEqual(receipt["forward"]["elapsed_seconds"], 300)
+ self.assertFalse(receipt["mutated"])
+ self.assertEqual(self.host.stops, 0)
+
+ def test_update_restores_complete_changed_executable_unit_and_data_after_late_failure(self):
+ self.host.fail_at = 55
+ code, receipt = self.run_transaction("update", "v6.4.6")
+ self.assertEqual(code, 1)
+ self.assertEqual(receipt["status"], "rolled_back")
+ self.assertEqual(receipt["failure"], "new-service-crash")
+ self.assertEqual(receipt["recovery"]["elapsed_seconds"], 300)
+ self.assertEqual(engine.estate_hash(self.paths), self.before)
+ self.assertEqual(self.host.runtime_version, "6.4.5")
+
+ def test_failure_at_299_seconds_still_collects_entire_recovery_and_cancellation(self):
+ self.host.fail_at = 299
+ self.host.cancel_in_recovery = True
+ previous = signal.getsignal(signal.SIGTERM)
+ code, receipt = self.run_transaction("update", "v6.4.6")
+ self.assertEqual(code, 1)
+ self.assertEqual(receipt["recovery"]["elapsed_seconds"], 300)
+ self.assertGreaterEqual(self.host.clock, 600)
+ self.assertFalse(self.host.cancel_in_recovery)
+ self.assertEqual(signal.getsignal(signal.SIGTERM), previous)
+ self.assertEqual(engine.estate_hash(self.paths), self.before)
+
+ def test_forward_cancel_owns_full_recovery(self):
+ self.host.cancel_at = 55
+ code, receipt = self.run_transaction("update", "v6.4.6")
+ self.assertEqual(code, 1)
+ self.assertEqual(receipt["failure"], "cancelled-forward")
+ self.assertEqual(receipt["status"], "rolled_back")
+ self.assertEqual(receipt["recovery"]["elapsed_seconds"], 300)
+
+ def test_failed_rollback_retains_estate_and_blocks_next_transaction(self):
+ self.host.fail_at = 55
+ self.host.rollback_failure = True
+ code, receipt = self.run_transaction("update", "v6.4.6")
+ self.assertEqual(code, 1)
+ self.assertEqual(receipt["status"], "rollback_failed")
+ self.assertTrue(receipt["snapshot_retained"])
+ next_attempt = self.root / "attempts" / ("b" * 64)
+ next_attempt.mkdir()
+ self.attempt = next_attempt
+ self.host.rollback_failure = False
+ code, receipt = self.run_transaction()
+ self.assertEqual(receipt["failure"], "prior-recovery-required")
+ self.assertFalse(receipt["mutated"])
+
+ def test_unhealthy_recovery_retains_original_generated_history(self):
+ self.host.unhealthy = True
+ code, receipt = self.run_transaction()
+ self.assertEqual(code, 0)
+ self.assertFalse(receipt["healthy_baseline"])
+ self.assertFalse((self.paths["data"] / "alerts/events.db").exists())
+ self.assertEqual((self.attempt / "snapshot/data/alerts/events.db").read_bytes(), b"synthetic opaque persistent file")
+ self.assertEqual(receipt["forward"]["elapsed_seconds"], 300)
+
+ def test_failed_unhealthy_recovery_is_not_a_healthy_rollback(self):
+ self.host.unhealthy = True
+ self.host.fail_at = 55
+ code, receipt = self.run_transaction()
+ self.assertEqual(code, 1)
+ self.assertEqual(receipt["status"], "recovery_required")
+ self.assertEqual(receipt["rollback"], "unavailable-unhealthy-baseline")
+ self.assertEqual(engine.estate_hash(self.paths), self.before)
+ self.assertFalse(self.host.running)
+
+ def test_update_refuses_unhealthy_baseline_without_stopping_or_pruning(self):
+ self.host.unhealthy = True
+ code, receipt = self.run_transaction("update", "v6.4.6")
+ self.assertEqual(code, 1)
+ self.assertEqual(receipt["failure"], "update-baseline-unhealthy")
+ self.assertEqual(self.host.stops, 0)
+ self.assertEqual(engine.estate_hash(self.paths), self.before)
+
+ def test_recovery_refuses_different_healthy_source(self):
+ code, receipt = self.run_transaction(version="v6.4.6")
+ self.assertEqual(code, 1)
+ self.assertEqual(receipt["failure"], "recovery-source-mismatch")
+ self.assertEqual(self.host.stops, 0)
+
+ def test_listener_readiness_does_not_spend_the_watch_window(self):
+ self.host.ready_until = 30
+ code, receipt = self.run_transaction()
+ self.assertEqual(code, 0)
+ self.assertEqual(receipt["forward"]["elapsed_seconds"], 300)
+ self.assertGreaterEqual(self.host.clock, 330)
+ self.assertEqual(self.host.starts, 1)
+
+ def test_missing_journal_fails_closed_and_retains_failure(self):
+ self.host.journal_failure = True
+ code, receipt = self.run_transaction()
+ self.assertEqual(code, 1)
+ self.assertEqual(receipt["status"], "rollback_failed")
+ self.assertEqual(receipt["recovery_failure"], "journal-observation")
+
+ def test_cohost_restart_is_not_silently_accepted_as_recovery(self):
+ self.host.co_host_change = True
+ code, receipt = self.run_transaction("update", "v6.4.6")
+ self.assertEqual(code, 1)
+ self.assertEqual(receipt["status"], "rollback_failed")
+ self.assertEqual(receipt["recovery_failure"], "cohost-identity-changed")
+
+ def test_snapshot_headroom_failure_never_deletes_persistent_files(self):
+ with patch.object(engine.shutil, "disk_usage", return_value=type("Usage", (), {"free": 0})()):
+ code, receipt = self.run_transaction()
+ self.assertEqual(code, 1)
+ self.assertEqual(receipt["status"], "rolled_back")
+ self.assertFalse(receipt["mutated"])
+ self.assertEqual(receipt["failure"], "snapshot-headroom")
+ self.assertEqual(engine.estate_hash(self.paths), self.before)
+
+ def test_linked_history_cannot_remove_files_outside_estate(self):
+ (self.paths["data"] / "alerts/events.db").unlink()
+ outside = self.root / "private-file"
+ outside.write_text("keep")
+ (self.paths["data"] / "alerts/events.db").symlink_to(outside)
+ self.host.unhealthy = True
+ code, receipt = self.run_transaction()
+ self.assertEqual(code, 1)
+ self.assertEqual(receipt["failure"], "history-symlink")
+ self.assertEqual(outside.read_text(), "keep")
+
+ def test_shared_host_lock_refusal_is_terminal_and_has_no_mutation(self):
+ lock = self.root / "host.lock"
+ fd = os.open(lock, os.O_CREAT | os.O_RDWR, 0o600)
+ self.addCleanup(os.close, fd)
+ engine.fcntl.flock(fd, engine.fcntl.LOCK_EX | engine.fcntl.LOCK_NB)
+ (self.attempt / "request.json").write_text(json.dumps(request()))
+ with patch.object(engine, "LOCK", lock), patch.object(engine.sys, "argv", ["transaction", str(self.attempt)]):
+ self.assertEqual(engine.main(), 1)
+ receipt = json.loads((self.attempt / "receipt.json").read_text())
+ self.assertEqual(receipt["status"], "refused")
+ self.assertFalse(receipt["mutated"])
+ self.assertEqual(engine.estate_hash(self.paths), self.before)
+
+
+class InputAndCommandTest(unittest.TestCase):
+ def test_valid_current_envelope(self):
+ engine.validate(request())
+ engine.validate(request("update"))
+
+ def test_no_candidate_prerelease_arbitrary_service_url_or_profile(self):
+ cases = [("mode", "shell"), ("version", "v6.4.6-rc.1"), ("control_sha", "main"),
+ ("local_url", "http://other-host:7655"), ("public_url", "http://demo.example/api/health"),
+ ("public_url", "https://user:secret@demo.example/api/health"),
+ ("public_url", "https://demo.example/api/health?token=secret"),
+ ("installer_sha256", "arbitrary")]
+ for name, value in cases:
+ with self.subTest(name=name, value=value):
+ req = request(); req[name] = value
+ with self.assertRaises(engine.Failure):
+ engine.validate(req)
+ req = request(); req["service"] = "license"
+ with self.assertRaises(engine.Failure):
+ engine.validate(req)
+ req = request(); req["profile"]["nodes"] = "2; restart caddy"
+ with self.assertRaises(engine.Failure):
+ engine.validate(req)
+
+ def test_http_observation_rejects_redirect_and_transport_failure(self):
+ host = engine.Host()
+ with patch.object(host, "command", return_value="302") as command:
+ self.assertEqual(host.status("https://demo.example/api/health"), "302")
+ self.assertNotIn("--location", command.call_args.args[0])
+ self.assertIn("--disable", command.call_args.args[0])
+ self.assertIn("/dev/null", command.call_args.args[0])
+ with patch.object(host, "command", side_effect=engine.Failure("command-failed")):
+ self.assertEqual(host.status("https://demo.example/api/health"), "000")
+
+ def test_malformed_or_unavailable_journal_is_not_no_crash(self):
+ host = engine.Host()
+ for text in ["not-json", '{"MESSAGE":"fine"}', '{"MESSAGE":[],"__CURSOR":"c"}']:
+ with self.subTest(text=text), patch.object(host, "command", return_value=text):
+ with self.assertRaises(engine.Failure):
+ host.check_journal({"pulse": "c"})
+ with patch.object(host, "command", return_value='{"MESSAGE":"panic: fixture","__CURSOR":"d"}'):
+ with self.assertRaisesRegex(engine.Failure, "new-service-crash"):
+ host.check_journal({"pulse": "c"})
+
+ def test_journal_cursor_advances_without_returning_private_message(self):
+ host = engine.Host(); cursors = {"pulse": "c"}
+ with patch.object(host, "command", return_value='{"MESSAGE":"private normal message","__CURSOR":"d"}'):
+ host.check_journal(cursors)
+ self.assertEqual(cursors, {"pulse": "d"})
+
+ def test_no_journal_cursor_and_malformed_service_state_fail_closed(self):
+ host = engine.Host()
+ with patch.object(host, "command", return_value=""):
+ with self.assertRaises(engine.Failure):
+ host.cursors()
+ with self.assertRaises(engine.Failure):
+ host.state()
+
+
+class DispatchLifecycleTest(unittest.TestCase):
+ def test_observer_budget_covers_child_and_recovery_and_no_wait_owned_by_ssh(self):
+ source = dispatcher.BOOTSTRAP
+ self.assertIn("RuntimeMaxSec=45min", source)
+ self.assertIn("TimeoutStopSec=20min", source)
+ self.assertIn("KillMode=mixed", source)
+ self.assertIn("+ 4000", source)
+ self.assertGreater(4000, 45 * 60 + 20 * 60)
+ self.assertNotIn("'--wait'", source)
+ self.assertNotIn("systemctl', 'stop'", source)
+
+ def test_bootstrap_retains_intent_and_repeated_or_uncertain_launch_does_not_replay(self):
+ # Execute the real bootstrap with only its filesystem/root/systemd/clock
+ # adapters rebound, including a lost submission response. No host tools.
+ for uncertain in (False, True):
+ with self.subTest(uncertain=uncertain), tempfile.TemporaryDirectory() as directory:
+ root = Path(directory) / "state"
+ source = dispatcher.BOOTSTRAP.replace("pathlib.Path('/var/lib/pulse-deploy/demo')", repr(root))
+ # repr(Path) is not a Python literal: bind just the fixed path.
+ source = dispatcher.BOOTSTRAP.replace("'/var/lib/pulse-deploy/demo'", repr(str(root)))
+ source = source.replace("root.stat().st_uid == 0", "root.stat().st_uid == os.getuid()")
+ clock = [0]
+ calls = []
+ payload = {"request": request(), "source": "c291cmNl", "installer": ""}
+
+ def launch(argv, **kwargs):
+ calls.append(argv)
+ attempt = next((root / "attempts").iterdir())
+ if uncertain:
+ raise subprocess.TimeoutExpired("systemd-run", 30)
+ (attempt / "receipt.json").write_text(json.dumps({"status": "committed"}))
+ return subprocess.CompletedProcess(argv, 0)
+
+ class Input:
+ def read(self, _limit):
+ return json.dumps(payload)
+
+ with patch.object(os, "geteuid", return_value=0), patch.object(subprocess, "run", side_effect=launch), \
+ patch.object(dispatcher.sys, "stdin", Input()), patch("time.monotonic", side_effect=lambda: clock[0]), \
+ patch("time.sleep", side_effect=lambda seconds: clock.__setitem__(0, clock[0] + seconds)), patch("builtins.print"):
+ for _ in range(2):
+ with self.assertRaises(SystemExit):
+ exec(compile(source, "fixed-bootstrap", "exec"), {})
+ self.assertEqual(len(calls), 1)
+ attempt = next((root / "attempts").iterdir())
+ self.assertTrue((attempt / "intent.json").exists())
+ self.assertEqual((attempt / "request.json").stat().st_mode & 0o777, 0o600)
+ self.assertEqual(attempt.stat().st_mode & 0o777, 0o700)
+ if uncertain:
+ self.assertEqual(json.loads((attempt / "launch.json").read_text())["state"], "uncertain")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/workflows/recover-demo-server.yml b/.github/workflows/recover-demo-server.yml
index c482081c59..b0dc92f2c0 100644
--- a/.github/workflows/recover-demo-server.yml
+++ b/.github/workflows/recover-demo-server.yml
@@ -15,7 +15,7 @@ concurrency:
jobs:
recover:
runs-on: ubuntu-24.04
- timeout-minutes: 45
+ timeout-minutes: 90
environment: demo-stable
env:
DEMO_EXPECTED_HOSTNAME: ${{ vars.DEMO_EXPECTED_HOSTNAME }}
@@ -117,16 +117,8 @@ jobs:
MOCK_UPDATE_INTERVAL: ${{ steps.runtime_profile.outputs.mock_update_interval }}
run: |
set -euo pipefail
- ssh -i ~/.ssh/id_ed25519 \
- -o IdentitiesOnly=yes \
- -o StrictHostKeyChecking=yes \
- -o UserKnownHostsFile=~/.ssh/known_hosts \
- "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \
- "bash -s -- $(printf '%q ' "$DEMO_EXPECTED_HOSTNAME" "$DEMO_LOCAL_BASE_URL" "$EXPECTED_VERSION" "$MOCK_NODES" "$MOCK_VMS_PER_NODE" "$MOCK_LXCS_PER_NODE" "$MOCK_DOCKER_HOSTS" "$MOCK_DOCKER_CONTAINERS" "$MOCK_GENERIC_HOSTS" "$MOCK_K8S_CLUSTERS" "$MOCK_K8S_NODES" "$MOCK_K8S_PODS" "$MOCK_K8S_DEPLOYMENTS" "$MOCK_SEED_DURATION" "$MOCK_SAMPLE_INTERVAL" "$MOCK_UPDATE_INTERVAL")" \
- < .github/scripts/recover-demo-runtime.sh \
- > demo-runtime-recovery.json
- jq -e '.schema_version == 1 and (.status == "recovered" or .status == "healthy_noop")' \
- demo-runtime-recovery.json
+ bash .github/scripts/recover-demo-runtime.sh > demo-runtime-recovery.json
+ jq -e '.schema_version == 2 and (.status == "committed" or .status == "healthy_noop") and .forward.elapsed_seconds >= 300' demo-runtime-recovery.json
echo "mutated=$(jq -r '.mutated' demo-runtime-recovery.json)" >> "$GITHUB_OUTPUT"
- name: Capture bounded Pulse failure diagnostics
@@ -199,22 +191,9 @@ jobs:
export PULSE_DEMO_AUTH_PASS="${DEMO_AUTH_PASS:-demo}"
./scripts/run_demo_public_browser_smoke.sh
- - name: Compensate failed mutated recovery
- if: failure() && steps.recovery.outputs.mutated == 'true'
- env:
- DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }}
- DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }}
- run: |
- set -euo pipefail
- BEFORE="$(ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \
- "systemctl show pulse --property=ActiveState,SubState,MainPID --value | paste -sd, -")"
- ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \
- "sudo systemctl stop pulse"
- AFTER="$(ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \
- "systemctl show pulse --property=ActiveState,SubState,MainPID --value | paste -sd, -")"
- jq -n --arg before "$BEFORE" --arg after "$AFTER" \
- '{schema_version: 1, status: "compensated_unavailable", before: $before, after: $after}' \
- > demo-runtime-rollback.json
+ # The retained systemd transaction already owns compensation. A later
+ # browser/network failure is evidence to reconcile, not authority to stop
+ # an unrelated or already-committed Pulse process in a second SSH session.
- name: Retain bounded recovery evidence
if: always()
@@ -223,7 +202,6 @@ jobs:
name: stable-demo-runtime-recovery-${{ github.run_id }}
path: |
demo-runtime-recovery.json
- demo-runtime-rollback.json
demo-runtime-diagnostics.txt
if-no-files-found: warn
retention-days: 14
diff --git a/.github/workflows/update-demo-server.yml b/.github/workflows/update-demo-server.yml
index 996f7bda56..214115ebbe 100644
--- a/.github/workflows/update-demo-server.yml
+++ b/.github/workflows/update-demo-server.yml
@@ -189,7 +189,7 @@ jobs:
needs: resolve
if: needs.resolve.outputs.skip != 'true'
runs-on: ubuntu-24.04
- timeout-minutes: 60
+ timeout-minutes: 90
environment: ${{ needs.resolve.outputs.environment_name }}
env:
DEMO_EXPECTED_HOSTNAME: ${{ vars.DEMO_EXPECTED_HOSTNAME }}
@@ -450,309 +450,41 @@ jobs:
echo "::error::Verification-only check found a demo version mismatch; refusing to update the host."
exit 1
- - name: Prepare demo host storage
- if: inputs.verify_only != true && steps.current.outputs.skip_current != 'true'
- env:
- DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }}
- DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }}
- WORKFLOW_OUTPUT_1: ${{ steps.config.outputs.service_name }}
- run: |
- set -euo pipefail
- SERVICE_NAME="${WORKFLOW_OUTPUT_1}"
- REMOTE_SCRIPT=$(cat <<'EOF'
- set -euo pipefail
- SERVICE_NAME="$1"
- CONFIG_DIR="/etc/${SERVICE_NAME:-pulse}"
- BACKUP_PARENT="$(dirname "$CONFIG_DIR")"
- BACKUP_BASENAME="$(basename "$CONFIG_DIR")"
- KEEP_BACKUPS=2
- BACKUP_MARGIN_KB=$((64 * 1024))
-
- echo "Demo host filesystem before cleanup:"
- df -h "$BACKUP_PARENT" /tmp || true
-
- mapfile -t old_backups < <(
- sudo find "$BACKUP_PARENT" -maxdepth 1 -type d -name "${BACKUP_BASENAME}.backup.*" -printf '%T@ %p\n' 2>/dev/null \
- | sort -nr \
- | awk -v keep="$KEEP_BACKUPS" 'NR > keep {sub(/^[^ ]+ /, ""); print}'
- )
- if ((${#old_backups[@]} > 0)); then
- printf 'Removing %d old demo backup directorie(s).\n' "${#old_backups[@]}"
- printf '%s\0' "${old_backups[@]}" | sudo xargs -0 rm -rf --
- fi
-
- if [[ -d "$CONFIG_DIR" ]]; then
- config_kb="$(sudo du -sk "$CONFIG_DIR" | awk 'NR == 1 {print $1}')"
- free_kb="$(df -Pk "$BACKUP_PARENT" | awk 'NR == 2 {print $4}')"
- required_kb=$((config_kb + BACKUP_MARGIN_KB))
- if ((free_kb < required_kb)); then
- mapfile -t remaining_backups < <(
- sudo find "$BACKUP_PARENT" -maxdepth 1 -type d -name "${BACKUP_BASENAME}.backup.*" -printf '%T@ %p\n' 2>/dev/null \
- | sort -n \
- | awk '{sub(/^[^ ]+ /, ""); print}'
- )
- for backup_dir in "${remaining_backups[@]}"; do
- printf 'Removing demo backup to restore install headroom: %s\n' "$backup_dir"
- sudo rm -rf -- "$backup_dir"
- free_kb="$(df -Pk "$BACKUP_PARENT" | awk 'NR == 2 {print $4}')"
- if ((free_kb >= required_kb)); then
- break
- fi
- done
- fi
- pruned_volatile=false
- if ((free_kb < required_kb)); then
- echo "Pruning demo volatile runtime stores to restore install headroom."
- if command -v systemctl >/dev/null 2>&1; then
- sudo systemctl stop "$SERVICE_NAME" || true
- fi
- mapfile -d '' volatile_paths < <(
- sudo find "$CONFIG_DIR" -xdev -type f \( \
- -name "metrics.db" -o \
- -name "metrics.db-shm" -o \
- -name "metrics.db-wal" -o \
- -name "notification_queue.db" -o \
- -name "notification_queue.db-shm" -o \
- -name "notification_queue.db-wal" -o \
- -name "deploy.db" -o \
- -name "deploy.db-shm" -o \
- -name "deploy.db-wal" \
- \) -print0 2>/dev/null || true
- )
- for volatile_path in "${volatile_paths[@]}"; do
- printf 'Removing demo volatile store: %s\n' "$(basename "$volatile_path")"
- sudo rm -f -- "$volatile_path"
- pruned_volatile=true
- done
- config_kb="$(sudo du -sk "$CONFIG_DIR" | awk 'NR == 1 {print $1}')"
- free_kb="$(df -Pk "$BACKUP_PARENT" | awk 'NR == 2 {print $4}')"
- required_kb=$((config_kb + BACKUP_MARGIN_KB))
- fi
- if ((free_kb < required_kb)); then
- echo "::error::Demo host does not have enough free space to back up $CONFIG_DIR before install."
- echo "Config: ${config_kb}KB, available: ${free_kb}KB, required: ${required_kb}KB"
- sudo find "$BACKUP_PARENT" -maxdepth 1 -type d -name "${BACKUP_BASENAME}.backup.*" -printf '%TY-%Tm-%Td %TH:%TM %p\n' 2>/dev/null | sort || true
- if [[ "$pruned_volatile" == "true" ]] && command -v systemctl >/dev/null 2>&1; then
- sudo systemctl start "$SERVICE_NAME" || true
- fi
- exit 1
- fi
- fi
-
- echo "Demo host filesystem after cleanup:"
- df -h "$BACKUP_PARENT" /tmp || true
- EOF
- )
- ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" "bash -s -- $(printf '%q ' "$SERVICE_NAME")" <<<"$REMOTE_SCRIPT"
-
- - name: Upload tagged installer
- if: inputs.verify_only != true && steps.current.outputs.skip_current != 'true'
- env:
- DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }}
- DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }}
- run: |
- set -euo pipefail
- scp -i ~/.ssh/id_ed25519 /tmp/pulse-install.sh "$DEMO_SERVER_USER@$DEMO_SERVER_HOST:/tmp/pulse-install.sh"
-
- - name: Update demo server
- if: inputs.verify_only != true && steps.current.outputs.skip_current != 'true'
- env:
- DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }}
- DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }}
- WORKFLOW_OUTPUT_1: ${{ needs.resolve.outputs.tag }}
- WORKFLOW_OUTPUT_2: ${{ steps.config.outputs.service_name }}
- run: |
- set -euo pipefail
- TAG="${WORKFLOW_OUTPUT_1}"
- SERVICE_NAME="${WORKFLOW_OUTPUT_2}"
- REMOTE_SCRIPT=$(cat <<'EOF'
- set -euo pipefail
- TAG="$1"
- SERVICE_NAME="$2"
- INSTALLER_ENV=()
- if [ -n "$SERVICE_NAME" ]; then
- INSTALLER_ENV+=("PULSE_SERVICE_NAME=$SERVICE_NAME")
- fi
- sudo env "${INSTALLER_ENV[@]}" bash /tmp/pulse-install.sh --version "$TAG"
- rm -f /tmp/pulse-install.sh
- EOF
- )
- ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \
- "bash -s -- $(printf '%q ' "$TAG" "$SERVICE_NAME")" <<<"$REMOTE_SCRIPT"
-
- - name: Restore demo runtime configuration
+ - name: Apply guarded demo transaction
if: inputs.verify_only != true
env:
DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }}
DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }}
- WORKFLOW_OUTPUT_1: ${{ steps.config.outputs.service_name }}
- WORKFLOW_OUTPUT_2: ${{ needs.resolve.outputs.mock_nodes }}
- WORKFLOW_OUTPUT_3: ${{ needs.resolve.outputs.mock_vms_per_node }}
- WORKFLOW_OUTPUT_4: ${{ needs.resolve.outputs.mock_lxcs_per_node }}
- WORKFLOW_OUTPUT_5: ${{ needs.resolve.outputs.mock_docker_hosts }}
- WORKFLOW_OUTPUT_6: ${{ needs.resolve.outputs.mock_docker_containers }}
- WORKFLOW_OUTPUT_7: ${{ needs.resolve.outputs.mock_generic_hosts }}
- WORKFLOW_OUTPUT_8: ${{ needs.resolve.outputs.mock_k8s_clusters }}
- WORKFLOW_OUTPUT_9: ${{ needs.resolve.outputs.mock_k8s_nodes }}
- WORKFLOW_OUTPUT_10: ${{ needs.resolve.outputs.mock_k8s_pods }}
- WORKFLOW_OUTPUT_11: ${{ needs.resolve.outputs.mock_k8s_deployments }}
- WORKFLOW_OUTPUT_12: ${{ needs.resolve.outputs.mock_seed_duration }}
- WORKFLOW_OUTPUT_13: ${{ needs.resolve.outputs.mock_sample_interval }}
- WORKFLOW_OUTPUT_14: ${{ needs.resolve.outputs.mock_update_interval }}
+ EXPECTED_VERSION: ${{ needs.resolve.outputs.tag }}
+ CURRENT_MATCHES: ${{ steps.current.outputs.skip_current }}
+ MOCK_NODES: ${{ needs.resolve.outputs.mock_nodes }}
+ MOCK_VMS_PER_NODE: ${{ needs.resolve.outputs.mock_vms_per_node }}
+ MOCK_LXCS_PER_NODE: ${{ needs.resolve.outputs.mock_lxcs_per_node }}
+ MOCK_DOCKER_HOSTS: ${{ needs.resolve.outputs.mock_docker_hosts }}
+ MOCK_DOCKER_CONTAINERS: ${{ needs.resolve.outputs.mock_docker_containers }}
+ MOCK_GENERIC_HOSTS: ${{ needs.resolve.outputs.mock_generic_hosts }}
+ MOCK_K8S_CLUSTERS: ${{ needs.resolve.outputs.mock_k8s_clusters }}
+ MOCK_K8S_NODES: ${{ needs.resolve.outputs.mock_k8s_nodes }}
+ MOCK_K8S_PODS: ${{ needs.resolve.outputs.mock_k8s_pods }}
+ MOCK_K8S_DEPLOYMENTS: ${{ needs.resolve.outputs.mock_k8s_deployments }}
+ MOCK_SEED_DURATION: ${{ needs.resolve.outputs.mock_seed_duration }}
+ MOCK_SAMPLE_INTERVAL: ${{ needs.resolve.outputs.mock_sample_interval }}
+ MOCK_UPDATE_INTERVAL: ${{ needs.resolve.outputs.mock_update_interval }}
run: |
set -euo pipefail
- SERVICE_NAME="${WORKFLOW_OUTPUT_1}"
- MOCK_NODES="${WORKFLOW_OUTPUT_2}"
- MOCK_VMS_PER_NODE="${WORKFLOW_OUTPUT_3}"
- MOCK_LXCS_PER_NODE="${WORKFLOW_OUTPUT_4}"
- MOCK_DOCKER_HOSTS="${WORKFLOW_OUTPUT_5}"
- MOCK_DOCKER_CONTAINERS="${WORKFLOW_OUTPUT_6}"
- MOCK_GENERIC_HOSTS="${WORKFLOW_OUTPUT_7}"
- MOCK_K8S_CLUSTERS="${WORKFLOW_OUTPUT_8}"
- MOCK_K8S_NODES="${WORKFLOW_OUTPUT_9}"
- MOCK_K8S_PODS="${WORKFLOW_OUTPUT_10}"
- MOCK_K8S_DEPLOYMENTS="${WORKFLOW_OUTPUT_11}"
- MOCK_SEED_DURATION="${WORKFLOW_OUTPUT_12}"
- MOCK_SAMPLE_INTERVAL="${WORKFLOW_OUTPUT_13}"
- MOCK_UPDATE_INTERVAL="${WORKFLOW_OUTPUT_14}"
- REMOTE_SCRIPT=$(cat <<'EOF'
- set -euo pipefail
- SERVICE_NAME="${1:-pulse}"
- DEMO_LOCAL_BASE_URL="$2"
- MOCK_NODES="$3"
- MOCK_VMS_PER_NODE="$4"
- MOCK_LXCS_PER_NODE="$5"
- MOCK_DOCKER_HOSTS="$6"
- MOCK_DOCKER_CONTAINERS="$7"
- MOCK_GENERIC_HOSTS="$8"
- MOCK_K8S_CLUSTERS="$9"
- MOCK_K8S_NODES="${10}"
- MOCK_K8S_PODS="${11}"
- MOCK_K8S_DEPLOYMENTS="${12}"
- MOCK_SEED_DURATION="${13}"
- MOCK_SAMPLE_INTERVAL="${14}"
- MOCK_UPDATE_INTERVAL="${15}"
- CONFIG_DIR="/etc/${SERVICE_NAME:-pulse}"
- SERVICE_USER="$(systemctl show "$SERVICE_NAME" --property=User --value 2>/dev/null || true)"
- SERVICE_GROUP="$(systemctl show "$SERVICE_NAME" --property=Group --value 2>/dev/null || true)"
- SERVICE_USER="${SERVICE_USER:-pulse}"
- SERVICE_GROUP="${SERVICE_GROUP:-$SERVICE_USER}"
+ MODE=update
+ if [ "$CURRENT_MATCHES" = "true" ]; then MODE=recover; fi
+ python3 .github/scripts/dispatch-demo-runtime.py "$MODE" > demo-runtime-transaction.json
+ jq -e '.schema_version == 2 and (.status == "committed" or .status == "healthy_noop") and .forward.elapsed_seconds >= 300' demo-runtime-transaction.json
- resolve_config_dir() {
- local fallback="$CONFIG_DIR"
- local unit_env=""
- local fragment=""
- unit_env="$(systemctl show "$SERVICE_NAME" --property=Environment --value 2>/dev/null || true)"
- for entry in $unit_env; do
- case "$entry" in
- PULSE_DATA_DIR=*)
- printf '%s\n' "${entry#PULSE_DATA_DIR=}"
- return 0
- ;;
- esac
- done
-
- fragment="$(systemctl show "$SERVICE_NAME" --property=FragmentPath --value 2>/dev/null || true)"
- if [ -n "$fragment" ] && sudo test -r "$fragment"; then
- local unit_data_dir
- unit_data_dir="$(sudo sed -n -E 's/^Environment="?PULSE_DATA_DIR=([^"]*)"?$/\1/p' "$fragment" | tail -1)"
- if [ -n "$unit_data_dir" ]; then
- printf '%s\n' "$unit_data_dir"
- return 0
- fi
- fi
-
- printf '%s\n' "$fallback"
- }
-
- CONFIG_DIR="$(resolve_config_dir)"
- ENV_FILE="$CONFIG_DIR/.env"
-
- set_env_value() {
- local key="$1"
- local value="$2"
- sudo mkdir -p "$CONFIG_DIR"
- if sudo test -f "$ENV_FILE"; then
- if sudo grep -Eq "^[[:space:]]*${key}=" "$ENV_FILE"; then
- sudo sed -i "s|^[[:space:]]*${key}=.*|${key}=${value}|" "$ENV_FILE"
- else
- printf '\n%s=%s\n' "$key" "$value" | sudo tee -a "$ENV_FILE" >/dev/null
- fi
- else
- printf '# Pulse demo runtime configuration\n%s=%s\n' "$key" "$value" | sudo tee "$ENV_FILE" >/dev/null
- fi
- }
-
- ensure_demo_fixture_entitlement() {
- local billing_file="$CONFIG_DIR/billing.json"
- local tmp_file
- tmp_file="$(mktemp)"
- sudo mkdir -p "$CONFIG_DIR"
-
- if sudo test -s "$billing_file" && sudo jq '
- .capabilities = (((.capabilities // []) + ["demo_fixtures"]) | unique)
- | .limits = (.limits // {})
- | .meters_enabled = (.meters_enabled // [])
- | .plan_version = (if (.plan_version // "") == "" then "community" else .plan_version end)
- | .subscription_state = (if (.subscription_state // "") == "" then "active" else .subscription_state end)
- | del(.integrity)
- ' "$billing_file" > "$tmp_file"; then
- :
- else
- cat > "$tmp_file" <<'JSON'
- {"capabilities":["demo_fixtures"],"limits":{},"meters_enabled":[],"plan_version":"community","subscription_state":"active"}
- JSON
- fi
-
- sudo tee "$billing_file" < "$tmp_file" >/dev/null
- rm -f "$tmp_file"
- sudo chown "$SERVICE_USER:$SERVICE_GROUP" "$billing_file" || sudo chown "$SERVICE_USER" "$billing_file" || true
- sudo chmod 600 "$billing_file"
- echo "Demo fixture entitlement ensured in governed demo billing state."
- }
-
- set_env_value DEMO_MODE true
- set_env_value PULSE_MOCK_MODE true
- set_env_value PULSE_MOCK_NODES "$MOCK_NODES"
- set_env_value PULSE_MOCK_VMS_PER_NODE "$MOCK_VMS_PER_NODE"
- set_env_value PULSE_MOCK_LXCS_PER_NODE "$MOCK_LXCS_PER_NODE"
- set_env_value PULSE_MOCK_DOCKER_HOSTS "$MOCK_DOCKER_HOSTS"
- set_env_value PULSE_MOCK_DOCKER_CONTAINERS "$MOCK_DOCKER_CONTAINERS"
- set_env_value PULSE_MOCK_GENERIC_HOSTS "$MOCK_GENERIC_HOSTS"
- set_env_value PULSE_MOCK_K8S_CLUSTERS "$MOCK_K8S_CLUSTERS"
- set_env_value PULSE_MOCK_K8S_NODES "$MOCK_K8S_NODES"
- set_env_value PULSE_MOCK_K8S_PODS "$MOCK_K8S_PODS"
- set_env_value PULSE_MOCK_K8S_DEPLOYMENTS "$MOCK_K8S_DEPLOYMENTS"
- set_env_value PULSE_MOCK_RANDOM_METRICS true
- set_env_value PULSE_MOCK_STOPPED_PERCENT 6
- # Never carry the isolated local-dev metrics-store opt-in onto the
- # persistent public demo data directory.
- set_env_value PULSE_MOCK_SEED_METRICS_STORE false
- set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"
- set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL"
- set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"
-
- ensure_demo_fixture_entitlement
-
- sudo chown "$SERVICE_USER:$SERVICE_GROUP" "$ENV_FILE" || sudo chown "$SERVICE_USER" "$ENV_FILE" || true
- sudo chmod 600 "$ENV_FILE"
- sudo systemctl daemon-reload
- sudo systemctl restart "$SERVICE_NAME"
-
- for attempt in $(seq 1 30); do
- if curl -fsS "${DEMO_LOCAL_BASE_URL}/api/health" >/dev/null; then
- echo "Demo service restarted with governed demo runtime configuration."
- exit 0
- fi
- sleep 2
- done
-
- echo "::error::Demo service did not become healthy after restoring demo runtime configuration."
- exit 1
- EOF
- )
- ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \
- "bash -s -- $(printf '%q ' "$SERVICE_NAME" "$DEMO_LOCAL_BASE_URL" "$MOCK_NODES" "$MOCK_VMS_PER_NODE" "$MOCK_LXCS_PER_NODE" "$MOCK_DOCKER_HOSTS" "$MOCK_DOCKER_CONTAINERS" "$MOCK_GENERIC_HOSTS" "$MOCK_K8S_CLUSTERS" "$MOCK_K8S_NODES" "$MOCK_K8S_PODS" "$MOCK_K8S_DEPLOYMENTS" "$MOCK_SEED_DURATION" "$MOCK_SAMPLE_INTERVAL" "$MOCK_UPDATE_INTERVAL")" <<<"$REMOTE_SCRIPT"
+ - name: Retain guarded demo transaction evidence
+ if: always() && inputs.verify_only != true
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: stable-demo-transaction-${{ github.run_id }}
+ path: demo-runtime-transaction.json
+ if-no-files-found: warn
+ retention-days: 14
- name: Verify demo runtime
env:
diff --git a/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md b/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md
new file mode 100644
index 0000000000..af5e4322da
--- /dev/null
+++ b/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md
@@ -0,0 +1,60 @@
+# Stable demo transaction and acceptance
+
+This continues the existing update/recovery route. It was built for a single
+demo workflow and first-success health; the actual host also runs Relay and
+Caddy. An immediate healthy sample does not cover a delayed failure, and a
+workflow queue does not exclude another repository's host deployment.
+
+## Implemented boundary
+
+- Both existing mutating workflows submit the same reviewed Python engine.
+ Update still requires the immutable stable activation and customer-promotion
+ lease, exact tagged installer and trusted release signing key. Verification
+ only never submits it. Recovery never installs a different executable.
+- The independently owned systemd child holds the existing Relay deployment
+ file lock through capture, installation/profile work, forward observation,
+ complete restoration and recovery observation. It does not restart Relay or
+ Caddy, upgrade a package, lift the Caddy hold or activate frozen commercial copy.
+- The complete quiescent Pulse executable/unit/drop-in/data estate is retained
+ privately before mutation. No backup or database is deleted for space. The
+ unhealthy demo-only route retains the original generated operational history
+ before its existing bounded-profile reset. No snapshot is automatically erased.
+- Readiness has a separate 60-second limit. Forward, healthy-noop and restored
+ paths require 300 seconds of actual local/public/Relay HTTP 200, expected
+ version, stable service PID/restarts and available new-journal observations.
+ Transport errors, redirects, missing state/journals and new crashes fail.
+ Recovery uses fresh cursors; a retained forward failure remains a failure.
+- Failed/unhealthy/uncertain recovery remains owned and blocks subsequent
+ mutation. A prior failed baseline is not a healthy rollback. No crash dump is
+ induced. Receipt fields contain fixed diagnostics and identities, never raw
+ journal/HTTP bodies, environment values or snapshot contents.
+- Source-bound intent precedes systemd submission. An identical request reads
+ its receipt and cannot start another child after a lost response. SSH loss
+ does not terminate the child. TERM/INT/HUP cannot abort owned recovery. The
+ 45-minute runtime, 20-minute stop, 4,000-second remote observer, 4,100-second
+ client and 90-minute workflow budgets cover both observation windows.
+
+## Available validation and remaining result
+
+`TestDemoTransactionConnectedRecoveryControls` executes the real engine and
+bootstrap using a real private filesystem estate and virtual-clock command,
+service, journal and HTTP adapters. Tests retain delayed 55/299-second failure,
+changed executable/unit/data rollback, cancellation, failed and unhealthy
+recovery, headroom refusal, shared-lock contention, source mismatch, journal
+failure, cohost restart, source-bound repeated/lost submission and no-op proof.
+The production window is not shortened. This is **not native acceptance**.
+
+Before treating this route as restored, the exact reviewed source must return
+native disposable systemd/SSH-loss receipts and signed published-installer
+forward/reverse data/identity results. Then reconcile actual demo workflow and
+public browser readback through the release owner; source tests do not establish
+an installed recovery. Delivery owns that continuation and the remaining ingress
+migration. Unrestricted root/Tailscale administration can still bypass a source
+transaction; this candidate is not host enforcement. Workers neither install
+policy/credentials nor replay a successful publication to test it.
+
+Retained failed/unknown estates require the operational owner's reconciliation;
+do not remove a lock, intent or snapshot merely to permit a fresh sample. A
+later frontend/network failure is not authority for a second SSH session to
+stop a committed or unrelated service. Keep the signed packet, failure and
+native/browser acceptance distinct in the release judgment.
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index efa5d4e8c8..78533e9a3f 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -15,6 +15,36 @@
## Purpose
+### Demo transactions share the service host, not just a workflow queue
+
+Stable demo update and recovery keep their existing publication/activation,
+exact-tag installer signing key, Tailscale, pinned SSH and environment boundaries.
+The mutating paths now share the Relay host's existing
+`/var/lib/pulse-deploy/relay/deploy.lock` and one retained systemd transaction.
+Every successful path, including a healthy no-op, needs a complete 300-second
+local/public/Relay health, version, service PID/restart and new-journal-crash
+window. Readiness is separate. Missing observations fail closed.
+
+Before installation or runtime-profile mutation, stop only Pulse and durably
+capture its executable, unit/drop-ins and entire persistent data estate. Refuse
+insufficient space rather than prune backups/databases. A failed healthy change
+restores that complete estate and observes another 300 seconds; the original
+operation still fails. An unhealthy recovery baseline is never claimed as a
+healthy rollback. Retain its original generated demo history, all snapshots and
+failed/uncertain terminal evidence privately. An unresolved estate blocks later
+mutation. No source-worker production capability, package upgrade, Relay copy
+activation or credential is added.
+
+SSH observes, but never owns or cancels, the systemd child. Durable intent
+precedes submission; the exact request identity is not replayed after a lost
+response. Forward cancellation initiates restoration; recovery ignores
+TERM/INT/HUP until its watch and terminal receipt are retained. Child, stop,
+observer and workflow deadlines cover both full windows. A later browser or
+network check must not stop a committed or unrelated service in another SSH
+session. These connected source controls do not establish native systemd,
+signed installed forward/reverse acceptance, ingress enforcement or customer
+recovery. See [demo transaction acceptance](../DEMO_RUNTIME_TRANSACTION.md).
+
### Superseded pull request validation
Build and Test and Core E2E cancel an older run for the same pull request when
@@ -732,6 +762,8 @@ release-latency optimization.
23. `.github/workflows/update-demo-server.yml`
23a. `.github/workflows/recover-demo-server.yml`
23b. `.github/scripts/recover-demo-runtime.sh`
+23c. `.github/scripts/demo-runtime-transaction.py`
+23d. `.github/scripts/dispatch-demo-runtime.py`
23c. `.github/scripts/resolve-demo-runtime-profile.sh`
24. `.github/workflows/validate-release-assets.yml`
25. `.github/workflows/install-sh-smoke.yml`
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index 5b59559345..69c3b883c3 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -4515,6 +4515,8 @@
".dockerignore",
".github/ISSUE_TEMPLATE/v6_rc_feedback.yml",
".github/scripts/check-demo-reachability.sh",
+ ".github/scripts/demo-runtime-transaction.py",
+ ".github/scripts/dispatch-demo-runtime.py",
".github/scripts/internal-api-test-seconds.txt",
".github/scripts/record-internal-api-test-seconds.py",
".github/scripts/recover-demo-runtime.sh",
@@ -4544,6 +4546,9 @@
".github/workflows/retry-release-convergence.yml",
".github/workflows/update-demo-server.yml",
".github/workflows/validate-release-assets.yml",
+ "Dockerfile",
+ "Makefile",
+ "VERSION",
"cmd/pulse-control-plane/main.go",
"cmd/pulse-control-plane/provider_msp.go",
"cmd/pulse-control-plane/provider_msp_backup.go",
@@ -4553,13 +4558,12 @@
"cmd/pulse-control-plane/provider_msp_recover.go",
"cmd/pulse-control-plane/provider_msp_status.go",
"docker-compose.yml",
- "Dockerfile",
"docs/MSP.md",
+ "docs/RELEASE_NOTES.md",
+ "docs/UPGRADE_v6.md",
"docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md",
"docs/release-control/v6/internal/RC_TO_GA_REHEARSAL_TEMPLATE.md",
"docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md",
- "docs/RELEASE_NOTES.md",
- "docs/UPGRADE_v6.md",
"frontend-modern/package-lock.json",
"frontend-modern/package.json",
"frontend-modern/public/docs/MSP.md",
@@ -4579,7 +4583,6 @@
"internal/cloudcp/provider_msp_backup.go",
"internal/cloudcp/provider_msp_recovery.go",
"internal/cloudcp/tenant_runtime_rollout.go",
- "Makefile",
"package-lock.json",
"package.json",
"pulse-enterprise:.github/workflows/build-pro-release.yml",
@@ -4654,17 +4657,16 @@
"scripts/uninstall-sensor-proxy.sh",
"scripts/validate-release.sh",
"scripts/verify-github-release-integrity.sh",
- "tests/integration/playwright.config.ts",
"tests/integration/QUICK_START.md",
"tests/integration/README.md",
+ "tests/integration/playwright.config.ts",
"tests/integration/scripts/bootstrap-hosted-mobile-onboarding.mjs",
"tests/integration/scripts/hosted-mobile-token-runtime.mjs",
"tests/integration/scripts/hosted-tenant-runtime.mjs",
"tests/integration/scripts/managed-dev-runtime.mjs",
"tests/integration/scripts/relay-mobile-token-helper.go",
"tests/integration/tests/helpers.ts",
- "tests/integration/tests/runtime-defaults.ts",
- "VERSION"
+ "tests/integration/tests/runtime-defaults.ts"
],
"release_cycle_artifact_globs": [
"docs/RELEASE_NOTES.md",
@@ -4749,6 +4751,8 @@
"match_files": [
".github/ISSUE_TEMPLATE/v6_rc_feedback.yml",
".github/scripts/check-demo-reachability.sh",
+ ".github/scripts/demo-runtime-transaction.py",
+ ".github/scripts/dispatch-demo-runtime.py",
".github/scripts/recover-demo-runtime.sh",
".github/scripts/resolve-demo-runtime-profile.sh",
".github/scripts/setup-demo-ssh.sh",
@@ -4771,11 +4775,12 @@
".github/workflows/retry-release-convergence.yml",
".github/workflows/update-demo-server.yml",
".github/workflows/validate-release-assets.yml",
+ "VERSION",
+ "docs/RELEASE_NOTES.md",
+ "docs/UPGRADE_v6.md",
"docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md",
"docs/release-control/v6/internal/RC_TO_GA_REHEARSAL_TEMPLATE.md",
"docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md",
- "docs/RELEASE_NOTES.md",
- "docs/UPGRADE_v6.md",
"frontend-modern/public/docs/UPGRADE_v6.md",
"frontend-modern/src/components/UpdateBanner.tsx",
"frontend-modern/src/components/WhatsNewCard.tsx",
@@ -4803,17 +4808,18 @@
"scripts/trigger-release-dry-run.sh",
"scripts/trigger-release.sh",
"scripts/trigger-stable-patch.sh",
- "scripts/verify-github-release-integrity.sh",
- "VERSION"
+ "scripts/verify-github-release-integrity.sh"
],
"allow_same_subsystem_tests": false,
"test_prefixes": [],
"exact_files": [
+ ".github/scripts/tests/test_demo_runtime_transaction.py",
"frontend-modern/src/components/__tests__/whatsNewModel.test.ts",
"frontend-modern/src/utils/__tests__/docsLinks.test.ts",
"pulse-enterprise:scripts/validate-pro-release-line_test.sh",
"pulse-pro:scripts/tests/test_validate_paid_runtime_distribution.py",
"scripts/installtests/build_release_assets_test.go",
+ "scripts/installtests/demo_transaction_test.go",
"scripts/release_control/internal/record_rc_to_ga_rehearsal_test.py",
"scripts/release_control/internal/release_preflight_test.py",
"scripts/release_control/mobile_release_gate_test.py",
diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go
index 94688451c0..fff9fd54fd 100644
--- a/scripts/installtests/build_release_assets_test.go
+++ b/scripts/installtests/build_release_assets_test.go
@@ -2306,7 +2306,11 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) {
if err != nil {
t.Fatalf("read demo runtime profile resolver: %v", err)
}
- workflow := string(workflowBytes) + "\n" + string(profileBytes)
+ transactionBytes, err := os.ReadFile(repoFile(".github", "scripts", "demo-runtime-transaction.py"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ workflow := string(workflowBytes) + "\n" + string(profileBytes) + "\n" + string(transactionBytes)
required := []string{
`- name: Tailscale`,
`uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3`,
@@ -2319,7 +2323,6 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) {
`workflow_call:`,
`verify_only:`,
`Waiting for activated release assets to be available`,
- `bash /tmp/pulse-install.sh --version "$TAG"`,
`Refuse mutation during verification-only checks`,
`uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0`,
`go run ./scripts/release_update_key.go public-key-ssh`,
@@ -2327,15 +2330,10 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) {
`Verify target host identity`,
`bash .github/scripts/setup-demo-ssh.sh`,
`Demo environment points at host $REMOTE_HOSTNAME but expected $DEMO_EXPECTED_HOSTNAME.`,
- `Prepare demo host storage`,
- `KEEP_BACKUPS=2`,
- `Removing demo backup to restore install headroom: %s`,
- `Pruning demo volatile runtime stores to restore install headroom.`,
- `sudo find "$CONFIG_DIR" -xdev -type f`,
- `-name "metrics.db"`,
- `Removing demo volatile store: %s`,
- `Demo host does not have enough free space to back up $CONFIG_DIR before install.`,
- `Restore demo runtime configuration`,
+ `Apply guarded demo transaction`,
+ `python3 .github/scripts/dispatch-demo-runtime.py "$MODE"`,
+ `Retain guarded demo transaction evidence`,
+ `.forward.elapsed_seconds >= 300`,
`Resolve target-compatible demo runtime profile`,
`git grep -q 'mockEagerHistoryPVEGuestLimit'`,
`git grep -q 'UpdateMetricCohort'`,
@@ -2361,22 +2359,13 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) {
`MOCK_SEED_DURATION=2h`,
`MOCK_SAMPLE_INTERVAL=5m`,
`MOCK_UPDATE_INTERVAL=15s`,
- `resolve_config_dir`,
- `set_env_value DEMO_MODE true`,
- `set_env_value PULSE_MOCK_MODE true`,
- `set_env_value PULSE_MOCK_NODES "$MOCK_NODES"`,
- `set_env_value PULSE_MOCK_VMS_PER_NODE "$MOCK_VMS_PER_NODE"`,
- `set_env_value PULSE_MOCK_LXCS_PER_NODE "$MOCK_LXCS_PER_NODE"`,
- `set_env_value PULSE_MOCK_DOCKER_HOSTS "$MOCK_DOCKER_HOSTS"`,
- `set_env_value PULSE_MOCK_K8S_PODS "$MOCK_K8S_PODS"`,
- `set_env_value PULSE_MOCK_SEED_METRICS_STORE false`,
- `set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"`,
- `set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL"`,
- `set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"`,
- `ensure_demo_fixture_entitlement`,
- `"demo_fixtures"`,
- `del(.integrity)`,
- `Demo fixture entitlement ensured in governed demo billing state.`,
+ `WINDOW = 300`,
+ `/var/lib/pulse-deploy/relay/deploy.lock`,
+ `snapshot-headroom`,
+ `restored-estate-identity`,
+ `demo_fixtures`,
+ `PULSE_MOCK_SEED_METRICS_STORE`,
+ `installer-identity`,
`/api/license/runtime-capabilities`,
`Mock mode enabled`,
`Demo server mock mode did not enable after entitlement sync`,
diff --git a/scripts/installtests/demo_transaction_test.go b/scripts/installtests/demo_transaction_test.go
new file mode 100644
index 0000000000..f53ed0abc0
--- /dev/null
+++ b/scripts/installtests/demo_transaction_test.go
@@ -0,0 +1,56 @@
+package installtests
+
+import (
+ "os"
+ "os/exec"
+ "strings"
+ "testing"
+)
+
+func TestDemoTransactionConnectedRecoveryControls(t *testing.T) {
+ cmd := exec.Command("python3", repoFile(".github", "scripts", "tests", "test_demo_runtime_transaction.py"))
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("demo connected transaction controls failed: %v\n%s", err, output)
+ }
+ if !strings.Contains(string(output), "\nOK\n") {
+ t.Fatalf("demo controls did not return a complete unittest verdict: %s", output)
+ }
+}
+
+func TestDemoTransactionKeepsSignedAdmissionAndVerificationOnlyReadOnly(t *testing.T) {
+ workflow, err := os.ReadFile(repoFile(".github", "workflows", "update-demo-server.yml"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ text := string(workflow)
+ start := strings.Index(text, "- name: Apply guarded demo transaction")
+ end := strings.Index(text[start:], "- name: Retain guarded demo transaction evidence")
+ if start < 0 || end < 0 {
+ t.Fatal("missing retained demo transaction entry")
+ }
+ entry := text[start : start+end]
+ if !strings.Contains(entry, "if: inputs.verify_only != true") {
+ t.Fatal("verification-only invocation must never submit a transaction")
+ }
+ for _, forbidden := range []string{"Pruning demo volatile runtime stores", "Removing demo backup to restore install headroom", "sudo systemctl restart"} {
+ if strings.Contains(text, forbidden) {
+ t.Fatalf("unguarded demo mutation remains: %s", forbidden)
+ }
+ }
+ for _, required := range []string{"Require exact committed activation marker for mutation", "release-activation.json", "Stable demo mutation refuses mutable, inactive, or prerelease tag", "PINNED_RELEASE_SSH_PUBLIC_KEY", "git show \"refs/tags/${TAG}:install.sh\"", "Verify public browser smoke"} {
+ if !strings.Contains(text, required) {
+ t.Fatalf("lost signed/admission/customer verification boundary: %s", required)
+ }
+ }
+ recovery, err := os.ReadFile(repoFile(".github", "workflows", "recover-demo-server.yml"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Contains(string(recovery), "sudo systemctl stop pulse") || strings.Contains(string(recovery), "Compensate failed mutated recovery") {
+ t.Fatal("later workflow failure must not stop a committed or unrelated service")
+ }
+ if !strings.Contains(string(recovery), ".forward.elapsed_seconds >= 300") || !strings.Contains(string(recovery), "if: always()") {
+ t.Fatal("recovery must validate and retain the actual terminal observation")
+ }
+}
From 751fb67523012658c305fd0dae12fba4cfe67c4c Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 01:12:47 +0100
Subject: [PATCH 04/86] Bind demo controls without broadening unrelated release
proof
Keep new source paths on their own connected proof policy after the full lookup suite exposed six broadened metadata expectations. Preserve terminal no-op cancellation evidence and the runtime file owner.
Contract-Neutral: Correct connected observation and verification routing within the existing demo operations and authority.
Change-source: pulse-maintainer
---
.github/scripts/demo-runtime-transaction.py | 9 ++++++---
.../tests/test_demo_runtime_transaction.py | 13 ++++++++++---
.../v6/internal/subsystems/registry.json | 19 +++++++++++++++----
scripts/installtests/demo_transaction_test.go | 5 ++++-
.../release_control/subsystem_lookup_test.py | 12 ++++++++++++
5 files changed, 47 insertions(+), 11 deletions(-)
diff --git a/.github/scripts/demo-runtime-transaction.py b/.github/scripts/demo-runtime-transaction.py
index 60193e378e..5fa81dda04 100644
--- a/.github/scripts/demo-runtime-transaction.py
+++ b/.github/scripts/demo-runtime-transaction.py
@@ -275,6 +275,8 @@ class Host:
pattern = re.compile(r"^[ \t]*" + re.escape(key) + r"=.*$", re.M)
text = pattern.sub(key + "=" + value, text) if pattern.search(text) else text.rstrip("\n") + "\n" + key + "=" + value + "\n"
env_file.write_text(text)
+ data_owner = PATHS["data"].stat()
+ os.chown(env_file, data_owner.st_uid, data_owner.st_gid)
os.chmod(env_file, 0o600)
billing = json.loads(billing_file.read_text())
if not isinstance(billing, dict) or not isinstance(billing.get("capabilities", []), list):
@@ -282,6 +284,7 @@ class Host:
billing["capabilities"] = sorted(set(billing.get("capabilities", []) + ["demo_fixtures"]))
billing.pop("integrity", None)
billing_file.write_text(json.dumps(billing))
+ os.chown(billing_file, data_owner.st_uid, data_owner.st_gid)
os.chmod(billing_file, 0o600)
if unhealthy:
for relative in DEMO_HISTORY:
@@ -408,14 +411,14 @@ class Transaction:
elif self.request["mode"] == "update":
raise Failure("update-baseline-unhealthy")
self.receipt["healthy_baseline"] = healthy_baseline
+ old_handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP)}
+ for sig in old_handlers:
+ signal.signal(sig, cancel)
# The no-op is observed too; an immediate healthy sample is not success.
if self.request["mode"] == "recover" and healthy_baseline and self.host.profile_matches(self.request):
self.watch("forward", original_version, self.host.cursors())
self.save("healthy_noop")
return 0
- old_handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP)}
- for sig in old_handlers:
- signal.signal(sig, cancel)
self.save("capturing")
stopped = True
self.host.stop()
diff --git a/.github/scripts/tests/test_demo_runtime_transaction.py b/.github/scripts/tests/test_demo_runtime_transaction.py
index 36bb810f44..6050a380c8 100644
--- a/.github/scripts/tests/test_demo_runtime_transaction.py
+++ b/.github/scripts/tests/test_demo_runtime_transaction.py
@@ -1,4 +1,3 @@
-import copy
import hashlib
import importlib.util
import json
@@ -195,6 +194,16 @@ class TransactionTest(unittest.TestCase):
self.assertFalse(receipt["mutated"])
self.assertEqual(self.host.stops, 0)
+ def test_noop_cancellation_retains_terminal_failure_without_stopping_service(self):
+ self.host.profile(request(), False)
+ self.host.cancel_at = 55
+ code, receipt = self.run_transaction()
+ self.assertEqual(code, 1)
+ self.assertEqual(receipt["status"], "refused")
+ self.assertEqual(receipt["failure"], "cancelled-forward")
+ self.assertEqual(self.host.stops, 0)
+ self.assertFalse(receipt["mutated"])
+
def test_update_restores_complete_changed_executable_unit_and_data_after_late_failure(self):
self.host.fail_at = 55
code, receipt = self.run_transaction("update", "v6.4.6")
@@ -404,8 +413,6 @@ class DispatchLifecycleTest(unittest.TestCase):
for uncertain in (False, True):
with self.subTest(uncertain=uncertain), tempfile.TemporaryDirectory() as directory:
root = Path(directory) / "state"
- source = dispatcher.BOOTSTRAP.replace("pathlib.Path('/var/lib/pulse-deploy/demo')", repr(root))
- # repr(Path) is not a Python literal: bind just the fixed path.
source = dispatcher.BOOTSTRAP.replace("'/var/lib/pulse-deploy/demo'", repr(str(root)))
source = source.replace("root.stat().st_uid == 0", "root.stat().st_uid == os.getuid()")
clock = [0]
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index 69c3b883c3..9cf02a0271 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -4751,8 +4751,6 @@
"match_files": [
".github/ISSUE_TEMPLATE/v6_rc_feedback.yml",
".github/scripts/check-demo-reachability.sh",
- ".github/scripts/demo-runtime-transaction.py",
- ".github/scripts/dispatch-demo-runtime.py",
".github/scripts/recover-demo-runtime.sh",
".github/scripts/resolve-demo-runtime-profile.sh",
".github/scripts/setup-demo-ssh.sh",
@@ -4813,13 +4811,11 @@
"allow_same_subsystem_tests": false,
"test_prefixes": [],
"exact_files": [
- ".github/scripts/tests/test_demo_runtime_transaction.py",
"frontend-modern/src/components/__tests__/whatsNewModel.test.ts",
"frontend-modern/src/utils/__tests__/docsLinks.test.ts",
"pulse-enterprise:scripts/validate-pro-release-line_test.sh",
"pulse-pro:scripts/tests/test_validate_paid_runtime_distribution.py",
"scripts/installtests/build_release_assets_test.go",
- "scripts/installtests/demo_transaction_test.go",
"scripts/release_control/internal/record_rc_to_ga_rehearsal_test.py",
"scripts/release_control/internal/release_preflight_test.py",
"scripts/release_control/mobile_release_gate_test.py",
@@ -5173,6 +5169,21 @@
"scripts/installtests/install_sh_test.go",
"scripts/installtests/safe_profile_migration_test.go"
]
+ },
+ {
+ "id": "demo-runtime-transaction",
+ "label": "demo shared-host transaction and retained recovery proof",
+ "match_prefixes": [],
+ "match_files": [
+ ".github/scripts/demo-runtime-transaction.py",
+ ".github/scripts/dispatch-demo-runtime.py"
+ ],
+ "allow_same_subsystem_tests": false,
+ "test_prefixes": [],
+ "exact_files": [
+ "scripts/installtests/demo_transaction_test.go",
+ ".github/scripts/tests/test_demo_runtime_transaction.py"
+ ]
}
],
"match_files": null
diff --git a/scripts/installtests/demo_transaction_test.go b/scripts/installtests/demo_transaction_test.go
index f53ed0abc0..ae800da558 100644
--- a/scripts/installtests/demo_transaction_test.go
+++ b/scripts/installtests/demo_transaction_test.go
@@ -25,8 +25,11 @@ func TestDemoTransactionKeepsSignedAdmissionAndVerificationOnlyReadOnly(t *testi
}
text := string(workflow)
start := strings.Index(text, "- name: Apply guarded demo transaction")
+ if start < 0 {
+ t.Fatal("missing retained demo transaction entry")
+ }
end := strings.Index(text[start:], "- name: Retain guarded demo transaction evidence")
- if start < 0 || end < 0 {
+ if end < 0 {
t.Fatal("missing retained demo transaction entry")
}
entry := text[start : start+end]
diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.py
index a4dc01c378..d77aece65c 100644
--- a/scripts/release_control/subsystem_lookup_test.py
+++ b/scripts/release_control/subsystem_lookup_test.py
@@ -100,6 +100,18 @@ def _contract_reference(contract_path: str, needle: str, runtime_path: str) -> d
class SubsystemLookupTest(unittest.TestCase):
+
+ def test_demo_transaction_sources_have_connected_proof_without_widening_release_metadata(self):
+ for path in (".github/scripts/demo-runtime-transaction.py", ".github/scripts/dispatch-demo-runtime.py"):
+ with self.subTest(path=path):
+ result = lookup_paths([path])
+ subsystem = next(item for item in result["subsystems"] if item["id"] == "deployment-installability")
+ self.assertEqual(subsystem["matched_verification_policies"][0]["id"], "demo-runtime-transaction")
+ self.assertEqual(sorted(subsystem["required_verification"]["exact_files"]), [
+ ".github/scripts/tests/test_demo_runtime_transaction.py",
+ "scripts/installtests/demo_transaction_test.go",
+ ])
+
def git(self, repo_root: Path, *args: str) -> None:
subprocess.run(
["git", *args],
From b3937a4803469bd555239eca1c4f2628862497ec Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 01:13:46 +0100
Subject: [PATCH 05/86] Use the actual lookup receipt shape in demo ownership
proof
Assert the existing file-level verification receipt so the new routing control exercises real ownership instead of assuming an unavailable summary field.
Contract-Neutral: Test-only assertion of existing subsystem lookup output; no runtime or authority change.
Change-source: pulse-maintainer
---
scripts/release_control/subsystem_lookup_test.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.py
index d77aece65c..1fae39c773 100644
--- a/scripts/release_control/subsystem_lookup_test.py
+++ b/scripts/release_control/subsystem_lookup_test.py
@@ -105,9 +105,11 @@ class SubsystemLookupTest(unittest.TestCase):
for path in (".github/scripts/demo-runtime-transaction.py", ".github/scripts/dispatch-demo-runtime.py"):
with self.subTest(path=path):
result = lookup_paths([path])
- subsystem = next(item for item in result["subsystems"] if item["id"] == "deployment-installability")
- self.assertEqual(subsystem["matched_verification_policies"][0]["id"], "demo-runtime-transaction")
- self.assertEqual(sorted(subsystem["required_verification"]["exact_files"]), [
+ self.assertEqual(result["unowned_runtime_files"], [])
+ self.assertEqual({item["subsystem"] for item in result["impacted_subsystems"]}, {"deployment-installability"})
+ requirement = result["files"][0]["matches"][0]["verification_requirement"]
+ self.assertEqual(requirement["id"], "demo-runtime-transaction")
+ self.assertEqual(sorted(requirement["exact_files"]), [
".github/scripts/tests/test_demo_runtime_transaction.py",
"scripts/installtests/demo_transaction_test.go",
])
From 8f42497b8f3bc2a4f46dc5d9f1d9cd867e731298 Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 01:16:42 +0100
Subject: [PATCH 06/86] Keep delayed alert recovery scoped to its resolved
occurrence
Capture the resolved alert before asynchronous callbacks and carry its start/end identity through monitor recovery. Cancel only that occurrence in grouping, cooldown and persistent firing rows, retaining newer incidents, operational links, destination receipts and failed attempts. Keep the explicit identifier-wide cancellation API and legacy callback IDs. Add snapshot, restart and connected queue-to-HTTP regression controls.
Change-source: pulse-maintainer
---
.../v6/internal/subsystems/agent-lifecycle.md | 11 ++
.../v6/internal/subsystems/alerts.md | 27 ++++
.../v6/internal/subsystems/monitoring.md | 21 +++
.../v6/internal/subsystems/notifications.md | 19 +++
internal/alerts/active_lifecycle.go | 6 +-
internal/alerts/alerts_test.go | 14 +-
internal/alerts/callbacks.go | 71 ++++++---
internal/alerts/canonical_lifecycle.go | 4 +-
internal/alerts/canonical_metric.go | 2 +-
internal/alerts/concurrency_test.go | 2 +-
internal/alerts/config_runtime.go | 2 +-
internal/alerts/guest.go | 2 +-
internal/alerts/metric_runtime.go | 2 +-
internal/alerts/operational_contract_test.go | 52 +++++++
internal/monitoring/monitor.go | 4 +-
.../monitoring/monitor_alert_handling_test.go | 136 ++++++++++++++++++
internal/monitoring/monitor_alerts.go | 47 ++++--
.../monitor_notification_startup_test.go | 39 +++++
internal/notifications/notifications.go | 32 ++++-
internal/notifications/queue.go | 31 +++-
.../queue_occurrence_cancellation_test.go | 110 ++++++++++++++
21 files changed, 570 insertions(+), 64 deletions(-)
create mode 100644 internal/notifications/queue_occurrence_cancellation_test.go
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index 4c18ec8692..4754a53d0d 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -8433,3 +8433,14 @@ update commands and credential authority unchanged.
`internal/api/onboarding_handlers.go` changed only the human-readable pairing
readiness messages so they name Settings > Pulse Mobile. No agent
registration, lifecycle or install path changed.
+
+### Monitor recovery callback ownership
+
+The monitor's external alert wiring uses a captured resolved-occurrence
+snapshot for notification cancellation. Re-enrolled or refiring resources keep
+their new occurrence's queued notification even when an older recovery callback
+arrives late under the same ID. Agent registration, deletion/tombstones,
+authorisation and continuity evidence are unchanged. The connected callback
+wiring control in `monitor_notification_startup_test.go` verifies the shared
+monitor boundary against real alert evaluation and local HTTP delivery; it does
+not claim host re-enrolment or installed-agent acceptance.
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index 4b002b9e4b..14bdd0b99f 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -3203,3 +3203,30 @@ upgrade prompt, and never load or save relay push settings. Licensed instances
keep the minimum-severity control and the link to Pulse Mobile settings, with
the 31 March 2027 retirement date in the copy. Webhook, email, Apprise and
dead-man destinations are unaffected.
+
+### Occurrence-qualified asynchronous recovery delivery
+
+Every resolution funnel passes the actual `ResolvedAlert` to the callback bus.
+`SetResolvedAlertCallback` captures a deep-cloned alert plus its resolution time
+before asynchronous execution, rather than reconstructing an occurrence from a
+reusable ID later. Legacy ID-only subscribers retain their public IDs; a panic
+in one recovery consumer does not prevent the other consumers from running.
+
+The live monitor uses this snapshot for AI recovery and notification delivery.
+`CancelResolvedAlert` removes only matching ID/start-time grouping entries and
+cooldown state. `CancelByAlertOccurrence` applies the same match to persistent
+pending, sending, failed and dead-letter firing rows, preserving newer
+occurrences, unrelated grouped members and their operational links. Missing
+start-time identity is a no-op, never a request for identifier-wide cancellation.
+The existing explicit ID-wide cancellation API remains available. Delivery
+gates, pending-only suppression counts, destination receipts, attempt history,
+quiet hours and best-effort treatment of already-started sends are unchanged.
+
+`operational_contract_test.go` verifies snapshot isolation across a
+blocked callback and changed ID-indexed history, legacy identity and panic
+isolation. `queue_occurrence_cancellation_test.go` verifies nanosecond-distinct
+recurrences, grouping/cooldown preservation, missing identity and restart
+readback for all four cancellable row states without deleting failed audits or
+new-occurrence destination receipts. The monitor's connected tests use real PBS
+evaluation, normal callbacks, the persistent queue and a local HTTP destination;
+these receipts are not native provider or installed-release acceptance.
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 6265060bec..6f5caeb5a0 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -4634,3 +4634,24 @@ proof lives in `internal/truenas/provider_test.go`
(`TestMaxTrueNASSystemTemperaturePrefersCoresOverSynthesisedAggregate`). This
is source-level proof, not reporter acceptance or appliance confirmation
(#2122).
+
+### Recovery callbacks retain the resolved occurrence
+
+`wireExternalAlertCallbacks` binds the occurrence-qualified recovery callback,
+not the legacy ID-only callback. The monitor uses its immutable alert/start/end
+snapshot for AI recovery and for exact notification cancellation, so a delayed
+recovery cannot remove a newer firing admitted under the same reusable ID.
+AI receives a separate clone. An ID-only websocket removal is omitted when the
+manager already exposes that ID as active; the regular state stream continues
+to own current alert truth. The legacy local ID adapter resolves a snapshot once
+and fails closed when no resolved record exists; it is not the live delivery
+binding. No browser payload, retention or notification routing contract changes.
+
+`monitor_alert_handling_test.go` and
+`monitor_notification_startup_test.go` covers a delayed ID-adapter cancellation
+with queue restart/readback, and a real PBS breach/recovery/new breach held at
+the callback boundary. Both require the replacement firing's exact ID/start to
+reach a loopback HTTP destination with one sent row and one cancelled old row.
+No clock, reducer map or configured cooldown is altered. This establishes local
+source-connected delivery, not the outstanding natural flapping cooldown check,
+a native appliance result or containing stable availability.
diff --git a/docs/release-control/v6/internal/subsystems/notifications.md b/docs/release-control/v6/internal/subsystems/notifications.md
index 83ced5ecfe..5b5b782ab2 100644
--- a/docs/release-control/v6/internal/subsystems/notifications.md
+++ b/docs/release-control/v6/internal/subsystems/notifications.md
@@ -1054,3 +1054,22 @@ regression establish frontend control preservation, not provider delivery.
Formatting-only follow-up retains this warning-level contract. The production-component
browser matrix was rerun after formatting at desktop and narrow widths, including
all/critical/warning save/reload, cancel, and warning webhook creation.
+
+### Resolution cancellation is occurrence-qualified
+
+The monitor's delayed recovery path calls `CancelResolvedAlert` with the
+captured ID/start-time snapshot. It removes only that occurrence's pending
+in-memory grouping members and cooldown record. Persistent
+`CancelByAlertOccurrence` shares the existing per-ID delivery gates and
+cancellation machinery, but matches both ID and start time: pending, sending,
+failed and dead-letter firing rows are cancelled or rewritten without removing
+a recurring incident or unrelated members/operational links in a mixed row.
+Missing occurrence identity is a no-op. The explicit identifier-wide API, audit
+history, destination receipts, retry budget, recovery rows and best-effort
+in-flight semantics remain unchanged.
+
+`queue_occurrence_cancellation_test.go` verifies all four row states after a
+fresh database reopen, nanosecond-distinct occurrences, operational-link and
+failed-attempt preservation, missing-identity safety, grouping/cooldown retention
+and the unchanged ID-wide API. Monitor connected controls additionally require
+HTTP acceptance of the surviving occurrence through the autonomous queue.
diff --git a/internal/alerts/active_lifecycle.go b/internal/alerts/active_lifecycle.go
index 6b90e6ec6d..3771f98205 100644
--- a/internal/alerts/active_lifecycle.go
+++ b/internal/alerts/active_lifecycle.go
@@ -115,7 +115,7 @@ func (m *Manager) clearAlert(alertID string) {
m.addRecentlyResolvedUnlocked(resolvedAlert)
- m.safeCallResolvedAlertCallback(alert, publicID, false)
+ m.safeCallResolvedAlertCallback(resolvedAlert, publicID, false)
log.Info().
Str("alertID", publicID).
@@ -738,7 +738,7 @@ func (m *Manager) resolveDiscreteRecoveryNoLock(resourceID, specKey, alertID str
resolvedAlert := m.newResolvedAlert(alert, time.Now(), nil)
m.addRecentlyResolvedWithPrimaryLock(resolvedAlert)
- m.safeCallResolvedAlertCallback(alert, alertID, true)
+ m.safeCallResolvedAlertCallback(resolvedAlert, alertID, true)
log.Info().
Str(strings.ToLower(resourceKind), resourceName).
@@ -790,7 +790,7 @@ func (m *Manager) clearAlertNoLock(alertID string) {
m.addRecentlyResolvedWithPrimaryLock(resolvedAlert)
- m.safeCallResolvedAlertCallback(alert, publicID, true)
+ m.safeCallResolvedAlertCallback(resolvedAlert, publicID, true)
log.Info().
Str("alertID", publicID).
diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go
index ccafc38ed5..68daa1b65c 100644
--- a/internal/alerts/alerts_test.go
+++ b/internal/alerts/alerts_test.go
@@ -7032,7 +7032,7 @@ func TestSafeCallResolvedAlertCallback(t *testing.T) {
receivedID = alertID
})
- m.safeCallResolvedAlertCallback(&Alert{ID: "test-alert-123"}, "test-alert-123", false)
+ m.safeCallResolvedAlertCallback(&ResolvedAlert{Alert: &Alert{ID: "test-alert-123"}}, "test-alert-123", false)
if receivedID != "test-alert-123" {
t.Errorf("expected alert ID 'test-alert-123', got %q", receivedID)
@@ -7051,7 +7051,7 @@ func TestSafeCallResolvedAlertCallback(t *testing.T) {
close(done)
})
- m.safeCallResolvedAlertCallback(&Alert{ID: "async-alert"}, "async-alert", true)
+ m.safeCallResolvedAlertCallback(&ResolvedAlert{Alert: &Alert{ID: "async-alert"}}, "async-alert", true)
select {
case <-done:
@@ -7078,7 +7078,7 @@ func TestSafeCallResolvedAlertCallback(t *testing.T) {
CanonicalState: buildCanonicalStateID("pve1:node1:101", "powered-state:powered-off"),
}
- m.safeCallResolvedAlertCallback(alert, alert.CanonicalState, false)
+ m.safeCallResolvedAlertCallback(&ResolvedAlert{Alert: alert}, alert.CanonicalState, false)
if receivedID != alert.CanonicalState {
t.Errorf("expected canonical alert ID %q, got %q", alert.CanonicalState, receivedID)
@@ -7091,8 +7091,8 @@ func TestSafeCallResolvedAlertCallback(t *testing.T) {
// No callback set
// Should not panic
- m.safeCallResolvedAlertCallback(&Alert{ID: "test-alert"}, "test-alert", false)
- m.safeCallResolvedAlertCallback(&Alert{ID: "test-alert"}, "test-alert", true)
+ m.safeCallResolvedAlertCallback(&ResolvedAlert{Alert: &Alert{ID: "test-alert"}}, "test-alert", false)
+ m.safeCallResolvedAlertCallback(&ResolvedAlert{Alert: &Alert{ID: "test-alert"}}, "test-alert", true)
})
t.Run("recovers from panic in sync callback", func(t *testing.T) {
@@ -7104,7 +7104,7 @@ func TestSafeCallResolvedAlertCallback(t *testing.T) {
})
// Should not panic the caller
- m.safeCallResolvedAlertCallback(&Alert{ID: "panic-test"}, "panic-test", false)
+ m.safeCallResolvedAlertCallback(&ResolvedAlert{Alert: &Alert{ID: "panic-test"}}, "panic-test", false)
})
t.Run("recovers from panic in async callback", func(t *testing.T) {
@@ -7117,7 +7117,7 @@ func TestSafeCallResolvedAlertCallback(t *testing.T) {
panic("async panic")
})
- m.safeCallResolvedAlertCallback(&Alert{ID: "async-panic"}, "async-panic", true)
+ m.safeCallResolvedAlertCallback(&ResolvedAlert{Alert: &Alert{ID: "async-panic"}}, "async-panic", true)
select {
case <-done:
diff --git a/internal/alerts/callbacks.go b/internal/alerts/callbacks.go
index e2378ecb0e..4a6a7a9d05 100644
--- a/internal/alerts/callbacks.go
+++ b/internal/alerts/callbacks.go
@@ -23,13 +23,14 @@ type LifecycleEvent struct {
type callbackBus struct {
mu sync.RWMutex
- onAlert func(alert *Alert)
- alertSubs map[int]func(alert *Alert)
- onAlertForAI func(alert *Alert)
- alertForAISubs map[int]func(alert *Alert)
- onResolved func(alertID string)
- resolvedSubs map[int]func(alertID string)
- lifecycleSubs map[int]func(event LifecycleEvent)
+ onAlert func(alert *Alert)
+ alertSubs map[int]func(alert *Alert)
+ onAlertForAI func(alert *Alert)
+ alertForAISubs map[int]func(alert *Alert)
+ onResolved func(alertID string)
+ resolvedSubs map[int]func(alertID string)
+ onResolvedAlert func(resolved *ResolvedAlert)
+ lifecycleSubs map[int]func(event LifecycleEvent)
onAcknowledged func(alert *Alert, user string)
onUnacknowledged func(alert *Alert, user string)
@@ -146,6 +147,18 @@ func (b *callbackBus) setResolvedCallback(cb func(alertID string)) {
b.onResolved = cb
}
+func (b *callbackBus) setResolvedAlertCallback(cb func(resolved *ResolvedAlert)) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ b.onResolvedAlert = cb
+}
+
+func (b *callbackBus) resolvedAlertCallback() func(resolved *ResolvedAlert) {
+ b.mu.RLock()
+ defer b.mu.RUnlock()
+ return b.onResolvedAlert
+}
+
func (b *callbackBus) subscribeResolvedCallback(cb func(alertID string)) func() {
if cb == nil {
return func() {}
@@ -324,6 +337,14 @@ func (m *Manager) SetResolvedCallback(cb func(alertID string)) {
m.callbacks.setResolvedCallback(cb)
}
+// SetResolvedAlertCallback installs the occurrence-qualified recovery consumer.
+// The snapshot is captured before asynchronous dispatch; consumers must not
+// look up a reusable alert ID to reconstruct the resolved occurrence later.
+// Legacy ID-only callbacks remain independent and retain their public IDs.
+func (m *Manager) SetResolvedAlertCallback(cb func(resolved *ResolvedAlert)) {
+ m.callbacks.setResolvedAlertCallback(cb)
+}
+
// SubscribeResolvedCallback registers an additional resolved-alert callback
// without replacing the legacy single callback slot. The returned function
// removes the subscription when called.
@@ -402,31 +423,43 @@ func (m *Manager) getEscalateCallback() func(alert *Alert, level int) {
// safeCallResolvedAlertCallback invokes onResolved with panic recovery while
// preserving canonical state as the internal identity and emitting the public
// alert ID to external callbacks for compatibility.
-func (m *Manager) safeCallResolvedAlertCallback(alert *Alert, fallbackID string, async bool) {
+func (m *Manager) safeCallResolvedAlertCallback(resolved *ResolvedAlert, fallbackID string, async bool) {
+ var alert *Alert
+ if resolved != nil {
+ alert = resolved.Alert
+ }
// Record the resolution before the callback guard: this funnel is the
// resolve seam for every lifecycle path, with or without subscribers.
m.recordAlertEvent(eventlog.TypeResolved, alert, fallbackID, "", "Alert resolved.", nil)
callbacks := m.getResolvedCallbacks()
- if len(callbacks) == 0 {
+ resolvedCallback := m.callbacks.resolvedAlertCallback()
+ if len(callbacks) == 0 && resolvedCallback == nil {
return
}
publicID := exportedAlertID(alert, fallbackID)
trackingKey := canonicalTrackingKeyForAlert(alert)
+ var snapshot *ResolvedAlert
+ if resolved != nil && alert != nil {
+ snapshot = &ResolvedAlert{Alert: cloneAlertForOutput(alert), ResolvedTime: resolved.ResolvedTime}
+ }
callbackFunc := func() {
- defer func() {
- if r := recover(); r != nil {
- log.Error().
- Interface("panic", r).
- Str("alertID", publicID).
- Str("trackingKey", trackingKey).
- Msg("Panic in onResolved callback")
- }
- }()
+ callSafely := func(callback func()) {
+ defer func() {
+ if r := recover(); r != nil {
+ log.Error().Interface("panic", r).Str("alertID", publicID).
+ Str("trackingKey", trackingKey).Msg("Panic in onResolved callback")
+ }
+ }()
+ callback()
+ }
for _, callback := range callbacks {
- callback(publicID)
+ callSafely(func() { callback(publicID) })
+ }
+ if resolvedCallback != nil && snapshot != nil {
+ callSafely(func() { resolvedCallback(snapshot) })
}
}
diff --git a/internal/alerts/canonical_lifecycle.go b/internal/alerts/canonical_lifecycle.go
index d1c8e738b5..b6f8386273 100644
--- a/internal/alerts/canonical_lifecycle.go
+++ b/internal/alerts/canonical_lifecycle.go
@@ -603,7 +603,7 @@ func (m *Manager) evaluateCanonicalLifecycleAlert(params canonicalLifecycleAlert
recoveryEvidenceRef,
)
m.addRecentlyResolvedWithPrimaryLock(resolvedAlert)
- m.safeCallResolvedAlertCallback(existing, storageKey, true)
+ m.safeCallResolvedAlertCallback(resolvedAlert, storageKey, true)
return result, true
}
@@ -863,7 +863,7 @@ func (m *Manager) evaluateCanonicalStatefulAlert(params canonicalStatefulAlertPa
recoveryEvidenceRef,
)
m.addRecentlyResolvedWithPrimaryLock(resolvedAlert)
- m.safeCallResolvedAlertCallback(existing, storageKey, true)
+ m.safeCallResolvedAlertCallback(resolvedAlert, storageKey, true)
return result, true
}
}
diff --git a/internal/alerts/canonical_metric.go b/internal/alerts/canonical_metric.go
index 1cceeaa0de..fcfa75baa5 100644
--- a/internal/alerts/canonical_metric.go
+++ b/internal/alerts/canonical_metric.go
@@ -407,7 +407,7 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
m.removeActiveAlertNoLock(storageKey)
m.saveActiveAlertsAsync("canonical metric resolution")
m.addRecentlyResolvedWithPrimaryLock(resolvedAlert)
- m.safeCallResolvedAlertCallback(existingAlert, storageKey, true)
+ m.safeCallResolvedAlertCallback(resolvedAlert, storageKey, true)
}
}
diff --git a/internal/alerts/concurrency_test.go b/internal/alerts/concurrency_test.go
index 7f9cc69bda..7d88d2db33 100644
--- a/internal/alerts/concurrency_test.go
+++ b/internal/alerts/concurrency_test.go
@@ -96,7 +96,7 @@ func TestManagerCallbacksConcurrentSetAndUse(t *testing.T) {
defer wg.Done()
<-start
for i := 0; i < iterations; i++ {
- manager.safeCallResolvedAlertCallback(&Alert{ID: "concurrency-alert"}, "concurrency-alert", false)
+ manager.safeCallResolvedAlertCallback(&ResolvedAlert{Alert: &Alert{ID: "concurrency-alert"}}, "concurrency-alert", false)
manager.safeCallAcknowledgedCallback(alert, "test-user")
manager.safeCallUnacknowledgedCallback(alert, "test-user")
manager.safeCallEscalateCallback(alert, 1)
diff --git a/internal/alerts/config_runtime.go b/internal/alerts/config_runtime.go
index 73b9fe2891..17d0ecf11e 100644
--- a/internal/alerts/config_runtime.go
+++ b/internal/alerts/config_runtime.go
@@ -608,7 +608,7 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
Str("alertID", alertID).
Msg("Alert auto-resolved after configuration change")
- m.safeCallResolvedAlertCallback(resolvedAlert.Alert, alertID, true)
+ m.safeCallResolvedAlertCallback(resolvedAlert, alertID, true)
}
}
diff --git a/internal/alerts/guest.go b/internal/alerts/guest.go
index 5eb502e55c..b185ea1d30 100644
--- a/internal/alerts/guest.go
+++ b/internal/alerts/guest.go
@@ -551,7 +551,7 @@ func (m *Manager) clearGuestPoweredOffAlert(guestID, name string) {
// Send recovery notification (async to avoid deadlock because callback acquires m.mu.RLock
// via ShouldSuppressResolvedNotification, and we currently hold m.mu.Lock)
- m.safeCallResolvedAlertCallback(alert, effectiveAlertID(alert, alertID), true)
+ m.safeCallResolvedAlertCallback(resolvedAlert, effectiveAlertID(alert, alertID), true)
// Log recovery
log.Info().
diff --git a/internal/alerts/metric_runtime.go b/internal/alerts/metric_runtime.go
index 6000fad5d8..6d1ad3cba2 100644
--- a/internal/alerts/metric_runtime.go
+++ b/internal/alerts/metric_runtime.go
@@ -612,7 +612,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
Float64("value", value).
Bool("wasAcknowledged", existingAlert.Acknowledged).
Msg("Alert resolved with hysteresis")
- m.safeCallResolvedAlertCallback(existingAlert, alertID, true)
+ m.safeCallResolvedAlertCallback(resolvedAlert, alertID, true)
}
func sanitizeAlertKey(label string) string {
diff --git a/internal/alerts/operational_contract_test.go b/internal/alerts/operational_contract_test.go
index e0e5c68a31..444cb7aba3 100644
--- a/internal/alerts/operational_contract_test.go
+++ b/internal/alerts/operational_contract_test.go
@@ -692,3 +692,55 @@ func TestMemoryUnavailablePreservesActiveAlertsUntilTrustedRecovery(t *testing.T
})
}
}
+
+func TestResolvedOccurrenceCallbackCapturesBeforeDispatch(t *testing.T) {
+ m := newTestManager(t)
+ start := time.Now().Add(-time.Hour)
+ end := start.Add(time.Minute)
+ resolved := &ResolvedAlert{Alert: &Alert{ID: "reusable", StartTime: start,
+ Metadata: map[string]interface{}{"generation": "old"}}, ResolvedTime: end}
+ entered, release := make(chan struct{}), make(chan struct{})
+ defer close(release)
+ m.SetResolvedCallback(func(id string) {
+ if id != "reusable" {
+ t.Errorf("legacy ID = %s", id)
+ }
+ close(entered)
+ <-release
+ })
+ got := make(chan *ResolvedAlert, 1)
+ m.SetResolvedAlertCallback(func(r *ResolvedAlert) { got <- r })
+ m.safeCallResolvedAlertCallback(resolved, resolved.Alert.ID, true)
+ select {
+ case <-entered:
+ case <-time.After(time.Second):
+ t.Fatal("legacy callback did not start")
+ }
+ // Both the source object and the ID-indexed history can change while the
+ // consumer waits. Neither is a source for the already-admitted snapshot.
+ resolved.Alert.StartTime = start.Add(time.Hour)
+ resolved.Alert.Metadata["generation"] = "new"
+ resolved.ResolvedTime = end.Add(time.Hour)
+ m.addRecentlyResolvedUnlocked(resolved)
+ // Unblock without closing twice in cleanup.
+ release <- struct{}{}
+ select {
+ case r := <-got:
+ if !r.Alert.StartTime.Equal(start) || !r.ResolvedTime.Equal(end) || r.Alert.Metadata["generation"] != "old" {
+ t.Fatalf("callback reconstructed the wrong occurrence: %+v", r)
+ }
+ case <-time.After(time.Second):
+ t.Fatal("no occurrence callback")
+ }
+}
+
+func TestResolvedOccurrenceCallbackSurvivesLegacyPanic(t *testing.T) {
+ m := newTestManager(t)
+ m.SetResolvedCallback(func(string) { panic("legacy consumer failure") })
+ called := false
+ m.SetResolvedAlertCallback(func(r *ResolvedAlert) { called = r.Alert.ID == "reusable" })
+ m.safeCallResolvedAlertCallback(&ResolvedAlert{Alert: &Alert{ID: "reusable"}}, "reusable", false)
+ if !called {
+ t.Fatal("legacy panic prevented occurrence-qualified recovery")
+ }
+}
diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go
index e2cf45886b..4d97279cda 100644
--- a/internal/monitoring/monitor.go
+++ b/internal/monitoring/monitor.go
@@ -2248,8 +2248,8 @@ func (m *Monitor) wireExternalAlertCallbacks(wsHub *websocket.Hub) {
callback(alert)
}
})
- m.alertManager.SetResolvedCallback(func(alertID string) {
- m.handleAlertResolved(alertID)
+ m.alertManager.SetResolvedAlertCallback(func(resolved *alerts.ResolvedAlert) {
+ m.handleResolvedAlert(resolved)
// Don't broadcast full state here - it causes a cascade with many guests.
// The frontend gets updated alerts through the regular broadcast ticker.
})
diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go
index c0b8b27947..312a2d89f0 100644
--- a/internal/monitoring/monitor_alert_handling_test.go
+++ b/internal/monitoring/monitor_alert_handling_test.go
@@ -1205,3 +1205,139 @@ func TestMonitorLifecycleRefireReopensRetainedOccurrence(t *testing.T) {
})
}
}
+
+type occurrenceHTTPReceipt struct {
+ Alerts []*alerts.Alert `json:"alerts"`
+}
+
+func occurrenceNotifier(t *testing.T, dir string, endpoint string) *notifications.NotificationManager {
+ t.Helper()
+ n := notifications.NewNotificationManagerWithDeferredQueue("", dir)
+ t.Cleanup(n.Stop)
+ if err := n.UpdateAllowedPrivateCIDRs("127.0.0.1/32,::1/128"); err != nil {
+ t.Fatal(err)
+ }
+ n.AddWebhook(notifications.WebhookConfig{ID: "ops", Enabled: true, URL: endpoint, Service: "generic"})
+ n.SetGroupingWindow(0)
+ n.SetNotifyOnResolve(false)
+ return n
+}
+
+func occurrenceManager(t *testing.T) (*alerts.Manager, models.PBSInstance) {
+ t.Helper()
+ a := alerts.NewManagerWithDataDir(t.TempDir())
+ t.Cleanup(a.Stop)
+ cfg := a.GetConfig()
+ cfg.Enabled, cfg.ActivationState, cfg.FlappingEnabled = true, alerts.ActivationActive, false
+ cfg.Schedule.QuietHours.Enabled = false
+ cfg.TimeThresholds["pbs"] = 0
+ cfg.PBSDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
+ a.UpdateConfig(cfg)
+ return a, models.PBSInstance{ID: "recurring-pbs", Name: "backup", Host: "pbs.invalid", Status: "online", ConnectionHealth: "healthy", CPU: 99}
+}
+
+func occurrenceEndpoint(t *testing.T) (string, <-chan occurrenceHTTPReceipt) {
+ t.Helper()
+ receipts := make(chan occurrenceHTTPReceipt, 8)
+ s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ defer r.Body.Close()
+ var receipt occurrenceHTTPReceipt
+ if err := json.NewDecoder(r.Body).Decode(&receipt); err != nil {
+ t.Errorf("decode HTTP acceptance: %v", err)
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ receipts <- receipt
+ w.WriteHeader(http.StatusOK)
+ }))
+ t.Cleanup(s.Close)
+ return s.URL, receipts
+}
+
+func awaitOccurrenceRows(t *testing.T, n *notifications.NotificationManager, count int) []*notifications.QueuedNotification {
+ t.Helper()
+ deadline := time.Now().Add(3 * time.Second)
+ for {
+ rows, err := n.GetQueue().GetPending(10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(rows) == count {
+ return rows
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("pending rows = %d, want %d", len(rows), count)
+ }
+ time.Sleep(time.Millisecond)
+ }
+}
+
+func assertOccurrenceHTTP(t *testing.T, n *notifications.NotificationManager, receipts <-chan occurrenceHTTPReceipt, current *alerts.Alert) {
+ t.Helper()
+ n.StartQueueProcessing()
+ select {
+ case got := <-receipts:
+ if len(got.Alerts) != 1 || got.Alerts[0].ID != current.ID || !got.Alerts[0].StartTime.Equal(current.StartTime) {
+ t.Fatalf("wrong HTTP occurrence: %+v", got)
+ }
+ t.Logf("HTTP 200 accepted current occurrence %s at %s", current.ID, current.StartTime.Format(time.RFC3339Nano))
+ case <-time.After(12 * time.Second):
+ t.Fatal("current firing did not reach the local HTTP destination")
+ }
+ deadline := time.Now().Add(3 * time.Second)
+ for {
+ stats, err := n.GetQueueStats()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if stats["sent"] == 1 && stats["cancelled"] == 1 && stats["pending"] == 0 && stats["sending"] == 0 {
+ t.Logf("persistent queue completion: %v", stats)
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("queue completion = %v", stats)
+ }
+ time.Sleep(time.Millisecond)
+ }
+ select {
+ case extra := <-receipts:
+ t.Fatalf("obsolete or duplicate delivery: %+v", extra)
+ case <-time.After(200 * time.Millisecond):
+ }
+}
+
+// This existing-interface control also runs against the exact predecessor.
+// It models a delayed ID callback after the next occurrence has entered the
+// real persistent queue. It does not assert native timing or cause a refire.
+func TestMonitorDelayedIDResolutionPreservesNewQueuedOccurrence(t *testing.T) {
+ endpoint, receipts := occurrenceEndpoint(t)
+ dir := t.TempDir()
+ n := occurrenceNotifier(t, dir, endpoint)
+ a, pbs := occurrenceManager(t)
+ a.CheckPBS(pbs)
+ active := a.GetActiveAlerts()
+ if len(active) != 1 {
+ t.Fatalf("PBS firing = %+v", active)
+ }
+ old := active[0].Clone()
+ pbs.CPU = 0
+ a.CheckPBS(pbs)
+ if r := a.GetResolvedAlert(old.ID); r == nil || !r.Alert.StartTime.Equal(old.StartTime) {
+ t.Fatal("missing resolved PBS snapshot")
+ }
+ current := old.Clone()
+ current.StartTime = current.StartTime.Add(time.Nanosecond)
+ n.SendAlert(old)
+ n.SendAlert(current)
+ awaitOccurrenceRows(t, n, 2)
+ m := &Monitor{alertManager: a, notificationMgr: n}
+ m.handleAlertResolved(old.ID)
+ rows, err := n.GetQueue().GetPending(10)
+ if err != nil || len(rows) != 1 || len(rows[0].Alerts) != 1 || !rows[0].Alerts[0].StartTime.Equal(current.StartTime) {
+ t.Fatalf("delayed recovery cancelled the replacement occurrence: rows=%+v err=%v", rows, err)
+ }
+ // Resume the autonomous processor only after reopening the same disk state.
+ n.Stop()
+ n = occurrenceNotifier(t, dir, endpoint)
+ assertOccurrenceHTTP(t, n, receipts, current)
+}
diff --git a/internal/monitoring/monitor_alerts.go b/internal/monitoring/monitor_alerts.go
index bb9a7af975..b84a4e2489 100644
--- a/internal/monitoring/monitor_alerts.go
+++ b/internal/monitoring/monitor_alerts.go
@@ -139,7 +139,9 @@ func (m *Monitor) SetAlertResolvedAICallback(callback func(*alerts.Alert)) {
if m.alertManager == nil {
return
}
+ m.mu.Lock()
m.alertResolvedAICallback = callback
+ m.mu.Unlock()
log.Info().Msg("alert-resolved AI callback registered")
}
@@ -220,41 +222,56 @@ func (m *Monitor) handleAlertFired(alert *alerts.Alert) {
}
func (m *Monitor) handleAlertResolved(alertID string) {
- var resolvedAlert *alerts.ResolvedAlert
+ // Compatibility for ID-only callers. The live monitor is wired to the
+ // occurrence snapshot callback, not a lookup after asynchronous dispatch.
+ if m.alertManager == nil {
+ return
+ }
+ m.handleResolvedAlert(m.alertManager.GetResolvedAlert(alertID))
+}
+
+func (m *Monitor) handleResolvedAlert(resolvedAlert *alerts.ResolvedAlert) {
+ if resolvedAlert == nil || resolvedAlert.Alert == nil {
+ return
+ }
+ alertID := resolvedAlert.Alert.ID
if m.wsHub != nil {
- m.wsHub.BroadcastAlertResolvedToTenant(m.GetOrgID(), alertID)
+ // The legacy websocket message has only an ID. Do not remove a current
+ // firing occurrence that is already visible under that reusable ID.
+ active := false
+ if m.alertManager != nil {
+ _, active = m.alertManager.DiagnoseAlertDelivery(alertID)
+ }
+ if !active {
+ m.wsHub.BroadcastAlertResolvedToTenant(m.GetOrgID(), alertID)
+ }
}
// Always trigger AI callback, regardless of notification suppression.
- if m.alertResolvedAICallback != nil {
- if resolvedAlert == nil {
- resolvedAlert = m.alertManager.GetResolvedAlert(alertID)
- }
- if resolvedAlert != nil && resolvedAlert.Alert != nil {
- go m.alertResolvedAICallback(resolvedAlert.Alert)
- }
+ m.mu.RLock()
+ aiCallback := m.alertResolvedAICallback
+ m.mu.RUnlock()
+ if aiCallback != nil {
+ go aiCallback(resolvedAlert.Alert.Clone())
}
// Handle notifications โ recovery notifications respect quiet hours.
// If the original alert would have been suppressed during quiet hours,
// the recovery notification is also suppressed to avoid noise.
if m.notificationMgr != nil {
- firingNeverDelivered := m.notificationMgr.CancelAlert(alertID)
+ firingNeverDelivered := m.notificationMgr.CancelResolvedAlert(resolvedAlert.Alert)
if m.notificationMgr.GetNotifyOnResolve() {
- if resolvedAlert == nil {
- resolvedAlert = m.alertManager.GetResolvedAlert(alertID)
- }
if resolvedAlert != nil && resolvedAlert.Alert != nil {
if firingNeverDelivered {
// The firing notification was still in the grouping window
// or waiting in the queue (e.g. quiet-hours replay) when the
- // alert resolved, and CancelAlert just cancelled it. A
+ // alert resolved, and exact cancellation just cancelled it. A
// recovery for an alert the user never saw fire is noise.
log.Info().
Str("alertID", alertID).
Msg("Resolved notification suppressed because the firing notification was cancelled before delivery")
- } else if m.alertManager.ShouldSuppressResolvedNotification(resolvedAlert.Alert) {
+ } else if m.alertManager != nil && m.alertManager.ShouldSuppressResolvedNotification(resolvedAlert.Alert) {
log.Info().
Str("alertID", alertID).
Msg("Resolved notification suppressed during quiet hours")
diff --git a/internal/monitoring/monitor_notification_startup_test.go b/internal/monitoring/monitor_notification_startup_test.go
index 34897b605a..0dc557f8a4 100644
--- a/internal/monitoring/monitor_notification_startup_test.go
+++ b/internal/monitoring/monitor_notification_startup_test.go
@@ -5,6 +5,7 @@ import (
"net/http"
"net/http/httptest"
"reflect"
+ "sync"
"sync/atomic"
"testing"
"time"
@@ -179,3 +180,41 @@ func TestNewRestoresSavedNotificationChoices(t *testing.T) {
})
}
}
+
+// Real PBS breach -> recovery -> new breach, with one callback deliberately
+// held at its asynchronous consumer boundary. No clocks or manager maps change.
+func TestMonitorResolvedSnapshotKeepsRefiringPBSDelivery(t *testing.T) {
+ endpoint, receipts := occurrenceEndpoint(t)
+ n := occurrenceNotifier(t, t.TempDir(), endpoint)
+ a, pbs := occurrenceManager(t)
+ m := &Monitor{alertManager: a, notificationMgr: n}
+ m.wireExternalAlertCallbacks(nil)
+ entered, release := make(chan struct{}), make(chan struct{})
+ var releaseOnce sync.Once
+ t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
+ unsubscribe := a.SubscribeResolvedCallback(func(string) { close(entered); <-release })
+ t.Cleanup(unsubscribe)
+ a.CheckPBS(pbs)
+ first := awaitOccurrenceRows(t, n, 1)[0].Alerts[0]
+ pbs.CPU = 0
+ a.CheckPBS(pbs)
+ select {
+ case <-entered:
+ case <-time.After(3 * time.Second):
+ t.Fatal("resolution callback did not start")
+ }
+ pbs.CPU = 95
+ a.CheckPBS(pbs)
+ awaitOccurrenceRows(t, n, 2)
+ active := a.GetActiveAlerts()
+ if len(active) != 1 || active[0].ID != first.ID || active[0].StartTime.Equal(first.StartTime) {
+ t.Fatalf("not a new recurring PBS occurrence: %+v", active)
+ }
+ current := active[0].Clone()
+ releaseOnce.Do(func() { close(release) })
+ remaining := awaitOccurrenceRows(t, n, 1)
+ if !remaining[0].Alerts[0].StartTime.Equal(current.StartTime) {
+ t.Fatal("delayed resolution kept the wrong occurrence")
+ }
+ assertOccurrenceHTTP(t, n, receipts, current)
+}
diff --git a/internal/notifications/notifications.go b/internal/notifications/notifications.go
index 17189a5043..7b93dbf590 100644
--- a/internal/notifications/notifications.go
+++ b/internal/notifications/notifications.go
@@ -1580,9 +1580,24 @@ func (n *NotificationManager) dispatchResolvedAlerts(alertList []*alerts.Alert)
// the grouping window or waiting in the queue), so sending a recovery
// notification would reference an alert that was never announced.
func (n *NotificationManager) CancelAlert(alertID string) bool {
+ return n.cancelAlert(alertID, nil)
+}
+
+// CancelResolvedAlert cancels firing work for one resolved occurrence, without
+// deleting a newer occurrence's grouping entries or delivery cooldown.
+// ID-wide cancellation remains available for explicit administrative callers.
+func (n *NotificationManager) CancelResolvedAlert(alert *alerts.Alert) bool {
+ if alert == nil || alert.ID == "" || alert.StartTime.IsZero() {
+ return false
+ }
+ return n.cancelAlert(alert.ID, &alert.StartTime)
+}
+
+func (n *NotificationManager) cancelAlert(alertID string, start *time.Time) bool {
n.mu.Lock()
queue := n.queue
- _, firingDelivered := n.lastNotified[alertID]
+ record, firingDelivered := n.lastNotified[alertID]
+ firingDelivered = firingDelivered && (start == nil || record.alertStart.Equal(*start))
removed := 0
if len(n.pendingAlerts) > 0 {
@@ -1591,7 +1606,7 @@ func (n *NotificationManager) CancelAlert(alertID string) bool {
if pending == nil {
continue
}
- if pending.ID == alertID {
+ if pending.ID == alertID && (start == nil || pending.StartTime.Equal(*start)) {
removed++
continue
}
@@ -1613,14 +1628,21 @@ func (n *NotificationManager) CancelAlert(alertID string) bool {
}
// Clean up cooldown record for resolved alert
- delete(n.lastNotified, alertID)
+ if start == nil || record.alertStart.Equal(*start) {
+ delete(n.lastNotified, alertID)
+ }
+ remaining := len(n.pendingAlerts)
n.mu.Unlock()
// Cancel any queued notifications containing this alert
cancelledPending := 0
if queue != nil {
var err error
- cancelledPending, err = queue.CancelByAlertIdentifiers([]string{alertID})
+ if start == nil {
+ cancelledPending, err = queue.CancelByAlertIdentifiers([]string{alertID})
+ } else {
+ cancelledPending, err = queue.CancelByAlertOccurrence(alertID, *start)
+ }
if err != nil {
log.Error().Err(err).Str("alertID", alertID).Msg("failed to cancel queued notifications")
}
@@ -1630,7 +1652,7 @@ func (n *NotificationManager) CancelAlert(alertID string) bool {
log.Debug().
Str("alertID", alertID).
- Int("remaining", len(n.pendingAlerts)).
+ Int("remaining", remaining).
Bool("firingNeverDelivered", firingNeverDelivered).
Msg("removed resolved alert from pending notifications and cooldown map")
diff --git a/internal/notifications/queue.go b/internal/notifications/queue.go
index 47c28fda14..9835972871 100644
--- a/internal/notifications/queue.go
+++ b/internal/notifications/queue.go
@@ -2150,6 +2150,30 @@ func calculateBackoff(attempt int) time.Duration {
// these do not contribute to the pending-only return count.
func (nq *NotificationQueue) CancelByAlertIdentifiers(alertIdentifiers []string) (int, error) {
alertIdentifiers = normalizeAlertIdentifiers(alertIdentifiers)
+ identifiers := make(map[string]struct{}, len(alertIdentifiers))
+ for _, id := range alertIdentifiers {
+ identifiers[id] = struct{}{}
+ }
+ return nq.cancelAlertNotifications(alertIdentifiers, func(alert *alerts.Alert) bool {
+ _, matched := identifiers[alert.ID]
+ return matched
+ })
+}
+
+// CancelByAlertOccurrence retires only the resolved occurrence's firing work.
+// Missing occurrence identity must never widen into ID-wide cancellation.
+// The same delivery gates, terminal-row handling and audit retention apply.
+func (nq *NotificationQueue) CancelByAlertOccurrence(alertID string, start time.Time) (int, error) {
+ alertID = strings.TrimSpace(alertID)
+ if alertID == "" || start.IsZero() {
+ return 0, nil
+ }
+ return nq.cancelAlertNotifications([]string{alertID}, func(alert *alerts.Alert) bool {
+ return alert.ID == alertID && alert.StartTime.Equal(start)
+ })
+}
+
+func (nq *NotificationQueue) cancelAlertNotifications(alertIdentifiers []string, matches func(*alerts.Alert) bool) (int, error) {
if len(alertIdentifiers) == 0 {
return 0, nil
}
@@ -2175,11 +2199,6 @@ func (nq *NotificationQueue) CancelByAlertIdentifiers(alertIdentifiers []string)
return 0, fmt.Errorf("failed to query notifications for cancellation: %w", err)
}
- alertIdentifierSet := make(map[string]struct{})
- for _, id := range alertIdentifiers {
- alertIdentifierSet[id] = struct{}{}
- }
-
type queuedAlertCancellation struct {
notificationID string
remaining []*alerts.Alert
@@ -2244,7 +2263,7 @@ func (nq *NotificationQueue) CancelByAlertIdentifiers(alertIdentifiers []string)
remainingAlerts = append(remainingAlerts, alert)
continue
}
- if _, exists := alertIdentifierSet[alert.ID]; exists {
+ if matches(alert) {
matchedAlertCount++
if alert.OperationalRecord != nil && alert.LatestTransition != nil {
removedLinkKeys[alert.OperationalRecord.ID+"\x00"+alert.LatestTransition.ID] = struct{}{}
diff --git a/internal/notifications/queue_occurrence_cancellation_test.go b/internal/notifications/queue_occurrence_cancellation_test.go
new file mode 100644
index 0000000000..713dbe0e38
--- /dev/null
+++ b/internal/notifications/queue_occurrence_cancellation_test.go
@@ -0,0 +1,110 @@
+package notifications
+
+import (
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+)
+
+func TestQueueOccurrenceCancellationPreservesRecurrenceAcrossRestart(t *testing.T) {
+ for _, status := range []NotificationQueueStatus{QueueStatusPending, QueueStatusSending, QueueStatusFailed, QueueStatusDLQ} {
+ t.Run(string(status), func(t *testing.T) {
+ dir := t.TempDir()
+ q, err := NewNotificationQueue(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = q.Stop() })
+ start := time.Now().UTC().Add(-time.Hour)
+ old := quietReplayAlert("reusable", alerts.AlertLevelWarning, start)
+ current := quietReplayAlert("reusable", alerts.AlertLevelCritical, start.Add(time.Nanosecond))
+ current.OperationalRecord.ID += "-new"
+ current.LatestTransition.ID += "-new"
+ peer := quietReplayAlert("peer", alerts.AlertLevelWarning, start)
+ for _, row := range []*QueuedNotification{
+ {ID: "old", Type: "webhook", Alerts: []*alerts.Alert{old}},
+ {ID: "new", Type: "webhook", Alerts: []*alerts.Alert{current}},
+ {ID: "mixed", Type: "webhook", Alerts: []*alerts.Alert{old, current, peer}},
+ {ID: "recovery", Type: "webhook_resolved", Alerts: []*alerts.Alert{old}},
+ {ID: "unknown", Type: "webhook", Alerts: []*alerts.Alert{{ID: old.ID}}},
+ } {
+ row.Config = []byte("{}")
+ row.DestinationID = "webhook:ops"
+ row.MaxAttempts = 3
+ if err := q.Enqueue(row); err != nil {
+ t.Fatal(err)
+ }
+ if err := q.UpdateStatus(row.ID, status, "retained-attempt"); err != nil {
+ t.Fatal(err)
+ }
+ if err := q.RecordAudit(row, false, "retained-attempt"); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := q.RecordDeliveryReceipt(current.ID, current.StartTime, "webhook:ops", time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ for _, identity := range []struct {
+ id string
+ start time.Time
+ }{{"", start}, {old.ID, time.Time{}}} {
+ if count, err := q.CancelByAlertOccurrence(identity.id, identity.start); err != nil || count != 0 {
+ t.Fatalf("missing identity widened cancellation: %d, %v", count, err)
+ }
+ }
+ count, err := q.CancelByAlertOccurrence(old.ID, old.StartTime)
+ wantCount := 0
+ if status == QueueStatusPending {
+ wantCount = 2
+ }
+ if err != nil || count != wantCount {
+ t.Fatalf("cancel count = %d, %v, want %d", count, err, wantCount)
+ }
+ if err := q.Stop(); err != nil {
+ t.Fatal(err)
+ }
+ q, err = NewNotificationQueue(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ oldRow, mixed := loadQuietReplay(t, q, "old"), loadQuietReplay(t, q, "mixed")
+ if oldRow.Status != QueueStatusCancelled || mixed.Status != status || len(mixed.Alerts) != 2 ||
+ !mixed.Alerts[0].StartTime.Equal(current.StartTime) || mixed.Alerts[1].ID != peer.ID || len(mixed.Links) != 2 ||
+ mixed.Links[0].OperationalRecordID != current.OperationalRecord.ID || mixed.Links[1].OperationalRecordID != peer.OperationalRecord.ID {
+ t.Fatalf("wrong retained rows/links: old=%+v mixed=%+v", oldRow, mixed)
+ }
+ for _, id := range []string{"new", "recovery", "unknown"} {
+ if row := loadQuietReplay(t, q, id); row.Status != status || len(row.Alerts) != 1 {
+ t.Fatalf("unrelated row %s changed: %+v", id, row)
+ }
+ }
+ if has, err := q.HasDeliveryReceipt(current.ID, current.StartTime, "webhook:ops"); err != nil || !has {
+ t.Fatalf("current occurrence lost its receipt: %t, %v", has, err)
+ }
+ var audits int
+ if err := q.db.QueryRow(`SELECT count(*) FROM notification_audit WHERE success = 0`).Scan(&audits); err != nil || audits != 5 {
+ t.Fatalf("retained failed attempts = %d, %v", audits, err)
+ }
+ })
+ }
+}
+
+func TestCancelResolvedAlertPreservesNewGroupingAndCooldown(t *testing.T) {
+ start := time.Now().Add(-time.Minute)
+ old := &alerts.Alert{ID: "reusable", StartTime: start}
+ current := &alerts.Alert{ID: old.ID, StartTime: start.Add(time.Nanosecond)}
+ record := notificationRecord{alertStart: current.StartTime, lastSent: time.Now(), level: alerts.AlertLevelCritical}
+ n := &NotificationManager{pendingAlerts: []*alerts.Alert{old, current}, lastNotified: map[string]notificationRecord{old.ID: record}}
+ if n.CancelResolvedAlert(&alerts.Alert{ID: old.ID}) || len(n.pendingAlerts) != 2 {
+ t.Fatal("missing occurrence cancelled pending work")
+ }
+ if !n.CancelResolvedAlert(old) || len(n.pendingAlerts) != 1 || n.pendingAlerts[0] != current || n.lastNotified[current.ID] != record {
+ t.Fatal("delayed recovery removed current grouping or cooldown state")
+ }
+ // Keep the explicit, identifier-wide cancellation contract for its callers.
+ n.CancelAlert(current.ID)
+ if len(n.pendingAlerts) != 0 || len(n.lastNotified) != 0 {
+ t.Fatal("ID-wide cancellation regressed")
+ }
+}
From 2c57a54ec4cba0ffd108465b74a582bf61b35bec Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 01:20:39 +0100
Subject: [PATCH 07/86] Preserve sending-row crash recovery in the occurrence
fixture
Assert surviving row status before restart, then expect the existing sending-to-pending crash recovery after reopening the queue. The initial source-dtamnmrn fixture failure is retained; runtime source and delivery policy are unchanged.
Change-source: pulse-maintainer
---
.../queue_occurrence_cancellation_test.go | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/internal/notifications/queue_occurrence_cancellation_test.go b/internal/notifications/queue_occurrence_cancellation_test.go
index 713dbe0e38..e2e5c7495b 100644
--- a/internal/notifications/queue_occurrence_cancellation_test.go
+++ b/internal/notifications/queue_occurrence_cancellation_test.go
@@ -61,6 +61,9 @@ func TestQueueOccurrenceCancellationPreservesRecurrenceAcrossRestart(t *testing.
if err != nil || count != wantCount {
t.Fatalf("cancel count = %d, %v, want %d", count, err, wantCount)
}
+ if row := loadQuietReplay(t, q, "mixed"); row.Status != status || len(row.Alerts) != 2 {
+ t.Fatalf("cancellation changed surviving row status: %+v", row)
+ }
if err := q.Stop(); err != nil {
t.Fatal(err)
}
@@ -68,14 +71,20 @@ func TestQueueOccurrenceCancellationPreservesRecurrenceAcrossRestart(t *testing.
if err != nil {
t.Fatal(err)
}
+ reopenStatus := status
+ if status == QueueStatusSending {
+ // Existing crash recovery requeues interrupted sends. Exact
+ // cancellation must preserve that behaviour for survivors.
+ reopenStatus = QueueStatusPending
+ }
oldRow, mixed := loadQuietReplay(t, q, "old"), loadQuietReplay(t, q, "mixed")
- if oldRow.Status != QueueStatusCancelled || mixed.Status != status || len(mixed.Alerts) != 2 ||
+ if oldRow.Status != QueueStatusCancelled || mixed.Status != reopenStatus || len(mixed.Alerts) != 2 ||
!mixed.Alerts[0].StartTime.Equal(current.StartTime) || mixed.Alerts[1].ID != peer.ID || len(mixed.Links) != 2 ||
mixed.Links[0].OperationalRecordID != current.OperationalRecord.ID || mixed.Links[1].OperationalRecordID != peer.OperationalRecord.ID {
t.Fatalf("wrong retained rows/links: old=%+v mixed=%+v", oldRow, mixed)
}
for _, id := range []string{"new", "recovery", "unknown"} {
- if row := loadQuietReplay(t, q, id); row.Status != status || len(row.Alerts) != 1 {
+ if row := loadQuietReplay(t, q, id); row.Status != reopenStatus || len(row.Alerts) != 1 {
t.Fatalf("unrelated row %s changed: %+v", id, row)
}
}
From 1c9440546afbb1d17432b10814d078ddb7a07aee Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 01:29:51 +0100
Subject: [PATCH 08/86] Use destination receipts for delayed partial-delivery
recovery
A cancelled retry and a lost RAM cooldown marker do not mean that no destination received the old firing. Keep quiet-hours and recovery controls, then let persistent occurrence/destination receipts select the actual recipients. Add an HTTP 200/503, restart and recurrence control so the newer firing survives and only the previously informed destination receives the old recovery.
Change-source: pulse-maintainer
---
.../v6/internal/subsystems/alerts.md | 6 +
.../v6/internal/subsystems/monitoring.md | 9 ++
.../v6/internal/subsystems/notifications.md | 5 +
.../monitoring/monitor_alert_handling_test.go | 116 ++++++++++++++++++
internal/monitoring/monitor_alerts.go | 25 ++--
5 files changed, 144 insertions(+), 17 deletions(-)
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index 14bdd0b99f..fefabf9828 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -3230,3 +3230,9 @@ readback for all four cancellable row states without deleting failed audits or
new-occurrence destination receipts. The monitor's connected tests use real PBS
evaluation, normal callbacks, the persistent queue and a local HTTP destination;
these receipts are not native provider or installed-release acceptance.
+
+Pending cancellation is not proof that every destination missed the firing.
+The monitor delegates recovery eligibility to persisted occurrence/destination
+receipts even when an old retry remains or restart has lost RAM cooldown state.
+The partial-destination monitor control establishes a real local HTTP 200/503
+split and preserves the newer firing and the old recipient's recovery.
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 6f5caeb5a0..9660e69b59 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -4655,3 +4655,12 @@ reach a loopback HTTP destination with one sent row and one cancelled old row.
No clock, reducer map or configured cooldown is altered. This establishes local
source-connected delivery, not the outstanding natural flapping cooldown check,
a native appliance result or containing stable availability.
+
+Recovery eligibility is destination-receipt-owned, not the cancellation helper's
+pending-work count or the RAM cooldown marker. After partial delivery/restart,
+a cancelled retry at one destination cannot suppress a recovery to another
+that actually accepted the same occurrence. The connected
+`TestMonitorDelayedPartialResolutionKeepsDestinationRecovery` retains a real
+HTTP 200/503 split, reopens the persistent queue, and requires the old accepted
+destination's recovery plus the new firing, with no recovery to the unannounced
+destination. Quiet hours and disabled recovery controls still apply.
diff --git a/docs/release-control/v6/internal/subsystems/notifications.md b/docs/release-control/v6/internal/subsystems/notifications.md
index 5b5b782ab2..068c67b93a 100644
--- a/docs/release-control/v6/internal/subsystems/notifications.md
+++ b/docs/release-control/v6/internal/subsystems/notifications.md
@@ -1073,3 +1073,8 @@ fresh database reopen, nanosecond-distinct occurrences, operational-link and
failed-attempt preservation, missing-identity safety, grouping/cooldown retention
and the unchanged ID-wide API. Monitor connected controls additionally require
HTTP acceptance of the surviving occurrence through the autonomous queue.
+
+The live monitor does not interpret pending-only cancellation counts as
+recipient acceptance. Per-occurrence/per-destination receipts remain the sole
+recovery admission proof, including partial delivery and lost RAM markers after
+restart; an unannounced destination receives no recovery.
diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go
index 312a2d89f0..33c98887b5 100644
--- a/internal/monitoring/monitor_alert_handling_test.go
+++ b/internal/monitoring/monitor_alert_handling_test.go
@@ -1207,6 +1207,7 @@ func TestMonitorLifecycleRefireReopensRetainedOccurrence(t *testing.T) {
}
type occurrenceHTTPReceipt struct {
+ Event string `json:"event"`
Alerts []*alerts.Alert `json:"alerts"`
}
@@ -1341,3 +1342,118 @@ func TestMonitorDelayedIDResolutionPreservesNewQueuedOccurrence(t *testing.T) {
n = occurrenceNotifier(t, dir, endpoint)
assertOccurrenceHTTP(t, n, receipts, current)
}
+
+// One destination accepted the old firing and another still has a retry. After
+// restart, that pending work (and the lost RAM cooldown marker) is not evidence
+// that no recipient saw it. Recovery must use exact persisted receipts.
+func TestMonitorDelayedPartialResolutionKeepsDestinationRecovery(t *testing.T) {
+ endpoint, receipts := occurrenceEndpoint(t)
+ var failedFirings, unwantedRecoveries atomic.Int32
+ broken := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ defer r.Body.Close()
+ var receipt occurrenceHTTPReceipt
+ if err := json.NewDecoder(r.Body).Decode(&receipt); err != nil {
+ t.Errorf("decode failing destination: %v", err)
+ }
+ if receipt.Event == "resolved" {
+ unwantedRecoveries.Add(1)
+ } else {
+ failedFirings.Add(1)
+ }
+ w.WriteHeader(http.StatusServiceUnavailable)
+ }))
+ t.Cleanup(broken.Close)
+ dir := t.TempDir()
+ open := func() *notifications.NotificationManager {
+ n := occurrenceNotifier(t, dir, endpoint)
+ n.SetNotifyOnResolve(true)
+ n.AddWebhook(notifications.WebhookConfig{ID: "unannounced", Enabled: true, URL: broken.URL, Service: "generic"})
+ return n
+ }
+ n := open()
+ a, pbs := occurrenceManager(t)
+ a.CheckPBS(pbs)
+ old := a.GetActiveAlerts()[0].Clone()
+ n.SendAlert(old)
+ n.StartQueueProcessing()
+ select {
+ case got := <-receipts:
+ if got.Event != "" || len(got.Alerts) != 1 || !got.Alerts[0].StartTime.Equal(old.StartTime) {
+ t.Fatalf("wrong initial firing: %+v", got)
+ }
+ case <-time.After(12 * time.Second):
+ t.Fatal("initial firing did not reach accepting destination")
+ }
+ deadline := time.Now().Add(3 * time.Second)
+ for {
+ stats, err := n.GetQueueStats()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if stats["sent"] == 1 && stats["pending"] == 1 && failedFirings.Load() > 0 {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("partial delivery not retained: %v", stats)
+ }
+ time.Sleep(time.Millisecond)
+ }
+ n.Stop()
+ pbs.CPU = 0
+ a.CheckPBS(pbs)
+ n = open() // deferred processor; prior destination receipts are now on disk only
+ pbs.CPU = 95
+ a.CheckPBS(pbs)
+ active := a.GetActiveAlerts()
+ if len(active) != 1 || active[0].ID != old.ID || active[0].StartTime.Equal(old.StartTime) {
+ t.Fatalf("not a new occurrence: %+v", active)
+ }
+ current := active[0].Clone()
+ n.SendAlert(current)
+ m := &Monitor{alertManager: a, notificationMgr: n}
+ m.handleAlertResolved(old.ID)
+ n.StartQueueProcessing()
+ seenFiring, seenRecovery := false, false
+ for i := 0; i < 2; i++ {
+ select {
+ case got := <-receipts:
+ if len(got.Alerts) != 1 || got.Alerts[0].ID != old.ID {
+ t.Fatalf("wrong delivery: %+v", got)
+ }
+ switch got.Event {
+ case "resolved":
+ if seenRecovery || !got.Alerts[0].StartTime.Equal(old.StartTime) {
+ t.Fatalf("wrong recovery occurrence: %+v", got)
+ }
+ seenRecovery = true
+ case "":
+ if seenFiring || !got.Alerts[0].StartTime.Equal(current.StartTime) {
+ t.Fatalf("wrong firing occurrence: %+v", got)
+ }
+ seenFiring = true
+ default:
+ t.Fatalf("unknown event: %+v", got)
+ }
+ case <-time.After(12 * time.Second):
+ t.Fatal("lost current firing or old accepted destination's recovery after restart")
+ }
+ }
+ deadline = time.Now().Add(3 * time.Second)
+ for {
+ stats, err := n.GetQueueStats()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if stats["sent"] == 3 && stats["cancelled"] == 1 && stats["pending"]+stats["sending"]+stats["failed"]+stats["dlq"] == 1 {
+ t.Logf("partial destination recovery retained exact old receipt and new firing: %v", stats)
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("partial recovery queue completion: %v", stats)
+ }
+ time.Sleep(time.Millisecond)
+ }
+ if unwantedRecoveries.Load() != 0 {
+ t.Fatal("unannounced destination received a recovery")
+ }
+}
diff --git a/internal/monitoring/monitor_alerts.go b/internal/monitoring/monitor_alerts.go
index b84a4e2489..826f9971b7 100644
--- a/internal/monitoring/monitor_alerts.go
+++ b/internal/monitoring/monitor_alerts.go
@@ -260,24 +260,15 @@ func (m *Monitor) handleResolvedAlert(resolvedAlert *alerts.ResolvedAlert) {
// If the original alert would have been suppressed during quiet hours,
// the recovery notification is also suppressed to avoid noise.
if m.notificationMgr != nil {
- firingNeverDelivered := m.notificationMgr.CancelResolvedAlert(resolvedAlert.Alert)
+ m.notificationMgr.CancelResolvedAlert(resolvedAlert.Alert)
if m.notificationMgr.GetNotifyOnResolve() {
- if resolvedAlert != nil && resolvedAlert.Alert != nil {
- if firingNeverDelivered {
- // The firing notification was still in the grouping window
- // or waiting in the queue (e.g. quiet-hours replay) when the
- // alert resolved, and exact cancellation just cancelled it. A
- // recovery for an alert the user never saw fire is noise.
- log.Info().
- Str("alertID", alertID).
- Msg("Resolved notification suppressed because the firing notification was cancelled before delivery")
- } else if m.alertManager != nil && m.alertManager.ShouldSuppressResolvedNotification(resolvedAlert.Alert) {
- log.Info().
- Str("alertID", alertID).
- Msg("Resolved notification suppressed during quiet hours")
- } else {
- go m.notificationMgr.SendResolvedAlert(resolvedAlert)
- }
+ if m.alertManager != nil && m.alertManager.ShouldSuppressResolvedNotification(resolvedAlert.Alert) {
+ log.Info().Str("alertID", alertID).Msg("Resolved notification suppressed during quiet hours")
+ } else {
+ // Only occurrence/destination receipts establish who saw the
+ // firing. A pending retry or lost in-memory cooldown marker
+ // after restart must not suppress recovery for other recipients.
+ go m.notificationMgr.SendResolvedAlert(resolvedAlert)
}
} else {
log.Info().
From 4df3e8974ec0e0931dd5edc877d29449c24c935a Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 01:36:36 +0100
Subject: [PATCH 09/86] Add disposable native acceptance for owned demo
recovery
Exercise actual systemd and Caddy with a changed executable, distinct persistent data, delayed synthetic failure, observer loss and TERM during full rollback. Keep 300-second clocks, exact-source and cleanup receipts; no production credential or worker system privilege is added. Update the connected recovery contract and policy assertions, and retain bounded service identity observations.
Change-source: pulse-maintainer
---
.github/scripts/demo-runtime-transaction.py | 11 +-
.github/scripts/dispatch-demo-runtime.py | 4 +-
.github/scripts/tests/demo_runtime_native.py | 218 ++++++++++++++++++
.../tests/test_demo_runtime_transaction.py | 50 ++++
.github/workflows/demo-runtime-native.yml | 43 ++++
.../v6/internal/DEMO_RUNTIME_TRANSACTION.md | 15 ++
.../subsystems/deployment-installability.md | 13 ++
.../v6/internal/subsystems/registry.json | 6 +-
scripts/installtests/demo_transaction_test.go | 34 +++
.../release_promotion_policy_test.py | 99 +++-----
.../release_control/subsystem_lookup_test.py | 3 +-
11 files changed, 425 insertions(+), 71 deletions(-)
create mode 100644 .github/scripts/tests/demo_runtime_native.py
create mode 100644 .github/workflows/demo-runtime-native.yml
diff --git a/.github/scripts/demo-runtime-transaction.py b/.github/scripts/demo-runtime-transaction.py
index 5fa81dda04..436e74b46d 100644
--- a/.github/scripts/demo-runtime-transaction.py
+++ b/.github/scripts/demo-runtime-transaction.py
@@ -224,7 +224,7 @@ class Host:
def check_journal(self, cursors):
for service, cursor in cursors.items():
- text = self.command(["journalctl", "-u", service, "--after-cursor", cursor, "--output=json", "--no-pager"])
+ text = self.command(["journalctl", "-u", service, "--after-cursor", cursor, "--output=json", "--quiet", "--no-pager"])
for line in text.splitlines():
try:
entry = json.loads(line)
@@ -335,13 +335,16 @@ class Transaction:
baseline = self.host.state()
started = self.host.now()
evidence = self.receipt[phase]
- evidence.update({"required_seconds": WINDOW, "samples": 0, "elapsed_seconds": 0})
+ evidence.update({"required_seconds": WINDOW, "samples": 0, "elapsed_seconds": 0,
+ "initial_services": baseline})
while True:
self.host.check_journal(cursors)
state = self.host.state()
+ evidence["last_services"] = state
if state != baseline or not self.healthy(state):
raise Failure("sustained-health")
- if self.host.version(self.request["local_url"]) != expected_version:
+ evidence["observed_version"] = self.host.version(self.request["local_url"])
+ if evidence["observed_version"] != expected_version:
raise Failure("runtime-version")
evidence["samples"] += 1
elapsed = self.host.now() - started
@@ -411,6 +414,8 @@ class Transaction:
elif self.request["mode"] == "update":
raise Failure("update-baseline-unhealthy")
self.receipt["healthy_baseline"] = healthy_baseline
+ self.receipt["baseline_services"] = states
+ self.receipt["baseline_version"] = original_version
old_handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP)}
for sig in old_handlers:
signal.signal(sig, cancel)
diff --git a/.github/scripts/dispatch-demo-runtime.py b/.github/scripts/dispatch-demo-runtime.py
index 499602fd17..a56e018ff0 100644
--- a/.github/scripts/dispatch-demo-runtime.py
+++ b/.github/scripts/dispatch-demo-runtime.py
@@ -38,7 +38,8 @@ if not attempt.exists():
os.chmod(attempt / name, 0o600)
stream.write(data); stream.flush(); os.fsync(stream.fileno())
# Intent is durable before starting the unit. An uncertain launch is never replayed.
- dfd = os.open(attempt, os.O_DIRECTORY); os.fsync(dfd); os.close(dfd)
+ for directory in (attempt, root / 'attempts', root, root.parent):
+ dfd = os.open(directory, os.O_DIRECTORY); os.fsync(dfd); os.close(dfd)
try:
child = subprocess.run(['systemd-run', '--quiet', '--collect',
'--unit=pulse-demo-' + identity[:32],
@@ -53,6 +54,7 @@ if not attempt.exists():
with open(attempt / 'launch.json', 'x') as stream:
os.chmod(attempt / 'launch.json', 0o600)
json.dump(launch, stream); stream.flush(); os.fsync(stream.fileno())
+ dfd = os.open(attempt, os.O_DIRECTORY); os.fsync(dfd); os.close(dfd)
os.close(fd)
# Closing or losing this observer never stops the independently owned unit.
deadline = time.monotonic() + 4000
diff --git a/.github/scripts/tests/demo_runtime_native.py b/.github/scripts/tests/demo_runtime_native.py
new file mode 100644
index 0000000000..b004b1cf05
--- /dev/null
+++ b/.github/scripts/tests/demo_runtime_native.py
@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+"""Secret-free, fresh public CI only. Never run this fixture on a service host."""
+import base64
+import hashlib
+import importlib.util
+import json
+import os
+from pathlib import Path
+import pwd
+import shutil
+import socket
+import subprocess
+import time
+
+SCRIPTS = Path(__file__).resolve().parents[1]
+RESULT = Path.cwd() / "demo-native-result.json"
+ROOT = Path("/var/lib/pulse-deploy/demo")
+OWNED = [Path("/etc/pulse"), Path("/opt/pulse"), Path("/var/lib/pulse-deploy"),
+ Path("/etc/systemd/system/pulse.service"), Path("/etc/systemd/system/pulse-relay.service"),
+ Path("/usr/local/share/ca-certificates/pulse-demo-native.crt")]
+
+
+def command(argv, timeout=120):
+ result = subprocess.run(argv, capture_output=True, timeout=timeout)
+ if result.returncode:
+ raise RuntimeError("fixture-command-failed")
+ return result.stdout
+
+
+def binary(version, fail_after=0, port=17655):
+ return ('''#!/usr/bin/python3
+import http.server,json,os,threading,time
+VERSION = %r
+def fail():
+ time.sleep(%r)
+ print('panic: synthetic disposable service fault',flush=True)
+ os._exit(1)
+if %r: threading.Thread(target=fail,daemon=True).start()
+class Handler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self):
+ body=json.dumps({'version':VERSION} if self.path=='/api/version' else {'status':'healthy'}).encode()
+ self.send_response(200); self.end_headers(); self.wfile.write(body)
+ def log_message(self,*args): pass
+http.server.HTTPServer(('127.0.0.1',%r),Handler).serve_forever()
+''' % (version, fail_after, fail_after, port)).encode()
+
+
+def unit(executable, user="pulse"):
+ return f"[Unit]\nDescription=Disposable demo native fixture\n[Service]\nUser={user}\nGroup={user}\nExecStart={executable}\nRestart=no\n[Install]\nWantedBy=multi-user.target\n"
+
+
+def installer(version, fail_after=0):
+ source = binary(version, fail_after)
+ return ("#!/usr/bin/python3\nfrom pathlib import Path\n"
+ "import base64\n"
+ f"p=Path('/opt/pulse/bin/pulse');p.write_bytes(base64.b64decode({base64.b64encode(source).decode()!r}));p.chmod(0o755)\n"
+ "p=Path('/etc/systemd/system/pulse.service');p.write_text(p.read_text()+'# candidate fixture\\n')\n"
+ f"Path('/etc/pulse/persistent-marker').write_text({'candidate data ' + version!r})\n").encode()
+
+
+def fixture_installer(version, fail_after=0):
+ # The real engine invokes bash like the published installer. This fixture
+ # deliberately changes executable, unit and persistent data without download.
+ script = installer(version, fail_after)
+ return ("#!/bin/bash\nset -eu\npython3 - <<'PY'\n" + script.decode().split("\n", 1)[1] + "\nPY\n").encode()
+
+
+def wait(predicate, limit):
+ deadline = time.monotonic() + limit
+ while time.monotonic() < deadline:
+ if predicate():
+ return
+ time.sleep(1)
+ raise RuntimeError("fixture-observation-timeout")
+
+
+def main():
+ result = {"schema_version": 1, "control_sha": os.environ.get("GITHUB_SHA"),
+ "run_id": os.environ.get("GITHUB_RUN_ID"), "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"),
+ "cases": [], "cleanup_complete": False, "signed_published_installer_acceptance": False}
+ processes = []
+ admitted = False
+ original_caddy = None
+ caddyfile = Path("/etc/caddy/Caddyfile")
+ try:
+ result["stage"] = "disposable-runner-admission"
+ # Environment fields are context, not permission: combine them with a
+ # fresh empty estate, effective root, disposable hosted VM and exact job.
+ if (os.geteuid() != 0 or os.environ.get("GITHUB_ACTIONS") != "true"
+ or os.environ.get("RUNNER_ENVIRONMENT") != "github-hosted"
+ or socket.gethostname() in {"pulse-relay", "pulse-license", "pulse-dev"}
+ or any(path.exists() or path.is_symlink() for path in OWNED)):
+ raise RuntimeError("not-an-empty-disposable-public-ci-runner")
+ admitted = True
+ result["stage"] = "prepare-native-fixtures"
+ original_caddy = caddyfile.read_bytes()
+ command(["systemctl", "show", "--property=Version"])
+ if not subprocess.run(["id", "pulse"], capture_output=True).returncode == 0:
+ command(["useradd", "--system", "--user-group", "--no-create-home", "pulse"])
+ owner = pwd.getpwnam("pulse")
+ Path("/opt/pulse/bin").mkdir(parents=True)
+ Path("/etc/pulse").mkdir(mode=0o755)
+ os.chown("/etc/pulse", owner.pw_uid, owner.pw_gid)
+ Path("/etc/pulse/.env").write_text("DEMO_MODE=true\n")
+ Path("/etc/pulse/billing.json").write_text('{"capabilities":[]}')
+ Path("/etc/pulse/persistent-marker").write_text("original data")
+ for path in Path("/etc/pulse").iterdir():
+ os.chown(path, owner.pw_uid, owner.pw_gid)
+ Path("/opt/pulse/bin/pulse").write_bytes(binary("1.0.0")); Path("/opt/pulse/bin/pulse").chmod(0o755)
+ Path("/opt/pulse/bin/relay").write_bytes(binary("1.0.0", port=17656)); Path("/opt/pulse/bin/relay").chmod(0o755)
+ for name, executable in (("pulse", "/opt/pulse/bin/pulse"), ("pulse-relay", "/opt/pulse/bin/relay")):
+ Path(f"/etc/systemd/system/{name}.service").write_text(unit(executable))
+ command(["systemctl", "daemon-reload"])
+ command(["systemctl", "enable", "--now", "pulse", "pulse-relay"])
+ cert = Path("/etc/caddy/demo-native.crt"); key = Path("/etc/caddy/demo-native.key")
+ command(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1",
+ "-subj", "/CN=localhost", "-addext", "subjectAltName=IP:127.0.0.1",
+ "-addext", "basicConstraints=critical,CA:TRUE",
+ "-keyout", str(key), "-out", str(cert)])
+ key.chmod(0o640); shutil.chown(key, group="caddy")
+ shutil.copyfile(cert, OWNED[-1]); command(["update-ca-certificates"])
+ caddyfile.write_text("{\n admin off\n auto_https off\n}\n" +
+ f"https://127.0.0.1:18443 {{\n tls {cert} {key}\n reverse_proxy 127.0.0.1:17655\n}}\n" +
+ f"https://127.0.0.1:18444 {{\n tls {cert} {key}\n reverse_proxy 127.0.0.1:17656\n}}\n")
+ command(["systemctl", "restart", "caddy"])
+ spec = importlib.util.spec_from_file_location("dispatcher", SCRIPTS / "dispatch-demo-runtime.py")
+ dispatcher = importlib.util.module_from_spec(spec); spec.loader.exec_module(dispatcher)
+ original = (SCRIPTS / "demo-runtime-transaction.py").read_bytes()
+ # Only the cohost endpoint is rebound. Transaction, paths, locking,
+ # systemd, HTTP/TLS, estate, clocks and watches remain real and unchanged.
+ endpoint = b'https://relay.pulserelay.pro/health'
+ assert original.count(endpoint) == 1
+ source = original.replace(endpoint, b'https://127.0.0.1:18444/health')
+ result["original_engine_sha256"] = hashlib.sha256(original).hexdigest()
+ result["fixture_engine_sha256"] = hashlib.sha256(source).hexdigest()
+ result["bootstrap_sha256"] = hashlib.sha256(dispatcher.BOOTSTRAP.encode()).hexdigest()
+ result["driver_sha256"] = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()
+ result["tools"] = {"systemd": command(["systemctl", "--version"]).decode().splitlines()[0],
+ "caddy": command(["caddy", "version"]).decode().strip()}
+ profile = {name: "2" for name in ("nodes", "vms_per_node", "lxcs_per_node", "docker_hosts", "docker_containers", "generic_hosts", "k8s_clusters", "k8s_nodes", "k8s_pods", "k8s_deployments")}
+ profile.update(seed_duration="2h", sample_interval="5m", update_interval="15s")
+
+ def submit(version, fault=0):
+ install = fixture_installer(version, fault)
+ request = {"mode": "update", "hostname": socket.gethostname(), "local_url": "http://127.0.0.1:17655",
+ "public_url": "https://127.0.0.1:18443/api/health", "version": "v" + version,
+ "profile": profile, "control_sha": os.environ["GITHUB_SHA"],
+ "run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": os.environ["GITHUB_RUN_ATTEMPT"],
+ "installer_sha256": hashlib.sha256(install).hexdigest()}
+ payload = {"request": request, "source": base64.b64encode(source).decode(), "installer": base64.b64encode(install).decode()}
+ identity = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
+ proc = subprocess.Popen(["python3", "-c", dispatcher.BOOTSTRAP], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ processes.append(proc); proc.stdin.write(json.dumps(payload).encode()); proc.stdin.close()
+ return proc, ROOT / "attempts" / identity, payload
+
+ # Readiness is real, including authenticated TLS through actual Caddy.
+ result["stage"] = "native-listener-readiness"
+ wait(lambda: subprocess.run(["curl", "--disable", "--silent", "--fail", "https://127.0.0.1:18443/api/health"], capture_output=True).returncode == 0, 60)
+ proc, attempt, payload = submit("1.0.1")
+ result["stage"] = "changed-executable-forward"
+ proc.wait(timeout=900)
+ receipt = json.loads((attempt / "receipt.json").read_text()); result["cases"].append(receipt)
+ assert proc.returncode == 0 and receipt["status"] == "committed" and receipt["forward"]["elapsed_seconds"] >= 300
+ assert Path("/etc/pulse/persistent-marker").read_text() == "candidate data 1.0.1"
+ # A later candidate fails at 55s; lose the observer while the owned
+ # systemd child continues, then TERM during actual full recovery.
+ proc, attempt, payload = submit("1.0.2", 55)
+ result["stage"] = "delayed-fault-observer-loss"
+ wait(lambda: (attempt / "receipt.json").exists() and json.loads((attempt / "receipt.json").read_text())["status"] == "applying", 60)
+ proc.terminate(); proc.wait(timeout=30)
+ command(["systemctl", "is-active", "pulse-demo-" + attempt.name[:32]])
+ wait(lambda: json.loads((attempt / "receipt.json").read_text())["status"] == "recovering", 180)
+ result["stage"] = "term-during-owned-recovery"
+ command(["systemctl", "kill", "--kill-whom=main", "--signal=TERM", "pulse-demo-" + attempt.name[:32]])
+ replay = subprocess.run(["python3", "-c", dispatcher.BOOTSTRAP], input=json.dumps(payload).encode(), capture_output=True, timeout=900)
+ receipt = json.loads((attempt / "receipt.json").read_text()); result["cases"].append(receipt)
+ assert replay.returncode == 1 and receipt["status"] == "rolled_back" and receipt["recovery"]["elapsed_seconds"] >= 300
+ assert receipt["failure"] in {"new-service-crash", "sustained-health"}
+ assert Path("/etc/pulse/persistent-marker").read_text() == "candidate data 1.0.1"
+ version = json.loads(command(["curl", "--disable", "--silent", "--fail", "http://127.0.0.1:17655/api/version"]))["version"]
+ assert version == "1.0.1"
+ result["stage"] = "complete"
+ result["passed"] = True
+ except Exception as error:
+ result["passed"] = False
+ result["failure_type"] = type(error).__name__
+ result["failure_code"] = str(error) if isinstance(error, RuntimeError) else "fixture-observation-or-assertion"
+ finally:
+ if admitted:
+ try:
+ for proc in processes:
+ if proc.poll() is None:
+ proc.terminate(); proc.wait(timeout=30)
+ # Stop only retained children belonging to this fresh fixture.
+ if (ROOT / "attempts").exists():
+ for attempt in (ROOT / "attempts").iterdir():
+ subprocess.run(["systemctl", "stop", "pulse-demo-" + attempt.name[:32]], capture_output=True, timeout=1300)
+ command(["systemctl", "disable", "--now", "pulse", "pulse-relay"])
+ command(["systemctl", "stop", "caddy"])
+ if original_caddy is not None:
+ caddyfile.write_bytes(original_caddy)
+ for path in OWNED:
+ if path.is_dir(): shutil.rmtree(path)
+ elif path.exists(): path.unlink()
+ for path in (Path("/etc/caddy/demo-native.crt"), Path("/etc/caddy/demo-native.key")):
+ if path.exists(): path.unlink()
+ command(["systemctl", "daemon-reload"]); command(["update-ca-certificates", "--fresh"])
+ result["cleanup_complete"] = not any(path.exists() for path in OWNED)
+ except Exception as cleanup:
+ result["cleanup_failure_type"] = type(cleanup).__name__
+ result["passed"] = False
+ RESULT.write_text(json.dumps(result, indent=2) + "\n")
+ RESULT.chmod(0o644)
+ return 0 if result.get("passed") and result["cleanup_complete"] else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.github/scripts/tests/test_demo_runtime_transaction.py b/.github/scripts/tests/test_demo_runtime_transaction.py
index 6050a380c8..c2fab87420 100644
--- a/.github/scripts/tests/test_demo_runtime_transaction.py
+++ b/.github/scripts/tests/test_demo_runtime_transaction.py
@@ -22,6 +22,7 @@ def load(name, filename):
engine = load("demo_transaction", "demo-runtime-transaction.py")
dispatcher = load("demo_dispatcher", "dispatch-demo-runtime.py")
+native = load("demo_native", "tests/demo_runtime_native.py")
def request(mode="recover"):
@@ -181,6 +182,13 @@ class TransactionTest(unittest.TestCase):
self.assertEqual(receipt["status"], "committed")
self.assertEqual(receipt["forward"]["elapsed_seconds"], 300)
self.assertEqual(receipt["forward"]["samples"], 61)
+ self.assertEqual(receipt["baseline_services"]["pulse"]["MainPID"], "100")
+ self.assertEqual(receipt["forward"]["initial_services"], receipt["forward"]["last_services"])
+ self.assertEqual(receipt["forward"]["observed_version"], "6.4.5")
+ self.assertEqual(self.host.install_count, 0)
+ for name in ("binary", "unit", "dropins"):
+ self.assertEqual(engine.estate_hash({name: self.paths[name]}),
+ engine.estate_hash({name: self.attempt / "snapshot" / name}))
self.assertEqual((self.paths["data"] / "alerts/events.db").read_bytes(), b"synthetic opaque persistent file")
self.assertTrue((self.attempt / "snapshot/data/persisted-history").exists())
self.assertNotIn("not-logged", json.dumps(receipt))
@@ -387,6 +395,13 @@ class InputAndCommandTest(unittest.TestCase):
host.check_journal(cursors)
self.assertEqual(cursors, {"pulse": "d"})
+ def test_available_empty_journal_window_has_no_non_json_header(self):
+ host = engine.Host(); cursors = {"pulse": "c"}
+ with patch.object(host, "command", return_value="") as command:
+ host.check_journal(cursors)
+ self.assertIn("--quiet", command.call_args.args[0])
+ self.assertEqual(cursors, {"pulse": "c"})
+
def test_no_journal_cursor_and_malformed_service_state_fail_closed(self):
host = engine.Host()
with patch.object(host, "command", return_value=""):
@@ -446,5 +461,40 @@ class DispatchLifecycleTest(unittest.TestCase):
self.assertEqual(json.loads((attempt / "launch.json").read_text())["state"], "uncertain")
+class NativeAdmissionTest(unittest.TestCase):
+ def test_native_driver_refuses_wrong_context_before_any_service_command(self):
+ with tempfile.TemporaryDirectory() as directory:
+ result_path = Path(directory) / "result.json"
+ existing = Path(directory) / "existing-estate"
+ existing.mkdir()
+ cases = [(1000, "true", "github-hosted", []),
+ (0, "false", "github-hosted", []),
+ (0, "true", "self-hosted", []),
+ (0, "true", "github-hosted", [existing])]
+ for uid, actions, environment, estate in cases:
+ with self.subTest(uid=uid, actions=actions, environment=environment, occupied=bool(estate)), \
+ patch.object(native, "RESULT", result_path), patch.object(native, "OWNED", estate), \
+ patch.object(os, "geteuid", return_value=uid), \
+ patch.dict(os.environ, {"GITHUB_ACTIONS": actions, "RUNNER_ENVIRONMENT": environment}), \
+ patch.object(native.socket, "gethostname", return_value="disposable-fixture"), \
+ patch.object(native, "command", side_effect=AssertionError("must not execute a service command")) as command:
+ self.assertEqual(native.main(), 1)
+ command.assert_not_called()
+ result = json.loads(result_path.read_text())
+ self.assertFalse(result["passed"])
+ self.assertFalse(result["cleanup_complete"])
+ self.assertFalse(result["signed_published_installer_acceptance"])
+ self.assertEqual(result["failure_code"], "not-an-empty-disposable-public-ci-runner")
+ self.assertTrue(existing.is_dir())
+
+ def test_synthetic_changed_executable_and_data_are_distinct_between_steps(self):
+ compile(native.binary("1.0.1"), "fixture-runtime", "exec")
+ compile(native.installer("1.0.1"), "fixture-installer", "exec")
+ self.assertNotEqual(native.binary("1.0.1"), native.binary("1.0.2", 55))
+ self.assertIn(b"candidate data 1.0.1", native.installer("1.0.1"))
+ self.assertIn(b"candidate data 1.0.2", native.installer("1.0.2", 55))
+ self.assertNotIn(b"curl", native.fixture_installer("1.0.1"))
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/.github/workflows/demo-runtime-native.yml b/.github/workflows/demo-runtime-native.yml
new file mode 100644
index 0000000000..a8928e5e5b
--- /dev/null
+++ b/.github/workflows/demo-runtime-native.yml
@@ -0,0 +1,43 @@
+name: Demo transaction native acceptance
+on:
+ pull_request:
+ paths:
+ - '.github/scripts/demo-runtime-transaction.py'
+ - '.github/scripts/dispatch-demo-runtime.py'
+ - '.github/scripts/tests/demo_runtime_native.py'
+ - '.github/workflows/demo-runtime-native.yml'
+ push:
+ branches: [main]
+ paths:
+ - '.github/scripts/demo-runtime-transaction.py'
+ - '.github/scripts/dispatch-demo-runtime.py'
+ - '.github/scripts/tests/demo_runtime_native.py'
+ - '.github/workflows/demo-runtime-native.yml'
+permissions:
+ contents: read
+concurrency:
+ group: demo-native-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+jobs:
+ native:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 35
+ steps:
+ - name: Check out exact source without retained credentials
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - name: Prepare Caddy only in the disposable public CI runner
+ run: |
+ sudo apt-get update -qq
+ sudo apt-get install -y --no-install-recommends caddy openssl
+ - name: Exercise real systemd Caddy and retained child recovery
+ run: sudo --preserve-env=GITHUB_ACTIONS,RUNNER_ENVIRONMENT,GITHUB_SHA,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT python3 .github/scripts/tests/demo_runtime_native.py
+ - name: Retain sanitised terminal receipts and cleanup
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: demo-native-${{ github.run_id }}
+ path: demo-native-result.json
+ if-no-files-found: error
+ retention-days: 14
diff --git a/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md b/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md
index af5e4322da..106e939ffc 100644
--- a/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md
+++ b/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md
@@ -44,6 +44,21 @@ recovery, headroom refusal, shared-lock contention, source mismatch, journal
failure, cohost restart, source-bound repeated/lost submission and no-op proof.
The production window is not shortened. This is **not native acceptance**.
+`demo-runtime-native.yml` now carries the next executable acceptance rather
+than requiring a worker to obtain system mode: exact PR/main source, a fresh
+secret-free public hosted runner, actual systemd/Caddy and disposable trusted
+TLS. The successful changed-executable step observes the full 300 seconds. A
+different executable and data then fail at 55 seconds; the original observer
+is lost, the same retained request observes its child without replay, TERM is
+sent during restoration, and the old executable/data must return through a
+full real-clock recovery window. Sanitised artifacts bind control/driver/
+engine/bootstrap identity, service PID/restart observations and cleanup. The
+fixture rebinds only Relay's fixed health URL to its local Caddy endpoint; the
+original and fixture hashes remain distinct. This is a synthetic installer,
+not signed published-installer or customer acceptance. Its source exists;
+**no native pass is asserted by this document**. Read the exact-head terminal
+job and artifact before judging that result.
+
Before treating this route as restored, the exact reviewed source must return
native disposable systemd/SSH-loss receipts and signed published-installer
forward/reverse data/identity results. Then reconcile actual demo workflow and
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index 78533e9a3f..f53a0ffd5d 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -45,6 +45,19 @@ session. These connected source controls do not establish native systemd,
signed installed forward/reverse acceptance, ingress enforcement or customer
recovery. See [demo transaction acceptance](../DEMO_RUNTIME_TRANSACTION.md).
+The secret-free public `demo-runtime-native.yml` checks the exact PR/push source
+on a fresh hosted runner, using actual systemd and Caddy with disposable TLS and
+synthetic executable/data fixtures. Its two real-window cases exercise a
+changed executable, a 55-second delayed synthetic failure, observer loss and
+TERM during full restoration. It records original/fixture source identities,
+actual bounded service observations and cleanup; a missing or failed record is
+not acceptance. Only the fixed Relay health URL is rebound to that fixture.
+This neither exercises a signed published installer nor proves customer or
+production recovery. The driver refuses a non-empty or non-disposable estate
+before service operations. Neither this driver nor worker source tests install
+host policy. Production activation still requires independent review and the
+operational owner's acceptance of the native result and published-source path.
+
### Superseded pull request validation
Build and Test and Core E2E cancel an older run for the same pull request when
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index 9cf02a0271..f3ec7681fb 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -4524,11 +4524,13 @@
".github/scripts/resolve-demo-runtime-profile.sh",
".github/scripts/select-internal-api-shard.sh",
".github/scripts/setup-demo-ssh.sh",
+ ".github/scripts/tests/demo_runtime_native.py",
".github/workflows/backfill-release-assets.yml",
".github/workflows/build-and-test.yml",
".github/workflows/build-release-candidate.yml",
".github/workflows/compile-release-payload.yml",
".github/workflows/create-release.yml",
+ ".github/workflows/demo-runtime-native.yml",
".github/workflows/dependency-advisory-watch.yml",
".github/workflows/deploy-demo-server.yml",
".github/workflows/helm-pages.yml",
@@ -5176,7 +5178,9 @@
"match_prefixes": [],
"match_files": [
".github/scripts/demo-runtime-transaction.py",
- ".github/scripts/dispatch-demo-runtime.py"
+ ".github/scripts/dispatch-demo-runtime.py",
+ ".github/scripts/tests/demo_runtime_native.py",
+ ".github/workflows/demo-runtime-native.yml"
],
"allow_same_subsystem_tests": false,
"test_prefixes": [],
diff --git a/scripts/installtests/demo_transaction_test.go b/scripts/installtests/demo_transaction_test.go
index ae800da558..d2d2be0683 100644
--- a/scripts/installtests/demo_transaction_test.go
+++ b/scripts/installtests/demo_transaction_test.go
@@ -57,3 +57,37 @@ func TestDemoTransactionKeepsSignedAdmissionAndVerificationOnlyReadOnly(t *testi
t.Fatal("recovery must validate and retain the actual terminal observation")
}
}
+
+func TestDemoNativeAcceptanceIsSecretFreeDisposableAndKeepsRealWindows(t *testing.T) {
+ data, err := os.ReadFile(repoFile(".github", "workflows", "demo-runtime-native.yml"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ text := string(data)
+ for _, required := range []string{
+ "pull_request:", "branches: [main]", "contents: read", "runs-on: ubuntu-24.04", "timeout-minutes: 35",
+ "persist-credentials: false", "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1",
+ "demo_runtime_native.py", "if: always()", "demo-native-result.json", "if-no-files-found: error",
+ } {
+ if !strings.Contains(text, required) {
+ t.Fatalf("lost disposable native CI/source or terminal-receipt boundary: %s", required)
+ }
+ }
+ for _, forbidden := range []string{"secrets.", "pull_request_target", "self-hosted", "contents: write", "id-token: write"} {
+ if strings.Contains(text, forbidden) {
+ t.Fatalf("native fixture gained credentials, privileged trigger or non-disposable target: %s", forbidden)
+ }
+ }
+ driver, err := os.ReadFile(repoFile(".github", "scripts", "tests", "demo_runtime_native.py"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, required := range []string{"not-an-empty-disposable-public-ci-runner", "dispatcher.BOOTSTRAP", "proc.terminate()", "--signal=TERM", "original_engine_sha256", "fixture_engine_sha256", "cleanup_complete", "signed_published_installer_acceptance", "elapsed_seconds\"] >= 300"} {
+ if !strings.Contains(string(driver), required) {
+ t.Fatalf("lost native fixture observation or limitation: %s", required)
+ }
+ }
+ if strings.Contains(string(driver), "WINDOW =") || strings.Contains(string(driver), "time.monotonic =") || strings.Contains(string(driver), "time.sleep =") {
+ t.Fatal("native acceptance must not shorten production windows or replace real clocks")
+ }
+}
diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.py
index 5f4278134e..a933737fd4 100644
--- a/scripts/release_control/release_promotion_policy_test.py
+++ b/scripts/release_control/release_promotion_policy_test.py
@@ -2749,73 +2749,42 @@ class ReleasePromotionPolicyTest(unittest.TestCase):
def test_stable_demo_recovery_is_fixed_current_bits_only(self) -> None:
workflow = read(".github/workflows/recover-demo-server.yml")
recovery = read(".github/scripts/recover-demo-runtime.sh")
+ transaction = read(".github/scripts/demo-runtime-transaction.py")
+ dispatcher = read(".github/scripts/dispatch-demo-runtime.py")
- self.assertIn("workflow_dispatch:", workflow)
- self.assertNotIn("workflow_call:", workflow)
- self.assertNotIn("inputs:", workflow)
- self.assertIn("environment: demo-stable", workflow)
- self.assertIn("contents: read", workflow)
- self.assertIn("cancel-in-progress: false", workflow)
- self.assertIn('gh api "repos/${GITHUB_REPOSITORY}/releases/latest"', workflow)
- self.assertIn("Resolve target-compatible recovery profile", workflow)
- self.assertIn("resolve-demo-runtime-profile.sh", workflow)
- self.assertIn(".github/scripts/recover-demo-runtime.sh", workflow)
- self.assertIn("Verify public health", workflow)
- self.assertIn("Verify public frontend parity", workflow)
- self.assertIn("./scripts/run_demo_public_browser_smoke.sh", workflow)
- self.assertIn("Compensate failed mutated recovery", workflow)
- self.assertIn("Capture bounded Pulse failure diagnostics", workflow)
- self.assertIn("demo-runtime-diagnostics.txt", workflow)
- self.assertIn("journalctl -u pulse --since '-30 minutes'", workflow)
- self.assertIn("journalctl -u pulse --since '-10 minutes'", workflow)
- self.assertIn("tail -400", workflow)
- self.assertIn("sudo systemctl stop pulse", workflow)
- self.assertIn("Retain bounded recovery evidence", workflow)
- self.assertNotIn("release-convergence.yml", workflow)
- self.assertNotIn("install.sh", workflow)
- self.assertNotIn("scp ", workflow)
+ for required in ("workflow_dispatch:", "environment: demo-stable", "contents: read",
+ "cancel-in-progress: false", 'gh api "repos/${GITHUB_REPOSITORY}/releases/latest"',
+ "Resolve target-compatible recovery profile", "resolve-demo-runtime-profile.sh",
+ ".github/scripts/recover-demo-runtime.sh", "Verify public health",
+ "Verify public frontend parity", "./scripts/run_demo_public_browser_smoke.sh",
+ "Capture bounded Pulse failure diagnostics", "demo-runtime-diagnostics.txt",
+ "journalctl -u pulse --since '-30 minutes'", "journalctl -u pulse --since '-10 minutes'",
+ "tail -400", "Retain bounded recovery evidence", ".forward.elapsed_seconds >= 300"):
+ self.assertIn(required, workflow)
+ for forbidden in ("workflow_call:", "inputs:", "release-convergence.yml", "install.sh", "scp ",
+ "Compensate failed mutated recovery", "sudo systemctl stop pulse"):
+ self.assertNotIn(forbidden, workflow)
+ self.assertIn('exec python3 .github/scripts/dispatch-demo-runtime.py recover', recovery)
+ self.assertIn("if mode == 'update' else b''", dispatcher)
+ self.assertIn("/var/lib/pulse-deploy/relay/deploy.lock", transaction)
+ self.assertIn('if self.request["mode"] == "update":', transaction)
+ self.assertIn('self.host.install(self.attempt, self.request)', transaction)
+ self.assertIn('raise Failure("recovery-source-mismatch")', transaction)
+ self.assertIn('copy_path(path, snapshot / label)', transaction)
+ self.assertIn('copy_path(snapshot / label, PATHS[label])', transaction)
+ self.assertIn('estate_hash(PATHS) != self.receipt["snapshot_sha256"]', transaction)
+ self.assertIn('WINDOW = 300', transaction)
+ self.assertIn('self.watch("recovery", original_version, cursors)', transaction)
+ self.assertIn('signal.signal(sig, signal.SIG_IGN)', transaction)
+ self.assertIn('self.save("rolled_back")', transaction)
+ self.assertIn('self.save("rollback_failed")', transaction)
+ for required in ("alerts/events.db", "alerts/alert-history.json.imported",
+ "alerts/alert-history.backup.json.imported", "ai_incidents.json", "snapshot_retained"):
+ self.assertIn(required, transaction)
+ self.assertNotIn('kill -QUIT', transaction + recovery + workflow)
+ self.assertNotIn('eval ', transaction + recovery)
+ self.assertNotIn("'--wait'", dispatcher)
- self.assertIn('if [ "$#" -ne 16 ]', recovery)
- self.assertIn('SERVICE_NAME="pulse"', recovery)
- self.assertIn('RELAY_SERVICE_NAME="pulse-relay"', recovery)
- self.assertIn('EXPECTED_BINARY="/opt/pulse/bin/pulse"', recovery)
- self.assertIn('EXPECTED_UNIT="/etc/systemd/system/pulse.service"', recovery)
- self.assertIn('set_env_value PULSE_MOCK_NODES "$MOCK_NODES"', recovery)
- self.assertIn('set_env_value PULSE_MOCK_VMS_PER_NODE "$MOCK_VMS_PER_NODE"', recovery)
- self.assertIn('set_env_value PULSE_MOCK_LXCS_PER_NODE "$MOCK_LXCS_PER_NODE"', recovery)
- self.assertIn('set_env_value PULSE_MOCK_DOCKER_HOSTS "$MOCK_DOCKER_HOSTS"', recovery)
- self.assertIn('set_env_value PULSE_MOCK_DOCKER_CONTAINERS "$MOCK_DOCKER_CONTAINERS"', recovery)
- self.assertIn('set_env_value PULSE_MOCK_GENERIC_HOSTS "$MOCK_GENERIC_HOSTS"', recovery)
- self.assertIn('set_env_value PULSE_MOCK_K8S_CLUSTERS "$MOCK_K8S_CLUSTERS"', recovery)
- self.assertIn('set_env_value PULSE_MOCK_K8S_NODES "$MOCK_K8S_NODES"', recovery)
- self.assertIn('set_env_value PULSE_MOCK_K8S_PODS "$MOCK_K8S_PODS"', recovery)
- self.assertIn('set_env_value PULSE_MOCK_K8S_DEPLOYMENTS "$MOCK_K8S_DEPLOYMENTS"', recovery)
- self.assertIn("set_env_value PULSE_MOCK_SEED_METRICS_STORE false", recovery)
- self.assertIn('grep -Fxq "PULSE_MOCK_SEED_METRICS_STORE=false"', recovery)
- self.assertIn('set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"', recovery)
- self.assertIn('set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL"', recovery)
- self.assertIn('set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"', recovery)
- self.assertIn('sudo systemctl restart "$SERVICE_NAME"', recovery)
- self.assertIn('sudo systemctl stop "$SERVICE_NAME"', recovery)
- self.assertIn('sudo kill -QUIT "$failed_pid"', recovery)
- self.assertIn("clear_demo_operational_history", recovery)
- self.assertIn("restore_demo_operational_history", recovery)
- self.assertIn("alerts/events.db", recovery)
- self.assertIn("alerts/alert-history.json.imported", recovery)
- self.assertIn("alerts/alert-history.backup.json.imported", recovery)
- self.assertIn("ai_incidents.json", recovery)
- self.assertIn("demo-operational-history.tar", recovery)
- self.assertNotIn('rm -f "/etc/pulse/', recovery)
- self.assertIn('AFTER_RELAY_PID" = "$BEFORE_RELAY_PID', recovery)
- self.assertIn('AFTER_BINARY_SHA" = "$BEFORE_BINARY_SHA', recovery)
- self.assertIn('AFTER_UNIT_SHA" = "$BEFORE_UNIT_SHA', recovery)
- self.assertIn('AFTER_DROPINS_SHA" = "$BEFORE_DROPINS_SHA', recovery)
- self.assertNotIn('AFTER_CONFIG_SHA" = "$BEFORE_CONFIG_SHA', recovery)
- self.assertIn("/etc/pulse/.env /etc/pulse/billing.json", recovery)
- self.assertIn("restore_runtime_config", recovery)
- self.assertIn("sudo cp --archive", recovery)
- self.assertNotIn("eval ", recovery)
- self.assertNotIn("sudo bash", recovery)
def test_blocked_record_tracks_current_target_and_candidate_version(self) -> None:
blocked_record_surface = {
diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.py
index 1fae39c773..8521b32545 100644
--- a/scripts/release_control/subsystem_lookup_test.py
+++ b/scripts/release_control/subsystem_lookup_test.py
@@ -102,7 +102,8 @@ def _contract_reference(contract_path: str, needle: str, runtime_path: str) -> d
class SubsystemLookupTest(unittest.TestCase):
def test_demo_transaction_sources_have_connected_proof_without_widening_release_metadata(self):
- for path in (".github/scripts/demo-runtime-transaction.py", ".github/scripts/dispatch-demo-runtime.py"):
+ for path in (".github/scripts/demo-runtime-transaction.py", ".github/scripts/dispatch-demo-runtime.py",
+ ".github/scripts/tests/demo_runtime_native.py", ".github/workflows/demo-runtime-native.yml"):
with self.subTest(path=path):
result = lookup_paths([path])
self.assertEqual(result["unowned_runtime_files"], [])
From d0d621a7c2d65c46dcf876986c30a20264fe9b08 Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 01:38:16 +0100
Subject: [PATCH 10/86] Exercise detector admission in partial-delivery
recovery proof
Drive firings through the existing monitor callback instead of sending them directly through the notifier. This records the detector notification state required by the unchanged recovery policy; keep that policy and all failed attempts intact.
Change-source: pulse-maintainer
---
.../monitoring/monitor_alert_handling_test.go | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go
index 33c98887b5..498bd35b8d 100644
--- a/internal/monitoring/monitor_alert_handling_test.go
+++ b/internal/monitoring/monitor_alert_handling_test.go
@@ -1372,9 +1372,17 @@ func TestMonitorDelayedPartialResolutionKeepsDestinationRecovery(t *testing.T) {
}
n := open()
a, pbs := occurrenceManager(t)
+ m := &Monitor{alertManager: a, notificationMgr: n}
+ // Use the live firing callback so the detector records notification policy
+ // state. Direct notifier calls would not establish LastNotified, and existing
+ // recovery policy would correctly suppress that unannounced detector alert.
+ a.SetAlertCallback(m.handleAlertFired)
a.CheckPBS(pbs)
old := a.GetActiveAlerts()[0].Clone()
- n.SendAlert(old)
+ if old.LastNotified == nil {
+ t.Fatal("detector did not admit the firing notification")
+ }
+ awaitOccurrenceRows(t, n, 2)
n.StartQueueProcessing()
select {
case got := <-receipts:
@@ -1402,6 +1410,8 @@ func TestMonitorDelayedPartialResolutionKeepsDestinationRecovery(t *testing.T) {
pbs.CPU = 0
a.CheckPBS(pbs)
n = open() // deferred processor; prior destination receipts are now on disk only
+ m = &Monitor{alertManager: a, notificationMgr: n}
+ a.SetAlertCallback(m.handleAlertFired)
pbs.CPU = 95
a.CheckPBS(pbs)
active := a.GetActiveAlerts()
@@ -1409,8 +1419,7 @@ func TestMonitorDelayedPartialResolutionKeepsDestinationRecovery(t *testing.T) {
t.Fatalf("not a new occurrence: %+v", active)
}
current := active[0].Clone()
- n.SendAlert(current)
- m := &Monitor{alertManager: a, notificationMgr: n}
+ awaitOccurrenceRows(t, n, 2)
m.handleAlertResolved(old.ID)
n.StartQueueProcessing()
seenFiring, seenRecovery := false, false
From c9a0f0bf6ae19cb4cb7a2ed20e2d7e435550dafb Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 01:39:20 +0100
Subject: [PATCH 11/86] Keep stored disk History independent of current
collection
Use stable disk-family chart entries so missing current SMART or temperature fields cannot hide stored observations, and matching snapshots do not remount charts or restart reads. Preserve existing identity, live collection status and entitlement boundaries; cover cancellation, measured zero, empty series and desktop/phone presentation.
Change-source: pulse-maintainer
---
.../subsystems/frontend-primitives.md | 12 +
.../internal/subsystems/storage-recovery.md | 26 ++
.../browser-tests/disk-history-collection.cjs | 303 ++++++++++++++++++
.../disk-history-collection.html | 12 +
.../browser-tests/disk-history-collection.tsx | 75 +++++
frontend-modern/browser-verification.json | 49 ++-
.../DiskDetail.history-collection.test.tsx | 260 +++++++++++++++
.../Storage/__tests__/DiskDetail.test.tsx | 13 +
.../__tests__/useDiskDetailModel.test.ts | 16 +
...DetailPresentation.branchcov0724pm.test.ts | 17 +-
.../storageBackups/diskDetailPresentation.ts | 82 +++--
11 files changed, 789 insertions(+), 76 deletions(-)
create mode 100644 frontend-modern/browser-tests/disk-history-collection.cjs
create mode 100644 frontend-modern/browser-tests/disk-history-collection.html
create mode 100644 frontend-modern/browser-tests/disk-history-collection.tsx
create mode 100644 frontend-modern/src/components/Storage/__tests__/DiskDetail.history-collection.test.tsx
diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md
index 5572c24252..6b766e3ef5 100644
--- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md
+++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md
@@ -20,6 +20,18 @@
## Purpose
+### Stable physical-disk History composition
+
+Disk detail composes shared HistoryChart owners from stable disk-family catalog
+entries, independently of the latest optional temperature/SMART fields. Matching
+snapshots must not recreate chart DOM, reset inspection or restart reads. Stored
+series retain their own sample provenance; current collection unavailability is
+still reported in Overview, and empty history is not a manufactured zero. The
+shared target/range owner continues to cancel old selections and reject late
+responses. `DiskDetail.history-collection.test.tsx` and the production
+`browser-tests/disk-history-collection.cjs` fixture verify these boundaries with
+synthetic data; neither establishes native NAS health or installed acceptance.
+
Storage pool Configuration key/value rows keep capacity absence distinct from
measured zero: unavailable bytes and percentages render as `n/a`, while valid
independent observations survive a missing total. Live partial snapshots must
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index 97ffabafe7..c0a85bd7b9 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -21,6 +21,32 @@
## Purpose
+### Disk History is independent of current collection
+
+The existing physical-disk History catalog is selected by disk family, not by
+whether the latest snapshot contains temperature or extended SMART values.
+Temperature and ATA reallocated-sector charts, or temperature and NVMe
+endurance/spare charts, query only the already-resolved canonical disk target.
+A missing current field must not hide stored observations. An empty series is
+shown as no history samples, never a fabricated zero or current-value fallback.
+Overview continues to report current collection availability; explicit
+unavailable/unsupported I/O still suppresses the existing live I/O section.
+
+Catalog entries retain their identity across matching snapshots so ordinary
+current-temperature changes cannot remount charts, drop inspection or restart
+History reads. The shared chart owner still invalidates reads and inspection
+when the explicit target or range changes; a late cancelled response cannot
+populate its replacement. This changes neither disk correlation nor collector,
+retention, permission or entitlement policy, and proves no appliance recovery.
+
+Verification: `DiskDetail.test.tsx` and `useDiskDetailModel.test.ts` pin the
+catalog and matching-snapshot boundary; `DiskDetail.history-collection.test.tsx`
+uses the actual chart owner to cover absent current fields, stored zero versus
+empty, NVMe/ATA catalogs, request counts, preserved inspection and late former
+target responses. `frontend-modern/browser-tests/disk-history-collection.cjs`
+checks the production components/client/CSS with synthetic API data on desktop
+Chromium and touch/dark phone WebKit, not native NAS or installed acceptance.
+
### Independent physical disk temperature
Disk Overview displays a finite positive reported temperature even when the
diff --git a/frontend-modern/browser-tests/disk-history-collection.cjs b/frontend-modern/browser-tests/disk-history-collection.cjs
new file mode 100644
index 0000000000..c8492ba1e4
--- /dev/null
+++ b/frontend-modern/browser-tests/disk-history-collection.cjs
@@ -0,0 +1,303 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const { chromium, webkit } = require('playwright');
+const parent = false;
+(async () => {
+ const root = '/workspace/frontend-modern';
+ process.chdir(root);
+ const artifacts = path.join(
+ root,
+ 'node_modules',
+ parent ? 'disk-history-parent' : 'disk-history-proof',
+ );
+ fs.mkdirSync(artifacts, { recursive: true });
+ const { createServer } = await import(path.join(root, 'node_modules/vite/dist/node/index.js'));
+ const server = await createServer({
+ root,
+ configFile: path.join(root, 'vite.config.ts'),
+ cacheDir: fs.mkdtempSync(path.join(artifacts, 'vite-')),
+ server: { host: '127.0.0.1', port: 5251, strictPort: true },
+ });
+ const playwright = require('playwright/package.json').version;
+ const expected = JSON.parse(
+ fs.readFileSync('/workspace/tests/integration/package-lock.json', 'utf8'),
+ ).packages['node_modules/@playwright/test'].version;
+ assert.equal(playwright, expected);
+ const observations = [];
+ let browser;
+ try {
+ await server.listen();
+ for (const [name, engine, phone] of [
+ ['chromium-desktop', chromium, false],
+ ['webkit-phone', webkit, true],
+ ]) {
+ browser = await engine.launch(
+ engine === chromium
+ ? { headless: true, channel: 'chromium', args: ['--no-sandbox'] }
+ : { headless: true },
+ );
+ const page = await browser.newPage({
+ viewport: phone ? { width: 390, height: 844 } : { width: 1365, height: 900 },
+ hasTouch: phone,
+ isMobile: phone,
+ });
+ const requests = [],
+ errors = [],
+ checks = [];
+ const observedAt = Date.now() - 60000;
+ page.on('pageerror', (error) => errors.push(error.message));
+ await page.route('**/*', (route) => {
+ const url = new URL(route.request().url());
+ if (url.origin !== 'http://127.0.0.1:5251') return route.abort();
+ if (!url.pathname.startsWith('/api/')) return route.continue();
+ if (url.pathname === '/api/metrics-store/history') {
+ const query = Object.fromEntries(url.searchParams);
+ requests.push(query);
+ const value =
+ query.metric === 'smart_temp'
+ ? query.resourceId.includes('nas-b')
+ ? 55
+ : 42
+ : query.metric === 'smart_percentage_used'
+ ? 12
+ : query.metric === 'smart_available_spare'
+ ? 97
+ : 0;
+ const empty = query.range === '1h' && query.metric === 'smart_temp';
+ return route.fulfill({
+ json: {
+ resourceType: query.resourceType,
+ resourceId: query.resourceId,
+ metric: query.metric,
+ range: query.range,
+ start: Date.now() - 60000,
+ end: Date.now(),
+ source: 'store',
+ points: empty ? [] : [{ timestamp: observedAt, value, min: value, max: value }],
+ },
+ });
+ }
+ return route.fulfill({
+ json:
+ url.pathname === '/api/license/runtime-capabilities'
+ ? {
+ capabilities: [],
+ limits: [],
+ max_history_days: 7,
+ hosted_mode: false,
+ runtime: { build: 'community' },
+ blocked_capabilities: [],
+ }
+ : { enabled: false, data: [] },
+ });
+ });
+ const check = async (label, predicate) => {
+ try {
+ await predicate();
+ checks.push({ label, passed: true });
+ } catch (error) {
+ checks.push({ label, passed: false, error: error.message });
+ }
+ };
+ const screenshot = async (state) => {
+ const file = `${name}-${state}.png`;
+ await page.screenshot({
+ path: path.join(artifacts, file),
+ fullPage: true,
+ animations: 'disabled',
+ });
+ observations.push({
+ browser: name,
+ state,
+ screenshot: file,
+ visibleCharts: await page.locator('canvas[role="img"]').allTextContents(),
+ selectedTabs: await page
+ .getByRole('tab')
+ .evaluateAll((tabs) =>
+ tabs.map((tab) => ({
+ label: tab.textContent,
+ selected: tab.getAttribute('aria-selected'),
+ border: getComputedStyle(tab).borderBottomColor,
+ })),
+ ),
+ overflow: await page.evaluate(
+ () => document.documentElement.scrollWidth > window.innerWidth,
+ ),
+ body: await page.locator('main').innerText(),
+ });
+ };
+ const thermal = () => page.getByRole('img', { name: 'Temperature chart' });
+ const description = async () =>
+ thermal().evaluate(
+ (el) => document.getElementById(el.getAttribute('aria-describedby')).textContent,
+ );
+ await page.goto('http://127.0.0.1:5251/browser-tests/disk-history-collection.html');
+ if (phone) await page.evaluate(() => document.documentElement.classList.add('dark'));
+ await page.getByRole('tab', { name: 'History' }).click();
+ await page.waitForTimeout(300);
+ await check('Stored ATA history is accessible before any current fields', async () => {
+ assert.equal(await page.locator('canvas[role="img"]').count(), 2);
+ assert.match(await description(), /42ยฐC/);
+ });
+ await check('History selection matches its visible content', async () => {
+ assert.equal(
+ await page.getByRole('tab', { name: 'History' }).getAttribute('aria-selected'),
+ 'true',
+ );
+ });
+ await screenshot('missing-current');
+ await page.getByRole('button', { name: 'Restore current fields' }).click();
+ await thermal().waitFor();
+ await page.getByRole('combobox', { name: 'Disk history range' }).selectOption('7d');
+ await page.waitForFunction(
+ () =>
+ document
+ .querySelector('canvas[aria-label="Temperature chart"]')
+ ?.getAttribute('aria-describedby') &&
+ document
+ .getElementById(
+ document
+ .querySelector('canvas[aria-label="Temperature chart"]')
+ .getAttribute('aria-describedby'),
+ )
+ .textContent.includes('42ยฐC'),
+ );
+ if (phone) await thermal().tap({ position: { x: 80, y: 60 } });
+ else {
+ await thermal().focus();
+ await thermal().press('End');
+ }
+ await page.waitForTimeout(100);
+ await check('Stored zero and exact temperature can be inspected', async () => {
+ assert.equal(await page.locator('[data-history-chart-tooltip]').count(), 2);
+ assert.match(
+ await page
+ .locator('[data-history-chart-tooltip]')
+ .allTextContents()
+ .then((parts) => parts.join(' ')),
+ /0 sectors/,
+ );
+ });
+ await thermal().evaluate((el) => (el.dataset.proofOwner = 'original'));
+ const reads = requests.length;
+ await page.getByRole('button', { name: 'Update current fields' }).click();
+ await page.waitForTimeout(200);
+ await check('Same-ID snapshot keeps mounted chart and makes no extra reads', async () => {
+ assert.equal(await thermal().getAttribute('data-proof-owner'), 'original');
+ assert.equal(requests.length, reads);
+ assert.equal(await page.getByRole('combobox').inputValue(), '7d');
+ });
+ await screenshot('updated-current');
+ await page.getByRole('button', { name: 'Lose current fields' }).click();
+ await page.waitForTimeout(200);
+ await check(
+ 'Missing current fields do not withdraw stored readings or invent live I/O',
+ async () => {
+ assert.equal(await thermal().count(), 1);
+ assert.equal(await thermal().getAttribute('data-proof-owner'), 'original');
+ assert.match(await description(), /42ยฐC/);
+ assert.equal(requests.length, reads);
+ assert.equal(await page.getByText('Live I/O (30m)').count(), 0);
+ },
+ );
+ await screenshot('missing-again');
+ if (!parent) {
+ await page.getByRole('tab', { name: 'Overview' }).click();
+ await check(
+ 'Overview reports unavailable collection rather than the old temperature',
+ async () => {
+ assert.equal(
+ await page
+ .getByText('Temperature is temporarily unavailable: collection deadline exceeded')
+ .isVisible(),
+ true,
+ );
+ assert.equal(await page.getByText('42ยฐC', { exact: true }).count(), 0);
+ },
+ );
+ await page.getByRole('tab', { name: 'History' }).click();
+ await page.getByRole('combobox').selectOption('1h');
+ await page.getByText('No history samples in this time range.').waitFor();
+ await check(
+ 'Empty temperature is not zero and reallocated zero remains stored',
+ async () => {
+ assert.doesNotMatch(await description(), /0ยฐC/);
+ assert.equal(await page.locator('canvas[role="img"]').count(), 2);
+ },
+ );
+ await screenshot('empty-temperature');
+ await page.getByRole('combobox').selectOption('7d');
+ await page.getByRole('button', { name: 'Select NVMe B' }).click();
+ await page.getByRole('img', { name: 'Available Spare chart' }).waitFor();
+ await page.waitForTimeout(200);
+ await check(
+ 'NVMe History uses explicit B target and its own metrics without live fields',
+ async () => {
+ assert.equal(await page.locator('canvas[role="img"]').count(), 3);
+ assert.equal(
+ await page.getByRole('img', { name: 'Reallocated Sectors chart' }).count(),
+ 0,
+ );
+ assert.match(await description(), /55ยฐC/);
+ const own = requests.filter((r) => r.resourceId === 'disk:nas-b:nvme0n1');
+ assert.equal(
+ own.some((r) => r.metric === 'smart_available_spare'),
+ true,
+ );
+ assert.equal(
+ own.every((r) => r.resourceType === 'disk'),
+ true,
+ );
+ },
+ );
+ await screenshot('nvme');
+ const beforeClear = requests.length;
+ await page.getByRole('button', { name: 'Clear identity' }).click();
+ await page.waitForTimeout(200);
+ await check(
+ 'Unresolvable identity withdraws History and does not guess a target',
+ async () => {
+ assert.equal(await page.getByRole('tab', { name: 'History' }).count(), 0);
+ assert.equal(await page.locator('canvas[role="img"]').count(), 0);
+ assert.equal(requests.length, beforeClear);
+ },
+ );
+ await screenshot('no-identity');
+ }
+ await check('No page errors, off-origin data reads or horizontal overflow', async () => {
+ assert.deepEqual(errors, []);
+ assert.equal(
+ await page.evaluate(() => document.documentElement.scrollWidth > window.innerWidth),
+ false,
+ );
+ assert.equal(
+ requests.some((r) => ['diskread', 'diskwrite', 'disk'].includes(r.metric)),
+ false,
+ );
+ });
+ observations.push({ browser: name, version: browser.version(), checks, requests, errors });
+ await browser.close();
+ browser = undefined;
+ }
+ } finally {
+ if (browser) await browser.close();
+ await server.close();
+ fs.writeFileSync(
+ path.join(artifacts, 'result.json'),
+ JSON.stringify({ parent, playwright, expected, observations }, null, 2),
+ );
+ }
+ const failed = observations.flatMap((o) => o.checks ?? []).filter((c) => !c.passed);
+ console.log(
+ JSON.stringify({
+ parent,
+ checks: observations.flatMap((o) => o.checks ?? []).length,
+ failed: failed.map((c) => c.label),
+ }),
+ );
+ if (!parent) assert.equal(failed.length, 0);
+})().catch((error) => {
+ console.error(error);
+ process.exitCode = 1;
+});
diff --git a/frontend-modern/browser-tests/disk-history-collection.html b/frontend-modern/browser-tests/disk-history-collection.html
new file mode 100644
index 0000000000..1d1a1db082
--- /dev/null
+++ b/frontend-modern/browser-tests/disk-history-collection.html
@@ -0,0 +1,12 @@
+
+
+
+
+
+ Disk History collection verification
+
+
+
+
+
+
diff --git a/frontend-modern/browser-tests/disk-history-collection.tsx b/frontend-modern/browser-tests/disk-history-collection.tsx
new file mode 100644
index 0000000000..8c467a1c9d
--- /dev/null
+++ b/frontend-modern/browser-tests/disk-history-collection.tsx
@@ -0,0 +1,75 @@
+import { createSignal } from 'solid-js';
+import { render } from 'solid-js/web';
+import { DiskDetail } from '../src/components/Storage/DiskDetail';
+import type { Resource } from '../src/types/resource';
+import '../src/index.css';
+
+// Only snapshots are synthetic: detail, history, client, inspection and CSS are production code.
+const disk = (missing = false, nvme = false, hot = false): Resource => ({
+ id: nvme ? 'disk-b' : 'disk-a',
+ name: nvme ? 'NVMe B' : 'Archive A',
+ displayName: nvme ? 'NVMe B' : 'Archive A',
+ type: 'physical_disk',
+ platformId: nvme ? 'nas-b' : 'nas-a',
+ platformType: 'truenas',
+ sourceType: 'api',
+ status: 'online',
+ lastSeen: Date.now(),
+ identity: { hostname: nvme ? 'nas-b' : 'nas-a' },
+ metricsTarget: {
+ resourceType: 'disk',
+ resourceId: nvme ? 'disk:nas-b:nvme0n1' : 'disk:nas-a:sda',
+ },
+ physicalDisk: {
+ devPath: nvme ? '/dev/nvme0n1' : '/dev/sda',
+ model: nvme ? 'NVMe B' : 'Archive HDD A',
+ serial: nvme ? 'SERIAL-B' : 'SERIAL-A',
+ diskType: nvme ? 'nvme' : 'hdd',
+ temperature: missing ? undefined : hot ? 65 : 42,
+ smart: missing
+ ? undefined
+ : nvme
+ ? { percentageUsed: 12, availableSpare: 97 }
+ : { reallocatedSectors: 0 },
+ collection: {
+ temperature: {
+ state: missing ? 'unavailable' : 'available',
+ reason: missing ? 'collection deadline exceeded' : undefined,
+ },
+ io: { state: 'unsupported', reason: 'per-member counters unavailable' },
+ },
+ },
+});
+const [snapshot, setSnapshot] = createSignal(disk(true));
+const controls = [
+ ['Restore current fields', () => setSnapshot(disk())],
+ ['Update current fields', () => setSnapshot(disk(false, false, true))],
+ ['Lose current fields', () => setSnapshot(disk(true))],
+ ['Select NVMe B', () => setSnapshot(disk(true, true))],
+ [
+ 'Clear identity',
+ () =>
+ setSnapshot({
+ ...disk(true),
+ id: '',
+ metricsTarget: undefined,
+ physicalDisk: { ...disk(true).physicalDisk!, serial: '', wwn: '' },
+ }),
+ ],
+] as const;
+render(
+ () => (
+
+
Physical disk History
+
+ {controls.map(([label, action]) => (
+
+ ))}
+
+
+
+ ),
+ document.getElementById('root')!,
+);
diff --git a/frontend-modern/browser-verification.json b/frontend-modern/browser-verification.json
index 79959c2dd1..7555748f01 100644
--- a/frontend-modern/browser-verification.json
+++ b/frontend-modern/browser-verification.json
@@ -1,27 +1,16 @@
{
"version": 1,
"result": "passed",
- "base_sha": "90c436c83d616eccbe544bf3737b13c3b6251e53",
- "verified_at": "2026-10-02T23:38:43Z",
+ "base_sha": "b6a42d8a4f329a1f3d7359d8d64b17c385006ed7",
+ "verified_at": "2026-10-03T00:38:42Z",
"changed_paths": [
- "frontend-modern/src/components/Infrastructure/ResourceDetailDrawer.tsx",
- "frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx",
- "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts",
- "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerHistoryState.ts",
- "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerState.ts",
- "frontend-modern/src/features/standalone/AvailabilityFleetView.tsx"
+ "frontend-modern/src/features/storageBackups/diskDetailPresentation.ts"
],
"content_sha256": {
- "frontend-modern/src/components/Infrastructure/ResourceDetailDrawer.tsx": "63429552c461f36d4f5af06ac2ecd612b2a84630bf82e2e5d8311653519585d1",
- "frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx": "0e313b23f06c2c5fa139f2a8f1acd70299a047ab20da88016ec1aa01b5a50b4d",
- "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerDerivedState.ts": "c7afd1510b7249021eb2af2a7dcd5fe71e5183bed019ade505489a2f46eb9635",
- "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerHistoryState.ts": "fa04600d900f51395493d1d3a51d578d84d13adb64cc5a159322a61877854bd0",
- "frontend-modern/src/components/Infrastructure/useResourceDetailDrawerState.ts": "1ac0ffb9ee287a9da04fbf24e9953dbcbf4e55de906241e7702266808da2243d",
- "frontend-modern/src/features/standalone/AvailabilityFleetView.tsx": "161617fb877870ce0729dcb8312762b81d6eb4bd5cb3835023fed543b88ff02d"
+ "frontend-modern/src/features/storageBackups/diskDetailPresentation.ts": "31c71a2d696f1b74d55b855220885619bbeebc0b75a1f34354cc5b794ba49375"
},
"routes": [
- "PBS resource detail History in existing platform tables",
- "Availability fleet open resource detail"
+ "Existing Storage physical disk detail History"
],
"viewports": [
{
@@ -34,19 +23,23 @@
}
],
"states": [
- "Same-ID replacement updates PBS heading, current CPU and history target without remounting the History body or losing the 7-day selection",
- "Lost metrics target withdraws the History body and returning target restores the selected tab",
- "Selecting another canonical resource starts on its own Overview and uses only its explicit History target",
- "An open availability check changes from Up to Down with the current fleet snapshot",
- "Selecting Check B after Check A changes header, endpoint and availability together",
- "Removal of the selected check closes details; reintroduction does not reopen it",
- "Desktop light Chromium and phone dark WebKit have no page overflow or uncaught page errors"
+ "Stored ATA temperature and measured-zero reallocated sectors render when current temperature and SMART are absent",
+ "Matching current snapshot updates keep the same chart owners and add zero History reads",
+ "Current field loss keeps stored samples and selected 7-day range; no unavailable live I/O is shown",
+ "Overview reports present collection unavailability instead of the earlier stored temperature",
+ "Empty temperature history is an explicit no-samples state, not a zero, while stored reallocated zero remains visible",
+ "NVMe uses its own explicit disk target and stored temperature/endurance/spare series without current SMART fields",
+ "No resolvable identity withdraws History and issues no guessed-target request",
+ "Keyboard desktop and touch phone inspection show real stored readings",
+ "All screenshots inspected with animations disabled; selected tab, light/dark layouts and page width agree with content",
+ "Playwright 1.56.1 matches the integration lock; no page errors or off-origin data reads"
],
"interactions": [
- "Select History and 7-day range",
- "Replace PBS snapshot, withdraw and restore metrics target",
- "Select PBS B and open its History",
- "Open Check A, publish failed replacement, then select Check B",
- "Remove and restore selected check"
+ "Open History from unavailable current Overview",
+ "Restore, update and remove current fields",
+ "Change 24h/7d/1h ranges",
+ "Inspect by keyboard or phone tap",
+ "Select NVMe B",
+ "Clear disk identity"
]
}
diff --git a/frontend-modern/src/components/Storage/__tests__/DiskDetail.history-collection.test.tsx b/frontend-modern/src/components/Storage/__tests__/DiskDetail.history-collection.test.tsx
new file mode 100644
index 0000000000..bfbd263fd4
--- /dev/null
+++ b/frontend-modern/src/components/Storage/__tests__/DiskDetail.history-collection.test.tsx
@@ -0,0 +1,260 @@
+import { cleanup, fireEvent, render, screen, waitFor } from '@solidjs/testing-library';
+import { createSignal } from 'solid-js';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import {
+ ChartsAPI,
+ type MetricsHistoryParams,
+ type SingleMetricHistoryResponse,
+} from '@/api/charts';
+import { DiskDetail } from '../DiskDetail';
+import type { Resource } from '@/types/resource';
+
+vi.mock('@/api/charts', () => ({ ChartsAPI: { getMetricsHistory: vi.fn() } }));
+vi.mock('@/stores/license', () => ({
+ isRangeLocked: () => false,
+ loadRuntimeCapabilities: vi.fn(),
+ maxHistoryDays: () => 7,
+}));
+vi.mock('@/stores/alertsActivation', () => ({
+ useAlertsActivation: () => ({ getDiskTemperatureThresholds: () => null }),
+}));
+const request = vi.mocked(ChartsAPI.getMetricsHistory);
+const context = {
+ clearRect: vi.fn(),
+ setTransform: vi.fn(),
+ beginPath: vi.fn(),
+ moveTo: vi.fn(),
+ lineTo: vi.fn(),
+ stroke: vi.fn(),
+ fillText: vi.fn(),
+ closePath: vi.fn(),
+ fill: vi.fn(),
+ arc: vi.fn(),
+ save: vi.fn(),
+ restore: vi.fn(),
+ setLineDash: vi.fn(),
+ measureText: () => ({ width: 20 }),
+};
+const disk = (missing = false, nvme = false): Resource => ({
+ id: 'disk-a',
+ type: 'physical_disk',
+ name: 'Archive disk',
+ displayName: 'Archive disk',
+ platformId: 'nas-a',
+ platformType: 'truenas',
+ sourceType: 'api',
+ status: 'online',
+ lastSeen: 1000,
+ identity: { hostname: 'nas-a' },
+ metricsTarget: { resourceType: 'disk', resourceId: 'disk:nas-a:sda' },
+ physicalDisk: {
+ devPath: nvme ? '/dev/nvme0n1' : '/dev/sda',
+ model: nvme ? 'NVMe' : 'Archive HDD',
+ serial: 'SERIAL-A',
+ diskType: nvme ? 'nvme' : 'hdd',
+ temperature: missing ? undefined : 42,
+ smart: missing
+ ? undefined
+ : nvme
+ ? { percentageUsed: 12, availableSpare: 97 }
+ : { reallocatedSectors: 0 },
+ collection: {
+ temperature: {
+ state: missing ? 'unavailable' : 'available',
+ reason: missing ? 'collection deadline exceeded' : undefined,
+ },
+ io: { state: 'unsupported', reason: 'per-member counters unavailable' },
+ },
+ },
+});
+const response = (
+ params: MetricsHistoryParams,
+ value = params.metric === 'smart_temp' ? 42 : 0,
+): SingleMetricHistoryResponse => ({
+ resourceType: params.resourceType,
+ resourceId: params.resourceId,
+ metric: params.metric!,
+ range: params.range!,
+ start: 1000,
+ end: 2000,
+ source: 'store',
+ points: [{ timestamp: 1000, value, min: value, max: value }],
+});
+const description = (canvas: HTMLElement) =>
+ document.getElementById(canvas.getAttribute('aria-describedby')!)!;
+const openHistory = () => fireEvent.click(screen.getByRole('tab', { name: 'History' }));
+
+beforeEach(() => {
+ request.mockReset().mockImplementation(async (params) => response(params));
+ vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue(
+ context as unknown as CanvasRenderingContext2D,
+ );
+ vi.stubGlobal(
+ 'ResizeObserver',
+ class {
+ observe() {}
+ disconnect() {}
+ },
+ );
+});
+afterEach(() => {
+ cleanup();
+ vi.restoreAllMocks();
+ vi.unstubAllGlobals();
+});
+
+describe('Disk History independently of current collection', () => {
+ it('loads stored ATA readings when current temperature and SMART are unavailable', async () => {
+ render(() => );
+ expect(screen.getByText(/Temperature is temporarily unavailable/)).toBeInTheDocument();
+ openHistory();
+ await waitFor(() =>
+ expect(screen.getByRole('img', { name: 'Temperature chart' })).toBeInTheDocument(),
+ );
+ const thermal = screen.getByRole('img', { name: 'Temperature chart' });
+ await waitFor(() => expect(description(thermal)).toHaveTextContent('42ยฐC'));
+ expect(
+ description(screen.getByRole('img', { name: 'Reallocated Sectors chart' })),
+ ).toHaveTextContent('0 sectors');
+ expect(
+ request.mock.calls.every(
+ ([params]) => params.resourceType === 'disk' && params.resourceId === 'disk:nas-a:sda',
+ ),
+ ).toBe(true);
+ expect(request.mock.calls.map(([params]) => params.metric)).not.toContain('diskread');
+ expect(screen.queryByText('Live I/O (30m)')).not.toBeInTheDocument();
+ });
+
+ it('loads stored NVMe temperature and endurance without current SMART fields', async () => {
+ render(() => );
+ openHistory();
+ await waitFor(() => expect(screen.getAllByRole('img')).toHaveLength(3));
+ expect(screen.getByRole('img', { name: 'Life Used chart' })).toBeInTheDocument();
+ expect(screen.getByRole('img', { name: 'Available Spare chart' })).toBeInTheDocument();
+ expect(
+ screen.queryByRole('img', { name: 'Reallocated Sectors chart' }),
+ ).not.toBeInTheDocument();
+ await waitFor(() =>
+ expect(description(screen.getByRole('img', { name: 'Temperature chart' }))).toHaveTextContent(
+ '42ยฐC',
+ ),
+ );
+ });
+
+ it('preserves mounted charts and inspection without extra reads on matching snapshots', async () => {
+ const [snapshot, setSnapshot] = createSignal(disk());
+ render(() => );
+ openHistory();
+ fireEvent.change(screen.getByRole('combobox', { name: 'Disk history range' }), {
+ target: { value: '7d' },
+ });
+ const thermal = screen.getByRole('img', { name: 'Temperature chart' });
+ await waitFor(() => expect(description(thermal)).toHaveTextContent('42ยฐC'));
+ fireEvent.focus(thermal);
+ expect(document.querySelector('[data-history-chart-tooltip]')).toHaveTextContent('42ยฐC');
+ const reads = request.mock.calls.length;
+ setSnapshot({ ...disk(), physicalDisk: { ...disk().physicalDisk!, temperature: 65 } });
+ expect(screen.getByRole('img', { name: 'Temperature chart' })).toBe(thermal);
+ expect(document.querySelector('[data-history-chart-tooltip]')).toHaveTextContent('42ยฐC');
+ expect(screen.getByRole('combobox', { name: 'Disk history range' })).toHaveValue('7d');
+ expect(request).toHaveBeenCalledTimes(reads);
+ });
+
+ it('keeps stored observations mounted after current fields disappear', async () => {
+ const [snapshot, setSnapshot] = createSignal(disk());
+ render(() => );
+ openHistory();
+ const thermal = screen.getByRole('img', { name: 'Temperature chart' });
+ await waitFor(() => expect(description(thermal)).toHaveTextContent('42ยฐC'));
+ const reads = request.mock.calls.length;
+ setSnapshot(disk(true));
+ expect(screen.getByRole('img', { name: 'Temperature chart' })).toBe(thermal);
+ expect(description(thermal)).toHaveTextContent('42ยฐC');
+ expect(screen.getByRole('img', { name: 'Reallocated Sectors chart' })).toBeInTheDocument();
+ expect(request).toHaveBeenCalledTimes(reads);
+ fireEvent.click(screen.getByRole('tab', { name: 'Overview' }));
+ expect(screen.getByText(/Temperature is temporarily unavailable/)).toBeVisible();
+ expect(screen.queryByText('42ยฐC')).not.toBeInTheDocument();
+ });
+
+ it('ignores an old target completion and uses only the explicit new disk target', async () => {
+ const pending: {
+ params: MetricsHistoryParams;
+ resolve: (value: SingleMetricHistoryResponse) => void;
+ }[] = [];
+ request.mockImplementation(
+ (params) => new Promise((resolve) => pending.push({ params, resolve })),
+ );
+ const [snapshot, setSnapshot] = createSignal(disk());
+ render(() => );
+ openHistory();
+ await waitFor(() =>
+ expect(
+ pending.filter(({ params }) => params.resourceId === 'disk:nas-a:sda').length,
+ ).toBeGreaterThan(0),
+ );
+ setSnapshot({
+ ...disk(),
+ metricsTarget: { resourceType: 'disk', resourceId: 'disk:nas-b:sda' },
+ });
+ await waitFor(() =>
+ expect(
+ pending.filter(({ params }) => params.resourceId === 'disk:nas-b:sda').length,
+ ).toBeGreaterThan(0),
+ );
+ for (const item of pending.filter(({ params }) => params.resourceId === 'disk:nas-b:sda'))
+ item.resolve(response(item.params, 55));
+ await waitFor(() =>
+ expect(description(screen.getByRole('img', { name: 'Temperature chart' }))).toHaveTextContent(
+ '55ยฐC',
+ ),
+ );
+ for (const item of pending.filter(({ params }) => params.resourceId === 'disk:nas-a:sda')) {
+ expect(item.params.signal?.aborted).toBe(true);
+ item.resolve(response(item.params, 99));
+ }
+ await Promise.resolve();
+ expect(description(screen.getByRole('img', { name: 'Temperature chart' }))).toHaveTextContent(
+ '55ยฐC',
+ );
+ expect(
+ description(screen.getByRole('img', { name: 'Temperature chart' })),
+ ).not.toHaveTextContent('99ยฐC');
+ });
+
+ it('distinguishes empty stored series from measured zero with no live fallback', async () => {
+ request.mockImplementation(async (params) => ({
+ ...response(params),
+ points: params.metric === 'smart_temp' ? [] : response(params, 0).points,
+ }));
+ render(() => );
+ openHistory();
+ await waitFor(() =>
+ expect(screen.getByText('No history samples in this time range.')).toBeInTheDocument(),
+ );
+ expect(
+ description(screen.getByRole('img', { name: 'Temperature chart' })),
+ ).not.toHaveTextContent('0ยฐC');
+ await waitFor(() =>
+ expect(
+ description(screen.getByRole('img', { name: 'Reallocated Sectors chart' })),
+ ).toHaveTextContent('0 sectors'),
+ );
+ });
+
+ it('makes no history request without a resolvable disk identity', () => {
+ render(() => (
+
+ ));
+ expect(screen.queryByRole('tab', { name: 'History' })).not.toBeInTheDocument();
+ expect(request).not.toHaveBeenCalled();
+ });
+});
diff --git a/frontend-modern/src/components/Storage/__tests__/DiskDetail.test.tsx b/frontend-modern/src/components/Storage/__tests__/DiskDetail.test.tsx
index 701ada6248..cfa8a0f5d0 100644
--- a/frontend-modern/src/components/Storage/__tests__/DiskDetail.test.tsx
+++ b/frontend-modern/src/components/Storage/__tests__/DiskDetail.test.tsx
@@ -146,4 +146,17 @@ describe('DiskDetail', () => {
screen.getByText('Detailed SMART attributes are not available for this disk.'),
).toHaveAttribute('role', 'status');
});
+
+ it('offers the stored disk-family catalog when current SMART and temperature disappear', () => {
+ const disk = buildDisk();
+ delete disk.physicalDisk!.temperature;
+ delete disk.physicalDisk!.smart;
+ disk.physicalDisk!.collection = { io: { state: 'unsupported' } };
+ render(() => );
+ fireEvent.click(screen.getByRole('tab', { name: 'History' }));
+ expect(screen.getAllByTestId('history-chart').map((chart) => chart.textContent)).toEqual([
+ 'disk:agent-tower:sda:smart_temp:24h',
+ 'disk:agent-tower:sda:smart_reallocated_sectors:24h',
+ ]);
+ });
});
diff --git a/frontend-modern/src/components/Storage/__tests__/useDiskDetailModel.test.ts b/frontend-modern/src/components/Storage/__tests__/useDiskDetailModel.test.ts
index 322e3024d2..fdf27e5f76 100644
--- a/frontend-modern/src/components/Storage/__tests__/useDiskDetailModel.test.ts
+++ b/frontend-modern/src/components/Storage/__tests__/useDiskDetailModel.test.ts
@@ -108,4 +108,20 @@ describe('useDiskDetailModel', () => {
'Disk I/O is unsupported: per-member counters unavailable',
]);
});
+
+ it('keeps catalog entries stable across matching current-field snapshots', () => {
+ const initial = buildDisk();
+ const [disk, setDisk] = createSignal(initial);
+ const { result } = renderHook(() => useDiskDetailModel({ disk }));
+ const charts = result.historyCharts();
+ setDisk({ ...initial, physicalDisk: { ...initial.physicalDisk!, temperature: 65 } });
+ expect(result.historyCharts()).toBe(charts);
+ setDisk({
+ ...initial,
+ physicalDisk: { ...initial.physicalDisk!, temperature: undefined, smart: undefined },
+ });
+ expect(result.historyCharts()).toBe(charts);
+ expect(result.historyResourceId()).toBe('agent-tower:sda');
+ expect(result.attributeCards()).toEqual([]);
+ });
});
diff --git a/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.branchcov0724pm.test.ts b/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.branchcov0724pm.test.ts
index 3b5af97539..39c67706d2 100644
--- a/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.branchcov0724pm.test.ts
+++ b/frontend-modern/src/features/storageBackups/__tests__/diskDetailPresentation.branchcov0724pm.test.ts
@@ -135,8 +135,8 @@ describe('diskDetailPresentation.branchcov0724pm', () => {
});
});
- describe('getDiskDetailHistoryCharts SATA reallocated-sectors arm (L192)', () => {
- it('emits only the smart_reallocated_sectors chart for a SATA disk with the field set', () => {
+ describe('getDiskDetailHistoryCharts disk-family catalog', () => {
+ it('includes stored temperature and reallocated sectors independently of current values', () => {
const charts = getDiskDetailHistoryCharts(
makeDiskData({
type: 'hdd',
@@ -144,8 +144,8 @@ describe('diskDetailPresentation.branchcov0724pm', () => {
smartAttributes: { reallocatedSectors: 3 },
}),
);
- expect(charts.map((c) => c.metric)).toEqual(['smart_reallocated_sectors']);
- expect(charts[0]).toMatchObject({
+ expect(charts.map((c) => c.metric)).toEqual(['smart_temp', 'smart_reallocated_sectors']);
+ expect(charts[1]).toMatchObject({
metric: 'smart_reallocated_sectors',
label: 'Reallocated Sectors',
unit: 'sectors',
@@ -160,8 +160,13 @@ describe('diskDetailPresentation.branchcov0724pm', () => {
expect(charts.find((c) => c.metric === 'smart_reallocated_sectors')).toBeUndefined();
});
- it('returns an empty chart list when smartAttributes is absent', () => {
- expect(getDiskDetailHistoryCharts(makeDiskData({ type: 'hdd' }))).toEqual([]);
+ it('keeps stored charts available when current SMART fields are absent', () => {
+ expect(
+ getDiskDetailHistoryCharts(makeDiskData({ type: 'hdd' })).map((chart) => chart.metric),
+ ).toEqual(['smart_temp', 'smart_reallocated_sectors']);
+ expect(
+ getDiskDetailHistoryCharts(makeDiskData({ type: 'NVMe' })).map((chart) => chart.metric),
+ ).toEqual(['smart_temp', 'smart_percentage_used', 'smart_available_spare']);
});
});
});
diff --git a/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts b/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts
index 87af184ce3..12cc6a8f12 100644
--- a/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts
+++ b/frontend-modern/src/features/storageBackups/diskDetailPresentation.ts
@@ -173,48 +173,46 @@ export function getDiskDetailAttributeCards(
return cards;
}
+const DISK_TEMPERATURE_HISTORY_CHART: DiskDetailHistoryChartConfig = {
+ metric: 'smart_temp',
+ label: 'Temperature',
+ unit: 'C',
+ color: '#ef4444',
+};
+
+const ATA_DISK_HISTORY_CHARTS: readonly DiskDetailHistoryChartConfig[] = [
+ DISK_TEMPERATURE_HISTORY_CHART,
+ {
+ metric: 'smart_reallocated_sectors',
+ label: 'Reallocated Sectors',
+ unit: 'sectors',
+ color: '#f59e0b',
+ },
+];
+
+const NVME_DISK_HISTORY_CHARTS: readonly DiskDetailHistoryChartConfig[] = [
+ DISK_TEMPERATURE_HISTORY_CHART,
+ {
+ metric: 'smart_percentage_used',
+ label: 'Life Used',
+ unit: '%',
+ color: '#f59e0b',
+ },
+ {
+ metric: 'smart_available_spare',
+ label: 'Available Spare',
+ unit: '%',
+ color: '#10b981',
+ },
+];
+
export function getDiskDetailHistoryCharts(
disk: PhysicalDiskPresentationData,
-): DiskDetailHistoryChartConfig[] {
- const attrs = disk.smartAttributes;
- const charts: DiskDetailHistoryChartConfig[] = [];
- const isNvme = disk.type?.toLowerCase() === 'nvme';
-
- if (disk.temperature > 0) {
- charts.push({
- metric: 'smart_temp',
- label: 'Temperature',
- unit: 'C',
- color: '#ef4444',
- });
- }
-
- if (!isNvme && attrs?.reallocatedSectors != null) {
- charts.push({
- metric: 'smart_reallocated_sectors',
- label: 'Reallocated Sectors',
- unit: 'sectors',
- color: '#f59e0b',
- });
- }
-
- if (isNvme && attrs?.percentageUsed != null) {
- charts.push({
- metric: 'smart_percentage_used',
- label: 'Life Used',
- unit: '%',
- color: '#f59e0b',
- });
- }
-
- if (isNvme && attrs?.availableSpare != null) {
- charts.push({
- metric: 'smart_available_spare',
- label: 'Available Spare',
- unit: '%',
- color: '#10b981',
- });
- }
-
- return charts;
+): readonly DiskDetailHistoryChartConfig[] {
+ // Missing current collection does not erase stored observations. Ask for the
+ // existing disk-family catalog and let the store return samples or an honest
+ // empty series; never synthesize a value from the current snapshot.
+ // Stable entries also keep Solid's reference-keyed For from remounting charts
+ // and restarting History reads on every ordinary disk snapshot replacement.
+ return disk.type?.toLowerCase() === 'nvme' ? NVME_DISK_HISTORY_CHARTS : ATA_DISK_HISTORY_CHARTS;
}
From 360721cc910b91b83bf39aa46cd130bf64266c1a Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 01:44:42 +0100
Subject: [PATCH 12/86] Bind demo cohost observation to the real Relay health
route
Current operational source exposes healthz and takes the shared deployment lock. Pin that route and make the disposable service return 404 for unknown paths so native acceptance cannot manufacture a healthy cohost observation.
Change-source: pulse-maintainer
---
.github/scripts/demo-runtime-transaction.py | 2 +-
.github/scripts/tests/demo_runtime_native.py | 9 ++++++---
.github/scripts/tests/test_demo_runtime_transaction.py | 3 +++
.../v6/internal/subsystems/deployment-installability.md | 3 +++
4 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/.github/scripts/demo-runtime-transaction.py b/.github/scripts/demo-runtime-transaction.py
index 436e74b46d..6f12dfebf2 100644
--- a/.github/scripts/demo-runtime-transaction.py
+++ b/.github/scripts/demo-runtime-transaction.py
@@ -27,7 +27,7 @@ PATHS = {"binary": Path("/opt/pulse/bin/pulse"),
"unit": Path("/etc/systemd/system/pulse.service"),
"dropins": Path("/etc/systemd/system/pulse.service.d"),
"data": Path("/etc/pulse")}
-RELAY_HEALTH = "https://relay.pulserelay.pro/health"
+RELAY_HEALTH = "https://relay.pulserelay.pro/healthz"
COUNT_KEYS = ("nodes", "vms_per_node", "lxcs_per_node", "docker_hosts",
"docker_containers", "generic_hosts", "k8s_clusters", "k8s_nodes",
"k8s_pods", "k8s_deployments")
diff --git a/.github/scripts/tests/demo_runtime_native.py b/.github/scripts/tests/demo_runtime_native.py
index b004b1cf05..0b7fc6a73b 100644
--- a/.github/scripts/tests/demo_runtime_native.py
+++ b/.github/scripts/tests/demo_runtime_native.py
@@ -38,11 +38,14 @@ def fail():
if %r: threading.Thread(target=fail,daemon=True).start()
class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
+ expected = {'/healthz'} if %r == 17656 else {'/api/health','/api/version'}
+ if self.path not in expected:
+ self.send_response(404); self.end_headers(); return
body=json.dumps({'version':VERSION} if self.path=='/api/version' else {'status':'healthy'}).encode()
self.send_response(200); self.end_headers(); self.wfile.write(body)
def log_message(self,*args): pass
http.server.HTTPServer(('127.0.0.1',%r),Handler).serve_forever()
-''' % (version, fail_after, fail_after, port)).encode()
+''' % (version, fail_after, fail_after, port, port)).encode()
def unit(executable, user="pulse"):
@@ -128,9 +131,9 @@ def main():
original = (SCRIPTS / "demo-runtime-transaction.py").read_bytes()
# Only the cohost endpoint is rebound. Transaction, paths, locking,
# systemd, HTTP/TLS, estate, clocks and watches remain real and unchanged.
- endpoint = b'https://relay.pulserelay.pro/health'
+ endpoint = b'https://relay.pulserelay.pro/healthz'
assert original.count(endpoint) == 1
- source = original.replace(endpoint, b'https://127.0.0.1:18444/health')
+ source = original.replace(endpoint, b'https://127.0.0.1:18444/healthz')
result["original_engine_sha256"] = hashlib.sha256(original).hexdigest()
result["fixture_engine_sha256"] = hashlib.sha256(source).hexdigest()
result["bootstrap_sha256"] = hashlib.sha256(dispatcher.BOOTSTRAP.encode()).hexdigest()
diff --git a/.github/scripts/tests/test_demo_runtime_transaction.py b/.github/scripts/tests/test_demo_runtime_transaction.py
index c2fab87420..86bc90e60c 100644
--- a/.github/scripts/tests/test_demo_runtime_transaction.py
+++ b/.github/scripts/tests/test_demo_runtime_transaction.py
@@ -347,6 +347,9 @@ class TransactionTest(unittest.TestCase):
class InputAndCommandTest(unittest.TestCase):
+ def test_cohost_check_uses_the_existing_relay_health_route(self):
+ self.assertEqual(engine.RELAY_HEALTH, "https://relay.pulserelay.pro/healthz")
+
def test_valid_current_envelope(self):
engine.validate(request())
engine.validate(request("update"))
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index f53a0ffd5d..c24547706f 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -24,6 +24,9 @@ The mutating paths now share the Relay host's existing
Every successful path, including a healthy no-op, needs a complete 300-second
local/public/Relay health, version, service PID/restart and new-journal-crash
window. Readiness is separate. Missing observations fail closed.
+The fixed Relay observation is its existing `/healthz` endpoint, not a guessed
+new route; disposable acceptance also rejects an unknown route rather than
+responding healthy to every URL.
Before installation or runtime-profile mutation, stop only Pulse and durably
capture its executable, unit/drop-ins and entire persistent data estate. Refuse
From 7ec246381fdf0a58c8b695a0982e34e04cb6545f Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 01:45:34 +0100
Subject: [PATCH 13/86] Check retained demo profile inside its owning
transaction
The broader release-policy check still expected the removed shell configuration step. Assert its real transaction entry and runtime profile/entitlement preservation while keeping activation, exact stable tag, network and public browser checks unchanged.
Contract-Neutral: Align verification with the already implemented demo transaction; no runtime or authority change.
Change-source: pulse-maintainer
---
.../release_promotion_policy_test.py | 28 +++++++++----------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.py
index a933737fd4..e27d53e1b9 100644
--- a/scripts/release_control/release_promotion_policy_test.py
+++ b/scripts/release_control/release_promotion_policy_test.py
@@ -2589,7 +2589,9 @@ class ReleasePromotionPolicyTest(unittest.TestCase):
self.assertIn("DEMO_EXPECTED_HOSTNAME", demo)
self.assertIn("Verify target host identity", demo)
self.assertIn("Demo environment points at host $REMOTE_HOSTNAME but expected $DEMO_EXPECTED_HOSTNAME.", demo)
- self.assertIn("Restore demo runtime configuration", demo)
+ self.assertIn("Apply guarded demo transaction", demo)
+ self.assertIn("dispatch-demo-runtime.py", demo)
+ self.assertIn("Retain guarded demo transaction evidence", demo)
self.assertIn("Resolve target-compatible demo runtime profile", demo)
self.assertIn("mockEagerHistoryPVEGuestLimit", demo_profile)
self.assertIn("UpdateMetricCohort", demo_profile)
@@ -2608,19 +2610,17 @@ class ReleasePromotionPolicyTest(unittest.TestCase):
self.assertIn("MOCK_SEED_DURATION=2h", demo_profile)
self.assertIn("MOCK_SAMPLE_INTERVAL=5m", demo_profile)
self.assertIn("MOCK_UPDATE_INTERVAL=15s", demo_profile)
- self.assertIn("resolve_config_dir", demo)
- self.assertIn("set_env_value DEMO_MODE true", demo)
- self.assertIn("set_env_value PULSE_MOCK_MODE true", demo)
- self.assertIn('set_env_value PULSE_MOCK_NODES "$MOCK_NODES"', demo)
- self.assertIn("set_env_value PULSE_MOCK_SEED_METRICS_STORE false", demo)
- self.assertIn('set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"', demo)
- self.assertIn('set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL"', demo)
- self.assertIn('set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"', demo)
- self.assertIn("ensure_demo_fixture_entitlement", demo)
- self.assertIn('"demo_fixtures"', demo)
- self.assertIn("del(.integrity)", demo)
- self.assertIn("Demo fixture entitlement ensured in governed demo billing state.", demo)
- self.assertIn("Demo service restarted with governed demo runtime configuration.", demo)
+ # Profile application moved into the owned transaction; keep its
+ # runtime settings and entitlement, not the superseded shell snippets.
+ transaction = read(".github/scripts/demo-runtime-transaction.py")
+ for required in ('"DEMO_MODE": "true"', '"PULSE_MOCK_MODE": "true"',
+ '"PULSE_MOCK_SEED_METRICS_STORE": "false"',
+ '"seed_duration": "trends_seed_duration"', '"sample_interval": "trends_sample_interval"',
+ '"PULSE_MOCK_" + key.upper()', '["demo_fixtures"]', 'billing.pop("integrity", None)',
+ 'self.host.profile(self.request, not healthy_baseline)', 'self.host.start()'):
+ self.assertIn(required, transaction)
+ self.assertIn("MOCK_NODES: ${{ needs.resolve.outputs.mock_nodes }}", demo)
+ self.assertIn("MOCK_UPDATE_INTERVAL: ${{ needs.resolve.outputs.mock_update_interval }}", demo)
self.assertIn("/api/license/runtime-capabilities", demo)
self.assertIn("Mock mode enabled", demo)
self.assertIn("Demo server mock mode did not enable after entitlement sync", demo)
From ee451f8c8c23dc64319dfee281b22d8d679f4d3e Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 02:23:58 +0100
Subject: [PATCH 14/86] Restore canonical ordering for demo registry entries
Candidate preflight rejected three path lists that used ASCII or insertion order instead of the registry's case-insensitive order. Sort only those lists and exercise the committed registry in the audit suite so fixture-only passes cannot miss the same mistake.
Contract-Neutral: Canonical registry path-list ordering and regression coverage only; ownership, verification requirements and runtime behaviour are unchanged.
Change-source: pulse-maintainer
---
.../v6/internal/subsystems/registry.json | 26 +++++++++----------
.../release_control/registry_audit_test.py | 26 +++++++++++++++++++
2 files changed, 39 insertions(+), 13 deletions(-)
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index f3ec7681fb..b58adc709b 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -4548,9 +4548,6 @@
".github/workflows/retry-release-convergence.yml",
".github/workflows/update-demo-server.yml",
".github/workflows/validate-release-assets.yml",
- "Dockerfile",
- "Makefile",
- "VERSION",
"cmd/pulse-control-plane/main.go",
"cmd/pulse-control-plane/provider_msp.go",
"cmd/pulse-control-plane/provider_msp_backup.go",
@@ -4560,12 +4557,13 @@
"cmd/pulse-control-plane/provider_msp_recover.go",
"cmd/pulse-control-plane/provider_msp_status.go",
"docker-compose.yml",
+ "Dockerfile",
"docs/MSP.md",
- "docs/RELEASE_NOTES.md",
- "docs/UPGRADE_v6.md",
"docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md",
"docs/release-control/v6/internal/RC_TO_GA_REHEARSAL_TEMPLATE.md",
"docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md",
+ "docs/RELEASE_NOTES.md",
+ "docs/UPGRADE_v6.md",
"frontend-modern/package-lock.json",
"frontend-modern/package.json",
"frontend-modern/public/docs/MSP.md",
@@ -4585,6 +4583,7 @@
"internal/cloudcp/provider_msp_backup.go",
"internal/cloudcp/provider_msp_recovery.go",
"internal/cloudcp/tenant_runtime_rollout.go",
+ "Makefile",
"package-lock.json",
"package.json",
"pulse-enterprise:.github/workflows/build-pro-release.yml",
@@ -4659,16 +4658,17 @@
"scripts/uninstall-sensor-proxy.sh",
"scripts/validate-release.sh",
"scripts/verify-github-release-integrity.sh",
+ "tests/integration/playwright.config.ts",
"tests/integration/QUICK_START.md",
"tests/integration/README.md",
- "tests/integration/playwright.config.ts",
"tests/integration/scripts/bootstrap-hosted-mobile-onboarding.mjs",
"tests/integration/scripts/hosted-mobile-token-runtime.mjs",
"tests/integration/scripts/hosted-tenant-runtime.mjs",
"tests/integration/scripts/managed-dev-runtime.mjs",
"tests/integration/scripts/relay-mobile-token-helper.go",
"tests/integration/tests/helpers.ts",
- "tests/integration/tests/runtime-defaults.ts"
+ "tests/integration/tests/runtime-defaults.ts",
+ "VERSION"
],
"release_cycle_artifact_globs": [
"docs/RELEASE_NOTES.md",
@@ -4775,12 +4775,11 @@
".github/workflows/retry-release-convergence.yml",
".github/workflows/update-demo-server.yml",
".github/workflows/validate-release-assets.yml",
- "VERSION",
- "docs/RELEASE_NOTES.md",
- "docs/UPGRADE_v6.md",
"docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md",
"docs/release-control/v6/internal/RC_TO_GA_REHEARSAL_TEMPLATE.md",
"docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md",
+ "docs/RELEASE_NOTES.md",
+ "docs/UPGRADE_v6.md",
"frontend-modern/public/docs/UPGRADE_v6.md",
"frontend-modern/src/components/UpdateBanner.tsx",
"frontend-modern/src/components/WhatsNewCard.tsx",
@@ -4808,7 +4807,8 @@
"scripts/trigger-release-dry-run.sh",
"scripts/trigger-release.sh",
"scripts/trigger-stable-patch.sh",
- "scripts/verify-github-release-integrity.sh"
+ "scripts/verify-github-release-integrity.sh",
+ "VERSION"
],
"allow_same_subsystem_tests": false,
"test_prefixes": [],
@@ -5185,8 +5185,8 @@
"allow_same_subsystem_tests": false,
"test_prefixes": [],
"exact_files": [
- "scripts/installtests/demo_transaction_test.go",
- ".github/scripts/tests/test_demo_runtime_transaction.py"
+ ".github/scripts/tests/test_demo_runtime_transaction.py",
+ "scripts/installtests/demo_transaction_test.go"
]
}
],
diff --git a/scripts/release_control/registry_audit_test.py b/scripts/release_control/registry_audit_test.py
index 1118e6c6fb..803a075b08 100644
--- a/scripts/release_control/registry_audit_test.py
+++ b/scripts/release_control/registry_audit_test.py
@@ -47,6 +47,32 @@ class RegistryAuditTest(unittest.TestCase):
args = parse_args(["--check", "--repo-scope", "pulse"])
self.assertEqual(args.repo_scope, ["pulse"])
+ def test_live_registry_path_lists_are_casefold_sorted(self) -> None:
+ # Fixture-only audit tests cannot catch ordering mistakes in the
+ # committed registry. Check its real path lists using the same
+ # case-insensitive contract as candidate preflight, not ASCII order.
+ registry = registry_audit.load_registry_payload()
+ for subsystem in registry["subsystems"]:
+ lists = {
+ field: subsystem[field]
+ for field in ("owned_prefixes", "owned_files")
+ }
+ verification = subsystem["verification"]
+ lists.update({
+ f"verification.{field}": verification[field]
+ for field in ("test_prefixes", "exact_files")
+ })
+ for policy in verification["path_policies"]:
+ lists.update({
+ f"{policy['id']}.{field}": policy[field]
+ for field in (
+ "match_prefixes", "match_files", "test_prefixes", "exact_files"
+ )
+ })
+ for field, paths in lists.items():
+ with self.subTest(subsystem=subsystem["id"], field=field):
+ self.assertEqual(paths, sorted(paths, key=str.casefold))
+
def test_trust_gate_regression_proofs_are_registered_to_exact_policies(self) -> None:
registry = registry_audit.load_registry_payload()
rules = {rule["id"]: rule for rule in registry["subsystems"]}
From 0dcd592ef2f8bc3a7fec0b46cc581e1d878b361f Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 02:28:48 +0100
Subject: [PATCH 15/86] Keep notification cooldowns ordered by alert occurrence
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
A late provider completion must retain its own firing receipt without replacing a newer occurrenceโs repeat-delivery cooldown. Preserve the latest completion and highest delivered severity within one occurrence, keeping genuine new occurrences and severity increases eligible. Exercise reverse HTTP completion, duplicate admission and durable recovery receipts alongside legacy ordering controls.
Change-source: pulse-maintainer
---
.../v6/internal/subsystems/notifications.md | 18 ++
.../cooldown_receipt_order_test.go | 163 ++++++++++++++++++
internal/notifications/notifications.go | 25 ++-
3 files changed, 201 insertions(+), 5 deletions(-)
create mode 100644 internal/notifications/cooldown_receipt_order_test.go
diff --git a/docs/release-control/v6/internal/subsystems/notifications.md b/docs/release-control/v6/internal/subsystems/notifications.md
index 068c67b93a..1ddb0616ee 100644
--- a/docs/release-control/v6/internal/subsystems/notifications.md
+++ b/docs/release-control/v6/internal/subsystems/notifications.md
@@ -15,6 +15,24 @@
## Purpose
+### Completion order cannot replace a newer occurrence's cooldown
+
+Firing completions still record an independent delivery receipt for their own
+occurrence and destination. The reusable-ID cooldown, however, belongs to the
+newest known occurrence: a late older completion or unknown legacy start cannot
+evict it. Within the same occurrence, successful completion times and delivered
+severity only advance. A genuinely newer occurrence begins its own severity
+history. No queue schema, retry budget, recovery selection or schedule changes.
+
+`TestNotificationCooldownReceiptOrdering` verifies older, unknown and equal-start
+completion ordering, including legacy zero starts. The connected
+`TestLateFiringHTTPReceiptPreservesCurrentCooldown` holds an older HTTP response
+while the newer firing completes through the persistent sender. It verifies that
+the ordinary repeat adds no queue row, a current severity increase still reaches
+the receiver, and both occurrence/destination receipts survive queue reopen.
+This is loopback source acceptance, not native delivery, restart cooldown
+persistence, a natural schedule cycle or resolution of the original flood reports.
+
### Queue recovery handler ownership after reload
Router monitor replacement must refresh the existing queue/DLQ handler as well
diff --git a/internal/notifications/cooldown_receipt_order_test.go b/internal/notifications/cooldown_receipt_order_test.go
new file mode 100644
index 0000000000..6650194479
--- /dev/null
+++ b/internal/notifications/cooldown_receipt_order_test.go
@@ -0,0 +1,163 @@
+package notifications
+
+import (
+ "io"
+ "net/http"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/stretchr/testify/require"
+)
+
+func TestNotificationCooldownReceiptOrdering(t *testing.T) {
+ start := time.Date(2026, 10, 3, 1, 0, 0, 123, time.UTC)
+ for _, tc := range []struct {
+ name string
+ priorStart time.Time
+ lateStart time.Time
+ lateSent time.Time
+ }{
+ {"older_occurrence", start, start.Add(-time.Nanosecond), start.Add(3 * time.Minute)},
+ {"unknown_legacy_occurrence", start, time.Time{}, start.Add(3 * time.Minute)},
+ {"earlier_same_occurrence_completion", start, start, start.Add(time.Minute)},
+ {"earlier_legacy_completion", time.Time{}, time.Time{}, start.Add(time.Minute)},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ m := NewNotificationManagerWithDeferredQueue("", t.TempDir())
+ t.Cleanup(m.Stop)
+ m.SetCooldown(0) // No ordinary repeats for the delivered occurrence.
+ m.SetGroupingConfig(true, 3600, false, false)
+ current := &alerts.Alert{ID: "cpu-order", Level: alerts.AlertLevelCritical, StartTime: tc.priorStart}
+ m.markAlertsNotified([]*alerts.Alert{current}, start.Add(2*time.Minute))
+ m.mu.RLock()
+ before := m.lastNotified[current.ID]
+ m.mu.RUnlock()
+ late := current.Clone()
+ late.StartTime = tc.lateStart
+ late.Level = alerts.AlertLevelWarning
+ m.markAlertsNotified([]*alerts.Alert{nil, late}, tc.lateSent)
+ m.mu.RLock()
+ after := m.lastNotified[current.ID]
+ m.mu.RUnlock()
+ require.Equal(t, before, after, "late receipt replaced the current cooldown")
+ m.SendAlert(current)
+ m.mu.RLock()
+ pending := len(m.pendingAlerts)
+ m.mu.RUnlock()
+ require.Zero(t, pending, "delivered current occurrence was admitted again")
+ })
+ }
+}
+
+// Use the actual persistent sender and loopback receiver. Two firing callbacks
+// for a reusable ID can complete in reverse occurrence order; no fabricated
+// successful-delivery marker or modified scheduling clock is needed here.
+func TestLateFiringHTTPReceiptPreservesCurrentCooldown(t *testing.T) {
+ oldArrived := make(chan struct{}, 1)
+ releaseOld := make(chan struct{})
+ var releaseOnce sync.Once
+ unblock := func() { releaseOnce.Do(func() { close(releaseOld) }) }
+ var mu sync.Mutex
+ var bodies []string
+ server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ body, err := io.ReadAll(r.Body)
+ if err != nil {
+ t.Errorf("read firing: %v", err)
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ mu.Lock()
+ bodies = append(bodies, string(body))
+ mu.Unlock()
+ if strings.Contains(string(body), "older firing") {
+ select {
+ case oldArrived <- struct{}{}:
+ default:
+ }
+ select {
+ case <-releaseOld:
+ case <-r.Context().Done():
+ return
+ }
+ }
+ w.WriteHeader(http.StatusOK)
+ }))
+ t.Cleanup(server.Close)
+ dir := t.TempDir()
+ webhook := WebhookConfig{ID: "ops", URL: server.URL + "/hook", Enabled: true, Service: "generic"}
+ m := NewNotificationManagerWithDataDir("", dir)
+ t.Cleanup(m.Stop)
+ t.Cleanup(unblock) // Release an in-flight provider before waiting for shutdown.
+ m.webhookClient = server.Client()
+ require.NoError(t, m.UpdateAllowedPrivateCIDRs("127.0.0.1/32"))
+ m.SetGroupingConfig(false, 0, false, false)
+ m.SetCooldown(60)
+ m.AddWebhook(webhook)
+ queue := m.GetQueue()
+ old := &alerts.Alert{ID: "recurring-cpu", ResourceName: "cpu-node", Type: "cpu",
+ Level: alerts.AlertLevelCritical, Message: "older firing", StartTime: time.Now().UTC().Add(-time.Minute)}
+ m.SendAlert(old)
+ select {
+ case <-oldArrived:
+ case <-time.After(10 * time.Second):
+ t.Fatal("old firing did not reach the HTTP receiver")
+ }
+ current := old.Clone()
+ current.StartTime = time.Now().UTC()
+ current.Level = alerts.AlertLevelWarning
+ current.Message = "current warning firing"
+ m.SendAlert(current)
+ wait := func(condition func() bool) {
+ t.Helper()
+ deadline := time.Now().Add(10 * time.Second)
+ for !condition() {
+ if time.Now().After(deadline) {
+ t.Fatal("notification completion timed out")
+ }
+ time.Sleep(time.Millisecond)
+ }
+ }
+ wait(func() bool {
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+ return m.lastNotified[current.ID].alertStart.Equal(current.StartTime)
+ })
+ unblock()
+ wait(func() bool {
+ stats, err := queue.GetQueueStats()
+ require.NoError(t, err)
+ return stats["sent"] == 2
+ })
+ // Sending synchronously admits queue work unless the cooldown rejects it.
+ // Assert admission itself, rather than hoping an excess send stays silent.
+ m.SendAlert(current)
+ stats, err := queue.GetQueueStats()
+ require.NoError(t, err)
+ require.Equal(t, map[string]int{"sent": 2}, stats, "late old completion admitted a duplicate current firing")
+ // Old critical severity must not leak into the newer warning occurrence.
+ current.Level = alerts.AlertLevelCritical
+ current.Message = "current critical firing"
+ m.SendAlert(current)
+ wait(func() bool {
+ stats, err := queue.GetQueueStats()
+ require.NoError(t, err)
+ return stats["sent"] == 3
+ })
+ mu.Lock()
+ count := len(bodies)
+ mu.Unlock()
+ require.Equal(t, 3, count)
+ m.Stop()
+ // The old completion still owns a durable recovery receipt, independently
+ // of which occurrence owns the transient repeat-delivery cooldown.
+ reopened := NewNotificationManagerWithDeferredQueue("", dir)
+ t.Cleanup(reopened.Stop)
+ for _, alert := range []*alerts.Alert{old, current} {
+ jobs := buildNotificationDeliveryJobs(EmailConfig{}, []WebhookConfig{webhook}, AppriseConfig{}, []*alerts.Alert{alert}, eventResolved, time.Now())
+ require.Len(t, reopened.filterResolvedJobsByDeliveryReceipt(jobs), 1, "firing receipt lost across reopen")
+ }
+ t.Log("Two reverse-order HTTP 200 completions retain the current cooldown; repeat adds no row, severity increase delivers, both occurrence receipts survive queue reopen.")
+}
diff --git a/internal/notifications/notifications.go b/internal/notifications/notifications.go
index 7b93dbf590..d508f53bbb 100644
--- a/internal/notifications/notifications.go
+++ b/internal/notifications/notifications.go
@@ -1291,14 +1291,29 @@ func (n *NotificationManager) markAlertsNotified(alertsToSend []*alerts.Alert, s
continue
}
level := alert.Level
- // Destinations and retries can complete out of order. A late warning
- // receipt must not re-enable critical delivery for this occurrence.
- if previous, ok := n.lastNotified[alert.ID]; ok && previous.alertStart.Equal(alert.StartTime) && notificationSeverityRank(previous.level) > notificationSeverityRank(level) {
- level = previous.level
+ lastSent := sentAt
+ if previous, ok := n.lastNotified[alert.ID]; ok {
+ // Destinations and retries can complete out of order. An older
+ // occurrence still owns its delivery receipt, but must not replace
+ // the current occurrence's repeat-delivery cooldown. Unknown legacy
+ // starts likewise cannot evict a known occurrence.
+ if previous.alertStart.After(alert.StartTime) {
+ continue
+ }
+ if previous.alertStart.Equal(alert.StartTime) {
+ // A late warning cannot re-enable critical delivery, and an
+ // earlier completion cannot shorten the same occurrence's hold.
+ if notificationSeverityRank(previous.level) > notificationSeverityRank(level) {
+ level = previous.level
+ }
+ if previous.lastSent.After(lastSent) {
+ lastSent = previous.lastSent
+ }
+ }
}
n.lastNotified[alert.ID] = notificationRecord{
level: level,
- lastSent: sentAt,
+ lastSent: lastSent,
alertStart: alert.StartTime,
}
}
From 1a1b59072b6ceb02e3faa42167c26f910ca7a652 Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 02:32:05 +0100
Subject: [PATCH 16/86] Point monitoring help at the current platform
navigation
Replace retired task-page advice with registered platform routes, TrueNAS tabs and Proxmox backups. Preserve saved connections when troubleshooting missing navigation and distinguish proxy reload failures from missing inventory.
Change-source: pulse-maintainer
---
docs/PBS.md | 5 +-
docs/TROUBLESHOOTING.md | 41 ++++++-
docs/TRUENAS.md | 43 ++++---
.../browser-tests/navigation-help.cjs | 110 ++++++++++++++++++
frontend-modern/public/docs/PBS.md | 5 +-
.../public/docs/TROUBLESHOOTING.md | 41 ++++++-
frontend-modern/public/docs/TRUENAS.md | 43 ++++---
scripts/tests/test_navigation_help_docs.py | 100 ++++++++++++++++
8 files changed, 338 insertions(+), 50 deletions(-)
create mode 100644 frontend-modern/browser-tests/navigation-help.cjs
create mode 100644 scripts/tests/test_navigation_help_docs.py
diff --git a/docs/PBS.md b/docs/PBS.md
index a377d2c602..2c8affcab3 100644
--- a/docs/PBS.md
+++ b/docs/PBS.md
@@ -152,7 +152,7 @@ It does **not** allow:
If you have multiple PBS servers, add each one separately in Settings. Pulse will:
- Monitor each server independently
-- Show backups from all servers in the unified Recovery view
+- Show backups from all servers under **Proxmox โ Backups** (`/proxmox/backups`)
- Deduplicate if the same backup appears via both PVE passthrough and direct PBS
---
@@ -301,7 +301,8 @@ or change retention to hide a display problem.
## Data Source Indicator
-In the Recovery view, PBS backups show a data source indicator:
+Under **Proxmox โ Backups** (`/proxmox/backups`), PBS backups show a data source
+indicator. There is no current top-level Recovery page:
- **"PBS"** badge alone = Direct PBS connection (full data)
- **"PBS via PVE"** = Passthrough via PVE storage (limited data)
diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md
index 9b11c341cd..f7f21b286a 100644
--- a/docs/TROUBLESHOOTING.md
+++ b/docs/TROUBLESHOOTING.md
@@ -413,7 +413,7 @@ consequential, manually redacted error.
### TrueNAS
#### "TrueNAS service unavailable"
-- Ensure TrueNAS was added in **Settings โ TrueNAS** with a valid HTTPS URL,
+- Ensure TrueNAS was added in **Settings โ Infrastructure โ Platform connections** with a valid HTTPS URL,
API key, and the username that owns the key.
- Check that the TrueNAS system is reachable from the Pulse server (default
HTTPS port).
@@ -424,7 +424,10 @@ consequential, manually redacted error.
#### TrueNAS pools/datasets not appearing
- TrueNAS data appears in the unified resource model and may take one configured
polling cycle (60 seconds by default) to appear.
-- Check **Infrastructure** (TrueNAS host), **Storage** (pools/datasets), and **Recovery** (snapshots/replication).
+- Open **TrueNAS โ Overview** for the appliance, **TrueNAS โ Storage**
+ (`/truenas/storage`) for pools, datasets and disks, and **TrueNAS โ Protection**
+ (`/truenas/protection`) for snapshots and replication. These are tabs within
+ TrueNAS, not separate top-level Storage or Recovery pages.
- For data that stops refreshing, use the [TrueNAS polling checks](TRUENAS.md#stale-truenas-data)
before testing or restarting. A stale badge is not proof of an invalid key,
and a successful connection test is not proof that collection has recovered.
@@ -432,8 +435,38 @@ consequential, manually redacted error.
### Navigation (v6)
#### Old bookmarks don't work
-- Legacy URLs (`/proxmox`, `/docker`, `/kubernetes`, `/hosts`, `/services`) are not supported in v6.
-- Update bookmarks to canonical routes. See [Migration Guide](MIGRATION_UNIFIED_NAV.md).
+
+Current Pulse uses platform navigation. `/proxmox`, `/docker` and `/kubernetes`
+are supported; do not replace them with the retired task-based routes.
+Open Pulse at its base URL and use the menu to find the relevant page:
+
+| Menu | Entry route |
+| --- | --- |
+| Proxmox | `/proxmox/overview` |
+| Docker | `/docker/overview` |
+| Kubernetes | `/kubernetes/overview` |
+| TrueNAS | `/truenas/overview` |
+| vSphere | `/vmware/overview` |
+| Machines | `/standalone/machines` |
+
+The short-lived top-level `/workloads`, `/storage` and `/recovery` layout is
+retired. `/infrastructure` now opens the default workspace, not the former
+unified host page. For old `/hosts` or `/services` bookmarks, select the current
+platform or Machines page instead of assuming an automatic redirect. PBS
+backups are under **Proxmox โ Backups**, and TrueNAS snapshots and replication
+are under **TrueNAS โ Protection**.
+
+Platform-connected hosts can appear under their platform rather than Machines.
+If a menu or expected resource is missing, check its saved connection and last
+successful collection in **Settings โ Infrastructure**; a missing page alone
+does not prove a host was deleted. Do not delete connections or re-enrol agents
+just to repair a bookmark.
+
+If menu navigation works but reloading the same URL returns a proxy 404, check
+the proxy's route handling using [Reverse Proxy Configuration](REVERSE_PROXY.md).
+For the current layout, see [FAQ](FAQ.md#how-is-navigation-organised-in-pulse-v6).
+The [unified-navigation migration](MIGRATION_UNIFIED_NAV.md) is historical,
+not a guide to the current menu.
### Relay / Mobile
diff --git a/docs/TRUENAS.md b/docs/TRUENAS.md
index ddd5ea8468..54dac2b6e6 100644
--- a/docs/TRUENAS.md
+++ b/docs/TRUENAS.md
@@ -4,8 +4,9 @@ Pulse v6 includes first-class monitoring for **TrueNAS SCALE** and **TrueNAS COR
## Quick Start
-1. Go to **Settings โ TrueNAS**.
-2. Click **Add Connection**.
+1. Go to **Settings โ Infrastructure**.
+2. Click **Add infrastructure** and choose **TrueNAS**. Existing connections
+ are listed under **Platform connections**.
3. Enter the TrueNAS URL (e.g., `https://truenas.local`), the API key, and the
username that owns the key.
4. Click **Test Connection** โ **Save**.
@@ -28,27 +29,27 @@ On your TrueNAS system:
## What Gets Monitored
-| Data | Unified Page | Details |
+| Data | Pulse page / tab | Details |
|---|---|---|
-| System info (hostname, version, uptime) | Infrastructure | CPU, memory, health status |
-| Virtual machines | TrueNAS Overview | State, CPU, memory, boot mode, devices, and security flags from the TrueNAS VM API |
-| Apps | TrueNAS Overview | Native app state, image/version, ports, volumes, networks, and runtime container details |
-| ZFS Pools | Storage | Total/used/free capacity, pool status (ONLINE/DEGRADED/FAULTED) |
-| ZFS Datasets | Storage | Used/available space, mount status, read-only flag |
-| Physical Disks | Storage | Model, serial, size, transport type, rotational flag, temperature, and native SMART failure/counter evidence when TrueNAS reports it |
-| ZFS Snapshots | Recovery | Dataset, creation time, size, referenced data |
-| Replication Tasks | Recovery | Source/target datasets, direction, last run status |
+| System info (hostname, version, uptime) | TrueNAS โ Overview | CPU, memory, health status |
+| Virtual machines | TrueNAS โ VMs | State, CPU, memory, boot mode, devices, and security flags from the TrueNAS VM API |
+| Apps | TrueNAS โ Apps | Native app state, image/version, ports, volumes, networks, and runtime container details |
+| ZFS Pools | TrueNAS โ Storage | Total/used/free capacity, pool status (ONLINE/DEGRADED/FAULTED) |
+| ZFS Datasets | TrueNAS โ Storage | Used/available space, mount status, read-only flag |
+| Physical Disks | TrueNAS โ Storage | Model, serial, size, transport type, rotational flag, temperature, and native SMART failure/counter evidence when TrueNAS reports it |
+| ZFS Snapshots | TrueNAS โ Protection | Dataset, creation time, size, referenced data |
+| Replication Tasks | TrueNAS โ Protection | Source/target datasets, direction, last run status |
| TrueNAS Alerts | Alerts | Native TrueNAS alert messages and severity levels |
## Unified Resource Mapping
TrueNAS resources are mapped into the unified resource model:
-- **TrueNAS host** โ appears as a resource with `source: truenas` on the **Infrastructure** page.
-- **TrueNAS VMs** โ appear as canonical `vm` workloads on the **TrueNAS** page.
-- **TrueNAS apps** โ appear as canonical `app-container` workloads on the **TrueNAS** page.
-- **ZFS pools and datasets** โ appear on the **Storage** page.
-- **ZFS snapshots and replication** โ appear on the **Recovery** page as recovery points.
+- **TrueNAS host** โ appears as a resource with `source: truenas` under **TrueNAS โ Overview** (`/truenas/overview`).
+- **TrueNAS VMs** โ appear as canonical `vm` workloads under **TrueNAS โ VMs** (`/truenas/vms`).
+- **TrueNAS apps** โ appear as canonical `app-container` workloads under **TrueNAS โ Apps** (`/truenas/apps`).
+- **ZFS pools, datasets and physical disks** โ appear under **TrueNAS โ Storage** (`/truenas/storage`).
+- **ZFS snapshots and replication** โ appear under **TrueNAS โ Protection** (`/truenas/protection`) as recovery points.
- **TrueNAS alerts** โ surfaced on the **Alerts** page alongside Proxmox and other platform alerts.
TrueNAS drive-health alerts that identify a specific disk remain disk-health
@@ -60,7 +61,11 @@ typed uncorrectable-error or spare-reserve argument, Pulse also projects that
value into the disk's SMART details. Pulse does not infer counters from alert
text, and current TrueNAS APIs do not expose every raw SMART attribute.
-Resources from TrueNAS can be filtered using the **source** filter on any page.
+The unified resource model is a backend contract, not a top-level
+Infrastructure, Storage or Recovery menu. TrueNAS tabs follow the inventory
+Pulse has collected; an absent tab is not proof of successful collection.
+For missing or stale data, use the [polling checks](#stale-truenas-data) before
+testing the connection or restarting.
## Multiple TrueNAS Systems
@@ -91,7 +96,7 @@ All endpoints require admin authentication.
### Testing and adding a connection (API)
-The **Settings โ TrueNAS** flow is the simplest option. For an API client,
+The **Settings โ Infrastructure โ Platform connections** flow is the simplest option. For an API client,
prepare the private Pulse header file as described in the
[API authentication guide](API.md). The Pulse API token authenticates to
Pulse; the separate TrueNAS API key belongs in the connection's JSON file.
@@ -260,4 +265,4 @@ Set `PULSE_ENABLE_TRUENAS=false` and restart Pulse. Existing connection data is
JSON-RPC transport, API-key, and TLS requirements
- [Configuration Guide](CONFIGURATION.md#truenas) โ environment variables and setup
- [ZFS Monitoring](ZFS_MONITORING.md) โ Proxmox-native ZFS pool monitoring
-- [Recovery](RECOVERY.md) โ TrueNAS snapshots in the recovery view
+- [Recovery data](RECOVERY.md) โ the snapshot and replication data shown under **TrueNAS โ Protection**
diff --git a/frontend-modern/browser-tests/navigation-help.cjs b/frontend-modern/browser-tests/navigation-help.cjs
new file mode 100644
index 0000000000..91028e5879
--- /dev/null
+++ b/frontend-modern/browser-tests/navigation-help.cjs
@@ -0,0 +1,110 @@
+// Production Docs/router/styles with the actual shipped Markdown; no backend.
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const { chromium, webkit } = require('playwright');
+
+(async () => {
+ const root = '/workspace/frontend-modern';
+ const artifacts = path.join(root, 'node_modules', 'navigation-help-browser');
+ fs.mkdirSync(artifacts, { recursive: true });
+ const { createServer } = await import(path.join(root, 'node_modules/vite/dist/node/index.js'));
+ const server = await createServer({
+ root,
+ configFile: path.join(root, 'vite.config.ts'),
+ cacheDir: path.join(root, 'node_modules', '.vite-navigation-help'),
+ server: { host: '127.0.0.1', port: 5247, strictPort: true },
+ optimizeDeps: { noDiscovery: true, include: [] },
+ });
+ const results = [];
+ let browser;
+ try {
+ await server.listen();
+ for (const [engine, width, tone] of [['chromium', 1280, 'light'], ['webkit', 390, 'dark']]) {
+ browser = await { chromium, webkit }[engine].launch(
+ engine === 'chromium'
+ ? { headless: true, channel: 'chromium', args: ['--no-sandbox'] }
+ : { headless: true },
+ );
+ const page = await browser.newPage({
+ viewport: { width, height: 900 },
+ ...(engine === 'webkit' ? { isMobile: true, hasTouch: true } : {}),
+ });
+ const errors = [];
+ page.on('pageerror', (e) => errors.push(e.message));
+ page.on('console', (m) => { if (m.type() === 'error') errors.push(m.text()); });
+ await page.goto('http://127.0.0.1:5247/browser-tests/docs-fragment-navigation.html?scenario=troubleshooting',
+ { waitUntil: 'domcontentloaded', timeout: 60_000 });
+ await page.getByRole('heading', { name: "Old bookmarks don't work", exact: true }).waitFor();
+ if (tone === 'dark') await page.evaluate(() => document.documentElement.classList.add('dark'));
+ const routes = page.locator('article table').filter({ hasText: '/standalone/machines' });
+ assert.equal(await routes.count(), 1);
+ assert.deepEqual(await routes.locator('tbody tr').allTextContents(), [
+ 'Proxmox/proxmox/overview', 'Docker/docker/overview', 'Kubernetes/kubernetes/overview',
+ 'TrueNAS/truenas/overview', 'vSphere/vmware/overview', 'Machines/standalone/machines',
+ ]);
+ const article = await page.locator('article').innerText();
+ assert.ok(article.includes('are supported; do not replace them with the retired task-based routes.'));
+ assert.ok(article.includes('Do not delete connections or re-enrol agents'));
+ await routes.scrollIntoViewIfNeeded();
+ assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth + 1));
+ await page.screenshot({ path: path.join(artifacts, `${engine}-routes.png`) });
+
+ await page.getByRole('link', { name: 'FAQ', exact: true }).click();
+ const faq = page.getByRole('heading', { name: 'How is navigation organised in Pulse v6?', exact: true });
+ await faq.waitFor();
+ await page.waitForFunction(() => {
+ const heading = document.getElementById('how-is-navigation-organised-in-pulse-v6');
+ return heading && heading.getBoundingClientRect().top >= 0 && heading.getBoundingClientRect().top < innerHeight;
+ });
+ assert.ok(page.url().endsWith('/docs/FAQ#how-is-navigation-organised-in-pulse-v6'));
+
+ await page.getByRole('link', { name: 'โ All documentation' }).click();
+ await page.getByRole('link', { name: 'TrueNAS SCALE and CORE', exact: true }).click();
+ const inventory = page.locator('article table').filter({ hasText: 'Pulse page / tab' });
+ await inventory.waitFor();
+ const destinations = await inventory.locator('tbody tr td:nth-child(2)').allTextContents();
+ assert.deepEqual(destinations, ['TrueNAS โ Overview', 'TrueNAS โ VMs', 'TrueNAS โ Apps',
+ 'TrueNAS โ Storage', 'TrueNAS โ Storage', 'TrueNAS โ Storage',
+ 'TrueNAS โ Protection', 'TrueNAS โ Protection', 'Alerts']);
+ const mapping = await page.locator('article').innerText();
+ for (const route of ['/truenas/overview', '/truenas/vms', '/truenas/apps', '/truenas/storage', '/truenas/protection'])
+ assert.ok(mapping.includes(route));
+ await inventory.scrollIntoViewIfNeeded();
+ const geometry = await inventory.evaluate((table) => ({
+ localScroll: table.parentElement.dataset.docTableScroll !== undefined,
+ viewport: innerWidth,
+ outer: document.documentElement.scrollWidth,
+ table: table.parentElement.scrollWidth,
+ wrapper: table.parentElement.clientWidth,
+ }));
+ assert.ok(geometry.localScroll);
+ assert.ok(geometry.outer <= geometry.viewport + 1);
+ if (width === 390) assert.ok(geometry.table > geometry.wrapper);
+ await page.screenshot({ path: path.join(artifacts, `${engine}-truenas.png`) });
+
+ await page.getByRole('link', { name: 'โ All documentation' }).click();
+ await page.getByRole('link', { name: 'Proxmox Backup Server', exact: true }).click();
+ const pbs = page.getByRole('heading', { name: 'Data Source Indicator', exact: true });
+ await pbs.waitFor();
+ const pbsText = await page.locator('article').innerText();
+ assert.ok(pbsText.includes('Proxmox โ Backups (/proxmox/backups)'));
+ assert.ok(pbsText.includes('There is no current top-level Recovery page'));
+ await pbs.scrollIntoViewIfNeeded();
+ assert.ok(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1));
+ await page.screenshot({ path: path.join(artifacts, `${engine}-pbs.png`) });
+ assert.deepEqual(errors, []);
+ results.push({ engine, version: browser.version(), width, tone, routes: 6, destinations,
+ geometry, faqFragment: true, pbsPlacement: true, errors });
+ await browser.close();
+ browser = undefined;
+ }
+ const result = { result: 'passed', playwright: require('playwright/package.json').version,
+ scope: 'shipped help rendering and navigation, not appliance collection', results };
+ fs.writeFileSync(path.join(artifacts, 'result.json'), JSON.stringify(result, null, 2) + '\n');
+ console.log(JSON.stringify(result));
+ } finally {
+ if (browser) await browser.close();
+ await server.close();
+ }
+})().catch((error) => { console.error(error); process.exitCode = 1; });
diff --git a/frontend-modern/public/docs/PBS.md b/frontend-modern/public/docs/PBS.md
index a377d2c602..2c8affcab3 100644
--- a/frontend-modern/public/docs/PBS.md
+++ b/frontend-modern/public/docs/PBS.md
@@ -152,7 +152,7 @@ It does **not** allow:
If you have multiple PBS servers, add each one separately in Settings. Pulse will:
- Monitor each server independently
-- Show backups from all servers in the unified Recovery view
+- Show backups from all servers under **Proxmox โ Backups** (`/proxmox/backups`)
- Deduplicate if the same backup appears via both PVE passthrough and direct PBS
---
@@ -301,7 +301,8 @@ or change retention to hide a display problem.
## Data Source Indicator
-In the Recovery view, PBS backups show a data source indicator:
+Under **Proxmox โ Backups** (`/proxmox/backups`), PBS backups show a data source
+indicator. There is no current top-level Recovery page:
- **"PBS"** badge alone = Direct PBS connection (full data)
- **"PBS via PVE"** = Passthrough via PVE storage (limited data)
diff --git a/frontend-modern/public/docs/TROUBLESHOOTING.md b/frontend-modern/public/docs/TROUBLESHOOTING.md
index 9b11c341cd..f7f21b286a 100644
--- a/frontend-modern/public/docs/TROUBLESHOOTING.md
+++ b/frontend-modern/public/docs/TROUBLESHOOTING.md
@@ -413,7 +413,7 @@ consequential, manually redacted error.
### TrueNAS
#### "TrueNAS service unavailable"
-- Ensure TrueNAS was added in **Settings โ TrueNAS** with a valid HTTPS URL,
+- Ensure TrueNAS was added in **Settings โ Infrastructure โ Platform connections** with a valid HTTPS URL,
API key, and the username that owns the key.
- Check that the TrueNAS system is reachable from the Pulse server (default
HTTPS port).
@@ -424,7 +424,10 @@ consequential, manually redacted error.
#### TrueNAS pools/datasets not appearing
- TrueNAS data appears in the unified resource model and may take one configured
polling cycle (60 seconds by default) to appear.
-- Check **Infrastructure** (TrueNAS host), **Storage** (pools/datasets), and **Recovery** (snapshots/replication).
+- Open **TrueNAS โ Overview** for the appliance, **TrueNAS โ Storage**
+ (`/truenas/storage`) for pools, datasets and disks, and **TrueNAS โ Protection**
+ (`/truenas/protection`) for snapshots and replication. These are tabs within
+ TrueNAS, not separate top-level Storage or Recovery pages.
- For data that stops refreshing, use the [TrueNAS polling checks](TRUENAS.md#stale-truenas-data)
before testing or restarting. A stale badge is not proof of an invalid key,
and a successful connection test is not proof that collection has recovered.
@@ -432,8 +435,38 @@ consequential, manually redacted error.
### Navigation (v6)
#### Old bookmarks don't work
-- Legacy URLs (`/proxmox`, `/docker`, `/kubernetes`, `/hosts`, `/services`) are not supported in v6.
-- Update bookmarks to canonical routes. See [Migration Guide](MIGRATION_UNIFIED_NAV.md).
+
+Current Pulse uses platform navigation. `/proxmox`, `/docker` and `/kubernetes`
+are supported; do not replace them with the retired task-based routes.
+Open Pulse at its base URL and use the menu to find the relevant page:
+
+| Menu | Entry route |
+| --- | --- |
+| Proxmox | `/proxmox/overview` |
+| Docker | `/docker/overview` |
+| Kubernetes | `/kubernetes/overview` |
+| TrueNAS | `/truenas/overview` |
+| vSphere | `/vmware/overview` |
+| Machines | `/standalone/machines` |
+
+The short-lived top-level `/workloads`, `/storage` and `/recovery` layout is
+retired. `/infrastructure` now opens the default workspace, not the former
+unified host page. For old `/hosts` or `/services` bookmarks, select the current
+platform or Machines page instead of assuming an automatic redirect. PBS
+backups are under **Proxmox โ Backups**, and TrueNAS snapshots and replication
+are under **TrueNAS โ Protection**.
+
+Platform-connected hosts can appear under their platform rather than Machines.
+If a menu or expected resource is missing, check its saved connection and last
+successful collection in **Settings โ Infrastructure**; a missing page alone
+does not prove a host was deleted. Do not delete connections or re-enrol agents
+just to repair a bookmark.
+
+If menu navigation works but reloading the same URL returns a proxy 404, check
+the proxy's route handling using [Reverse Proxy Configuration](REVERSE_PROXY.md).
+For the current layout, see [FAQ](FAQ.md#how-is-navigation-organised-in-pulse-v6).
+The [unified-navigation migration](MIGRATION_UNIFIED_NAV.md) is historical,
+not a guide to the current menu.
### Relay / Mobile
diff --git a/frontend-modern/public/docs/TRUENAS.md b/frontend-modern/public/docs/TRUENAS.md
index ddd5ea8468..54dac2b6e6 100644
--- a/frontend-modern/public/docs/TRUENAS.md
+++ b/frontend-modern/public/docs/TRUENAS.md
@@ -4,8 +4,9 @@ Pulse v6 includes first-class monitoring for **TrueNAS SCALE** and **TrueNAS COR
## Quick Start
-1. Go to **Settings โ TrueNAS**.
-2. Click **Add Connection**.
+1. Go to **Settings โ Infrastructure**.
+2. Click **Add infrastructure** and choose **TrueNAS**. Existing connections
+ are listed under **Platform connections**.
3. Enter the TrueNAS URL (e.g., `https://truenas.local`), the API key, and the
username that owns the key.
4. Click **Test Connection** โ **Save**.
@@ -28,27 +29,27 @@ On your TrueNAS system:
## What Gets Monitored
-| Data | Unified Page | Details |
+| Data | Pulse page / tab | Details |
|---|---|---|
-| System info (hostname, version, uptime) | Infrastructure | CPU, memory, health status |
-| Virtual machines | TrueNAS Overview | State, CPU, memory, boot mode, devices, and security flags from the TrueNAS VM API |
-| Apps | TrueNAS Overview | Native app state, image/version, ports, volumes, networks, and runtime container details |
-| ZFS Pools | Storage | Total/used/free capacity, pool status (ONLINE/DEGRADED/FAULTED) |
-| ZFS Datasets | Storage | Used/available space, mount status, read-only flag |
-| Physical Disks | Storage | Model, serial, size, transport type, rotational flag, temperature, and native SMART failure/counter evidence when TrueNAS reports it |
-| ZFS Snapshots | Recovery | Dataset, creation time, size, referenced data |
-| Replication Tasks | Recovery | Source/target datasets, direction, last run status |
+| System info (hostname, version, uptime) | TrueNAS โ Overview | CPU, memory, health status |
+| Virtual machines | TrueNAS โ VMs | State, CPU, memory, boot mode, devices, and security flags from the TrueNAS VM API |
+| Apps | TrueNAS โ Apps | Native app state, image/version, ports, volumes, networks, and runtime container details |
+| ZFS Pools | TrueNAS โ Storage | Total/used/free capacity, pool status (ONLINE/DEGRADED/FAULTED) |
+| ZFS Datasets | TrueNAS โ Storage | Used/available space, mount status, read-only flag |
+| Physical Disks | TrueNAS โ Storage | Model, serial, size, transport type, rotational flag, temperature, and native SMART failure/counter evidence when TrueNAS reports it |
+| ZFS Snapshots | TrueNAS โ Protection | Dataset, creation time, size, referenced data |
+| Replication Tasks | TrueNAS โ Protection | Source/target datasets, direction, last run status |
| TrueNAS Alerts | Alerts | Native TrueNAS alert messages and severity levels |
## Unified Resource Mapping
TrueNAS resources are mapped into the unified resource model:
-- **TrueNAS host** โ appears as a resource with `source: truenas` on the **Infrastructure** page.
-- **TrueNAS VMs** โ appear as canonical `vm` workloads on the **TrueNAS** page.
-- **TrueNAS apps** โ appear as canonical `app-container` workloads on the **TrueNAS** page.
-- **ZFS pools and datasets** โ appear on the **Storage** page.
-- **ZFS snapshots and replication** โ appear on the **Recovery** page as recovery points.
+- **TrueNAS host** โ appears as a resource with `source: truenas` under **TrueNAS โ Overview** (`/truenas/overview`).
+- **TrueNAS VMs** โ appear as canonical `vm` workloads under **TrueNAS โ VMs** (`/truenas/vms`).
+- **TrueNAS apps** โ appear as canonical `app-container` workloads under **TrueNAS โ Apps** (`/truenas/apps`).
+- **ZFS pools, datasets and physical disks** โ appear under **TrueNAS โ Storage** (`/truenas/storage`).
+- **ZFS snapshots and replication** โ appear under **TrueNAS โ Protection** (`/truenas/protection`) as recovery points.
- **TrueNAS alerts** โ surfaced on the **Alerts** page alongside Proxmox and other platform alerts.
TrueNAS drive-health alerts that identify a specific disk remain disk-health
@@ -60,7 +61,11 @@ typed uncorrectable-error or spare-reserve argument, Pulse also projects that
value into the disk's SMART details. Pulse does not infer counters from alert
text, and current TrueNAS APIs do not expose every raw SMART attribute.
-Resources from TrueNAS can be filtered using the **source** filter on any page.
+The unified resource model is a backend contract, not a top-level
+Infrastructure, Storage or Recovery menu. TrueNAS tabs follow the inventory
+Pulse has collected; an absent tab is not proof of successful collection.
+For missing or stale data, use the [polling checks](#stale-truenas-data) before
+testing the connection or restarting.
## Multiple TrueNAS Systems
@@ -91,7 +96,7 @@ All endpoints require admin authentication.
### Testing and adding a connection (API)
-The **Settings โ TrueNAS** flow is the simplest option. For an API client,
+The **Settings โ Infrastructure โ Platform connections** flow is the simplest option. For an API client,
prepare the private Pulse header file as described in the
[API authentication guide](API.md). The Pulse API token authenticates to
Pulse; the separate TrueNAS API key belongs in the connection's JSON file.
@@ -260,4 +265,4 @@ Set `PULSE_ENABLE_TRUENAS=false` and restart Pulse. Existing connection data is
JSON-RPC transport, API-key, and TLS requirements
- [Configuration Guide](CONFIGURATION.md#truenas) โ environment variables and setup
- [ZFS Monitoring](ZFS_MONITORING.md) โ Proxmox-native ZFS pool monitoring
-- [Recovery](RECOVERY.md) โ TrueNAS snapshots in the recovery view
+- [Recovery data](RECOVERY.md) โ the snapshot and replication data shown under **TrueNAS โ Protection**
diff --git a/scripts/tests/test_navigation_help_docs.py b/scripts/tests/test_navigation_help_docs.py
new file mode 100644
index 0000000000..3008fe45ee
--- /dev/null
+++ b/scripts/tests/test_navigation_help_docs.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""Keep operator navigation guidance tied to the shipped routes and tab labels.
+
+These checks cover documentation accuracy, not collection or appliance health.
+"""
+
+from pathlib import Path
+import re
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def doc_text(name: str) -> str:
+ return (ROOT / "docs" / name).read_text(encoding="utf-8")
+
+
+def source(name: str) -> str:
+ return (ROOT / "frontend-modern/src" / name).read_text(encoding="utf-8")
+
+
+class NavigationHelpDocsTest(unittest.TestCase):
+ def test_entry_routes_match_the_registered_platform_routes(self):
+ guide = doc_text("TROUBLESHOOTING.md").split("#### Old bookmarks don't work", 1)[1]
+ guide = guide.split("### Relay / Mobile", 1)[0]
+ links = source("routing/resourceLinks.ts")
+ app = source("App.tsx")
+ for label, prefix in (("Proxmox", "PROXMOX"), ("Docker", "DOCKER"),
+ ("Kubernetes", "KUBERNETES"), ("TrueNAS", "TRUENAS"),
+ ("vSphere", "VMWARE"), ("Machines", "STANDALONE")):
+ with self.subTest(platform=label):
+ path = re.search(rf"export const {prefix}_PATH = '([^']+)';", links).group(1)
+ tab = re.search(rf"export const {prefix}_DEFAULT_TAB = '([^']+)';", links).group(1)
+ self.assertIn(f"| {label} | `{path}/{tab}` |", guide)
+ self.assertIn(f'
Date: Sat, 3 Oct 2026 02:35:18 +0100
Subject: [PATCH 17/86] Activate paired firing work in the receipt-ordering
fixture
The queue discovers one concurrent batch at a time; the first HTTP control incorrectly expected another batch to start while the held response was unfinished. Admit both snapshots through the deferred bootstrap owner and then use normal queue activation. Preserve the runtime policy and timeout, and verify each received message explicitly.
Change-source: pulse-maintainer
---
.../v6/internal/subsystems/notifications.md | 4 ++-
.../cooldown_receipt_order_test.go | 29 ++++++++++++++-----
2 files changed, 24 insertions(+), 9 deletions(-)
diff --git a/docs/release-control/v6/internal/subsystems/notifications.md b/docs/release-control/v6/internal/subsystems/notifications.md
index 1ddb0616ee..2903cb9f5a 100644
--- a/docs/release-control/v6/internal/subsystems/notifications.md
+++ b/docs/release-control/v6/internal/subsystems/notifications.md
@@ -27,7 +27,9 @@ history. No queue schema, retry budget, recovery selection or schedule changes.
`TestNotificationCooldownReceiptOrdering` verifies older, unknown and equal-start
completion ordering, including legacy zero starts. The connected
`TestLateFiringHTTPReceiptPreservesCurrentCooldown` holds an older HTTP response
-while the newer firing completes through the persistent sender. It verifies that
+while the newer firing completes through the persistent sender. Both snapshots
+are admitted before ordinary queue activation, so one discovered batch can
+complete concurrently without adding another dispatcher. It verifies that
the ordinary repeat adds no queue row, a current severity increase still reaches
the receiver, and both occurrence/destination receipts survive queue reopen.
This is loopback source acceptance, not native delivery, restart cooldown
diff --git a/internal/notifications/cooldown_receipt_order_test.go b/internal/notifications/cooldown_receipt_order_test.go
index 6650194479..d5d3f66c07 100644
--- a/internal/notifications/cooldown_receipt_order_test.go
+++ b/internal/notifications/cooldown_receipt_order_test.go
@@ -88,7 +88,7 @@ func TestLateFiringHTTPReceiptPreservesCurrentCooldown(t *testing.T) {
t.Cleanup(server.Close)
dir := t.TempDir()
webhook := WebhookConfig{ID: "ops", URL: server.URL + "/hook", Enabled: true, Service: "generic"}
- m := NewNotificationManagerWithDataDir("", dir)
+ m := NewNotificationManagerWithDeferredQueue("", dir)
t.Cleanup(m.Stop)
t.Cleanup(unblock) // Release an in-flight provider before waiting for shutdown.
m.webhookClient = server.Client()
@@ -100,16 +100,20 @@ func TestLateFiringHTTPReceiptPreservesCurrentCooldown(t *testing.T) {
old := &alerts.Alert{ID: "recurring-cpu", ResourceName: "cpu-node", Type: "cpu",
Level: alerts.AlertLevelCritical, Message: "older firing", StartTime: time.Now().UTC().Add(-time.Minute)}
m.SendAlert(old)
- select {
- case <-oldArrived:
- case <-time.After(10 * time.Second):
- t.Fatal("old firing did not reach the HTTP receiver")
- }
current := old.Clone()
current.StartTime = time.Now().UTC()
current.Level = alerts.AlertLevelWarning
current.Message = "current warning firing"
m.SendAlert(current)
+ // Workers run concurrently within one discovered batch, but discovery waits
+ // for that batch to finish. Admit both snapshots before normal activation so
+ // the HTTP ordering is possible without a second dispatcher or clock edits.
+ m.StartQueueProcessing()
+ select {
+ case <-oldArrived:
+ case <-time.After(10 * time.Second):
+ t.Fatal("old firing did not reach the HTTP receiver")
+ }
wait := func(condition func() bool) {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
@@ -147,9 +151,18 @@ func TestLateFiringHTTPReceiptPreservesCurrentCooldown(t *testing.T) {
return stats["sent"] == 3
})
mu.Lock()
- count := len(bodies)
+ received := append([]string(nil), bodies...)
mu.Unlock()
- require.Equal(t, 3, count)
+ require.Len(t, received, 3)
+ for _, message := range []string{"older firing", "current warning firing", "current critical firing"} {
+ count := 0
+ for _, body := range received {
+ if strings.Contains(body, message) {
+ count++
+ }
+ }
+ require.Equal(t, 1, count, "receiver must see exactly one %s", message)
+ }
m.Stop()
// The old completion still owns a durable recovery receipt, independently
// of which occurrence owns the transient repeat-delivery cooldown.
From 0027f7c9d979b2f69295ba4117f31d6107d6c200 Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 02:35:45 +0100
Subject: [PATCH 18/86] Verify help table cells with the production style
configuration
Compare individual route cells rather than Markdown whitespace, and resolve Tailwind from the frontend module so desktop and phone proof inspect the actual styled Docs page.
Change-source: pulse-maintainer
---
frontend-modern/browser-tests/navigation-help.cjs | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/frontend-modern/browser-tests/navigation-help.cjs b/frontend-modern/browser-tests/navigation-help.cjs
index 91028e5879..2cd5994658 100644
--- a/frontend-modern/browser-tests/navigation-help.cjs
+++ b/frontend-modern/browser-tests/navigation-help.cjs
@@ -6,6 +6,7 @@ const { chromium, webkit } = require('playwright');
(async () => {
const root = '/workspace/frontend-modern';
+ process.chdir(root);
const artifacts = path.join(root, 'node_modules', 'navigation-help-browser');
fs.mkdirSync(artifacts, { recursive: true });
const { createServer } = await import(path.join(root, 'node_modules/vite/dist/node/index.js'));
@@ -39,9 +40,13 @@ const { chromium, webkit } = require('playwright');
if (tone === 'dark') await page.evaluate(() => document.documentElement.classList.add('dark'));
const routes = page.locator('article table').filter({ hasText: '/standalone/machines' });
assert.equal(await routes.count(), 1);
- assert.deepEqual(await routes.locator('tbody tr').allTextContents(), [
- 'Proxmox/proxmox/overview', 'Docker/docker/overview', 'Kubernetes/kubernetes/overview',
- 'TrueNAS/truenas/overview', 'vSphere/vmware/overview', 'Machines/standalone/machines',
+ const entryRoutes = await routes.locator('tbody tr').evaluateAll((rows) =>
+ rows.map((row) => Array.from(row.cells, (cell) => cell.textContent.trim())),
+ );
+ assert.deepEqual(entryRoutes, [
+ ['Proxmox', '/proxmox/overview'], ['Docker', '/docker/overview'],
+ ['Kubernetes', '/kubernetes/overview'], ['TrueNAS', '/truenas/overview'],
+ ['vSphere', '/vmware/overview'], ['Machines', '/standalone/machines'],
]);
const article = await page.locator('article').innerText();
assert.ok(article.includes('are supported; do not replace them with the retired task-based routes.'));
From 63b24757d6b9e52729488a6c5a8f4fffcdb5f23f Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 02:37:27 +0100
Subject: [PATCH 19/86] Accept readable wrapped help tables as well as local
scrolling
Phone tables can fit through wrapping. Check their container bound, or actual local scroll movement for wide tables, rather than requiring overflow.
Change-source: pulse-maintainer
---
frontend-modern/browser-tests/navigation-help.cjs | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/frontend-modern/browser-tests/navigation-help.cjs b/frontend-modern/browser-tests/navigation-help.cjs
index 2cd5994658..8acc827772 100644
--- a/frontend-modern/browser-tests/navigation-help.cjs
+++ b/frontend-modern/browser-tests/navigation-help.cjs
@@ -85,7 +85,15 @@ const { chromium, webkit } = require('playwright');
}));
assert.ok(geometry.localScroll);
assert.ok(geometry.outer <= geometry.viewport + 1);
- if (width === 390) assert.ok(geometry.table > geometry.wrapper);
+ if (width === 390 && geometry.table > geometry.wrapper + 1) {
+ const movement = await inventory.evaluate((table) => {
+ table.parentElement.scrollLeft = 180;
+ return table.parentElement.scrollLeft;
+ });
+ assert.ok(movement > 0, 'wide tables must remain locally scrollable');
+ } else {
+ assert.ok(geometry.table <= geometry.wrapper + 1, 'wrapped table must fit its container');
+ }
await page.screenshot({ path: path.join(artifacts, `${engine}-truenas.png`) });
await page.getByRole('link', { name: 'โ All documentation' }).click();
From 6f5bf963704b187844da05bbec9e4a86ba6c9068 Mon Sep 17 00:00:00 2001
From: "pulse-triage[bot]"
<249995291+pulse-triage[bot]@users.noreply.github.com>
Date: Sat, 3 Oct 2026 02:45:14 +0100
Subject: [PATCH 20/86] Withdraw History readings when access is denied
Treat final HTTP 401/403 as access failures rather than transient refresh outages. Clear displayed samples and inspection, invalidate retained query ranges and remounts, and keep bounded access guidance through scoped recovery. Preserve ordinary outage retention, current-value labelling, abort ownership and license gates with runtime and real-client desktop/touch browser controls.
Change-source: pulse-maintainer
---
.../subsystems/frontend-primitives.md | 28 +-
.../subsystems/performance-and-scalability.md | 12 +-
.../v6/internal/subsystems/registry.json | 2 +
.../browser-tests/history-access.cjs | 325 ++++++++++++++++++
.../browser-tests/history-access.html | 12 +
.../browser-tests/history-access.tsx | 57 +++
frontend-modern/browser-verification.json | 53 +--
.../Workloads/GuestDrawerHistory.tsx | 8 +-
.../GuestDrawerHistory.refresh.test.tsx | 102 ++++++
.../components/shared/HistoryChartOverlay.tsx | 4 +-
.../shared/__tests__/HistoryChart.test.tsx | 52 ++-
.../__tests__/useHistoryChartState.test.tsx | 42 +++
.../components/shared/useHistoryChartState.ts | 21 +-
.../createNonSuspendingQuery.test.tsx | 128 +++++++
.../src/hooks/createNonSuspendingQuery.ts | 15 +-
.../utils/__tests__/apiAccessError.test.ts | 35 ++
frontend-modern/src/utils/apiAccessError.ts | 12 +
17 files changed, 874 insertions(+), 34 deletions(-)
create mode 100644 frontend-modern/browser-tests/history-access.cjs
create mode 100644 frontend-modern/browser-tests/history-access.html
create mode 100644 frontend-modern/browser-tests/history-access.tsx
create mode 100644 frontend-modern/src/utils/__tests__/apiAccessError.test.ts
create mode 100644 frontend-modern/src/utils/apiAccessError.ts
diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md
index 6255d4a7d0..f99fcd1271 100644
--- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md
+++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md
@@ -353,7 +353,30 @@ PBS browser runner verifies delayed range and withdrawn-target reads at desktop
and phone widths; neither proof establishes installed collection or #1723 relief.
Failed same-source history refreshes keep valid previously loaded observations
-visible with an explicit warning, rather than hiding the entire chart. Initial
+visible with an explicit warning, rather than hiding the entire chart. That
+retention applies to transient transport/server failures, never a final HTTP
+`401` or `403`. Both the batch drawer and shared canvas reader withdraw stored
+samples and inspection on access failure; denied history is not an empty,
+collecting, or stale-but-still-readable result. Their guidance is bounded by
+numeric status, not transport bodies or guessed diagnostic text. The shared
+canvas announces its error, and the drawer keeps its scoped, focusable retry.
+Only a successful replacement read restores observations.
+
+The retained-value query helper also withdraws its active value on a final
+`401`/`403` and clears its shared remount cache. Other resource/range entries
+were read under the same access context, so they cannot be resurrected after
+that context is denied. Advancing the cache generation rejects cache writes
+from already-running pre-denial reads. A superseded denied completion cannot
+clear the current target or its cache. This does not broadcast a session reset
+or claim revocation of independently mounted consumers; their current response
+still owns their presentation. Transient failures keep the existing retained
+value, and a newly successful read may populate the cache normally. Verification
+uses runtime assertions in `createNonSuspendingQuery.test.tsx`,
+`GuestDrawerHistory.refresh.test.tsx`, `useHistoryChartState.test.tsx`,
+`HistoryChart.test.tsx`, and `browser-tests/history-access.cjs` (both production
+readers/client, real same-origin 403/503 responses, cache/range/remount and
+fresh-read recovery on desktop Chromium and touch WebKit). These controls do
+not establish appliance authorisation, collection or released acceptance. Initial
or uncached replacement failures show unavailable history, never borrowed points
or a collecting claim. The existing target/range owns every manual refresh;
locked or absent targets expose no refresh control. The control remains mounted
@@ -674,6 +697,9 @@ that manual-channel consequence must stay visible at desktop and narrow widths.
108. `frontend-modern/src/components/shared/summaryInteractionA11y.ts`
109. `frontend-modern/src/components/shared/SummaryRowActionButton.tsx`
110. `frontend-modern/src/hooks/createNonSuspendingQuery.ts`
+ Shared API read-access classification is owned by
+ `frontend-modern/src/utils/apiAccessError.ts`; it recognises only final
+ numeric 401/403 statuses and never renders transport diagnostics.
111a. `frontend-modern/src/utils/storageSummaryCache.ts`
111. `frontend-modern/src/components/shared/TableCardHeader.tsx`
112. `frontend-modern/src/components/shared/UpgradeLink.tsx`
diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
index 8df6300caa..80cd898e5a 100644
--- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
+++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
@@ -921,7 +921,17 @@ change may globally weaken the Task 03 lifecycle-state idempotency invariant.
single-observation caption rather than a collecting claim or invented trend.
Successful empty reads say `No stored history in this range`. Only
metrics-store samples may contribute chart geometry, bounds, or hover
- inspection, including valid measured zero. This presentation adds no fetch,
+ inspection, including valid measured zero. Same-source transient failures
+ retain those observations with a warning, but final HTTP 401/403 responses
+ withdraw the batch charts, inspection and its retained remount entries.
+ The shared query cache advances its generation so a pre-denial read cannot
+ restore a former cached range. Access guidance comes from numeric status,
+ never raw transport text; the existing target/range-scoped retry remains
+ focusable while pending, and only a successful read restores history.
+ Runtime denial/cache/recovery controls are in
+ `GuestDrawerHistory.refresh.test.tsx` and `createNonSuspendingQuery.test.tsx`,
+ with real client 403/503 browser controls in `browser-tests/history-access.cjs`.
+ These are client presentation proofs, not native authorisation or collection. This presentation adds no fetch,
point expansion, polling or retention budget. Verify sparse and mixed-series
geometry and failed-refresh/source-change behaviour in
`GuestDrawerHistory.sparse.test.tsx`, with rendered phone/desktop coverage in
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index 5b59559345..e7361f0193 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -5812,6 +5812,7 @@
"match_prefixes": [],
"match_files": [
"frontend-modern/src/hooks/createNonSuspendingQuery.ts",
+ "frontend-modern/src/utils/apiAccessError.ts",
"frontend-modern/src/utils/storageSummaryCache.ts"
],
"allow_same_subsystem_tests": false,
@@ -5819,6 +5820,7 @@
"exact_files": [
"frontend-modern/src/components/Infrastructure/__tests__/ResourceDetailDrawer.history.test.tsx",
"frontend-modern/src/hooks/__tests__/createNonSuspendingQuery.test.tsx",
+ "frontend-modern/src/utils/__tests__/apiAccessError.test.ts",
"frontend-modern/src/utils/__tests__/storageSummaryCache.test.ts"
]
},
diff --git a/frontend-modern/browser-tests/history-access.cjs b/frontend-modern/browser-tests/history-access.cjs
new file mode 100644
index 0000000000..0a2c6261e2
--- /dev/null
+++ b/frontend-modern/browser-tests/history-access.cjs
@@ -0,0 +1,325 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const { chromium, webkit } = require('playwright');
+const parent = false;
+(async () => {
+ const root = parent
+ ? '/workspace/tmp/history-access-parent/frontend-modern'
+ : '/workspace/frontend-modern';
+ process.chdir(root);
+ const artifacts = `/workspace/tmp/history-access-${parent ? 'parent' : 'final'}-proof`;
+ fs.mkdirSync(artifacts, { recursive: true });
+ const { createServer } = await import(path.join(root, 'node_modules/vite/dist/node/index.js'));
+ const server = await createServer({
+ root,
+ configFile: path.join(root, 'vite.config.ts'),
+ cacheDir: fs.mkdtempSync(path.join(artifacts, 'vite-')),
+ server: { host: '127.0.0.1', port: 5261, strictPort: true },
+ });
+ const playwright = require('playwright/package.json').version;
+ const expected = JSON.parse(
+ fs.readFileSync('/workspace/tests/integration/package-lock.json', 'utf8'),
+ ).packages['node_modules/@playwright/test'].version;
+ assert.equal(playwright, expected);
+ const observations = [];
+ let browser;
+ try {
+ await server.listen();
+ for (const [name, engine, phone] of [
+ ['chromium-desktop', chromium, false],
+ ['webkit-phone', webkit, true],
+ ]) {
+ browser = await engine.launch(
+ engine === chromium
+ ? { headless: true, channel: 'chromium', args: ['--no-sandbox'] }
+ : { headless: true },
+ );
+ const page = await browser.newPage({
+ viewport: phone ? { width: 390, height: 844 } : { width: 1365, height: 900 },
+ hasTouch: phone,
+ isMobile: phone,
+ });
+ page.setDefaultTimeout(15000);
+ await page.clock.install({ time: new Date('2026-10-03T12:00:00Z') });
+ let mode = 'success',
+ value = 42;
+ const requests = [],
+ errors = [],
+ offOrigin = [],
+ held = [],
+ checks = [];
+ page.on('pageerror', (error) => errors.push(error.message));
+ await page.route('**/*', async (route) => {
+ const url = new URL(route.request().url());
+ if (url.origin !== 'http://127.0.0.1:5261') {
+ offOrigin.push(url.origin);
+ return route.abort();
+ }
+ if (!url.pathname.startsWith('/api/')) return route.continue();
+ if (url.pathname === '/api/metrics-store/history') {
+ const query = Object.fromEntries(url.searchParams);
+ requests.push({ ...query, responseMode: mode });
+ if (mode === 'hold') await new Promise((resolve) => held.push(resolve));
+ try {
+ if (mode === '503' || mode === '403')
+ return await route.fulfill({
+ status: Number(mode),
+ json: { error: 'fixture-only diagnostic body' },
+ });
+ const points = [0, 1].map((index) => ({
+ timestamp: 1791028740000 + index * 60000,
+ value: value - 1 + index,
+ min: value - 1 + index,
+ max: value - 1 + index,
+ }));
+ return await route.fulfill({
+ json: {
+ resourceType: query.resourceType,
+ resourceId: query.resourceId,
+ range: query.range,
+ metric: query.metric,
+ start: 1791028740000,
+ end: 1791028800000,
+ source: 'store',
+ ...(query.metric ? { points } : { metrics: { cpu: points, temperature: points } }),
+ },
+ });
+ } catch {
+ /* An explicitly superseded browser read may already be aborted. */
+ }
+ return;
+ }
+ return route.fulfill({
+ json:
+ url.pathname === '/api/license/runtime-capabilities'
+ ? {
+ capabilities: [],
+ limits: [],
+ max_history_days: 7,
+ hosted_mode: false,
+ runtime: { build: 'community' },
+ blocked_capabilities: [],
+ }
+ : { enabled: false, data: [] },
+ });
+ });
+ const check = async (label, predicate) => {
+ try {
+ await predicate();
+ checks.push({ label, passed: true });
+ } catch (error) {
+ checks.push({ label, passed: false, error: error.message });
+ }
+ };
+ const screenshot = async (state) => {
+ const file = `${name}-${state}.png`;
+ await page.screenshot({
+ path: path.join(artifacts, file),
+ fullPage: true,
+ animations: 'disabled',
+ });
+ observations.push({
+ browser: name,
+ state,
+ screenshot: file,
+ body: await page.locator('main').innerText(),
+ overflow: await page.evaluate(() => document.documentElement.scrollWidth > innerWidth),
+ });
+ };
+ const disk = page.getByTestId('disk'),
+ service = page.getByTestId('service');
+ const thermal = disk.getByRole('img', { name: 'Temperature chart' });
+ const description = () =>
+ thermal.evaluate(
+ (el) => document.getElementById(el.getAttribute('aria-describedby')).textContent,
+ );
+ const storedPaths = () => service.locator('[data-testid="guest-history-plot"] path').count();
+ await page.goto('http://127.0.0.1:5261/browser-tests/history-access.html');
+ if (phone) await page.evaluate(() => document.documentElement.classList.add('dark'));
+ await service.getByRole('slider', { name: 'Inspect Utilization history' }).waitFor();
+ await page.waitForFunction(
+ () =>
+ document.querySelector('[data-testid="disk"] canvas')?.getAttribute('aria-describedby') &&
+ document
+ .getElementById(
+ document
+ .querySelector('[data-testid="disk"] canvas')
+ .getAttribute('aria-describedby'),
+ )
+ .textContent.includes('42ยฐC'),
+ );
+ await service.getByRole('combobox', { name: 'History range' }).selectOption('24h');
+ await service.getByRole('slider', { name: 'Inspect Utilization history' }).waitFor();
+ if (phone) await thermal.tap({ position: { x: 100, y: 80 } });
+ else {
+ await thermal.focus();
+ await thermal.press('End');
+ }
+ await check('Both readers show only their successful stored samples', async () => {
+ assert.match(await description(), /42ยฐC/);
+ assert.equal(await storedPaths(), 2);
+ assert.equal(await disk.locator('[data-history-chart-tooltip]').count(), 1);
+ });
+ await screenshot('loaded');
+ mode = '503';
+ await page.clock.fastForward(30_000);
+ await service
+ .getByRole('status', { name: 'History refresh status' })
+ .getByText(/previously loaded/)
+ .waitFor();
+ await check('503 retains observations with a warning, not a collection claim', async () => {
+ assert.match(await description(), /42ยฐC/);
+ assert.equal(await storedPaths(), 2);
+ assert.match(
+ await disk.getByRole('status', { name: 'History refresh status' }).innerText(),
+ /last successful result/,
+ );
+ });
+ await screenshot('transient');
+ // Re-establish active inspection immediately before access changes.
+ if (phone) await thermal.tap({ position: { x: 100, y: 80 } });
+ else {
+ await thermal.focus();
+ await thermal.press('End');
+ }
+ mode = '403';
+ const beforeDenied = requests.length;
+ await page.clock.fastForward(30_000);
+ await page.waitForTimeout(250);
+ await check('The real API client receives a final 403 for both exact targets', async () => {
+ const denied = requests.slice(beforeDenied).filter((r) => r.responseMode === '403');
+ assert(denied.some((r) => r.resourceId === 'disk:nas-a:sda'));
+ assert(denied.some((r) => r.resourceId === 'pbs-service-a' && !r.metric));
+ });
+ await check('403 withdraws canvas values, tooltip and announced inspection', async () => {
+ assert.doesNotMatch(await description(), /42ยฐC/);
+ assert.equal(await disk.locator('[data-history-chart-tooltip]').count(), 0);
+ assert.match(await disk.getByRole('alert').innerText(), /Access denied/);
+ assert.equal(
+ await disk.getByRole('status', { name: 'History refresh status' }).innerText(),
+ '',
+ );
+ });
+ await check('403 withdraws batch charts, sliders and current-only legends', async () => {
+ assert.equal(await storedPaths(), 0);
+ assert.equal(await service.getByRole('slider').count(), 0);
+ assert.equal(await service.locator('[data-history-current]').count(), 0);
+ assert.match(
+ await service.getByRole('status', { name: 'History refresh status' }).innerText(),
+ /Access denied/,
+ );
+ });
+ await screenshot('denied');
+ mode = 'hold';
+ const retry = service.getByRole('button', { name: 'Retry history' });
+ const beforeRetry = requests.length;
+ await retry.focus();
+ await retry.click();
+ await retry.press('Enter');
+ await page.waitForTimeout(150);
+ await check(
+ 'One busy keyboard-safe retry remains scoped and shows no denied samples',
+ async () => {
+ assert.equal(
+ requests.slice(beforeRetry).filter((r) => r.resourceId === 'pbs-service-a').length,
+ 1,
+ );
+ assert.equal(await retry.getAttribute('aria-disabled'), 'true');
+ assert.equal(await retry.evaluate((el) => el === document.activeElement), true);
+ assert.equal(await storedPaths(), 0);
+ },
+ );
+ await screenshot('retry-pending');
+ await service.getByRole('combobox', { name: 'History range' }).selectOption('1h');
+ await page.waitForTimeout(150);
+ await check('An earlier cached range cannot resurrect denied observations', async () =>
+ assert.equal(await storedPaths(), 0),
+ );
+ await service.getByRole('button', { name: 'Hide service History' }).click();
+ await service.getByRole('button', { name: 'Show service History' }).click();
+ await page.waitForTimeout(150);
+ await check('A remount cannot resurrect the earlier cached range', async () =>
+ assert.equal(await storedPaths(), 0),
+ );
+ await screenshot('remount-pending');
+ mode = 'success';
+ value = 12;
+ held.splice(0).forEach((resolve) => resolve());
+ await page.clock.fastForward(30_000);
+ await page.waitForFunction(() =>
+ document
+ .getElementById(
+ document.querySelector('[data-testid="disk"] canvas').getAttribute('aria-describedby'),
+ )
+ .textContent.includes('12ยฐC'),
+ );
+ await service.getByRole('slider', { name: 'Inspect Utilization history' }).waitFor();
+ await check(
+ 'Only a new successful response restores readings and clears access errors',
+ async () => {
+ assert.equal(await storedPaths(), 2);
+ assert.equal(await disk.getByRole('alert').count(), 0);
+ assert.equal(
+ await service.getByRole('status', { name: 'History refresh status' }).innerText(),
+ '',
+ );
+ assert.doesNotMatch(await description(), /42ยฐC/);
+ assert(
+ requests.some(
+ (r) =>
+ r.resourceId === 'pbs-service-a' &&
+ r.range === '1h' &&
+ r.responseMode === 'success',
+ ),
+ );
+ },
+ );
+ await screenshot('recovered');
+ await check(
+ 'No diagnostic body, page error, off-origin request or horizontal overflow',
+ async () => {
+ assert.doesNotMatch(
+ await page.locator('main').innerText(),
+ /fixture-only diagnostic body/,
+ );
+ assert.deepEqual(errors, []);
+ assert.deepEqual(offOrigin, []);
+ assert.equal(
+ await page.evaluate(() => document.documentElement.scrollWidth > innerWidth),
+ false,
+ );
+ },
+ );
+ observations.push({
+ browser: name,
+ version: browser.version(),
+ checks,
+ requests,
+ errors,
+ offOrigin,
+ });
+ await browser.close();
+ browser = undefined;
+ }
+ } finally {
+ if (browser) await browser.close();
+ await server.close();
+ fs.writeFileSync(
+ path.join(artifacts, 'result.json'),
+ JSON.stringify({ parent, playwright, expected, observations }, null, 2),
+ );
+ }
+ const failed = observations.flatMap((o) => o.checks ?? []).filter((c) => !c.passed);
+ console.log(
+ JSON.stringify({
+ parent,
+ checks: observations.flatMap((o) => o.checks ?? []).length,
+ failed: failed.map((c) => c.label),
+ }),
+ );
+ if (!parent) assert.equal(failed.length, 0);
+})().catch((error) => {
+ console.error(error);
+ process.exitCode = 1;
+});
diff --git a/frontend-modern/browser-tests/history-access.html b/frontend-modern/browser-tests/history-access.html
new file mode 100644
index 0000000000..c41d942a61
--- /dev/null
+++ b/frontend-modern/browser-tests/history-access.html
@@ -0,0 +1,12 @@
+
+
+
+
+
+ History access failure verification
+
+
+
+
+
+
diff --git a/frontend-modern/browser-tests/history-access.tsx b/frontend-modern/browser-tests/history-access.tsx
new file mode 100644
index 0000000000..a1450ea206
--- /dev/null
+++ b/frontend-modern/browser-tests/history-access.tsx
@@ -0,0 +1,57 @@
+// Production readers/client/styles with synthetic same-origin HTTP responses.
+// This fixture proves presentation, not appliance authorisation or collection.
+import { Show, createSignal } from 'solid-js';
+import { render } from 'solid-js/web';
+import type { HistoryTimeRange } from '../src/api/charts';
+import {
+ GuestDrawerHistory,
+ GuestDrawerHistoryRangeSelect,
+} from '../src/components/Workloads/GuestDrawerHistory';
+import { HistoryChart } from '../src/components/shared/HistoryChart';
+import { HOST_METRICS_HISTORY_GROUPS } from '../src/components/shared/hostMetricsHistoryModel';
+import '../src/index.css';
+
+function Fixture() {
+ const [range, setRange] = createSignal('1h');
+ const [visible, setVisible] = createSignal(true);
+ return (
+
+
History access failure verification
+
+
Disk History
+
+
+
+
PBS service History
+
+
+
+
+
+
+
+
+
+ );
+}
+render(() => , document.getElementById('root')!);
diff --git a/frontend-modern/browser-verification.json b/frontend-modern/browser-verification.json
index 7555748f01..353d9f0ce2 100644
--- a/frontend-modern/browser-verification.json
+++ b/frontend-modern/browser-verification.json
@@ -1,16 +1,25 @@
{
"version": 1,
"result": "passed",
- "base_sha": "b6a42d8a4f329a1f3d7359d8d64b17c385006ed7",
- "verified_at": "2026-10-03T00:38:42Z",
+ "base_sha": "a9533eb82a29700ede2d4ad609812ddd01e5f287",
+ "verified_at": "2026-10-03T01:44:27.336168Z",
"changed_paths": [
- "frontend-modern/src/features/storageBackups/diskDetailPresentation.ts"
+ "frontend-modern/src/components/Workloads/GuestDrawerHistory.tsx",
+ "frontend-modern/src/components/shared/HistoryChartOverlay.tsx",
+ "frontend-modern/src/components/shared/useHistoryChartState.ts",
+ "frontend-modern/src/hooks/createNonSuspendingQuery.ts",
+ "frontend-modern/src/utils/apiAccessError.ts"
],
"content_sha256": {
- "frontend-modern/src/features/storageBackups/diskDetailPresentation.ts": "31c71a2d696f1b74d55b855220885619bbeebc0b75a1f34354cc5b794ba49375"
+ "frontend-modern/src/components/Workloads/GuestDrawerHistory.tsx": "c4370a69495c8ccea01a0d92076e4a051e4d06abec0e675e2fb3547d46e87bcb",
+ "frontend-modern/src/components/shared/HistoryChartOverlay.tsx": "14c0699b827ec6eb4d85cf9d8fc76d61a0af3dd6565d2396a3f22a03429af104",
+ "frontend-modern/src/components/shared/useHistoryChartState.ts": "5a9b022656ac7316710675a0742e9b2908c9114ffbed67d6f6962d361bb65bbf",
+ "frontend-modern/src/hooks/createNonSuspendingQuery.ts": "df293d0484612a6d31dd9f3fa6af2a1160e1e98d66965730a45829b7048a4ba7",
+ "frontend-modern/src/utils/apiAccessError.ts": "3b34a498959a77014af3c842677fc6646b5e7e6a8b9b332ca8acf91254c01730"
},
"routes": [
- "Existing Storage physical disk detail History"
+ "Existing PBS/Machine/resource drawer History batch reader",
+ "Existing Storage physical disk shared History reader"
],
"viewports": [
{
@@ -23,23 +32,25 @@
}
],
"states": [
- "Stored ATA temperature and measured-zero reallocated sectors render when current temperature and SMART are absent",
- "Matching current snapshot updates keep the same chart owners and add zero History reads",
- "Current field loss keeps stored samples and selected 7-day range; no unavailable live I/O is shown",
- "Overview reports present collection unavailability instead of the earlier stored temperature",
- "Empty temperature history is an explicit no-samples state, not a zero, while stored reallocated zero remains visible",
- "NVMe uses its own explicit disk target and stored temperature/endurance/spare series without current SMART fields",
- "No resolvable identity withdraws History and issues no guessed-target request",
- "Keyboard desktop and touch phone inspection show real stored readings",
- "All screenshots inspected with animations disabled; selected tab, light/dark layouts and page width agree with content",
- "Playwright 1.56.1 matches the integration lock; no page errors or off-origin data reads"
+ "Real same-origin 503 responses retain stored samples with explicit warnings in both production readers",
+ "Real final HTTP 403 responses withdraw samples, tooltip, keyboard announcement, sliders and formerly displayed current-only legends",
+ "Access-denied guidance fits light desktop Chromium and dark touch WebKit without horizontal overflow",
+ "The existing focusable scoped retry remains mounted and ignores duplicate activation while busy",
+ "Earlier cached ranges and remounts cannot resurrect denied stored readings while a replacement is pending",
+ "Only a new successful read restores History and clears denial; old values do not return",
+ "Every final screenshot inspected; fixed Playwright 1.56.1 equals integration lock, no page errors or off-origin data requests"
],
"interactions": [
- "Open History from unavailable current Overview",
- "Restore, update and remove current fields",
- "Change 24h/7d/1h ranges",
- "Inspect by keyboard or phone tap",
- "Select NVMe B",
- "Clear disk identity"
+ "Inspect by desktop keyboard and phone tap",
+ "Poll through 503 then 403",
+ "Retry twice while busy",
+ "Change 24h/1h range",
+ "Unmount and remount batch History",
+ "Resume with newly successful observations"
+ ],
+ "limitations": [
+ "Direct production readers/client with synthetic HTTP responses, not a complete native PBS/NAS drawer, session revocation, appliance collection or release acceptance.",
+ "401 is covered by runtime regressions; the real API client redirects on 401, not bypassed for browser proof.",
+ "Independently supplied current readings remain distinctly current during a new pending query; they are not stored History."
]
}
diff --git a/frontend-modern/src/components/Workloads/GuestDrawerHistory.tsx b/frontend-modern/src/components/Workloads/GuestDrawerHistory.tsx
index 427b7ec2f6..4eed3c6ba5 100644
--- a/frontend-modern/src/components/Workloads/GuestDrawerHistory.tsx
+++ b/frontend-modern/src/components/Workloads/GuestDrawerHistory.tsx
@@ -30,6 +30,7 @@ import {
} from '@/components/shared/historyChartModel';
import { createNonSuspendingQuery } from '@/hooks/createNonSuspendingQuery';
import { isRangeLocked, loadRuntimeCapabilities, maxHistoryDays } from '@/stores/license';
+import { getAPIReadAccessErrorMessage } from '@/utils/apiAccessError';
import {
GUEST_DRAWER_HISTORY_DEFAULT_RANGE,
@@ -686,9 +687,10 @@ export const GuestDrawerHistory: Component = (props) =>
>
- {hasHistoryPoints()
- ? 'History refresh failed. Showing previously loaded history.'
- : 'Failed to load history data'}
+ {getAPIReadAccessErrorMessage(historyQuery.error()) ||
+ (hasHistoryPoints()
+ ? 'History refresh failed. Showing previously loaded history.'
+ : 'Failed to load history data')}