From f8a52fcaf53ba1bf90a3caa38342a8b668f8f4a7 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 03:42:08 +0100 Subject: [PATCH] Bound passive resident-memory sampling without inventing readings Keep the sampled counters and process identity behind a read-only deadline. Hanging readers and a missing timeout utility produce an unavailable result, while privacy, PID-reuse and measured-zero controls remain. Preserve the ten-second test assertion and clean up the test-owned shell on timeout. Contract-Neutral: Documentation and copied-recipe verification only; no runtime collection, memory accounting, service lifecycle or API contract changes. Change-source: pulse-maintainer --- docs/TROUBLESHOOTING.md | 62 ++++++++++------- .../public/docs/TROUBLESHOOTING.md | 62 ++++++++++------- .../tests/test_memory_troubleshooting_docs.py | 66 +++++++++++++++++-- 3 files changed, 139 insertions(+), 51 deletions(-) diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index df28fe2b71..3f8e67ef8d 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -415,32 +415,50 @@ authorised to read these counters; a failed read is unavailable, not zero. # systemd / Proxmox LXC: Pulse resident-memory sample ( set -eu - date -u +'%Y-%m-%dT%H:%M:%SZ' - pid=$(systemctl show pulse --property=MainPID --value) - case "$pid" in - ''|0|*[!0-9]*) printf 'No running Pulse PID; sample unavailable.\n' >&2; exit 1 ;; - esac - identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=) - if [ -z "$identity" ]; then - printf 'Process identity unavailable.\n' >&2; exit 1 + if timeout --kill-after=1s 8s bash <<'PULSE_MEMORY_SAMPLE' +set -eu +date -u +'%Y-%m-%dT%H:%M:%SZ' +pid=$(systemctl show pulse --property=MainPID --value) +case "$pid" in + ''|0|*[!0-9]*) printf 'No running Pulse PID; sample unavailable.\n' >&2; exit 1 ;; +esac +identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=) +if [ -z "$identity" ]; then + printf 'Process identity unavailable.\n' >&2; exit 1 +fi +counters=$(awk ' + $1 == "VmRSS:" || $1 == "RssAnon:" || $1 == "RssFile:" || + $1 == "RssShmem:" || $1 == "VmSwap:" { + if (NF != 3 || $2 !~ /^[0-9]+$/ || $3 != "kB" || seen[$1]++) exit 1 + print; fields++ + } + END { if (fields != 5) exit 1 } +' "/proc/$pid/status") +current_pid=$(systemctl show pulse --property=MainPID --value) +current_identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=) +if [ "$pid" != "$current_pid" ] || [ "$identity" != "$current_identity" ]; then + printf 'Pulse changed during collection; discard this sample.\n' >&2; exit 1 +fi +printf 'Pulse process (PID and UTC start): %s\n%s\n' "$identity" "$counters" +PULSE_MEMORY_SAMPLE + then + : + else + status=$? + printf 'Resident-memory sample unavailable; retain the error and stop sampling.\n' >&2 + exit "$status" fi - counters=$(awk ' - $1 == "VmRSS:" || $1 == "RssAnon:" || $1 == "RssFile:" || - $1 == "RssShmem:" || $1 == "VmSwap:" { - if (NF != 3 || $2 !~ /^[0-9]+$/ || $3 != "kB" || seen[$1]++) exit 1 - print; fields++ - } - END { if (fields != 5) exit 1 } - ' "/proc/$pid/status") - current_pid=$(systemctl show pulse --property=MainPID --value) - current_identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=) - if [ "$pid" != "$current_pid" ] || [ "$identity" != "$current_identity" ]; then - printf 'Pulse changed during collection; discard this sample.\n' >&2; exit 1 - fi - printf 'Pulse process (PID and UTC start): %s\n%s\n' "$identity" "$counters" ) ``` +The recipe requires GNU `timeout`: it gives the read-only sampling commands +eight seconds, with a one-second termination grace, rather than leaving a hung +reader running indefinitely. It does not stop or restart Pulse. A missing +utility, timeout or reader error means the sample is unavailable; keep the +failure, and do not remove the deadline or rerun it against an unresponsive +installation. No complete process/counter sample is printed before every check +succeeds; the initial UTC timestamp alone is not a memory result. + Linux labels these values `kB`, meaning 1,024 bytes; divide by 1,024 for MiB. `VmRSS` is resident process memory, split into anonymous (`RssAnon`), file-backed (`RssFile`) and shared-memory (`RssShmem`) pages. `VmSwap` is swapped diff --git a/frontend-modern/public/docs/TROUBLESHOOTING.md b/frontend-modern/public/docs/TROUBLESHOOTING.md index df28fe2b71..3f8e67ef8d 100644 --- a/frontend-modern/public/docs/TROUBLESHOOTING.md +++ b/frontend-modern/public/docs/TROUBLESHOOTING.md @@ -415,32 +415,50 @@ authorised to read these counters; a failed read is unavailable, not zero. # systemd / Proxmox LXC: Pulse resident-memory sample ( set -eu - date -u +'%Y-%m-%dT%H:%M:%SZ' - pid=$(systemctl show pulse --property=MainPID --value) - case "$pid" in - ''|0|*[!0-9]*) printf 'No running Pulse PID; sample unavailable.\n' >&2; exit 1 ;; - esac - identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=) - if [ -z "$identity" ]; then - printf 'Process identity unavailable.\n' >&2; exit 1 + if timeout --kill-after=1s 8s bash <<'PULSE_MEMORY_SAMPLE' +set -eu +date -u +'%Y-%m-%dT%H:%M:%SZ' +pid=$(systemctl show pulse --property=MainPID --value) +case "$pid" in + ''|0|*[!0-9]*) printf 'No running Pulse PID; sample unavailable.\n' >&2; exit 1 ;; +esac +identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=) +if [ -z "$identity" ]; then + printf 'Process identity unavailable.\n' >&2; exit 1 +fi +counters=$(awk ' + $1 == "VmRSS:" || $1 == "RssAnon:" || $1 == "RssFile:" || + $1 == "RssShmem:" || $1 == "VmSwap:" { + if (NF != 3 || $2 !~ /^[0-9]+$/ || $3 != "kB" || seen[$1]++) exit 1 + print; fields++ + } + END { if (fields != 5) exit 1 } +' "/proc/$pid/status") +current_pid=$(systemctl show pulse --property=MainPID --value) +current_identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=) +if [ "$pid" != "$current_pid" ] || [ "$identity" != "$current_identity" ]; then + printf 'Pulse changed during collection; discard this sample.\n' >&2; exit 1 +fi +printf 'Pulse process (PID and UTC start): %s\n%s\n' "$identity" "$counters" +PULSE_MEMORY_SAMPLE + then + : + else + status=$? + printf 'Resident-memory sample unavailable; retain the error and stop sampling.\n' >&2 + exit "$status" fi - counters=$(awk ' - $1 == "VmRSS:" || $1 == "RssAnon:" || $1 == "RssFile:" || - $1 == "RssShmem:" || $1 == "VmSwap:" { - if (NF != 3 || $2 !~ /^[0-9]+$/ || $3 != "kB" || seen[$1]++) exit 1 - print; fields++ - } - END { if (fields != 5) exit 1 } - ' "/proc/$pid/status") - current_pid=$(systemctl show pulse --property=MainPID --value) - current_identity=$(TZ=UTC ps -p "$pid" -o pid=,lstart=) - if [ "$pid" != "$current_pid" ] || [ "$identity" != "$current_identity" ]; then - printf 'Pulse changed during collection; discard this sample.\n' >&2; exit 1 - fi - printf 'Pulse process (PID and UTC start): %s\n%s\n' "$identity" "$counters" ) ``` +The recipe requires GNU `timeout`: it gives the read-only sampling commands +eight seconds, with a one-second termination grace, rather than leaving a hung +reader running indefinitely. It does not stop or restart Pulse. A missing +utility, timeout or reader error means the sample is unavailable; keep the +failure, and do not remove the deadline or rerun it against an unresponsive +installation. No complete process/counter sample is printed before every check +succeeds; the initial UTC timestamp alone is not a memory result. + Linux labels these values `kB`, meaning 1,024 bytes; divide by 1,024 for MiB. `VmRSS` is resident process memory, split into anonymous (`RssAnon`), file-backed (`RssFile`) and shared-memory (`RssShmem`) pages. `VmSwap` is swapped diff --git a/scripts/tests/test_memory_troubleshooting_docs.py b/scripts/tests/test_memory_troubleshooting_docs.py index fd9e76c4ae..1728f7d4df 100644 --- a/scripts/tests/test_memory_troubleshooting_docs.py +++ b/scripts/tests/test_memory_troubleshooting_docs.py @@ -13,9 +13,11 @@ import os from pathlib import Path import re import shutil +import signal import subprocess import sys import tempfile +import time import unittest @@ -39,7 +41,7 @@ def recipe() -> str: READER = '''#!/usr/bin/env python3 -import json, os, pathlib, subprocess, sys +import json, os, pathlib, subprocess, sys, time name = pathlib.Path(sys.argv[0]).name args = sys.argv[1:] root = pathlib.Path(os.environ['RECIPE_FIXTURE']) @@ -48,6 +50,8 @@ previous = [json.loads(line) for line in log.read_text().splitlines()] if log.ex count = sum(call['name'] == name for call in previous) with log.open('a') as stream: stream.write(json.dumps({'name': name, 'args': args}) + '\\n') +if name == os.environ.get('HANG_TOOL'): + time.sleep(30) if name == 'date': assert args == ['-u', '+%Y-%m-%dT%H:%M:%SZ'] print('2026-10-06T00:00:00Z') @@ -69,6 +73,25 @@ else: ''' +def run_owned_recipe(copied: str, env: dict[str, str]) -> subprocess.CompletedProcess: + """Keep the existing ten-second assertion; clean up our shell on every exit.""" + with subprocess.Popen(["bash", "-c", copied], env=env, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True) as process: + try: + stdout, stderr = process.communicate(timeout=10) + except subprocess.TimeoutExpired: + # Never target a service or another test's group. TimeoutExpired + # remains an error, not a passing or empty sample. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate(timeout=5) + raise + return subprocess.CompletedProcess(process.args, process.returncode, stdout, stderr) + + class MemoryTroubleshootingDocsTest(unittest.TestCase): def exercise(self, *, status: str = COUNTERS, copied: str | None = None, **settings: str): @@ -81,9 +104,9 @@ class MemoryTroubleshootingDocsTest(unittest.TestCase): (directory / "status").write_text(status, encoding="utf-8") env = dict(os.environ, PATH=f"{directory}:{os.environ['PATH']}", RECIPE_FIXTURE=str(directory), REAL_AWK=shutil.which("awk"), **settings) - result = subprocess.run(["bash", "-c", copied if copied is not None else recipe()], - env=env, text=True, capture_output=True, timeout=10) - calls = [json.loads(line) for line in (directory / "calls.jsonl").read_text().splitlines()] + result = run_owned_recipe(copied if copied is not None else recipe(), env) + log = directory / "calls.jsonl" + calls = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else [] return result, calls def assert_no_memory_claim(self, result): @@ -167,6 +190,34 @@ class MemoryTroubleshootingDocsTest(unittest.TestCase): self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout.count("0 kB"), 5) + def test_hung_service_reader_is_unavailable_before_the_test_deadline(self): + started = time.monotonic() + result, calls = self.exercise(HANG_TOOL="systemctl") + self.assertLess(time.monotonic() - started, 10) + self.assertEqual(result.returncode, 124, result.stderr) + self.assert_no_memory_claim(result) + self.assertIn("sample unavailable", result.stderr) + self.assertEqual([call["name"] for call in calls], ["date", "systemctl"]) + + def test_hung_counter_reader_never_publishes_identity_or_partial_memory(self): + started = time.monotonic() + result, calls = self.exercise(HANG_TOOL="awk") + self.assertLess(time.monotonic() - started, 10) + self.assertEqual(result.returncode, 124, result.stderr) + self.assert_no_memory_claim(result) + self.assertIn("sample unavailable", result.stderr) + self.assertEqual([call["name"] for call in calls], ["date", "systemctl", "ps", "awk"]) + + def test_missing_deadline_utility_is_not_a_sample_or_an_unbounded_fallback(self): + copied = recipe().replace("timeout --kill-after=1s 8s bash", + "pulse-missing-memory-deadline --kill-after=1s 8s bash") + self.assertNotEqual(copied, recipe()) + result, calls = self.exercise(copied=copied) + self.assertEqual(result.returncode, 127, result.stderr) + self.assert_no_memory_claim(result) + self.assertIn("sample unavailable", result.stderr) + self.assertEqual(calls, []) + @unittest.skipUnless(sys.platform.startswith("linux"), "Linux /proc recipe") def test_real_linux_status_of_an_owned_disposable_process(self): with tempfile.TemporaryDirectory() as temporary: @@ -178,8 +229,7 @@ class MemoryTroubleshootingDocsTest(unittest.TestCase): command.chmod(0o700) env = dict(os.environ, PATH=f"{directory}:{os.environ['PATH']}", RECIPE_CHILD_PID=str(child.pid)) - result = subprocess.run(["bash", "-c", recipe()], env=env, - text=True, capture_output=True, timeout=10) + result = run_owned_recipe(recipe(), env) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("Pulse process (PID and UTC start)", result.stdout) for field in ("VmRSS", "RssAnon", "RssFile", "RssShmem", "VmSwap"): @@ -197,11 +247,13 @@ class MemoryTroubleshootingDocsTest(unittest.TestCase): "1,024 bytes", "approximate and not an atomic snapshot", "not specifically the Go heap", "different run", "do not measure Pulse RSS", "do not assume its PID 1 is Pulse", + "requires GNU `timeout`", "initial UTC timestamp alone", + "do not remove the deadline", "does not stop or restart Pulse", "Do not restart Pulse, drop caches, force garbage collection", "Do not post a heap dump", "existing screenshots remain useful"): self.assertIn(phrase, guide) self.assertNotRegex(recipe(), - r"\b(?:sudo|restart|kill|rm|truncate|sqlite3|curl|printenv|cat)\b" + r"(?