From ae5df19b621ca1985e2c93dee6042b33cf0c04c8 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Fri, 21 Aug 2026 08:49:05 +0100 Subject: [PATCH] Give release backend tests hosted-runner headroom The internal/api race suite now routinely exceeds the old 20-minute package timeout on hosted runners while passing. Set a governed 30-minute package timeout and 40-minute release job ceiling, pin the relationship with contract tests, and refresh the rc.4 packet with the fixes landed since preparation. --- .github/workflows/create-release.yml | 6 ++- Makefile | 3 +- .../subsystems/deployment-installability.md | 6 +++ docs/releases/RELEASE_NOTES_v6.3.0-rc.4.md | 38 +++++++++++-------- docs/releases/V6_CHANGELOG_v6.3.0-rc.4.md | 13 +++++++ .../installtests/build_release_assets_test.go | 25 ++++++++++++ scripts/tests/test-hot-dev-runtime.sh | 10 +++++ 7 files changed, 84 insertions(+), 17 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 2897cee00..f4da4f6fd 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -328,7 +328,11 @@ jobs: - frontend_checks if: ${{ needs.prepare.outputs.historical_asset_backfill_only != 'true' }} runs-on: ubuntu-24.04 - timeout-minutes: 30 + # The race-enabled internal/api package has measured at 24 minutes on the + # hosted runner. Keep the job ceiling above the Makefile's 30-minute + # per-package timeout so setup and result collection cannot become the + # effective release gate. + timeout-minutes: 40 env: FRONTEND_DIST: frontend-modern/dist steps: diff --git a/Makefile b/Makefile index 257a2b868..13329cfae 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ FRONTEND_DIR := frontend-modern FRONTEND_DIST := $(FRONTEND_DIR)/dist FRONTEND_EMBED_DIR := internal/api/frontend-modern GO_TEST_PACKAGES := ./cmd/... ./internal/... ./pkg/... ./scripts/... ./tests/... +GO_TEST_TIMEOUT ?= 30m # Build everything (including all agent binaries) all: frontend backend build-agents @@ -106,7 +107,7 @@ control-plane: test: @./scripts/ensure_test_assets.sh @echo "Running backend tests..." - go test -race -timeout 20m $$(go list $(GO_TEST_PACKAGES)) + go test -race -timeout $(GO_TEST_TIMEOUT) $$(go list $(GO_TEST_PACKAGES)) # Run integration tests (requires Ollama at OLLAMA_URL or 127.0.0.1:11434) test-integration: diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index c0d059e7d..d73296859 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -505,6 +505,12 @@ upgrade, update, release, or artifact-selection behavior. persistent run directory and remove that bounded scratch directory on exit; a small WSL `/tmp` tmpfs must not turn release qualification into a false product failure. + The canonical backend race gate must likewise budget for hosted-runner + variance without weakening test coverage. `make test` owns a 30-minute + per-package timeout, and the release workflow's backend job owns a larger + 40-minute ceiling so checkout, toolchain setup, and result collection cannot + become a tighter implicit deadline. The release-promotion contract test must + pin both sides of that headroom relationship. The rehearsal diagnostic spec is opt-in by design, so both the hosted rehearsal and its worker profile must set `PULSE_E2E_DIAGNOSTIC=1`; invoking that spec while leaving it skipped is not browser proof. diff --git a/docs/releases/RELEASE_NOTES_v6.3.0-rc.4.md b/docs/releases/RELEASE_NOTES_v6.3.0-rc.4.md index d12cc864e..83ea261f6 100644 --- a/docs/releases/RELEASE_NOTES_v6.3.0-rc.4.md +++ b/docs/releases/RELEASE_NOTES_v6.3.0-rc.4.md @@ -9,24 +9,17 @@ correctness fixes. This candidate adds estate-first platform summaries and search, an operator- visible notification delivery log, a supported least-privilege agent profile, and safer Docker-in-LXC discovery. It also prevents failed settings or AI-state -reads from silently overwriting preserved data. +reads from silently overwriting preserved data, restores distinct alert-history +recurrences, and hardens agent operation on QNAP and hosts with stalled mounts. ## Highlights -- Patrol guides operators through one ranked decision at a time; Actions now - has its own primary review workspace. -- Read-only observers extend Patrol coverage between full model investigations - without granting mutation authority. -- Approved actions gain agent preflight and stable refusal telemetry; large - installations gain compressed APIs and indexed lookups. -- Platform pages now lead with estate totals, status facets, and search that - share the same predicates as their underlying tables. -- Notification settings show the outcome of real delivery attempts instead of - relying on test sends as a proxy for live delivery health. -- Docker-in-LXC discovery is explicitly controlled and backs off against slow - or failing Proxmox hosts instead of creating a probe storm. -- Unified Agent installs can opt into a supported least-privilege profile with - narrowly scoped elevation for the capabilities that require it. +- Patrol guides operators through ranked decisions, read-only observation, and + governed Actions without expanding mutation authority. +- Estate-first search, faster APIs, delivery logs, resilient alert history, and + longer notification retries improve daily operations. +- Unified Agent hardening covers least privilege, Docker-in-LXC, QNAP storage, + stalled mounts, action preflight, and credential repair. ## Added @@ -119,6 +112,21 @@ reads from silently overwriting preserved data. cooldown was recorded and never read, so suppression ended as soon as the measurement window drained and a resource oscillating just under the threshold was never damped at all. +- Per-guest overrides that do not set powered-off severity continue to inherit + the global severity instead of silently being stamped as warning. +- Each alert recurrence now creates its own history row after the preceding + occurrence resolves, while genuine short observation gaps still coalesce. +- Notification delivery retries now cover roughly three minutes by default, so + a destination restarting with monitored infrastructure can recover before + the notification is dead-lettered. +- QNAP installs place the agent runtime and download staging on the persistent + data volume instead of requiring tens of MiB on the RAM-backed root. +- An unreachable network mount no longer freezes the entire host disk collector + or delays agent shutdown; skipped network filesystems are filtered before + usage probes and remaining probes are bounded. +- Unix Repair Authentication commands pin the existing agent ID and hostname, + matching Windows behavior and preventing a repair from creating a suffixed + duplicate identity. ## Security diff --git a/docs/releases/V6_CHANGELOG_v6.3.0-rc.4.md b/docs/releases/V6_CHANGELOG_v6.3.0-rc.4.md index f3359fd16..045ee3a06 100644 --- a/docs/releases/V6_CHANGELOG_v6.3.0-rc.4.md +++ b/docs/releases/V6_CHANGELOG_v6.3.0-rc.4.md @@ -80,6 +80,19 @@ prerelease and rolls back to stable `v6.2.1`._ post-install command enablement, and surfaces failures without host churn. - In-progress Proxmox backups remain in progress rather than being shown as completed. +- Per-guest overrides without an explicit powered-off severity inherit the + global severity rather than being normalized to warning. +- Resolved alert occurrences remain immutable in history, and each later + recurrence receives its own row without weakening short-gap coalescing. +- Notification retries span roughly three minutes by default, allowing a + destination restart to recover before delivery is dead-lettered. +- QNAP installs stage and run the Unified Agent from the persistent data volume + instead of consuming the RAM-backed system root. +- Network filesystems are skipped before disk-usage syscalls by default, and + remaining mount probes are bounded so one stalled mount cannot freeze host + collection or shutdown. +- Unix credential-repair commands retain the existing agent ID and hostname so + the repair converges on the current record instead of creating a duplicate. ## Security diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index e18bff201..5070dea23 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -2341,6 +2341,31 @@ func TestCreateReleasePublishesPrivateProRuntime(t *testing.T) { } } +func TestReleaseBackendRaceGateHasHostedRunnerHeadroom(t *testing.T) { + makefileBytes, err := os.ReadFile(repoFile("Makefile")) + if err != nil { + t.Fatalf("read Makefile: %v", err) + } + if !strings.Contains(string(makefileBytes), "GO_TEST_TIMEOUT ?= 30m") { + t.Fatal("backend race suite must keep a 30-minute per-package timeout for hosted-runner variance") + } + if !strings.Contains(string(makefileBytes), "go test -race -timeout $(GO_TEST_TIMEOUT)") { + t.Fatal("make test must apply the governed backend race-suite timeout") + } + + workflowBytes, err := os.ReadFile(repoFile(".github", "workflows", "create-release.yml")) + if err != nil { + t.Fatalf("read create-release.yml: %v", err) + } + backendJob := workflowJobBlock(t, string(workflowBytes), "backend_tests") + if !strings.Contains(backendJob, "timeout-minutes: 40") { + t.Fatal("release backend job must leave setup and result-collection headroom above the package timeout") + } + if !strings.Contains(backendJob, "run: make test") { + t.Fatal("release backend job must consume the governed Makefile race-suite timeout") + } +} + func TestHelmAgentRuntimePointsAtRealImage(t *testing.T) { // The helm chart's agent.enabled=true workload used to default to // ghcr.io/rcourtman/pulse-agent — an image that was never published. diff --git a/scripts/tests/test-hot-dev-runtime.sh b/scripts/tests/test-hot-dev-runtime.sh index 0c2ed9275..c76e2b579 100755 --- a/scripts/tests/test-hot-dev-runtime.sh +++ b/scripts/tests/test-hot-dev-runtime.sh @@ -5,6 +5,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" HOT_DEV_RUNTIME_LIB="${ROOT_DIR}/scripts/lib/hot-dev-runtime.sh" HOT_DEV="${ROOT_DIR}/scripts/hot-dev.sh" +MAKEFILE="${ROOT_DIR}/Makefile" if [[ ! -f "${HOT_DEV_RUNTIME_LIB}" ]]; then echo "hot-dev-runtime.sh not found at ${HOT_DEV_RUNTIME_LIB}" >&2 @@ -333,6 +334,14 @@ test_go_module_security_dependency_floors() { assert_contains "Go module floor keeps x/sys aligned with security module graph" "${output}" "golang.org/x/sys v0.47.0" } +test_backend_race_suite_keeps_hosted_runner_timeout_headroom() { + local output + output="$(sed -n '1,125p' "${MAKEFILE}")" + + assert_contains "backend race suite exposes the governed package timeout" "${output}" "GO_TEST_TIMEOUT ?= 30m" + assert_contains "make test applies the governed package timeout" "${output}" 'go test -race -timeout $(GO_TEST_TIMEOUT)' +} + source "${HOT_DEV_RUNTIME_LIB}" test_pulse_process_count_handles_zero_matches_under_pipefail test_pulse_process_count_counts_matching_processes @@ -346,6 +355,7 @@ test_hot_dev_lab_agent_mode_enables_lan_and_guest_docker_inventory_defaults test_hot_dev_remembers_explicit_lab_agent_mode_for_later_managed_starts test_hot_dev_browser_urls_distinguish_bind_and_browser_hosts test_go_module_security_dependency_floors +test_backend_race_suite_keeps_hosted_runner_timeout_headroom if (( failures > 0 )); then echo "FAIL: ${failures} hot-dev runtime assertions failed" >&2