From 3a4a3fd62bb8d36660f6f0756ed8b58338f39d83 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Mon, 7 Sep 2026 09:45:31 +0100 Subject: [PATCH] Preserve native filesystem evidence and Patrol action history Expose confined, identity-bound filesystem observations through the shared resource pipeline so investigations can distinguish an exhausted container mount from unrelated host capacity. Keep unavailable measurements explicit. Isolate alert-history reads from durable writes and reuse one chronological fold across polling. Catch up through bounded durable event IDs so simultaneous readers do not replay every retained snapshot. Retain expired actions when investigation outcomes move back to needs attention, and keep attached Assistant context focused. Record live storage diagnosis, healthy and dependency controls, approved and rejected Docker outcomes, source-bound browser proof and exact test limits. Missing-access continuity, VM dispatch completion and remaining Assistant orchestration defects stay open in the redesign plan. --- .../PATROL_ASSISTANT_CUSTOMER_JOURNEY.md | 318 +++++++++++++++++- docs/release-control/v6/internal/status.json | 20 +- .../v6/internal/subsystems/agent-lifecycle.md | 9 + .../v6/internal/subsystems/ai-runtime.md | 16 + .../v6/internal/subsystems/alerts.md | 24 ++ .../v6/internal/subsystems/api-contracts.md | 9 + .../v6/internal/subsystems/monitoring.md | 25 ++ .../subsystems/patrol-intelligence.md | 13 + .../subsystems/performance-and-scalability.md | 9 + .../v6/internal/subsystems/registry.json | 43 ++- .../internal/subsystems/storage-recovery.md | 8 + .../internal/subsystems/unified-resources.md | 17 + frontend-modern/browser-verification.json | 78 ++--- .../AI/Chat/__tests__/AIChat.test.tsx | 12 + .../src/components/AI/Chat/index.tsx | 14 +- .../src/components/AI/FindingsPanel.tsx | 2 + .../src/components/patrol/ApprovalSection.tsx | 16 +- .../patrol/__tests__/ApprovalSection.test.tsx | 30 +- .../src/types/__tests__/resource.test.ts | 30 ++ frontend-modern/src/types/resource.ts | 17 + internal/ai/chat/agentic_prompt.go | 30 +- internal/ai/chat/service_tooling_test.go | 17 +- internal/ai/tools/data_types.go | 46 +-- internal/ai/tools/filesystem_evidence_test.go | 57 ++++ internal/ai/tools/tools_query.go | 5 +- internal/alerts/event_emission.go | 3 + internal/alerts/eventlog/eventlog.go | 120 +++++-- .../alerts/eventlog/reader_isolation_test.go | 169 ++++++++++ internal/alerts/history_projection.go | 186 ++++++---- .../alerts/history_projection_parity_test.go | 81 +++++ internal/alerts/manager.go | 4 + ..._handlers_investigation_additional_test.go | 56 +++ internal/api/patrol_action_reconciliation.go | 2 + internal/dockeragent/agent.go | 6 + internal/dockeragent/collect.go | 1 + .../dockeragent/collect_tmpfs_live_test.go | 16 + internal/dockeragent/container_filesystems.go | 67 ++++ .../dockeragent/container_filesystems_test.go | 78 +++++ internal/filesystemprobe/observer.go | 144 ++++++++ internal/filesystemprobe/observer_linux.go | 133 ++++++++ .../filesystemprobe/observer_linux_test.go | 83 +++++ internal/filesystemprobe/observer_other.go | 14 + internal/filesystemprobe/observer_test.go | 81 +++++ internal/models/converters.go | 2 + internal/models/deepcopy.go | 2 + internal/models/models.go | 2 + internal/models/models_frontend.go | 2 + .../docker_filesystem_evidence_test.go | 68 ++++ internal/monitoring/monitor_agents.go | 2 + internal/unifiedresources/adapters.go | 2 + internal/unifiedresources/clone.go | 2 + internal/unifiedresources/store.go | 1 + .../store_history_index_test.go | 61 ++++ internal/unifiedresources/types.go | 2 + internal/unifiedresources/views.go | 8 + pkg/agents/docker/report.go | 64 ++-- pkg/agents/filesystem/report.go | 53 +++ pkg/agents/filesystem/report_test.go | 40 +++ .../canonical_completion_guard_test.py | 6 + .../release_control/subsystem_lookup_test.py | 4 + 60 files changed, 2201 insertions(+), 229 deletions(-) create mode 100644 internal/ai/tools/filesystem_evidence_test.go create mode 100644 internal/alerts/eventlog/reader_isolation_test.go create mode 100644 internal/dockeragent/container_filesystems.go create mode 100644 internal/dockeragent/container_filesystems_test.go create mode 100644 internal/filesystemprobe/observer.go create mode 100644 internal/filesystemprobe/observer_linux.go create mode 100644 internal/filesystemprobe/observer_linux_test.go create mode 100644 internal/filesystemprobe/observer_other.go create mode 100644 internal/filesystemprobe/observer_test.go create mode 100644 internal/monitoring/docker_filesystem_evidence_test.go create mode 100644 internal/unifiedresources/store_history_index_test.go create mode 100644 pkg/agents/filesystem/report.go create mode 100644 pkg/agents/filesystem/report_test.go diff --git a/docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md b/docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md index 390b1ad7b..1e180397b 100644 --- a/docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md +++ b/docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md @@ -64,8 +64,8 @@ reproduction evidence, not a representative customer success rate. | 1. Product contract and baseline | Map the current loop and sources of judgment. Record telemetry populations and gaps. | Every identified decision has an owner. Activity is not labelled usefulness. | Complete for this redesign scope. Contract, ownership decisions and baseline limits are recorded. | | 2. Shared evidence | Preserve canonical risk reasons and SMART counters, source/time semantics and history across tools/turns. | Regression tests preserve unknown versus zero and all canonical evidence. Real responses can inspect the same facts as the product. | Implemented and qualified for the named shared-evidence defects. Canonical disk detail, risk and cadence pass real data-path proof. Affected package and concurrency checks pass. Integrated CI later exposed remaining query and allocation regressions. The final bounded query-reuse correction passes complete selected exact-base worker comparisons and full metrics/database and focused race checks. Final landing CI passed and PRs #1928 and #1929 merged. Real-model interpretation failures remain tracked in step 5. | | 3. Diagnostic orchestration | Correct proposal-as-proof. Audit triage budgets, unmatched-signal evaluation, assessment completion and investigation cutoffs. | No code-written causal conclusion. No quality inferred from tool, flag or finding counts. Each retained pass has an objective reason. Safety boundaries and incomplete outcomes remain explicit. | Proposal promotion and capture inference were removed in c5d2f56dda. Commit 668af3fe6b removes investigation success-call floors, checkpoint instructions and generic call-count wrap-up rules. The detection slice removes contextless follow-up passes, flag/report-count policy and first-finding completion modes. Full chat and AI suites, focused API and conversation race tests pass. Real-model/action outcome qualification remains open. | -| 4. Issue through verified outcome | Follow existing issue/investigation/action records into Assistant, approval, execution and independent readback. | Accepted proposal is visibly distinct from execution and verification. Rejected or unsupported actions do not become success. Uncertainty can survive an action proposal. | Existing foundation, full journey qualification pending. | -| 5. Ground-truth qualification and landing | Extend existing qualification tooling only where necessary. Exercise healthy/unhealthy, dependency, missing-access, storage/backup and approved/rejected action cases. Inspect the final browser journey at desktop and narrow widths. | Record exact source/model/permissions, evidence, decisions, faults/misses, latency and verification. Fix in-scope failures, pass appropriate proofs and land scoped commits. | Partial. Regression, controlled browser and live collector evidence are recorded below. Real-model diagnosis, linked approval/action outcomes and installed collector qualification remain open. | +| 4. Issue through verified outcome | Follow existing issue/investigation/action records into Assistant, approval, execution and independent readback. | Accepted proposal is visibly distinct from execution and verification. Rejected or unsupported actions do not become success. Uncertainty can survive an action proposal. | Approved/rejected Docker execution paths passed named live oracles. Expired-state and attached-context browser repairs passed the named current-runtime matrices. Missing-command-access remains a recorded contract gap. | +| 5. Ground-truth qualification and landing | Extend existing qualification tooling only where necessary. Exercise healthy/unhealthy, dependency, missing-access, storage/backup and approved/rejected action cases. Inspect the final browser journey at desktop and narrow widths. | Record exact source/model/permissions, evidence, decisions, faults/misses, latency and verification. Fix in-scope failures, pass appropriate proofs and land scoped commits. | Partial. Installed native filesystem evidence and real storage diagnosis now pass semantic review, alongside healthy/dependency and approved/rejected Docker cases. Current-source browser and shared history race proofs pass. Landing, missing-access continuity and VM lifecycle qualification remain open. | Use one shared runtime and the existing qualification runner, not a second product intelligence engine or a new parallel lifecycle. Preserve independent @@ -2255,3 +2255,317 @@ intermittent bootstrap connection screen. The complete final matrix passed after removing redundant immediate navigations from the proof driver, without claiming a bootstrap fix. Source bindings are in `frontend-modern/browser-verification.json`. + +### Next implementation: resource filesystem observations + +PR #1935 merged as `560dbf314c4fc3744f52aa4e5a6a204cafe3aa7d`. +Its API race suite and other CI checks passed. The log-level-parser benchmark +failed twice in CI, at +10.04 and +10.23 percent. Ten alternating exact-base worker samples did not +reproduce that regression (5.449 ns versus 5.357 ns, p=.436), with unchanged +parser source and unchanged thresholds. This is not a new product-readiness claim. +The repeated CI observation remains open. No further retry was requested. + +The next slice is in progress and is not qualified. Its required work is: + +1. Collect filesystem capacity, available blocks and finite inode inventory at + the resource's actual mountpoints. Preserve observation time and native + source. Keep configuration, image layers, filesystem capacity and resource + quotas distinct. A failed read carries no numeric usage payload. +2. Bind native Linux reads to the exact inspected container process and its + runtime cgroup, confine path resolution to the process root, and revalidate + container identity after collection. Remote or unattested process namespaces + remain unavailable. No container binary or general command permission is used. +3. Bound stalled filesystem reads without accumulating repeated kernel calls. + Carry observations through report ingestion, snapshot cloning, the unified + resource and both shared model query paths. Replace old observations on a + later failed or absent report instead of presenting stale usage as current. +4. Review accumulated investigation instructions that prescribe restart or peer + investigation. Preserve objective tool/proposal/permission contracts while + leaving diagnosis and evidence selection to the model. +5. Qualify actual storage diagnosis and recovery against the bounded tmpfs + oracle, then check healthy, dependency, missing-access and action regressions. + Review the complete causal claims and recommendations, independently of the + existing lexical score. Backup and independent-environment coverage remain + separate open requirements. + +Browser matrix for the final source and current build: at 1440, 900 and 390 by +1000, inspect `/patrol` finding details, investigation messages with measured +and unavailable filesystem evidence, Assistant handoff and its final diagnosis, +and the adjacent container details. Exercise loading/error presentation, +disclosures, keyboard controls, scrolling/overflow, dismissal/focus return and +reload. The approved/rejected action-history journey must remain intact. No +browser or real-model pass is claimed by this implementation plan. + +Native observer race tests passed in 1.031s and the shared report tests in +1.010s on pulse-dev with Go 1.26.8. Test binaries cross-compile for Linux arm64, +Linux 386 and Darwin arm64. Targeted collection identity, report ingestion, +stale-observation replacement and both model-query projections passed. + +The first real collector fixture ran as the unprivileged worker account. Docker +API access succeeded, but opening the root-owned process namespace failed with +permission denied. No numeric usage was emitted and independent cleanup passed. +The separate positive test ran the precompiled collector fixture as UID 0, +matching Tower's native development-agent privilege, without changing any +installed service or permissions. It measured the exact 8,388,608-byte tmpfs: +8,380,416 bytes available at baseline, zero under the independent pressure fault, +and 8,380,416 after recovery. Container health changed healthy/unhealthy/healthy. +The restored-baseline sample remained healthy with 8,372,224 bytes available. +Teardown, second cleanup no-op and inventory restoration all passed. + +The native fixture binary SHA-256 was +`c275a530f332237b1c1fff06e31a0194929f9929edcba4aa09423da168869b1a`. +The full root-native log SHA-256 is +`83b62f1cab9b5a6d597447b672c964bee26bd9187801dd80b0069c1251ba3e5f`, +retained at workspace `tmp/patrol-filesystem-evidence/live/native-live.log`. +This establishes native collection on that worker privilege profile. It does +not establish an installed homelab agent, model diagnosis or wider deployment +support. The prompt change and final query description still need their final +regressions and all runtime/browser/model qualification above. + +The first installed runtime attempt exposed two additional qualification defects. +The native observer was incorrectly gated by `CollectDiskMetrics`, although the +unified agent disables that flag to avoid expensive image-layer sizing. Native +mountpoint observations now run independently, with a regression for that exact +configuration. No installed-agent diagnosis pass is inferred from the earlier +collector-only proof. + +The real backend also stalled agent reports and resource reads while the Patrol +attention page reconstructed alert history. A goroutine capture showed history +occupying the event store's only database connection, a durable lifecycle append +waiting for it while holding the alert manager lock, and monitoring/API reads +waiting for that lock. The existing query sorted full event snapshots before +selecting each page. The owning event store now separates read-only WAL readers +from its serialized writer and selects chronological page IDs before loading +snapshots. The actual database query plans show the former temporary sort and +the replacement time-index scan plus primary-key payload reads. The new disk +regression holds a read snapshot open while requiring a durable write to finish, +rejects a mutation through the reader and verifies committed data from a new +snapshot. Event-store race tests and alert history/recovery tests passed. + +Run `q-20260906-220538-71a19f74` stopped at preflight because the runner had +retained monitor mode. The corrected approval-mode run +`q-20260906-220615-f15cef02` detected the intended unhealthy resource, but its +detection took 163.581 seconds during the database contention. It was cancelled +before investigation qualification completed. Cancellation interrupted the +recovery measurement, so no recovery pass is claimed. The independent final +teardown removed both owned containers and their network, the second cleanup +was a no-op and the original inventory was restored. Tower's original runtime, +persistent binary and token were restored, the temporary command token revoked +and its absence verified, and this transaction's backups/candidate removed. +The production agent retained PID 752388 throughout. The first browser attempts +reached the slow connection state and are failures, not final browser proof. +The native-agent wiring and event-store fixes still require fresh runtime, +model, browser and delivery qualification. + +A diagnostic startup stack subsequently identified a distinct canonical resource +history scan. Incident catch-up called global `GetRecentChanges` while the +resource store lacked a leading observation-time index. That query occupied the +store connection while initial registry change emission waited. The actual +database has the canonical non-null `observed_at` schema. Its query plan showed +a table scan and temporary sort. The store now creates the missing global +chronological index, with an existing-database upgrade/query-plan/ordering test. +This is a second fix requiring a new backend build and live qualification. +Legacy timestamp fallback query plans remain outside this canonical-schema +proof and need separate migration qualification. + +Run `q-20260906-224128-a59214cb` passed preflight and independently established +the disposable storage fault, then was cancelled during collection convergence. +The local hot-development watcher completed an earlier build and replaced the +backend during the run, despite the later verification lock. The replacement +used Go 1.27.1 and had SHA-256 +`6eb9b4e30e7fa4a11d71accbfe9b9922133e1d885b5910aec0e267a54e37fd7f`, +which differs from the pinned worker artifact. This run reached no model turn +and supplies no diagnosis evidence. Both owned containers and their network +were removed, repeat cleanup was a no-op and the original inventory was restored. +Final qualification must wait for the pinned artifact and verify that no earlier +build is still able to replace it. + +The second Tower transaction was fully rolled back: original development binary +and token restored, temporary command token revoked with absence verified, owned +backups/candidate removed, and production PID 752388 unchanged. The restored +development PID was 2398103. The rollback also persisted paused Patrol and monitor +autonomy through the authenticated API. + +Review of the replacement history query found that its unconditional time-index +hint also forced retained-history scans for replay watermarks and individual +alerts. A read-only empty-tail probe of the actual event database returned no +rows in both cases, but the forced time scan took 0.14939 seconds versus 0.00006 +seconds for a primary-key seek in the native SQLite observer. These timings are +a diagnostic comparison, not a Go runtime performance qualification. The owning +page query now seeks durable IDs for an explicit replay watermark and uses the +alert-specific chronological index for an alert bound. Full chronological walks +retain their time-index paging. An EXPLAIN regression uses the actual composed +query and rejects full retained-history scans for both filtered cases. The +queued backend build is superseded by this source change and requires fresh +event-store/history proof before a final artifact can be qualified. + +The focused actual-query plan regression passed locally on Go 1.27.1 in 0.791 +seconds. This was a single non-race test for quick feedback against the warm +development cache. It does not replace the pending pinned Go 1.26.8 worker race, +history and artifact qualification. + +The final worker event-store race suite passed in 3.981 seconds and the targeted +alert-history/recovery checks passed in 1.274 seconds. The resulting Darwin Pro +artifact SHA-256 is +`bef2a3cce5a18f0d2c14badc4ba14f3ffffe454777ccd83f9b641a893b80ea38`. +After deployment, the running executable inode matched that artifact. The global +resource-history index was present. Authenticated preflight confirmed Gemini +3.8 Flash in all three selectors, read-only control and paused Patrol. Resource +listing took 0.647 seconds. The first attention summary took 21.265 seconds and +a repeat took 0.968 seconds. The first-read delay remains a measured startup +latency limit, not a qualified latency improvement. + +The completion audit also confirmed that the existing Proxmox bulk lifecycle +evaluation denies approval and proves planning only. VM execution and independent +verification remain required by the candidate-lane contract. The explicitly +disposable FreeBSD agent-lab VM 110 on delly is stopped and available as a bounded +fixture. Pulse identifies it as `vm-e8cc8be82e584c58`, but reports the node command +agent disconnected. The existing separate development unit is inactive and +disabled, while the production unit is active. No VM lifecycle pass is claimed +from inventory, planning or the Docker action results. + +Final browser review must also cover an expired unexecuted action: Patrol's +investigation, durable finding record and Assistant context must say needs +attention, while the linked action retains its precise expired state. Original +model prose remains retained evidence. An attached finding suppresses generic +empty-chat starters and unrelated recent sessions. Exercise attached and cleared +context, empty and existing conversations, close/reopen, keyboard focus, reload, +and expired/completed/rejected action links at 1440, 900 and 390 pixels wide. +The prior storage browser pass exposed both defects and does not qualify these +repairs. + +Installed native filesystem case `q-20260906-232102-5316b904` passed the runner and +independent semantic review. The model queried the actual 8,388,608-byte tmpfs +with zero free/available bytes, then obtained ENOSPC logs and attributed the +unhealthy container to that mount. Its recommendation distinguished temporary +restart of ephemeral storage from long-term size/retention changes. The case +recorded a proposal only. Recovery proof came from independent filler removal, +which restored 8,347,648 available bytes and healthy status. The healthy neighbour +remained unaffected and teardown/inventory restoration passed. Detection took +31.121 seconds of model time, collection took 24.825 seconds, and the runner +recorded 107.490 seconds end-to-end. The scorecard's US$0.01331325 is Watch cost, +not a claim about complete investigation billing. + +Fresh current-prompt cases `q-20260907-073028-5b5ab1f1` (dependency) and +`q-20260907-073212-807e363b` (healthy control) passed with cleanup. Semantic review +confirmed that investigation identified the stopped upstream dependency and +proposed starting that dependency, while healthy Watch produced no finding. +Approved action `q-20260907-073416-1d33864c` retained exact finding/investigation/ +action and plan-hash linkage and reached independently verified healthy recovery. +The uncertain diagnostic prose is retained, so executed recovery is not presented +as proof that the model knew the original cause. + +Rejected restart `q-20260907-073601-2d6b13ae` passed with an authoritative rejected +action, no execution and unchanged inventory after teardown. Tower transaction +r3 restored its exact original runtime and persistent binaries and token, +revoked the temporary command token and removed its backups. Production agent +PID 752388 remained unchanged. + +Missing-command-access replay `q-20260907-074158-110e8868` safely detected the +unhealthy container, retained needs-attention and did not execute. It failed +the approved-remediation scenario, as expected for unavailable execution, and +is not a remediation pass. Investigation reported the absent agent and unknown +internal cause, but still captured a restart proposal that the canonical broker +subsequently refused. The stored submission error says no action was created. +This exposes a remaining contract gap: action availability needs to reach the +model before its conclusion, so it can explain the unavailable recovery path +without relying on a later orchestration failure. Do not describe this as a +qualified seamless missing-access journey. Fixture teardown and independent +safety/recovery oracles passed. + +VM110 qualification on 2026-09-07 remains incomplete. The existing development +unit pointed to an old control-plane address and had an invalid token. Each +attempt restored the original binary/token and inactive, disabled unit. A +temporary runtime-only service override pointed the bounded test at this Mac, +and fresh command registration then passed. Read-only Assistant correctly +withheld control. Under temporarily enabled approval-required control, Gemini +created exact-target start plan `act_72dde1ef1b7b67884e66f6c4608faae8` and +explicitly said it had not executed. The first plan attempt returned SQLITE_BUSY +and the model retried successfully. The test then approved the canonical plan. +Execution returned HTTP500 with the durable action left executing and its +attempt receipt-pending. No lifecycle pass is claimed. VM110 was independently +confirmed stopped after cleanup, the original agent and read-only control were +restored, all temporary tokens and runtime overrides were removed, and production +agent PID1565 stayed unchanged. The owning durable action execution/reconciliation +contract needs investigation before another lifecycle qualification. This is +a product failure found by qualification, not evidence of successful recovery. + +The Mac hot-dev verification lock from the previous session had expired before +the morning source edits. A local automatic build replaced the previous runtime +at 07:33:58 UTC. Morning Docker case receipts still prove their observed +behavior but must not be attributed to the earlier pinned Darwin artifact. A +fresh live lock and explicit worker artifact deployment now bind final browser +qualification. The last restart also exposed a multi-minute bootstrap delay. +Healthy request timings after startup do not qualify that bootstrap latency. + +The stranded VM qualification action was closed through the canonical operator +force-fail endpoint after independent stopped-state and inactive-agent checks. +The audit retains an inconclusive failed outcome. It was not deleted, retried +or converted into successful execution. + +Final browser artifact `e223f46accfc35102d671dd21793b5508bd64cd1dac4784c0d41c0f50b8f55b2` +contains native filesystem/history changes, expired-outcome hydration and +action-presence-based Patrol history, plus contextual Assistant starter +visibility. The pinned worker API regression passes, including expired +hydration without invented verification. Frontend type-check and 417 tests +covering Assistant, FindingsPanel and ApprovalSection pass. The artifact is +source-bound to the final runtime changes, while subsequent edits add tests and +qualification documentation only. Full staged-hook verification remains the +last landing check. + +Final Playwright interaction matrices passed at 1440x1000, 900x1000 and +390x1000 on `/patrol` and exact expired/completed/rejected `/actions?action=...` +links. Pixel review covered settled dialog placement, nested tool-output +scrolling, native filesystem provenance/counters, retained action history, +independent verification, keyboard disclosures, reload, Escape and explicit +close. Attached empty Assistant context hides unrelated starters. New session +clears context and restores welcome/recent sessions. Loading an existing +session keeps the transcript accessible. Source hashes and exact routes are in +`frontend-modern/browser-verification.json`. Browser proof covers these named +changes, not whole-product readiness. + +Existing-session inspection exposed another canonical orchestration residual: +`internal/ai/chat/agentic.go` replaces retained assistant prose with an internal +FSM verification instruction even though it withholds that instruction from +the live callback. A planned `pulse_control` action triggered this write gate +despite no execution. The next orchestration slice must use actual action +execution/verification facts and keep internal provider instructions out of +customer transcripts. This is not qualified by the context-visibility fix. + +A subsequent live check on 2026-09-07 invalidated the earlier assumption that +history paging alone was sufficient. `/api/ai/patrol/attention/summary` exceeded +30 seconds. A goroutine capture showed 107 full-history walkers, with two reading +retained snapshot payloads and the others waiting for the bounded reader pool. +The same complete history fold was being repeated independently for every poll. +The current change shares a derived chronological fold at a durable event-ID +boundary, reads only its new tail, and rebuilds after retention, delayed older +events or store replacement. Tombstones and live overlays retain their original +semantics. New regression, rebuild and current-runtime browser proof are required +before this slice lands. The previous artifact and browser receipt remain +historical evidence rather than proof of this additional change. + +The follow-up history change passed the full eventlog race suite (3.827 seconds), +focused history/projection/migration/parity race tests (4.048 seconds), and then +the complete alerts package race suite (23.785 seconds) on the non-root worker. +The replacement Darwin Pro artifact is +`1f1f71d2fd77b89f43a980d1e990a4010d2ec467502551555d5ab702f1aa71e6`. +Source hashes bind all six changed alerts files and the unchanged final frontend. + +After installation, the first attention summary completed in 4.695 seconds. +Eighteen authenticated requests with at most six concurrent callers all returned +HTTP200. The first six took 4.457 to 4.514 seconds, the remaining twelve took +0.319 to 0.705 seconds, and the final stack check found zero history walkers. +This is bounded functional recovery evidence, not a performance benchmark. +The interactive Mac load was above five and wider load qualification remains a +worker responsibility. No fault, provider call or infrastructure action was +needed for this concurrency check. + +Both complete Playwright scripts passed again against that replacement artifact: +`/patrol` storage evidence, expired action and Assistant attached/new/existing +states, plus completed and rejected action handoffs. Viewports were 1440x1000, +900x1000 and 390x1000. Current pixels were reviewed for native measurement output, +nested scrolling, action state, independent verification, drawer content, policy +expansion, reachable controls and dialog placement. Reload, keyboard expansion, +Escape and close checks passed. The current browser receipt supersedes the earlier +artifact binding for this slice. Missing-access continuity, VM dispatch and +Assistant orchestration failures remain separate open qualification defects. diff --git a/docs/release-control/v6/internal/status.json b/docs/release-control/v6/internal/status.json index 357352464..f2de43d66 100644 --- a/docs/release-control/v6/internal/status.json +++ b/docs/release-control/v6/internal/status.json @@ -10201,7 +10201,7 @@ }, { "id": "patrol-assistant-customer-outcome-qualification", - "summary": "The redesign goal remains open. The plan, historical receipts and exact source bindings are in docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md. Patrol owns investigation and Assistant continues the same issue. Observations, hypotheses, accepted proposals, execution and independently verified outcomes remain distinct. The recorded baseline of 127 paid installations, 71 Patrol-enabled, 23 with Assistant calls and fourteen verified resolutions from one installation does not establish representative customer success. Schema17 outcome/provider/cost fields had no adoption. Shared evidence/history/risk and removal of proposal-as-proof and proxy completion policy landed through PR1928/1929. PR1934 merged canonical tool/transcript identity. PR1935 contains subsequent canonical history, measurement-presence, command-connectivity, tmpfs context, exact Gemini pricing, retained broker errors and result-bearing transcript corrections. Enterprise broker refusal handling merged in PR22. Real Gemini Watch, healthy-control and client-to-dependency cases passed. With explicit authority for a temporary current development agent and scoped token, approved restart q-20260906-193013-12d25545 passed exact plan/origin binding, explicit approval, execution and independent recovery. Rejected restart q-20260906-193207-7095dad5 passed exact rejection and independent non-execution. All test resources were removed, original development binaries/token restored, temporary tokens revoked, scheduled Patrol paused in monitor mode and production agent PID preserved. Storage collection failure was traced to qualification pagination beyond the API page cap of 100 and corrected with full-package regression proof. Installed storage q-20260906-193613-556ef23d passed the existing scorecard but FAILED semantic diagnosis review: the model falsely attributed an exhausted container tmpfs to Tower/array capacity despite available mount configuration. A capacity read required approval. Storage remains unqualified, and lexical/identifier scoring must not be treated as causal correctness. Next storage work needs canonical authorized filesystem evidence and independent semantic review without benchmark-specific diagnosis rules or weakened command approval. Real browser review additionally exposed an embedded investigation record left fix_queued after verified recovery and hidden action history on resolved findings. Current action reconciliation refreshes the durable record through the canonical builder, preserves prose/evidence/rollback, repairs missed transitions without duplicate publication, and retains completed action history and resolved Assistant context. Final worker regressions and source-bound runtime/browser proof pass at 1440, 900 and 390 widths, including completed/rejected history and read-only Assistant handoff. The exact staged hook and PR1935 landing gate integration. Other residuals include canonical incident-memory listing/aliases and failed-read propagation, unsupported filters, typed compatibility ID lookup, legacy direction availability, Docker-host history, responsive mount details, backup coverage and broader model qualification. Independent volunteered Pro environments remain a wider-readiness gate. Autonomous modes remain unqualified. The earlier Claude refusal was not retried.", + "summary": "The redesign goal remains open. The executable plan and historical source-bound receipts are in docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md. Product ownership, evidence/history/risk corrections and removal of proposal-as-proof and proxy completion rules have landed through PR1928/1929/1934/1935, with enterprise broker refusal handling in PR22. The telemetry baseline of 127 paid installations, 71 Patrol-enabled and 23 with Assistant usage does not establish linked customer effectiveness. Native container filesystem observations now preserve exact bytes, inodes, source, observation time and explicit unavailable errors through collection, ingestion, canonical resources and model tools. The installed native collector and real Gemini storage case q-20260906-232102-5316b904 passed independent semantic review: the model identified the exhausted 8 MiB container tmpfs and ENOSPC, rather than misattributing host array capacity. Recovery was independently observed after fixture filler removal, not after executing the proposal. Fresh healthy, dependency, approved-restart and rejected-restart cases passed their named oracles and cleanup. Tower's original binaries/token were restored, temporary tokens revoked and production agent preserved. Shared event-history WAL isolation, bounded indexed paging and a global resource-history index pass targeted race/query-plan proof. Browser review found expired actions still marked queued, generic starters beside attached Assistant context, and linked action history hidden by outcome-based visibility. Repairs pass final source-bound browser qualification for linked completed/rejected/expired actions and attached/cleared Assistant context at 1440, 900 and 390 pixels. The current slice is awaiting staged-hook verification and landing. Missing-access q-20260907-074158-110e8868 safely refused execution and retained unknown cause, but remains an unqualified journey: proposal capture precedes canonical availability validation, leaving the model unable to incorporate the refusal in its conclusion. The next canonical work must expose current action availability to model decisions and preserve definitive submission truth through the shared journey. Disposable VM110 received an exact Gemini-generated plan and explicit canonical approval. Execution returned HTTP500 and left receipt-pending durable state. The stopped VM, original inactive agent and read-only control were restored, temporary tokens revoked, and the action was closed through operator force-fail with inconclusive evidence. VM execution remains unqualified and its durable dispatch/reconciliation path is the next required root investigation. An additional reproduced orchestration gap replaces retained Assistant prose with an internal verification instruction and treats planned controls as executed writes. The canonical action lifecycle, not generic tool-use state, must own execution/verification truth. Residuals include responsive mount details, canonical incident-memory listing/aliases and failed-read propagation, unsupported filters, typed compatibility ID lookup, legacy direction availability, Docker-host history, backup coverage, broader model qualification and startup latency. Autonomous modes remain unqualified. Independent volunteered Pro environments remain a wider-readiness gate. The explicit Claude subscription refusal was not retried. Do not mark the goal or candidate complete from a scorecard or this one homelab. A later current-runtime attention-summary request timed out after 30 seconds. The captured stacks showed 107 independent full-history walkers. The shared incremental history fold now passes durable replay boundary, retention and concurrent-reader regressions, full alerts/eventlog race suites, eighteen bounded live attention requests and repeated current-runtime browser matrices. Exact staged landing checks remain pending.", "owner": "project-owner", "status": "planned", "recorded_at": "2026-09-05", @@ -10213,6 +10213,7 @@ ], "subsystem_ids": [ "ai-runtime", + "alerts", "api-contracts", "frontend-primitives", "monitoring", @@ -10385,6 +10386,7 @@ ], "subsystem_ids": [ "ai-runtime", + "alerts", "api-contracts", "frontend-primitives", "monitoring", @@ -10399,7 +10401,21 @@ ] } ], - "work_claims": [], + "work_claims": [ + { + "id": "patrol-filesystem-evidence-coverage-gap-patrol-assistant-customer-outcome-qualification", + "agent_id": "patrol-filesystem-evidence", + "summary": "Finish filesystem evidence, shared expired-action outcome repair and contextual Assistant qualification.", + "target_id": "v6-product-lane-expansion", + "claimed_at": "2026-09-07T07:33:08Z", + "heartbeat_at": "2026-09-07T07:33:08Z", + "expires_at": "2026-09-07T09:33:08Z", + "work_item": { + "kind": "coverage-gap", + "id": "patrol-assistant-customer-outcome-qualification" + } + } + ], "open_decisions": [], "source_of_truth_file": "docs/release-control/v6/internal/SOURCE_OF_TRUTH.md", "resolved_decisions": [ diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 2567589c5..2988204d4 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -15,6 +15,15 @@ ## Purpose +Optional container filesystem observations add no command or lifecycle +authority. The collector's existing disk-metrics option controls them. Native +reads require the exact local container process identity and current namespace +access. Inaccessible root-owned namespaces remain unavailable to unprivileged +collectors even when Docker API reads work. No failure widens privileges or +falls back to container exec. A post-read inspect rejects observations from a +restarted/replaced process. These counters do not modify token scopes, helper +operations, enrollment, removal, configuration or agent identity. + Docker mount reports include tmpfs configuration from `HostConfig.Tmpfs` through the existing optional mount array. This adds collection evidence only. It does not change admission, enrollment, execution permissions or agent diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 73258760a..fc0acb27b 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -25,6 +25,22 @@ that same result. Successful reads retain their content and execution provenance ## Purpose +Patrol investigation's mode contract states its non-interactive read/proposal +boundary and asks for diagnosis, supporting evidence and remaining uncertainty. +It does not prescribe restarting an unhealthy container or require a peer +investigation before admitting uncertainty. The model chooses useful evidence +and justified remediation. An advertised action recommendation is submitted +through patrol_propose_action. A proposal is neither execution nor proof of +causal correctness, and core policy retains execution authority. + +The shared app-container query returns native filesystem observations from the +canonical resource or its typed view. Configuration mounts and measured +filesystem usage remain distinct. Filesystem capacity is not a resource quota. +Unavailable observations omit the usage payload rather than supplying zero. +The query contract describes those Pulse-specific semantics without a storage +diagnosis recipe. Real-model requalification is required after this prompt and +evidence change and remains recorded in the customer-journey qualification. + Action reconciliation refreshes the durable product investigation record from the authoritative session/action even when the finding outcome already matches. The same builder owns initial completion and later refresh. Completion replaces diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index acb62df48..297254d6b 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -434,6 +434,30 @@ default construction path still restores. ## Shared Boundaries +History reads must not occupy the connection used to commit alert lifecycle +transitions. Disk-backed event logs use a bounded read-only WAL pool alongside +the serialized durable writer. An open history snapshot cannot block a durable +append or acquire mutation authority. In-memory test stores retain their single +connection because their private database is connection-local. + +Chronological replay selects bounded event IDs in time-index order before +loading snapshots and details. Pages preserve occurrence-time and ID ordering, +type filters, durable-ID watermarks and caller limits without sorting all +retained payloads for every page. A full history read remains complete, and +history reconstruction must not truncate silently at the public query limit. + +Normal history polling shares one manager-owned chronological fold and advances +it only through newly committed event IDs. Its inclusive replay boundary keeps +concurrent appends out of the current pass. The event log remains authoritative: +retention removal advances a revision in the same transaction, late historical +events force chronological reconstruction, and replacing or disabling the store +discards the fold. Clear-history tombstones still remove preceding occurrences. +Live active-alert overlays and returned objects are independent clones. Explicit +Since queries preserve their existing event-window semantics without evicting +the normal full-history fold. Repeated attention requests must not replay all +retained snapshots independently or accumulate behind the history connection pool. + + PBS node-status availability is distinct from connectivity. While a connected PBS carries monitoring-owned `NodeMetricsUnavailable` evidence, CPU and memory evaluation must not treat zero-valued placeholders as recovery. Existing diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index c611f6b40..bcfd119b3 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -479,6 +479,15 @@ enums locally. ## Shared Boundaries +Canonical Patrol action hydration maps an expired action to `needs_attention` +and preserves its exact action reference, original investigation prose and +retained finding evidence. An expired proposal creates no execution or +verification evidence. Hydration repairs the investigation and durable finding +record from the same authoritative audit and does not publish duplicate updates. +The API regression for expired hydration covers missed callbacks and unchanged +historical evidence. The linked action remains discoverable after its outcome +stops being queued. + ### Independent Docker update readback `dockerContainerUpdateExecutionResult` must not promote replacement-ID equality diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 7d242a91b..6a59869d1 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -17,6 +17,31 @@ ## Purpose +Container filesystem observations are native, resource-scoped reads. The shared +`pkg/agents/filesystem` contract keeps measurement time, mountpoint, source, +filesystem type and optional usage together. Unavailable reads have an error +and no usage. Capacity/free/available counters describe the filesystem visible +there, not a container quota, image layer or host-wide capacity diagnosis. +Finite inode inventory is optional independently of byte capacity. + +The Docker collector requires a local Unix runtime endpoint, an exact full +container ID, a matching process cgroup and a stable inspected PID/start time. +Linux reads through the held process-root descriptor using confined openat2 +resolution and fstatfs, without executing container programs. A remote daemon, +unattested or inaccessible namespace, unsupported kernel or changed process +produces unavailable evidence. This path does not grant ptrace, root, helper +or command privileges. Ordinary root-owned containers may therefore remain +unavailable to an unprivileged collector even when its Docker socket is usable. + +One native probe per container may remain in flight, with a bounded wait and +an observer-wide limit. Timeout does not start replacement syscalls or reuse +late measurements. Report ingestion and snapshot conversion clone all nested +usage fields. A later failed or absent observation replaces prior capacity. +Proof surfaces are the filesystemprobe native/timeout tests, shared report +tests, container collection boundary tests, ingestion regression and the +opt-in owned tmpfs collector fixture. Installed-agent and model qualification +are tracked separately in PATROL_ASSISTANT_CUSTOMER_JOURNEY.md. + Docker mount collection preserves both native `Mounts` records and entries reported only in `HostConfig.Tmpfs`. Existing reported destinations remain authoritative. Additional tmpfs destinations are ordered deterministically, diff --git a/docs/release-control/v6/internal/subsystems/patrol-intelligence.md b/docs/release-control/v6/internal/subsystems/patrol-intelligence.md index 37c512dea..ebf19b5e0 100644 --- a/docs/release-control/v6/internal/subsystems/patrol-intelligence.md +++ b/docs/release-control/v6/internal/subsystems/patrol-intelligence.md @@ -2622,3 +2622,16 @@ reasoning and real remediation in `docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md`. The repeatable browser proof is `scripts/check-patrol-assistant-journey.mjs`. A passing scripted response does not establish a useful customer outcome or model qualification. + +### Expired action continuity (2026-09-07) + +An expired, unexecuted canonical action maps the investigation outcome to +`needs_attention`. It creates no recovery verification evidence. The durable +finding record and investigation retain the expired action identity and +original model evidence. Findings and the inline action history render a linked +action from that identity independently of the outcome vocabulary. Changing +from queued to needs-attention must not hide its Actions link or Assistant +handoff. The attached Assistant context suppresses unrelated empty-chat +starters and recent sessions. Starting a new conversation clears that context +and restores the ordinary welcome surface. Regression and current-browser +qualification are recorded in the customer-journey qualification document. diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 9556fd2d8..ab7b9ae9d 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -15,6 +15,15 @@ ## Purpose +The PR #1935 log-level parser benchmark remains an unresolved environment-bound +observation. Two CI comparisons on unchanged parser source report +10.04 and ++10.23 percent for the empty-string case, with stable base/candidate binaries. +Ten alternating exact-base worker samples report 5.449 versus 5.357 ns (p=.436) +and unchanged allocations. Identical source does not invalidate the CI timing, +and the worker result does not turn it into a pass. No further rerun or threshold +relaxation is justified by that reproduction. The implementation merged as +560dbf314c4fc3744f52aa4e5a6a204cafe3aa7d, independently of this open observation. + The Docker/app-container history families `dockercontainer` and `docker` use separate physical `.observed` series for new disk capacity and block-I/O measurements. Older disk series lack the required presence/capacity semantics diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index f04bef0d7..dfebfb8d5 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -1619,6 +1619,7 @@ "internal/config/host_continuity_test.go", "internal/models/metrics_types_test.go", "internal/monitoring/availability_probe_agent_test.go", + "internal/monitoring/docker_filesystem_evidence_test.go", "internal/monitoring/docker_metric_presence_test.go", "internal/monitoring/monitor_host_agent_removal_lifecycle_test.go", "internal/monitoring/monitor_host_agents_test.go", @@ -1663,6 +1664,7 @@ "internal/agenttarget/config_test.go", "internal/agenttls/config_test.go", "internal/dockeragent/agent_internal_test.go", + "internal/dockeragent/container_filesystems_test.go", "internal/dockeragent/multi_target_test.go", "internal/dockeragent/report_size_test.go", "internal/kubernetesagent/agent_new_test.go", @@ -2757,6 +2759,8 @@ "internal/alerts/config/evaluation_windows_test.go", "internal/alerts/config_validation_test.go", "internal/alerts/eventlog/eventlog_snapshot_test.go", + "internal/alerts/eventlog/eventlog_test.go", + "internal/alerts/eventlog/reader_isolation_test.go", "internal/alerts/external_probe_test.go", "internal/alerts/filter_evaluation_test.go", "internal/alerts/guest_snapshot_test.go", @@ -5831,12 +5835,14 @@ "contract": "docs/release-control/v6/internal/subsystems/monitoring.md", "owned_prefixes": [ "internal/availabilityprobe/", + "internal/filesystemprobe/", "internal/fleethealth/", "internal/maintenancesentinel/", "internal/monitoring/", "internal/servicediscovery/", "internal/storagehealth/", "internal/truenas/", + "pkg/agents/filesystem/", "pkg/diskinventory/", "pkg/pbs/" ], @@ -5847,6 +5853,7 @@ "internal/config/guest_metadata.go", "internal/config/host_continuity.go", "internal/dockeragent/collect.go", + "internal/dockeragent/container_filesystems.go", "internal/dockeragent/docker_client.go", "internal/dockeragent/swarm.go", "internal/kubernetesagent/agent.go", @@ -5972,6 +5979,7 @@ "test_prefixes": [], "exact_files": [ "internal/config/host_continuity_test.go", + "internal/monitoring/docker_filesystem_evidence_test.go", "internal/monitoring/docker_metric_presence_test.go", "internal/monitoring/issue1485_unraid_lifecycle_test.go", "internal/monitoring/issue1595_collection_trust_test.go", @@ -6095,7 +6103,9 @@ "internal/dockeragent/agent_cpu_test.go", "internal/dockeragent/agent_internal_test.go", "internal/dockeragent/blockio_presence_test.go", + "internal/dockeragent/collect_tmpfs_live_test.go", "internal/dockeragent/collect_tmpfs_test.go", + "internal/dockeragent/container_filesystems_test.go", "internal/dockeragent/swarm_coverage_test.go" ] }, @@ -6155,6 +6165,7 @@ "internal/models/issue1639_pbs_collision_test.go", "internal/models/metrics_types_test.go", "internal/models/state_host_test.go", + "internal/monitoring/docker_filesystem_evidence_test.go", "internal/monitoring/docker_metric_presence_test.go", "internal/monitoring/issue1595_collection_trust_test.go", "internal/monitoring/monitor_full_coverage_test.go", @@ -6163,7 +6174,8 @@ "internal/monitoring/monitor_package_updates_test.go", "internal/unifiedresources/adapter_coverage_test.go", "internal/unifiedresources/registry_test.go", - "pkg/agents/docker/blockio_presence_test.go" + "pkg/agents/docker/blockio_presence_test.go", + "pkg/agents/filesystem/report_test.go" ] }, { @@ -6400,6 +6412,27 @@ "internal/monitoring/truenas_poller_test.go", "internal/unifiedresources/code_standards_test.go" ] + }, + { + "id": "filesystem-observations", + "label": "Native namespace-bound filesystem observation and report proof", + "match_prefixes": [ + "internal/filesystemprobe/", + "pkg/agents/filesystem/" + ], + "match_files": [ + "internal/dockeragent/container_filesystems.go" + ], + "allow_same_subsystem_tests": false, + "test_prefixes": [], + "exact_files": [ + "internal/dockeragent/collect_tmpfs_live_test.go", + "internal/dockeragent/container_filesystems_test.go", + "internal/filesystemprobe/observer_linux_test.go", + "internal/filesystemprobe/observer_test.go", + "internal/monitoring/docker_filesystem_evidence_test.go", + "pkg/agents/filesystem/report_test.go" + ] } ], "match_files": null @@ -8077,7 +8110,9 @@ "test_prefixes": [], "exact_files": [ "frontend-modern/src/utils/__tests__/resourceIdentity.test.ts", + "internal/ai/tools/filesystem_evidence_test.go", "internal/hostagent/issue1595_sas_collection_test.go", + "internal/monitoring/docker_filesystem_evidence_test.go", "internal/monitoring/issue1595_collection_trust_test.go", "internal/truenas/contract_test.go", "internal/unifiedresources/availability_projection_test.go", @@ -8108,6 +8143,8 @@ "test_prefixes": [], "exact_files": [ "frontend-modern/src/stores/__tests__/websocket-unified.test.ts", + "internal/ai/tools/filesystem_evidence_test.go", + "internal/monitoring/docker_filesystem_evidence_test.go", "internal/monitoring/issue1595_collection_trust_test.go", "internal/unifiedresources/adapter_coverage_test.go", "internal/unifiedresources/adapters_test.go", @@ -8519,6 +8556,7 @@ "internal/unifiedresources/resolved_host_set_test.go", "internal/unifiedresources/resource_operator_state_policy_test.go", "internal/unifiedresources/snapshot_source_filter_test.go", + "internal/unifiedresources/store_history_index_test.go", "internal/unifiedresources/store_test.go" ] }, @@ -8571,6 +8609,8 @@ "allow_same_subsystem_tests": false, "test_prefixes": [], "exact_files": [ + "internal/ai/tools/filesystem_evidence_test.go", + "internal/monitoring/docker_filesystem_evidence_test.go", "internal/unifiedresources/availability_link_test.go", "internal/unifiedresources/canonical_id_pins_test.go", "internal/unifiedresources/clone_test.go", @@ -8585,6 +8625,7 @@ "internal/unifiedresources/resolved_host_set_test.go", "internal/unifiedresources/resource_operator_state_policy_test.go", "internal/unifiedresources/snapshot_source_filter_test.go", + "internal/unifiedresources/store_history_index_test.go", "internal/unifiedresources/store_test.go" ] } diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 65aff7bc4..3e9821818 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -21,6 +21,14 @@ ## Purpose +Container filesystem evidence is a read-only observation at a named resource +mountpoint. It grants no storage mutation, recovery or host-capacity authority. +A full container tmpfs does not establish host-array exhaustion. Unknown byte +or inode inventory remains unavailable, and no image-layer count substitutes +for observed filesystem capacity. Shared queries preserve native source and +time for model interpretation. Qualification must compare the diagnosis with +the independent storage fault, not merely match resource IDs or ENOSPC terms. + Container image-layer sizes do not establish filesystem capacity. Docker resource metrics omit that invalid ratio, and retained queries exclude legacy ambiguous disk observations while preserving new valid provider measurements. diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index fe099ec83..55ae8585f 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -1,5 +1,13 @@ # Unified Resources Contract +Canonical cross-resource history reads must have an index on observation time. +Incident reconstruction requests recent changes across the estate, so indexes +whose first column is a resource ID, kind or source do not satisfy that access +path. Existing databases acquire the global chronological index on open. +Qualification checks the query plan and chronological limits after reopening a +database that predates the index. A bounded recent-history request must not scan +and sort the complete canonical change table while startup and ingestion wait. + ## Contract Metadata ```json @@ -15,6 +23,15 @@ ## Purpose +Canonical Docker resource metadata carries the shared filesystem observation +contract unchanged. Adapters, retained resource clones and typed views own +their nested usage values. Per-mount filesystem capacity never populates a +container-wide disk percentage or quota, and unavailable observations never +retain earlier numeric usage. The source-specific Docker facet remains the +owner of these observations. Shared model queries project the same observation +objects for both Patrol and Assistant. Ingestion and model-query filesystem +regressions cover measured zero, unavailable reads and replacement of old data. + Action review distinguishes the recorded plan from live or executed facts. The shared decision packet labels its state and expiry as planning-time evidence, including when opened from a resolved Patrol finding. Potential blast radius diff --git a/frontend-modern/browser-verification.json b/frontend-modern/browser-verification.json index 133aba2ea..71ee2dd7c 100644 --- a/frontend-modern/browser-verification.json +++ b/frontend-modern/browser-verification.json @@ -1,40 +1,25 @@ { "version": 1, - "base_sha": "3853124a391ed451f75402f07951c0142e6b5ad8", - "verified_at": "2026-09-06T20:40:07.804772Z", + "base_sha": "560dbf314c4fc3744f52aa4e5a6a204cafe3aa7d", + "verified_at": "2026-09-07T08:40:47Z", "result": "passed", "changed_paths": [ - "frontend-modern/src/components/DemoBanner.tsx" + "frontend-modern/src/components/AI/Chat/index.tsx", + "frontend-modern/src/components/AI/FindingsPanel.tsx", + "frontend-modern/src/components/patrol/ApprovalSection.tsx", + "frontend-modern/src/types/resource.ts" ], "content_sha256": { - "frontend-modern/src/components/DemoBanner.tsx": "13fd8dea552d97f1df866438f7ba38eec01f25907dc68a0bec672e1793c3fc97" + "frontend-modern/src/components/AI/Chat/index.tsx": "a31459ed9eeda0e96a192c0b6b471746c1a6da70cd8c68ab75b833ca763071f8", + "frontend-modern/src/components/AI/FindingsPanel.tsx": "92f1fd73740cb71c868c2d7054323d43f93ebc53dbb5cacdc1f828c5c2f2a868", + "frontend-modern/src/components/patrol/ApprovalSection.tsx": "b04e131f5a82d538f37aa9cb3aadaf0f7303266cfc318ff04a42a8f7f6a4f6ba", + "frontend-modern/src/types/resource.ts": "54217a487ce295d433a9e016bf655ab8572a903383ae0516fdd52e5004524ea7" }, - "backend_content_sha256": { - "internal/agentcapabilities/transcript.go": "356c4ca201470407988ff9b2c1fb848619ed38e9d8db844e7390adce0f93ec19", - "internal/ai/chat/types.go": "1f624daf7e511eb971e2d81b787dcd72b2a82d0e0ecd775aa4e580c59d915382", - "internal/ai/service.go": "25dca2a70a985e8ab07443444a9f05f01a569c62ff7bc068a0b587d500831de6", - "internal/api/chat_service_adapter.go": "6d0ab14456b1901c5020a408de95796ece1aa8d1057dd58737ea2637777d6ccb", - "internal/ai/cost/pricing.go": "7b64bc881311ee0a7c1c8a319fcd162974f5a307ced1679e614ff20ac1ac532c", - "internal/ai/tools/tools_query.go": "3e074b204a8c8c4f8b66eaf147c269a2e57739bfc69d03ea36ee8c47cf8b908a", - "internal/ai/tools/tools_propose.go": "43d720c78a010b72f53f7e4e9e1b7b2a763e7f1ac555edb921b7cdca3e82fd51", - "internal/ai/patrol_findings.go": "d5eeb386f025cca338ac328b1d4ec7a2bf51150023454e2b61670f196fcf0356", - "internal/api/ai_handlers.go": "f8c9b24dc684346da4bddab066540fd43ca4999f5fd379c4dc34b5293f78394c", - "internal/api/patrol_action_reconciliation.go": "bc5da1a8050b94271dcdc88841a0ce3329e1773bd01c8746068391a71740ffa2", - "internal/monitoring/monitor.go": "63c8ef4867c07c96b4cd7c4316b5b8646f11b7e9e43a5233471956f76f33d553", - "internal/monitoring/system_alerts.go": "53ed4f02784636363da273883415331066d8b2484ca50e6b506e69990abb149d" - }, - "enterprise_base_sha": "3d9f4e3051d38027355a2a1f36b8c7f672a09b65", - "enterprise_content_sha256": { - "internal/investigation/orchestrator.go": "d56fd512dc47f8a2453559e89863da5d24dffc0a5977abb8f1ea7a82655cd9e8" - }, - "binary_sha256": "0c19f9b9265eff2214e79ab6b2e4a6b19a5645eb467c994c44af70b3d6593317", "routes": [ - "/qualification (isolated Overview component on :5199)", - "/patrol (Activity, All history)", - "/actions?action=act_dcc3b52e5451810e49466daf9a6fccb0", - "/actions?action=act_ee0b736f0430e472e896a456ba3cb6eb", - "Pulse Assistant contextual panel from resolved Patrol findings", - "/qualification (isolated DemoBanner component on :5198)" + "/patrol", + "/actions?action=act_185eebf5dfc24652e6332881dbe94f6c", + "/actions?action=act_7687850d214e9c5f7038a436c6cc2ff7", + "/actions?action=act_e5e1f06da3dc42215a97360731d7e36b" ], "viewports": [ { @@ -51,23 +36,26 @@ } ], "states": [ - "Incoming merged alert delivery diagnosis ordering: hold older request, add alert to start newer request, render current notifications-disabled state, release older ready response, verify current state and both cards remain. Existing Patrol and login proof is retained in the prior committed receipt and runtime source is unchanged.", - "Real persisted approved/verified and rejected findings remain reviewable after resolution. Durable investigation outcome agrees with authoritative action, with original prose retained. No stale Fix Queued status. Exact action links remain available.", - "Completed and Rejected action headers, State when planned and Plan expiry copy, inert settled action controls, independently verified recovery and explicit unavailable rollback. Policy, evidence and delivery disclosures expand and collapse.", - "Assistant handoff shows the exact finding with completed/rejected action context and Chat: Read-only. Browser provider readiness POST is deliberately blocked, so its visible route error is a rendering check and does not retest the provider. No prompt is submitted.", - "Incoming main DemoBanner install link: visible in demo policy, absent outside demo, link hover/focus, exact external setup destination opened in a new tab with noopener/noreferrer, keyboard dismissal and reload persistence. Real backend mock mode remains off. External destination content is intercepted because only navigation is under test." + "Activity with All findings including resolved records", + "Expired action with needs-attention investigation and retained linked action", + "Completed action with independent verification", + "Rejected action without execution controls", + "Expanded investigation thread with raw filesystem evidence and nested output scroll", + "Assistant empty conversation with attached finding", + "New Assistant conversation with context cleared and ordinary starters restored", + "Existing Assistant conversation loaded from recent sessions", + "Reloaded exact action review" ], "interactions": [ - "scripts/check-alert-diagnosis-ordering.mjs passed at 1440, 900 and 390 by 1000. Actual pixels inspected at desktop and narrow sizes. This is scripted component evidence, not proof of installed delivery or recipient receipt. Screenshots in /tmp/pulse-alert-diagnosis-ordering/.", - "Final Pro binary and final frontend content exercised in Playwright at 1440, 900 and 390 by 1000. Keyboard open/review, safety disclosure, exact action navigation, direct deep-link reload, policy/evidence/delivery disclosure keyboard toggles, Escape and close-button dismissal, review focus return where retained, Assistant open/close, scrolling and page overflow checks. Desktop/intermediate/phone pixels inspected including deepest evidence and Assistant overlay.", - "Private artifacts: tmp/patrol-gemini-38/live-action-browser/. Final complete matrix passed after removing redundant back-to-back full-page navigations from the proof driver. Earlier proof attempts hit an intermittent bootstrap connection screen. No bootstrap fix or general availability claim is made. API writes blocked except login.", - "After integrating main eef4ea21e73aedaa69380574b1fdf4d9dbdee3c8, repeated the complete real Patrol/Actions/Assistant matrix on binary 0c19f9b9265eff2214e79ab6b2e4a6b19a5645eb467c994c44af70b3d6593317. All three widths passed and pixels were reinspected. Incoming DemoBanner separately passed at the same widths using an isolated Vite cache and actual project styling. Prior frontend hashes remain byte-identical and are retained above. Merged monitoring regressions, API action tests, 37 component tests and type-check pass." + "Keyboard focus and Enter open the finding review and investigation thread", + "Expand tool results and scroll inside the native filesystem observation output", + "Discuss with Assistant retains the finding and expired action context without unrelated starters", + "Start new Assistant session clears context and restores welcome and recent sessions", + "Load existing session and inspect its transcript", + "Follow exact Actions links, inspect planned state and recorded independent verification", + "Expand policy evidence, evidence details and delivery identifiers with keyboard", + "Reload exact action URLs, close with Escape and the explicit close button", + "Inspect desktop, intermediate and narrow placement, nested scrolling, footer reachability and document overflow" ], - "prior_frontend_content_sha256": { - "frontend-modern/src/components/AI/FindingsPanel.tsx": "f506a26757b4c0ea3adf3f77af10214bfd31578b7122d3904a9b3a7272d1e146", - "frontend-modern/src/components/patrol/ApprovalSection.tsx": "6a18d67d5d3d0a8335589bdf199c340eb4775aea3442dc094971d7f924a0dcc5", - "frontend-modern/src/features/actions/ActionDecisionPacket.tsx": "2f1fd68ec333e7f9e8792d74ba8d7e6a2e95755c82b9c7121d847469a31f931b", - "frontend-modern/src/features/actions/ActionReviewDialog.tsx": "49e12cfd44686bd657ddddfb167c5956ec693b6f5d45d9d43c1e3a2e858a6c7f", - "frontend-modern/src/features/alerts/useAlertOverviewState.ts": "64d0b891e7ad228e8590da859dc25e825b6164c8cf76a01983a219d6cd079b23" - } + "notes": "Real bundled development runtime SHA256 1f1f71d2fd77b89f43a980d1e990a4010d2ec467502551555d5ab702f1aa71e6. Playwright scripts and screenshots are in workspace tmp/patrol-filesystem-evidence/{storage-browser,action-browser}. Final pixels were reviewed after transitions settled. Browser proof covers the named changes, not full product readiness. Independent qualification still fails missing-access continuity and VM execution. An existing Assistant verification-policy message leaking into historical transcript is retained as a separate canonical orchestration gap. Mock mode is off. Browser scripts allow only login and selected-provider readiness POSTs. Repeated after the shared incremental alert-history change. Eighteen bounded concurrent attention-summary requests completed and the final goroutine check found no history walkers. This functional check is not a workstation performance benchmark." } diff --git a/frontend-modern/src/components/AI/Chat/__tests__/AIChat.test.tsx b/frontend-modern/src/components/AI/Chat/__tests__/AIChat.test.tsx index ce78df075..6928e7768 100644 --- a/frontend-modern/src/components/AI/Chat/__tests__/AIChat.test.tsx +++ b/frontend-modern/src/components/AI/Chat/__tests__/AIChat.test.tsx @@ -32,6 +32,8 @@ const { onUseModelRoute?: (modelId: string, messageId?: string) => void; queuedFollowUps?: QueuedFollowUp[]; queuedFollowUpsPaused?: boolean; + onSuggestedPrompt?: (prompt: string) => void; + recentSessions?: unknown[]; onEditQueuedFollowUp?: (id: string) => void; onCancelQueuedFollowUp?: (id: string) => void; }> = []; @@ -330,6 +332,8 @@ vi.mock('../ChatMessages', () => ({ onUseModelRoute?: (modelId: string, messageId?: string) => void; queuedFollowUps?: QueuedFollowUp[]; queuedFollowUpsPaused?: boolean; + onSuggestedPrompt?: (prompt: string) => void; + recentSessions?: unknown[]; onEditQueuedFollowUp?: (id: string) => void; onCancelQueuedFollowUp?: (id: string) => void; }) => { @@ -1765,6 +1769,9 @@ describe('AIChat', () => { renderChat(); expect(screen.getByLabelText('Assistant context')).toBeInTheDocument(); + const messagesProps = mockChatMessagesProps.at(-1); + expect(messagesProps?.onSuggestedPrompt).toBeUndefined(); + expect(messagesProps?.recentSessions).toEqual([]); expect(screen.queryByTestId('assistant-workflow-starters')).not.toBeInTheDocument(); expect(screen.getByText('Pulse Patrol')).toBeInTheDocument(); expect(screen.getByText('High CPU usage on web-server')).toBeInTheDocument(); @@ -1776,6 +1783,11 @@ describe('AIChat', () => { expect(screen.queryByText('systemctl restart workload.service')).not.toBeInTheDocument(); }); + it('offers ordinary conversation starters when no finding context is attached', () => { + renderChat(); + expect(mockChatMessagesProps.at(-1)?.onSuggestedPrompt).toBeTypeOf('function'); + }); + it('renders safe briefing actions as links when a route is attached', () => { mockAiChatStore.context = { autonomousMode: false, diff --git a/frontend-modern/src/components/AI/Chat/index.tsx b/frontend-modern/src/components/AI/Chat/index.tsx index 7482f70bb..b3a45eb8c 100644 --- a/frontend-modern/src/components/AI/Chat/index.tsx +++ b/frontend-modern/src/components/AI/Chat/index.tsx @@ -5006,15 +5006,21 @@ export const AIChat: Component = (props) => { onUseModelRoute={switchToModelRoute} queuedFollowUps={chat.queuedFollowUps()} queuedFollowUpsPaused={chat.queuedFollowUpsPaused()} - onSuggestedPrompt={(prompt) => { - void chat.sendMessage(prompt, undefined, undefined); - }} + onSuggestedPrompt={ + contextBriefing() + ? undefined + : (prompt) => { + void chat.sendMessage(prompt, undefined, undefined); + } + } onEditQueuedFollowUp={editQueuedFollowUp} onCancelQueuedFollowUp={(id) => { chat.cancelQueuedFollowUp(id); focusComposer(); }} - recentSessions={selectQuickResumeSessions(sessions(), chat.sessionId())} + recentSessions={ + contextBriefing() ? [] : selectQuickResumeSessions(sessions(), chat.sessionId()) + } onLoadSession={handleLoadSession} /> diff --git a/frontend-modern/src/components/AI/FindingsPanel.tsx b/frontend-modern/src/components/AI/FindingsPanel.tsx index f788613e2..693870b94 100644 --- a/frontend-modern/src/components/AI/FindingsPanel.tsx +++ b/frontend-modern/src/components/AI/FindingsPanel.tsx @@ -2070,6 +2070,7 @@ export const FindingsPanel: Component = (props) => { {/* Inline Approval Section (replaces manual approval JSX) */} = (props) => { = (props) => { const [investigation] = createResource( - () => ({ findingId: props.findingId, outcome: props.investigationOutcome }), - async ({ findingId, outcome }) => { - if (!outcome || !FIX_RELATED_OUTCOMES.has(outcome)) return null; + () => ({ + findingId: props.findingId, + outcome: props.investigationOutcome, + hasAction: props.hasAction, + }), + async ({ findingId, outcome, hasAction }) => { + if (!hasAction && (!outcome || !FIX_RELATED_OUTCOMES.has(outcome))) return null; try { return await AIAPI.getInvestigation(findingId); } catch { @@ -114,7 +119,10 @@ export const ApprovalSection: Component = (props) => { } }); const shouldShow = createMemo(() => - Boolean(props.investigationOutcome && FIX_RELATED_OUTCOMES.has(props.investigationOutcome)), + Boolean( + props.hasAction || + (props.investigationOutcome && FIX_RELATED_OUTCOMES.has(props.investigationOutcome)), + ), ); const handleDiscuss = (event: Event) => { diff --git a/frontend-modern/src/components/patrol/__tests__/ApprovalSection.test.tsx b/frontend-modern/src/components/patrol/__tests__/ApprovalSection.test.tsx index e04bef736..95ad8d254 100644 --- a/frontend-modern/src/components/patrol/__tests__/ApprovalSection.test.tsx +++ b/frontend-modern/src/components/patrol/__tests__/ApprovalSection.test.tsx @@ -67,7 +67,11 @@ describe('ApprovalSection typed action handoff', () => { window.history.replaceState({}, '', '/'); }); - const renderSection = (investigationOutcome: string, findingStatus = 'active') => + const renderSection = ( + investigationOutcome: string, + findingStatus = 'active', + hasAction = false, + ) => render(() => ( { )} @@ -138,6 +143,29 @@ describe('ApprovalSection typed action handoff', () => { ); }); + it('retains an expired action and Assistant handoff when the issue needs attention', async () => { + getInvestigationMock.mockResolvedValue({ + ...investigation(actionReference('expired')), + outcome: 'needs_attention', + }); + + renderSection('needs_attention', 'active', true); + + expect(await screen.findByRole('link', { name: /view outcome in actions/i })).toHaveAttribute( + 'href', + '/actions?action=act-1', + ); + expect(screen.getByText('Expired')).toBeInTheDocument(); + expect(screen.queryByText('Outcome verified')).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: /discuss with assistant/i })); + expect(openMock).toHaveBeenCalledWith( + expect.objectContaining({ + handoffContext: expect.stringContaining('expired'), + autonomousMode: false, + }), + ); + }); + it('fails closed when only a legacy investigation artifact remains', async () => { getInvestigationMock.mockResolvedValue({ ...investigation(), diff --git a/frontend-modern/src/types/__tests__/resource.test.ts b/frontend-modern/src/types/__tests__/resource.test.ts index 5ed8dfcc1..1da67b54a 100644 --- a/frontend-modern/src/types/__tests__/resource.test.ts +++ b/frontend-modern/src/types/__tests__/resource.test.ts @@ -44,6 +44,36 @@ function createResource(overrides: Partial = {}): Resource { } describe('Resource Type Guards', () => { + it('keeps mount exhaustion and unavailable observations separate from aggregate disk usage', () => { + const resource = createResource({ + type: 'app-container', + disk: { current: 20 }, + docker: { + filesystems: [ + { + mountpoint: '/cache', + source: 'linux-proc-root-statfs', + observedAt: '2026-09-07T08:00:00Z', + type: 'tmpfs', + usage: { capacityBytes: 8388608, freeBytes: 0, availableBytes: 0 }, + }, + { + mountpoint: '/restricted', + source: 'linux-proc-root-statfs', + observedAt: '2026-09-07T08:00:00Z', + error: 'namespace access unavailable', + }, + ], + }, + }); + + expect(getDiskPercent(resource)).toBe(20); + const observations = resource.docker?.filesystems; + expect(observations?.[0].usage?.availableBytes).toBe(0); + expect(observations?.[1].usage).toBeUndefined(); + expect(observations?.[1].error).toBe('namespace access unavailable'); + }); + it('retains Docker helper collection completeness on the host facet', () => { const resource = createResource({ type: 'docker-host', diff --git a/frontend-modern/src/types/resource.ts b/frontend-modern/src/types/resource.ts index 8fe58ef6e..cb60a9329 100644 --- a/frontend-modern/src/types/resource.ts +++ b/frontend-modern/src/types/resource.ts @@ -787,6 +787,22 @@ export interface ResourceVirtualMachineMeta { vcpus?: number; } +// Native filesystem counters at a resource mountpoint. These are not resource +// quotas. Failed observations omit usage, while measured zero stays numeric. +export interface FilesystemObservation { + mountpoint: string; + source: string; + observedAt: string; + type?: string; + usage?: { + capacityBytes: number; + freeBytes: number; + availableBytes: number; + inodes?: { capacity: number; free: number }; + }; + error?: string; +} + // Docker runtime, container, and Swarm service projection emitted by the // canonical adapter. Host resources use the runtime fields on the Docker // platform hosts table; `app-container` resources use the container fields @@ -881,6 +897,7 @@ export interface ResourceDockerMeta { mode?: string; rw?: boolean; }>; + filesystems?: FilesystemObservation[]; networks?: Array<{ name?: string; ipv4?: string; diff --git a/internal/ai/chat/agentic_prompt.go b/internal/ai/chat/agentic_prompt.go index e973ddcbf..02bac7579 100644 --- a/internal/ai/chat/agentic_prompt.go +++ b/internal/ai/chat/agentic_prompt.go @@ -31,29 +31,13 @@ state-changing action; such calls will be blocked.` modeContext = ` EXECUTION MODE: Patrol investigation This is a non-interactive investigation of one finding. You cannot ask the user questions or -directly change infrastructure. Gather evidence with read-only tools and conclude with your -diagnosis. Tool function names are exact: call only a top-level name from the advertised tool -manifest. Values inside a tool's action or operation schema are arguments to that tool, never -function names of their own. If your diagnosis concludes that an advertised remediation is safe -and supported by the evidence, call patrol_propose_action before the final summary; that governed proposal is not -execution or approval. An unknown root cause does not by itself rule out a reversible, advertised -initial remediation when the evidence confirms the operational symptom and supports a bounded -reason for trying it. In particular, a running but currently unhealthy app container supports a -governed restart proposal when restart is advertised and the evidence reveals no restart-specific -hazard; empty logs, approval-blocked deeper inspection, or unknown root cause are not by themselves -reasons to withhold that proposal. The proposal hands the exact decision to core policy or the -operator. This does not permit an early symptom-only proposal: before proposing on the symptom -resource or concluding that root cause is unknown, use available canonical query, discovery, or -topology evidence to test at least one plausible causal peer or dependency whenever a cross-resource -cause remains plausible. Only after that test may empty logs or blocked deeper inspection support a -bounded symptom-resource proposal. Reconcile the actual resource entries returned by later topology, -query, or discovery calls before concluding that no peer or dependency is implicated. A result saying -that no Docker services exist covers Swarm services only and never rules out ordinary Docker container -peers or dependencies; inspect the topology container entries instead. If your Recommendation or Conclusion tells the operator to try, consider, -or perform an advertised remediation, you must call patrol_propose_action for that exact action; -never leave it only as prose. Core policy independently decides whether it may execute. If the -evidence does not support any advertised remediation, state the uncertainty and conclude without -proposing. Every direct state-changing call will be blocked.` +change infrastructure. Use the available read-only tools to investigate, then conclude with +your diagnosis, supporting evidence and remaining uncertainty. Decide which evidence is useful +and whether an advertised remediation is justified. +If you recommend an advertised remediation, call patrol_propose_action for that action before +your final summary. A proposal records your recommendation for core policy or the operator to +handle. It is not approval, execution, verification or proof of the diagnosis. If you cannot +justify a remediation, conclude without a proposal.` case tools.ProfileInteractiveAssistant: fallthrough default: diff --git a/internal/ai/chat/service_tooling_test.go b/internal/ai/chat/service_tooling_test.go index 6b263b4c2..a8a2a0827 100644 --- a/internal/ai/chat/service_tooling_test.go +++ b/internal/ai/chat/service_tooling_test.go @@ -1040,20 +1040,9 @@ func TestExecutionProfilePromptModes(t *testing.T) { if !strings.Contains(prompt, "EXECUTION MODE: Patrol investigation") { t.Fatalf("investigation profile prompt wrong: %q", prompt) } - if !strings.Contains(prompt, "call patrol_propose_action") || - !strings.Contains(prompt, "you must call patrol_propose_action for that exact action") || - !strings.Contains(prompt, "Core policy independently decides whether it may execute") || - !strings.Contains(prompt, "unknown root cause does not by itself rule out") || - !strings.Contains(prompt, "running but currently unhealthy app container") || - !strings.Contains(prompt, "approval-blocked deeper inspection") || - !strings.Contains(prompt, "hands the exact decision to core policy") || - !strings.Contains(prompt, "does not permit an early symptom-only proposal") || - !strings.Contains(prompt, "plausible causal peer or dependency") || - !strings.Contains(prompt, "Reconcile the actual resource entries returned by later topology") || - !strings.Contains(prompt, "no Docker services exist covers Swarm services only") || - !strings.Contains(prompt, "Tool function names are exact") || - !strings.Contains(prompt, "action or operation schema are arguments") { - t.Fatalf("investigation prompt must turn supported remediation into a governed proposal, got %q", prompt) + if strings.Contains(prompt, "EXECUTION MODE: Autonomous") || + strings.Contains(prompt, "EXECUTION MODE: Controlled") { + t.Fatalf("investigation inherited interactive execution authority: %q", prompt) } } diff --git a/internal/ai/tools/data_types.go b/internal/ai/tools/data_types.go index 8344ec73d..31f14762b 100644 --- a/internal/ai/tools/data_types.go +++ b/internal/ai/tools/data_types.go @@ -4,6 +4,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" ) // GovernedResourceMetadata carries canonical policy metadata for AI-facing @@ -662,28 +663,29 @@ type TopologySummary struct { // ResourceResponse is returned by pulse_get_resource type ResourceResponse struct { GovernedResourceMetadata - Type string `json:"type"` // "agent", "vm", "system-container", "app-container", "docker-host", "storage" - ID string `json:"id"` - Name string `json:"name"` - Status string `json:"status"` - Platform string `json:"platform,omitempty"` - Node string `json:"node,omitempty"` - Host string `json:"host,omitempty"` - CPU ResourceCPU `json:"cpu"` - Memory ResourceMemory `json:"memory"` - Disk *ResourceDisk `json:"disk,omitempty"` - OS string `json:"os,omitempty"` - Tags []string `json:"tags"` - Networks []NetworkInfo `json:"networks"` - Ports []PortInfo `json:"ports"` - Mounts []MountInfo `json:"mounts"` - Labels map[string]string `json:"labels"` - LastBackup *time.Time `json:"last_backup,omitempty"` - Image string `json:"image,omitempty"` - Health string `json:"health,omitempty"` - HealthcheckTargets []string `json:"healthcheck_targets"` - RestartCount int `json:"restart_count,omitempty"` - UpdateAvailable bool `json:"update_available,omitempty"` + Type string `json:"type"` // "agent", "vm", "system-container", "app-container", "docker-host", "storage" + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Platform string `json:"platform,omitempty"` + Node string `json:"node,omitempty"` + Host string `json:"host,omitempty"` + CPU ResourceCPU `json:"cpu"` + Memory ResourceMemory `json:"memory"` + Disk *ResourceDisk `json:"disk,omitempty"` + OS string `json:"os,omitempty"` + Tags []string `json:"tags"` + Networks []NetworkInfo `json:"networks"` + Ports []PortInfo `json:"ports"` + Mounts []MountInfo `json:"mounts"` + Filesystems []filesystem.Observation `json:"filesystems,omitempty"` + Labels map[string]string `json:"labels"` + LastBackup *time.Time `json:"last_backup,omitempty"` + Image string `json:"image,omitempty"` + Health string `json:"health,omitempty"` + HealthcheckTargets []string `json:"healthcheck_targets"` + RestartCount int `json:"restart_count,omitempty"` + UpdateAvailable bool `json:"update_available,omitempty"` } func EmptyResourceResponse() ResourceResponse { diff --git a/internal/ai/tools/filesystem_evidence_test.go b/internal/ai/tools/filesystem_evidence_test.go new file mode 100644 index 000000000..7324c4f23 --- /dev/null +++ b/internal/ai/tools/filesystem_evidence_test.go @@ -0,0 +1,57 @@ +package tools + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" +) + +func TestQueryPreservesFilesystemScopeAndUnavailableMeasurements(t *testing.T) { + for _, provider := range []bool{false, true} { + snapshot := commandEvidenceSnapshot() + snapshot.DockerHosts[0].Containers[0].Filesystems = []filesystem.Observation{ + {Mountpoint: "/cache", Source: "linux-proc-root-statfs", ObservedAt: time.Now().UTC(), Type: "tmpfs", Usage: &filesystem.Usage{CapacityBytes: 8 << 20}}, + {Mountpoint: "/data", Source: "linux-proc-root-statfs", ObservedAt: time.Now().UTC(), Error: "permission denied"}, + } + registry := unifiedresources.NewRegistry(nil) + registry.IngestSnapshot(snapshot) + cfg := ExecutorConfig{ReadState: registry, ControlLevel: ControlLevelReadOnly} + if provider { + cfg.UnifiedResourceProvider = ®istryUnifiedQueryProvider{registry} + } + executor := NewPulseToolExecutor(cfg) + id := snapshot.DockerHosts[0].Containers[0].Name + if provider { + id = registry.ListByType(unifiedresources.ResourceTypeAppContainer)[0].ID + } + result, err := executor.executeQuery(context.Background(), map[string]interface{}{"action": "get", "resource_type": "app-container", "resource_id": id}) + if err != nil || result.IsError { + t.Fatalf("provider=%v: %v %+v", provider, err, result) + } + var decoded map[string]any + if err := json.Unmarshal([]byte(result.Content[0].Text), &decoded); err != nil { + t.Fatal(err) + } + rows, ok := decoded["filesystems"].([]any) + if !ok || len(rows) != 2 { + t.Fatalf("filesystem evidence missing: %+v", decoded) + } + full := rows[0].(map[string]any) + unavailable := rows[1].(map[string]any) + usage := full["usage"].(map[string]any) + if full["mountpoint"] != "/cache" || full["type"] != "tmpfs" || usage["capacityBytes"] != float64(8<<20) || usage["availableBytes"] != float64(0) { + t.Fatalf("filesystem scope/measurement changed: %+v", full) + } + if _, exists := unavailable["usage"]; exists || unavailable["error"] != "permission denied" { + t.Fatalf("unavailable read became measurement: %+v", unavailable) + } + if _, exists := decoded["disk"]; exists { + t.Fatal("resource query invented container-wide disk usage") + } + t.Logf("FILESYSTEM_MODEL_EVIDENCE provider=%v %s", provider, result.Content[0].Text) + } +} diff --git a/internal/ai/tools/tools_query.go b/internal/ai/tools/tools_query.go index 2fbe1bc4d..4273b5df6 100644 --- a/internal/ai/tools/tools_query.go +++ b/internal/ai/tools/tools_query.go @@ -12,6 +12,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/agentcapabilities" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" "github.com/rs/zerolog/log" ) @@ -2151,7 +2152,7 @@ func (e *PulseToolExecutor) registerQueryTools() { e.registry.registerBuiltin(RegisteredTool{ Definition: Tool{ Name: agentcapabilities.PulseQueryToolName, - Description: `Query and search canonical infrastructure resources. Start here to discover systems, workloads, storage, and disks by name. Actions: search, get, config, topology, list, health. Health returns the connection overview by default, or the canonical resource projection when resource_id is provided. command_agent_connected describes live command transport, independently of monitoring collection or freshness. Missing connection fields were not observed. can_execute describes connected transport with control enabled, not approval for a particular operation.`, + Description: `Query and search canonical infrastructure resources. Start here to discover systems, workloads, storage, and disks by name. Actions: search, get, config, topology, list, health. For app-container get, filesystems reports observed capacity at each mountpoint, not container quotas. An observation error has no usage payload. Mounts describe configuration. Health returns the connection overview by default, or the canonical resource projection when resource_id is provided. command_agent_connected describes live command transport, independently of monitoring collection or freshness. Missing connection fields were not observed. can_execute describes connected transport with control enabled, not approval for a particular operation.`, InputSchema: InputSchema{ Type: "object", Properties: map[string]PropertySchema{ @@ -5064,6 +5065,7 @@ func (e *PulseToolExecutor) executeGetResource(_ context.Context, args map[strin ReadWrite: m.RW, }) } + response.Filesystems = filesystem.Clone(resource.Docker.Filesystems) } if reg, ok := resolvedAppContainerRegistration(resource); ok { @@ -5126,6 +5128,7 @@ func (e *PulseToolExecutor) executeGetResource(_ context.Context, args map[strin response.Image = container.Image() response.Health = container.Health() response.HealthcheckTargets = container.HealthcheckTargets() + response.Filesystems = container.Filesystems() response.CPU = ResourceCPU{ Percent: container.CPUPercent(), } diff --git a/internal/alerts/event_emission.go b/internal/alerts/event_emission.go index 002ae74c2..d10f79802 100644 --- a/internal/alerts/event_emission.go +++ b/internal/alerts/event_emission.go @@ -42,6 +42,9 @@ func (m *Manager) SetEventLog(store *eventlog.Store) { return } previous := m.eventLog.Swap(store) + m.historyProjectionMu.Lock() + m.historyProjection = nil + m.historyProjectionMu.Unlock() authoritative := store != nil && m.historyManager != nil && !m.historyManager.StorageFileExists() && !m.historyManager.ImportedStorageFileExists() && diff --git a/internal/alerts/eventlog/eventlog.go b/internal/alerts/eventlog/eventlog.go index ceeda6eb6..5caa8f575 100644 --- a/internal/alerts/eventlog/eventlog.go +++ b/internal/alerts/eventlog/eventlog.go @@ -90,6 +90,8 @@ type Filter struct { // id they have fully applied so a walk visits only the un-projected tail // instead of the whole log. AfterID int64 + // ThroughID fixes an inclusive replay boundary while new events arrive. + ThroughID int64 } const ( @@ -107,6 +109,7 @@ const ( // Close are no-ops and Query returns no events. type Store struct { db *sql.DB + readDB *sql.DB dbPath string events chan Event stop chan struct{} @@ -168,14 +171,16 @@ func openDSN(dbPath, dsn string) (*Store, error) { if err != nil { return nil, fmt.Errorf("open alert event log: %w", err) } - // Single connection: the write path is one goroutine, and WAL keeps the - // rare reads from blocking behind it for long. + // Serialize durable transactions on their own connection. Disk-backed WAL + // readers use a separate, read-only pool so history cannot occupy the only + // connection while an alert transition holds the manager's state lock. db.SetMaxOpenConns(1) db.SetMaxIdleConns(1) db.SetConnMaxLifetime(0) s := &Store{ db: db, + readDB: db, dbPath: dbPath, events: make(chan Event, appendBufferSize), stop: make(chan struct{}), @@ -186,6 +191,21 @@ func openDSN(dbPath, dsn string) (*Store, error) { db.Close() return nil, fmt.Errorf("init alert event log schema: %w", err) } + if dbPath != ":memory:" { + reader, err := sql.Open("sqlite", sqliteDSN(dbPath)+"&mode=ro&_pragma=query_only(1)") + if err != nil { + db.Close() + return nil, fmt.Errorf("open alert event reader: %w", err) + } + reader.SetMaxOpenConns(2) + reader.SetMaxIdleConns(2) + if err := reader.Ping(); err != nil { + reader.Close() + db.Close() + return nil, fmt.Errorf("initialize alert event reader: %w", err) + } + s.readDB = reader + } s.wg.Add(1) go s.writeLoop() @@ -600,11 +620,49 @@ func (s *Store) pruneOld() { func (s *Store) pruneEventsBefore(cutoff time.Time) { cutoffValue := cutoff.UTC().Format(time.RFC3339Nano) - if _, err := s.db.Exec(`DELETE FROM alert_events WHERE occurred_at < ?`, cutoffValue); err != nil { + tx, err := s.db.Begin() + if err != nil { + log.Error().Err(err).Msg("alert event log prune failed") + return + } + defer tx.Rollback() + result, err := tx.Exec(`DELETE FROM alert_events WHERE occurred_at < ?`, cutoffValue) + if err == nil { + var deleted int64 + deleted, err = result.RowsAffected() + if err == nil && deleted > 0 { + _, err = tx.Exec(`INSERT INTO alert_store_meta (key, value) VALUES ('retention_revision', '1') + ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1`) + } + } + if err == nil { + err = tx.Commit() + } + if err != nil { log.Error().Err(err).Msg("alert event log prune failed") } } +// ReplayBoundary identifies the durable append tail and any removal of retained +// events. Projections can reuse a fold and read only new IDs. Retention changes +// invalidate that fold, including deletion of older events below its cursor. +type ReplayBoundary struct { + LastID int64 + RetentionRevision int64 +} + +func (s *Store) ReplayBoundary() (ReplayBoundary, error) { + var boundary ReplayBoundary + if s == nil { + return boundary, fmt.Errorf("event log is not enabled") + } + err := s.readDB.QueryRow(`SELECT + COALESCE((SELECT MAX(id) FROM alert_events), 0), + COALESCE((SELECT CAST(value AS INTEGER) FROM alert_store_meta WHERE key = 'retention_revision'), 0) + `).Scan(&boundary.LastID, &boundary.RetentionRevision) + return boundary, err +} + // Flush blocks until every diagnostic event appended before the call has been // written. It reports failed writes and timeouts rather than claiming a lost // batch was successfully flushed. @@ -692,7 +750,7 @@ func (s *Store) Query(filter Filter) ([]Event, error) { query += " ORDER BY occurred_at DESC, id DESC LIMIT ?" args = append(args, limit) - rows, err := s.db.Query(query, args...) + rows, err := s.readDB.Query(query, args...) if err != nil { return nil, fmt.Errorf("query alert event log: %w", err) } @@ -708,8 +766,8 @@ func (s *Store) Query(filter Filter) ([]Event, error) { // WalkOldest visits every matching event oldest first. It uses bounded keyset // pages so a complete history projection is not truncated by Query's public -// safety cap and does not hold the store's single database connection for the -// full scan. A positive filter Limit caps the total visits; zero walks all +// safety cap or retain a reader connection for the full scan. +// A positive filter Limit caps the total visits; zero walks all // matching rows. func (s *Store) WalkOldest(filter Filter, visit func(Event) error) error { if s == nil { @@ -719,7 +777,6 @@ func (s *Store) WalkOldest(filter Filter, visit func(Event) error) error { return fmt.Errorf("event visitor is required") } - baseWhere, baseArgs := eventFilterWhere(filter) visited := 0 cursorOccurredAt := "" var cursorID int64 @@ -736,21 +793,9 @@ func (s *Store) WalkOldest(filter Filter, visit func(Event) error) error { } } - where := append([]string(nil), baseWhere...) - args := append([]any(nil), baseArgs...) - if cursorOccurredAt != "" { - where = append(where, "(occurred_at > ? OR (occurred_at = ? AND id > ?))") - args = append(args, cursorOccurredAt, cursorOccurredAt, cursorID) - } + query, args := historyPageQuery(filter, cursorOccurredAt, cursorID, pageLimit) - query := "SELECT id, occurred_at, event_type, alert_id, resource_id, resource_name, alert_type, level, reason, message, details, snapshot FROM alert_events" - if len(where) > 0 { - query += " WHERE " + strings.Join(where, " AND ") - } - query += " ORDER BY occurred_at ASC, id ASC LIMIT ?" - args = append(args, pageLimit) - - rows, err := s.db.Query(query, args...) + rows, err := s.readDB.Query(query, args...) if err != nil { return fmt.Errorf("walk alert event log: %w", err) } @@ -779,6 +824,30 @@ func (s *Store) WalkOldest(filter Filter, visit func(Event) error) error { } } +// historyPageQuery selects bounded IDs before reading wide event payloads. +// A replay watermark must seek by durable ID, not scan the time index from the +// beginning on every catch-up. Alert-specific reads use their chronological +// index. Full-history walks seek by event time between pages. +func historyPageQuery(filter Filter, cursorOccurredAt string, cursorID int64, pageLimit int) (string, []any) { + where, args := eventFilterWhere(filter) + if cursorOccurredAt != "" { + where = append(where, "(occurred_at, id) > (?, ?)") + args = append(args, cursorOccurredAt, cursorID) + } + index := " INDEXED BY idx_alert_events_time" + if filter.AfterID > 0 { + index = " NOT INDEXED" // SQLite seeks the INTEGER PRIMARY KEY for id > ?. + } else if strings.TrimSpace(filter.AlertID) != "" { + index = " INDEXED BY idx_alert_events_alert" + } + query := "WITH page AS MATERIALIZED (SELECT id, occurred_at FROM alert_events" + index + if len(where) > 0 { + query += " WHERE " + strings.Join(where, " AND ") + } + query += " ORDER BY occurred_at ASC, id ASC LIMIT ?) SELECT e.id, e.occurred_at, e.event_type, e.alert_id, e.resource_id, e.resource_name, e.alert_type, e.level, e.reason, e.message, e.details, e.snapshot FROM page JOIN alert_events AS e ON e.id = page.id ORDER BY page.occurred_at ASC, page.id ASC" + return query, append(args, pageLimit) +} + func eventFilterWhere(filter Filter) ([]string, []any) { where := make([]string, 0, 4) args := make([]any, 0, 6) @@ -812,6 +881,10 @@ func eventFilterWhere(filter Filter) ([]string, []any) { where = append(where, "id > ?") args = append(args, filter.AfterID) } + if filter.ThroughID > 0 { + where = append(where, "id <= ?") + args = append(args, filter.ThroughID) + } return where, args } @@ -864,6 +937,11 @@ func (s *Store) Close() { s.closeOnce.Do(func() { close(s.stop) s.wg.Wait() + if s.readDB != s.db { + if err := s.readDB.Close(); err != nil { + log.Error().Err(err).Msg("alert event reader close failed") + } + } if err := s.db.Close(); err != nil { log.Error().Err(err).Msg("alert event log close failed") } diff --git a/internal/alerts/eventlog/reader_isolation_test.go b/internal/alerts/eventlog/reader_isolation_test.go new file mode 100644 index 000000000..8ab7360db --- /dev/null +++ b/internal/alerts/eventlog/reader_isolation_test.go @@ -0,0 +1,169 @@ +package eventlog + +import ( + "context" + "fmt" + "strings" + "testing" + "time" +) + +func TestHistoryReaderCannotBlockDurableLifecycleWrite(t *testing.T) { + store, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer store.Close() + store.readDB.SetMaxOpenConns(1) + tx, err := store.readDB.BeginTx(context.Background(), nil) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + var before int + if err := tx.QueryRow("SELECT COUNT(*) FROM alert_events").Scan(&before); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + done <- store.AppendDurable(Event{OccurredAt: time.Now(), Type: TypeFired, AlertID: "during-history-read"}) + }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("an open history snapshot blocked the durable writer") + } + var during int + if err := tx.QueryRow("SELECT COUNT(*) FROM alert_events").Scan(&during); err != nil || during != before { + t.Fatalf("reader snapshot changed: count=%d before=%d error=%v", during, before, err) + } + if _, err := tx.Exec("DELETE FROM alert_events"); err == nil { + t.Fatal("history reader accepted a mutation") + } + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + events, err := store.Query(Filter{AlertID: "during-history-read"}) + if err != nil || len(events) != 1 { + t.Fatalf("committed event unavailable to new reader: count=%d error=%v", len(events), err) + } +} + +func TestHistoryReplayBoundaryExcludesLaterAppendsAndTracksRetention(t *testing.T) { + store, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer store.Close() + at := time.Now().UTC().Add(-time.Hour) + if err := store.AppendDurable(Event{OccurredAt: at, Type: TypeFired, AlertID: "old"}); err != nil { + t.Fatal(err) + } + before, err := store.ReplayBoundary() + if err != nil || before.LastID == 0 { + t.Fatalf("initial boundary: %+v, %v", before, err) + } + if err := store.AppendDurable(Event{OccurredAt: at.Add(time.Minute), Type: TypeResolved, AlertID: "new"}); err != nil { + t.Fatal(err) + } + var ids []int64 + if err := store.WalkOldest(Filter{ThroughID: before.LastID}, func(e Event) error { + ids = append(ids, e.ID) + return nil + }); err != nil || len(ids) != 1 || ids[0] != before.LastID { + t.Fatalf("bounded replay: %v, %v", ids, err) + } + appended, err := store.ReplayBoundary() + if err != nil || appended.LastID <= before.LastID || appended.RetentionRevision != before.RetentionRevision { + t.Fatalf("append boundary: %+v, %v", appended, err) + } + store.pruneEventsBefore(at.Add(time.Second)) + pruned, err := store.ReplayBoundary() + if err != nil || pruned.LastID != appended.LastID || pruned.RetentionRevision <= appended.RetentionRevision { + t.Fatalf("old-row removal must invalidate history without changing its newest ID: %+v, %v", pruned, err) + } + store.pruneEventsBefore(at.Add(time.Second)) + unchanged, err := store.ReplayBoundary() + if err != nil || unchanged != pruned { + t.Fatalf("empty prune changed boundary: %+v, %v", unchanged, err) + } +} + +func TestDiskHistoryWalkPreservesFilteredPagesAndCursor(t *testing.T) { + store, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer store.Close() + at := time.Now().UTC().Add(-time.Hour) + events := make([]Event, 0, 2406) + for i := 0; i < 1203; i++ { + for _, kind := range []string{TypeResolved, TypeNotificationSuppressed} { + events = append(events, Event{OccurredAt: at, Type: kind, AlertID: fmt.Sprintf("event-%04d", i)}) + } + } + if err := store.ImportEvents(events); err != nil { + t.Fatal(err) + } + var seen []Event + if err := store.WalkOldest(Filter{Types: []string{TypeResolved}, AfterID: 2, Limit: 1100}, func(e Event) error { + seen = append(seen, e) + return nil + }); err != nil { + t.Fatal(err) + } + if len(seen) != 1100 || seen[0].AlertID != "event-0001" || seen[1099].AlertID != "event-1100" { + t.Fatalf("filtered page walk lost ordering or bounds: count=%d", len(seen)) + } + for i := 1; i < len(seen); i++ { + if seen[i].ID <= seen[i-1].ID || seen[i].Type != TypeResolved { + t.Fatalf("page repeated or admitted a filtered event at %d", i) + } + } +} + +func TestFilteredHistoryPagesSeekTheirExplicitBounds(t *testing.T) { + store, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer store.Close() + for _, tc := range []struct { + name string + filter Filter + seek string + }{ + {"replay watermark", Filter{AfterID: 50000, Types: []string{TypeFired, TypeResolved}}, "SEARCH alert_events USING INTEGER PRIMARY KEY (rowid>?)"}, + {"one alert", Filter{AlertID: "incident", Types: []string{TypeFired, TypeResolved}}, "SEARCH alert_events USING INDEX idx_alert_events_alert (alert_id=?)"}, + } { + t.Run(tc.name, func(t *testing.T) { + query, args := historyPageQuery(tc.filter, "", 0, maxQueryLimit) + rows, err := store.readDB.Query("EXPLAIN QUERY PLAN "+query, args...) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + seek := false + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + if strings.Contains(detail, "SCAN alert_events") { + t.Fatalf("filtered replay scans retained history: %s", detail) + } + seek = seek || strings.Contains(detail, tc.seek) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if !seek { + t.Fatalf("history page did not seek its explicit bound: %s", tc.seek) + } + }) + } +} diff --git a/internal/alerts/history_projection.go b/internal/alerts/history_projection.go index d59994c12..77dc8d2a7 100644 --- a/internal/alerts/history_projection.go +++ b/internal/alerts/history_projection.go @@ -10,6 +10,7 @@ package alerts import ( "encoding/json" + "errors" "sort" "time" @@ -38,64 +39,7 @@ func (m *Manager) AlertHistoryFromEvents(since time.Time, limit int) ([]Alert, b return nil, false } - // Fold oldest to newest through the store's bounded-page walker. The - // ordinary Query API intentionally caps responses at 1,000 rows, which is - // appropriate for callers but not for reconstructing a complete history - // window on a noisy installation. - occurrences := make(map[string]*historyOccurrence) - order := make([]string, 0) - err := store.WalkOldest(eventlog.Filter{ - Types: []string{ - eventlog.TypeFired, - eventlog.TypeRefired, - eventlog.TypeResolved, - eventlog.TypeAcknowledged, - eventlog.TypeUnacknowledged, - eventlog.TypeSnoozed, - eventlog.TypeUnsnoozed, - eventlog.TypeEscalated, - eventlog.TypeHistoryImported, - eventlog.TypeHistoryCleared, - }, - Since: since, - }, func(event eventlog.Event) error { - if event.Type == eventlog.TypeHistoryCleared { - // The user cleared history: everything before the tombstone - // leaves the projection. The log itself stays append-only. - occurrences = make(map[string]*historyOccurrence) - order = order[:0] - return nil - } - if len(event.Snapshot) == 0 { - return nil - } - var snapshot Alert - if err := json.Unmarshal(event.Snapshot, &snapshot); err != nil { - return nil - } - key := historyOccurrenceKey(event.AlertID, &snapshot) - occ, exists := occurrences[key] - if !exists { - occ = &historyOccurrence{alert: snapshot, firstEvent: event.OccurredAt} - occurrences[key] = occ - order = append(order, key) - } else { - occ.alert = mergeHistoryAlertSnapshots(occ.alert, snapshot) - } - occ.lastEvent = event.OccurredAt - if event.Type == eventlog.TypeResolved { - occ.resolved = true - // The JSON history's resolve path stamps the entry's LastSeen - // with the resolution time so the row reflects the true - // duration; mirror that. - if event.OccurredAt.After(occ.alert.LastSeen) { - occ.alert.LastSeen = event.OccurredAt - } - } else { - occ.resolved = false - } - return nil - }) + occurrences, order, err := m.historyOccurrences(store, since) if err != nil { return nil, false } @@ -155,3 +99,129 @@ func (m *Manager) AlertHistoryFromEvents(since time.Time, limit int) ([]Alert, b } return m.applyCurrentNodeDisplayNames(canonicalizeAlertHistoryForOutput(results)), true } + +// historyProjection is a disposable fold of the durable log, never a second +// source of truth. Its cursor is bounded by a committed event ID. Old imports, +// retention and store replacement rebuild it using the original chronological +// fold. A moving Since window keeps its original event-filter semantics and +// does not evict the full-history fold used by attention and normal polling. +type historyProjection struct { + store *eventlog.Store + boundary eventlog.ReplayBoundary + lastTimeKey string + occurrences map[string]*historyOccurrence + order []string +} + +var errHistoryEventBeforeCursor = errors.New("new history event precedes the folded chronology") + +func newHistoryProjection(store *eventlog.Store) *historyProjection { + return &historyProjection{store: store, occurrences: make(map[string]*historyOccurrence)} +} + +func (p *historyProjection) fold(since time.Time, throughID int64) error { + if throughID == 0 || throughID == p.boundary.LastID { + return nil + } + return p.store.WalkOldest(eventlog.Filter{ + Types: []string{ + eventlog.TypeFired, eventlog.TypeRefired, eventlog.TypeResolved, + eventlog.TypeAcknowledged, eventlog.TypeUnacknowledged, + eventlog.TypeSnoozed, eventlog.TypeUnsnoozed, eventlog.TypeEscalated, + eventlog.TypeHistoryImported, eventlog.TypeHistoryCleared, + }, + Since: since, AfterID: p.boundary.LastID, ThroughID: throughID, + }, func(event eventlog.Event) error { + // Match the log's (occurred_at, id) ordering. Equal timestamps are + // safe because all appended IDs exceed the preceding boundary. + timeKey := event.OccurredAt.UTC().Format(time.RFC3339Nano) + if timeKey < p.lastTimeKey { + return errHistoryEventBeforeCursor + } + p.lastTimeKey = timeKey + if event.Type == eventlog.TypeHistoryCleared { + // The user cleared history: everything before the tombstone + // leaves the projection. The log itself stays append-only. + p.occurrences = make(map[string]*historyOccurrence) + p.order = p.order[:0] + return nil + } + if len(event.Snapshot) == 0 { + return nil + } + var snapshot Alert + if err := json.Unmarshal(event.Snapshot, &snapshot); err != nil { + return nil + } + key := historyOccurrenceKey(event.AlertID, &snapshot) + occ, exists := p.occurrences[key] + if !exists { + occ = &historyOccurrence{alert: snapshot, firstEvent: event.OccurredAt} + p.occurrences[key] = occ + p.order = append(p.order, key) + } else { + occ.alert = mergeHistoryAlertSnapshots(occ.alert, snapshot) + } + occ.lastEvent = event.OccurredAt + if event.Type == eventlog.TypeResolved { + occ.resolved = true + // The JSON history's resolve path stamps the entry's LastSeen + // with the resolution time so the row reflects the true + // duration; mirror that. + if event.OccurredAt.After(occ.alert.LastSeen) { + occ.alert.LastSeen = event.OccurredAt + } + } else { + occ.resolved = false + } + return nil + }) +} + +func (m *Manager) historyOccurrences(store *eventlog.Store, since time.Time) (map[string]*historyOccurrence, []string, error) { + m.historyProjectionMu.Lock() + defer m.historyProjectionMu.Unlock() + for { + boundary, err := store.ReplayBoundary() + if err != nil { + m.historyProjection = nil + return nil, nil, err + } + p := m.historyProjection + if !since.IsZero() || p == nil || p.store != store || + p.boundary.RetentionRevision != boundary.RetentionRevision || p.boundary.LastID > boundary.LastID { + p = newHistoryProjection(store) + } + err = p.fold(since, boundary.LastID) + if errors.Is(err, errHistoryEventBeforeCursor) { + p = newHistoryProjection(store) + err = p.fold(since, boundary.LastID) + } + if err != nil { + m.historyProjection = nil + return nil, nil, err + } + after, err := store.ReplayBoundary() + if err != nil { + m.historyProjection = nil + return nil, nil, err + } + if after.RetentionRevision != boundary.RetentionRevision { + m.historyProjection = nil + continue + } + p.boundary = boundary + if since.IsZero() { + m.historyProjection = p + } + // The live-state overlay and callers may mutate their result. Never + // let those changes contaminate the durable fold or another reader. + occurrences := make(map[string]*historyOccurrence, len(p.occurrences)) + for key, occurrence := range p.occurrences { + copy := *occurrence + copy.alert = *occurrence.alert.Clone() + occurrences[key] = © + } + return occurrences, append([]string(nil), p.order...), nil + } +} diff --git a/internal/alerts/history_projection_parity_test.go b/internal/alerts/history_projection_parity_test.go index ac683c310..039b75d0b 100644 --- a/internal/alerts/history_projection_parity_test.go +++ b/internal/alerts/history_projection_parity_test.go @@ -10,12 +10,93 @@ package alerts import ( "encoding/json" "fmt" + "reflect" + "sync" "testing" "time" "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog" ) +func TestHistoryProjectionIncrementalReadersPreserveChronologyAndIsolation(t *testing.T) { + m := newHistoryParityManager(t) + store := m.eventLogStore() + at := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + appendEvent := func(kind, id string, occurred time.Time, value float64) { + t.Helper() + snapshot, err := json.Marshal(Alert{ID: id, ResourceID: "vm-" + id, StartTime: at, LastSeen: occurred, Value: value}) + if err != nil { + t.Fatal(err) + } + if err := store.AppendDurable(eventlog.Event{Type: kind, AlertID: id, OccurredAt: occurred, Snapshot: snapshot}); err != nil { + t.Fatal(err) + } + } + read := func() []Alert { + t.Helper() + result, ok := m.AlertHistoryFromEvents(time.Time{}, 0) + if !ok { + t.Fatal("history unavailable") + } + return result + } + compareFresh := func() { + t.Helper() + incremental := read() + m.historyProjectionMu.Lock() + m.historyProjection = nil + m.historyProjectionMu.Unlock() + if fresh := read(); !reflect.DeepEqual(incremental, fresh) { + t.Fatalf("incremental history differs from a complete chronological replay:\n%+v\n%+v", incremental, fresh) + } + } + appendEvent(eventlog.TypeFired, "one", at, 90) + first := read() + first[0].Value = -1 + if got := read(); got[0].Value != 90 { + t.Fatal("caller mutation contaminated the durable history fold") + } + appendEvent(eventlog.TypeResolved, "one", at.Add(2*time.Minute), 30) + compareFresh() + // A delayed earlier event must be placed before the resolution, not + // overwrite it just because its durable insertion ID is newer. + appendEvent(eventlog.TypeFired, "one", at.Add(time.Minute), 99) + compareFresh() + if got := read(); got[0].Value != 30 { + t.Fatalf("late event replaced the later resolution: %+v", got) + } + if err := store.AppendDurable(eventlog.Event{Type: eventlog.TypeHistoryCleared, OccurredAt: at.Add(3 * time.Minute)}); err != nil { + t.Fatal(err) + } + if got := read(); len(got) != 0 { + t.Fatalf("clear left cached occurrences: %+v", got) + } + appendEvent(eventlog.TypeFired, "two", at.Add(4*time.Minute), 85) + compareFresh() + if _, ok := m.AlertHistoryFromEvents(at.Add(4*time.Minute), 5); !ok { + t.Fatal("windowed history unavailable") + } + want := read() + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { + defer wg.Done() + got, ok := m.AlertHistoryFromEvents(time.Time{}, 5) + if !ok || !reflect.DeepEqual(got, want) { + t.Errorf("concurrent history: %+v, available=%v", got, ok) + } + if len(got) > 0 { + got[0].Value = -2 + } + }() + } + wg.Wait() + if got := read(); !reflect.DeepEqual(got, want) { + t.Fatal("parallel callers contaminated the retained fold") + } +} + func newHistoryParityManager(t *testing.T) *Manager { t.Helper() m := newTestManager(t) diff --git a/internal/alerts/manager.go b/internal/alerts/manager.go index 2cecbea62..d26364874 100644 --- a/internal/alerts/manager.go +++ b/internal/alerts/manager.go @@ -119,6 +119,10 @@ type Manager struct { // absent or has been durably imported. Reads keep using JSON while migration // is incomplete or the event store reports a write failure. eventHistoryAuthoritative atomic.Bool + // One derived history fold per event store. Readers share catch-up work, + // while live alert overlays remain fresh and outside this mutex. + historyProjectionMu sync.Mutex + historyProjection *historyProjection // activeStateAuthoritative is true only when events.db owns restart state. // active-alerts.json remains an atomic recovery mirror, never a competing // source while the SQLite projection is healthy. diff --git a/internal/api/ai_handlers_investigation_additional_test.go b/internal/api/ai_handlers_investigation_additional_test.go index bd48a1202..b00f92d77 100644 --- a/internal/api/ai_handlers_investigation_additional_test.go +++ b/internal/api/ai_handlers_investigation_additional_test.go @@ -829,3 +829,59 @@ func TestAgentCommandAdapter_FindAgentForTarget(t *testing.T) { t.Fatalf("expected empty agent when multiple connected, got %q", got) } } + +func TestPatrolExpiredActionHydrationRemovesQueuedOutcomeWithoutRewritingEvidence(t *testing.T) { + investigations := newTestInvestigationStore() + investigation := investigations.Create("finding-1", "session-1") + investigation.Status = aicontracts.InvestigationStatusCompleted + investigation.Outcome = aicontracts.OutcomeFixQueued + investigation.Summary = "Cause uncertain. Restart proposed for review." + investigation.EvidenceIDs = []string{"observed-health"} + investigations.Update(investigation) + svc := ai.NewService(nil, nil) + svc.SetStateProvider(&MockStateProvider{}) + patrol := svc.GetPatrolService() + findings := patrol.GetFindings() + findings.Add(&ai.Finding{ID: "finding-1", ResourceID: "vm:42", Title: "Unhealthy service", Severity: ai.FindingSeverityWarning, + InvestigationStatus: string(investigation.Status), InvestigationOutcome: string(aicontracts.OutcomeNeedsAttention)}) + // Reproduce the persisted mismatch after an outcome already reconciled. + record := ai.BuildFindingInvestigationRecord(findings.Get("finding-1"), investigation) + record.Rollback = []string{"Retained rollback evidence"} + record.Impact = "Original service impact absent from the later finding projection." + findings.UpdateInvestigationRecord("finding-1", record) + audits := unifiedresources.NewMemoryStore() + audit := unifiedresources.ActionAuditRecord{ + ID: "act-1", CreatedAt: time.Now().UTC(), UpdatedAt: time.Now().UTC(), State: unifiedresources.ActionStateExpired, + Request: unifiedresources.ActionRequest{RequestID: "proposal-1", ResourceID: "vm:42", CapabilityName: "restart", RequestedBy: "pulse_patrol"}, + Plan: unifiedresources.ActionPlan{ActionID: "act-1", RequestID: "proposal-1", Allowed: true}, + Origin: &unifiedresources.ActionOrigin{Surface: patrolActionOriginSurface, FindingID: "finding-1", InvestigationID: investigation.ID, ProposalID: "proposal-1"}, + } + if _, _, err := audits.CreateActionAudit(audit, nil); err != nil { + t.Fatal(err) + } + handler := &AISettingsHandler{defaultAIService: svc, + investigationStores: map[string]aicontracts.InvestigationStore{"default": investigations}, + resourceStoreProvider: func(string) (unifiedresources.ResourceStore, error) { return audits, nil }, + } + var published []*ai.Finding + patrol.SetUnifiedFindingCallback(func(f *ai.Finding) bool { published = append(published, f); return true }) + published = nil // Ignore the initial synchronization when registering the callback. + handler.hydratePatrolInvestigationAction("default", investigation) + got := findings.Get("finding-1").InvestigationRecord + if got.Outcome != aicontracts.OutcomeNeedsAttention || got.Action == nil || got.Action.State != "expired" { + t.Fatalf("durable record did not reconcile: %#v", got) + } + if got.Conclusion != investigation.Summary || got.Impact != record.Impact || got.Confidence != record.Confidence || !reflect.DeepEqual(got.Rollback, record.Rollback) || !reflect.DeepEqual(got.Evidence, record.Evidence) { + t.Fatalf("retained investigation evidence changed: %#v", got) + } + if len(got.Verification) != 0 { + t.Fatalf("unexecuted expired action invented verification: %#v", got.Verification) + } + if len(published) != 1 || published[0].InvestigationRecord.Outcome != aicontracts.OutcomeNeedsAttention { + t.Fatalf("reconciled record was not published: record=%#v published=%#v", got, published) + } + handler.hydratePatrolInvestigationAction("default", investigation) + if len(published) != 1 { + t.Fatal("duplicate hydration republished unchanged record") + } +} diff --git a/internal/api/patrol_action_reconciliation.go b/internal/api/patrol_action_reconciliation.go index 5f44c3572..e38ea247b 100644 --- a/internal/api/patrol_action_reconciliation.go +++ b/internal/api/patrol_action_reconciliation.go @@ -146,6 +146,8 @@ func patrolOutcomeForActionAudit(audit unifiedresources.ActionAuditRecord) aicon switch audit.State { case unifiedresources.ActionStatePlanned, unifiedresources.ActionStatePending, unifiedresources.ActionStateApproved, unifiedresources.ActionStateExecuting: return aicontracts.OutcomeFixQueued + case unifiedresources.ActionStateExpired: + return aicontracts.OutcomeNeedsAttention case unifiedresources.ActionStateRejected: return aicontracts.OutcomeFixRejected case unifiedresources.ActionStateFailed, unifiedresources.ActionStateCompleted: diff --git a/internal/dockeragent/agent.go b/internal/dockeragent/agent.go index 62baaef1d..0c268a91a 100644 --- a/internal/dockeragent/agent.go +++ b/internal/dockeragent/agent.go @@ -22,8 +22,10 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/agenthelper" "github.com/rcourtman/pulse-go-rewrite/internal/agenttarget" "github.com/rcourtman/pulse-go-rewrite/internal/agenttls" + "github.com/rcourtman/pulse-go-rewrite/internal/filesystemprobe" "github.com/rcourtman/pulse-go-rewrite/internal/utils" agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" "github.com/rs/zerolog" ) @@ -130,6 +132,10 @@ func setAgentHeaders(req *http.Request, token string) { // Agent collects Docker / Podman metrics and posts them to Pulse. type Agent struct { + filesystemObserver filesystemprobe.Observer + // Optional native-read seam for collection boundary tests. Production uses + // the bounded filesystemObserver above. + observeFilesystems func(context.Context, filesystemprobe.ContainerRequest) ([]filesystem.Observation, error) cfg Config docker dockerClient helperInventory ContainerInventory diff --git a/internal/dockeragent/collect.go b/internal/dockeragent/collect.go index 02223dc3a..d2e0356f6 100644 --- a/internal/dockeragent/collect.go +++ b/internal/dockeragent/collect.go @@ -761,6 +761,7 @@ func (a *Agent) collectContainer(ctx context.Context, summary containertypes.Sum RootFilesystemBytes: rootFsBytes, BlockIO: blockIO, Mounts: mounts, + Filesystems: a.collectContainerFilesystems(containerCtx, summary.ID, inspect, mounts), } if a.runtime == RuntimePodman { diff --git a/internal/dockeragent/collect_tmpfs_live_test.go b/internal/dockeragent/collect_tmpfs_live_test.go index ca0bb459b..4e4bdc13e 100644 --- a/internal/dockeragent/collect_tmpfs_live_test.go +++ b/internal/dockeragent/collect_tmpfs_live_test.go @@ -128,6 +128,22 @@ func TestCollectContainerStorageFaultLive(t *testing.T) { if matches != 1 { t.Fatalf("expected one storage mount in report, got %d", matches) } + measured := false + for _, observation := range report.Filesystems { + if observation.Mountpoint != destination { + continue + } + measured = true + if observation.Error != "" || observation.Usage == nil || observation.Type != "tmpfs" || observation.Usage.CapacityBytes != 8<<20 || observation.ObservedAt.IsZero() { + t.Fatalf("missing native tmpfs capacity: %+v", observation) + } + if (phase == "storage-full") != (observation.Usage.AvailableBytes == 0) { + t.Fatalf("%s tmpfs availability contradicts independent oracle: %+v", phase, observation) + } + } + if !measured { + t.Fatal("collected report omitted the affected filesystem observation") + } t.Logf("%s raw_mount_count=%d native_tmpfs_options=%q", phase, len(inspect.Mounts), options) } t.Logf("%s %s report=%s", phase, alias, encoded) diff --git a/internal/dockeragent/container_filesystems.go b/internal/dockeragent/container_filesystems.go new file mode 100644 index 000000000..11ad404f1 --- /dev/null +++ b/internal/dockeragent/container_filesystems.go @@ -0,0 +1,67 @@ +package dockeragent + +import ( + "context" + "errors" + "net/url" + "path" + "sort" + "time" + + containertypes "github.com/moby/moby/api/types/container" + "github.com/rcourtman/pulse-go-rewrite/internal/filesystemprobe" + agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" +) + +func (a *Agent) collectContainerFilesystems(ctx context.Context, id string, inspect containertypes.InspectResponse, mounts []agentsdocker.ContainerMount) []filesystem.Observation { + // Native filesystem counters are ordinary resource evidence. The separate + // CollectDiskMetrics switch controls Docker's expensive image-layer sizing, + // which the unified agent deliberately disables. + paths := []string{"/"} + seen := map[string]bool{"/": true} + for _, mount := range mounts { + if !seen[mount.Destination] { + paths = append(paths, mount.Destination) + seen[mount.Destination] = true + } + } + sort.Strings(paths) + failed := func(err error) []filesystem.Observation { + out := make([]filesystem.Observation, len(paths)) + for i, mount := range paths { + out[i] = filesystem.Observation{Mountpoint: mount, Source: filesystemprobe.Source, ObservedAt: time.Now().UTC(), Error: err.Error()} + } + return out + } + if inspect.State == nil || (!inspect.State.Running && !inspect.State.Paused) || inspect.State.Pid <= 0 { + return failed(errors.New("container filesystem namespace is not running")) + } + endpoint, err := url.Parse(a.docker.DaemonHost()) + if err != nil || endpoint.Scheme != "unix" || endpoint.Host != "" || !path.IsAbs(endpoint.Path) { + return failed(errors.New("filesystem observations require a local runtime socket and an attested local container process")) + } + if inspect.ID != id { + return failed(errors.New("container inspect identity does not match the collected resource")) + } + observe := a.observeFilesystems + if observe == nil { + observe = a.filesystemObserver.Observe + } + observations, err := observe(ctx, filesystemprobe.ContainerRequest{ + PID: inspect.State.Pid, ContainerID: id, Runtime: string(a.runtime), Mountpoints: paths, + }) + if err != nil { + return failed(err) + } + // Reinspect after the native read. Never attach the old namespace's counters + // to a container that restarted during collection, even if its ID is stable. + current, err := a.docker.ContainerInspect(ctx, id) + if err != nil { + return failed(errors.New("container identity could not be revalidated after filesystem observation")) + } + if current.ID != id || current.State == nil || current.State.Pid != inspect.State.Pid || current.State.StartedAt != inspect.State.StartedAt || (!current.State.Running && !current.State.Paused) { + return failed(errors.New("container process changed during filesystem observation")) + } + return observations +} diff --git a/internal/dockeragent/container_filesystems_test.go b/internal/dockeragent/container_filesystems_test.go new file mode 100644 index 000000000..011d6c3bd --- /dev/null +++ b/internal/dockeragent/container_filesystems_test.go @@ -0,0 +1,78 @@ +package dockeragent + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + "time" + + containertypes "github.com/moby/moby/api/types/container" + "github.com/rcourtman/pulse-go-rewrite/internal/filesystemprobe" + agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" +) + +func TestCollectContainerFilesystemIdentityBoundary(t *testing.T) { + for _, scenario := range []string{"measured", "remote daemon", "wrong inspect", "process restarted", "revalidation failed", "native unavailable", "layer sizing disabled"} { + t.Run(scenario, func(t *testing.T) { + id := strings.Repeat("a", 64) + inspect := baseInspect() + inspect.ID = id + inspect.State.Pid = 42 + inspect.State.Running = true + inspect.State.StartedAt = time.Now().Format(time.RFC3339Nano) + client := &fakeDockerClient{daemonHost: "unix:///var/run/docker.sock"} + client.containerInspectFn = func(context.Context, string) (containertypes.InspectResponse, error) { + if scenario == "revalidation failed" { + return containertypes.InspectResponse{}, errors.New("gone") + } + current := inspect + state := *inspect.State + current.State = &state + if scenario == "process restarted" { + current.State.Pid++ + } + return current, nil + } + calls := 0 + a := &Agent{cfg: Config{CollectDiskMetrics: scenario != "layer sizing disabled"}, docker: client, runtime: RuntimeDocker, + observeFilesystems: func(_ context.Context, req filesystemprobe.ContainerRequest) ([]filesystem.Observation, error) { + calls++ + if req.ContainerID != id || req.PID != 42 || req.Runtime != "docker" || !reflect.DeepEqual(req.Mountpoints, []string{"/", "/cache"}) { + t.Fatalf("wrong namespace coordinates: %+v", req) + } + if scenario == "native unavailable" { + return nil, errors.New("namespace unavailable") + } + return []filesystem.Observation{{Mountpoint: "/cache", Source: filesystemprobe.Source, ObservedAt: time.Now(), Type: "tmpfs", Usage: &filesystem.Usage{CapacityBytes: 8 << 20}}}, nil + }, + } + if scenario == "remote daemon" { + client.daemonHost = "tcp://remote.invalid:2376" + } + if scenario == "wrong inspect" { + inspect.ID = strings.Repeat("b", 64) + } + got := a.collectContainerFilesystems(context.Background(), id, inspect, []agentsdocker.ContainerMount{{Destination: "/cache"}, {Destination: "/cache"}}) + if scenario == "measured" || scenario == "layer sizing disabled" { + if calls != 1 || len(got) != 1 || got[0].Usage == nil || got[0].Usage.AvailableBytes != 0 { + t.Fatalf("exhaustion lost: %+v", got) + } + return + } + if len(got) != 2 { + t.Fatalf("missing failure scope: %+v", got) + } + for _, row := range got { + if row.Usage != nil || row.Error == "" { + t.Fatalf("unverified observation became capacity: %+v", row) + } + } + if (scenario == "remote daemon" || scenario == "wrong inspect") && calls != 0 { + t.Fatal("unattested target reached native probe") + } + }) + } +} diff --git a/internal/filesystemprobe/observer.go b/internal/filesystemprobe/observer.go new file mode 100644 index 000000000..a4aa8587d --- /dev/null +++ b/internal/filesystemprobe/observer.go @@ -0,0 +1,144 @@ +// Package filesystemprobe reads container filesystem counters without executing +// container-controlled programs or exposing file contents. +package filesystemprobe + +import ( + "context" + "errors" + "fmt" + "path" + "strings" + "sync" + "time" + + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" +) + +const ( + Source = "linux-proc-root-statfs" + maxMountpoints = 128 + maxInFlight = 32 + probeTimeout = 3 * time.Second +) + +type ContainerRequest struct { + PID int + ContainerID string + Runtime string + Mountpoints []string +} + +// Observer bounds potentially uninterruptible kernel reads. A timed-out probe +// keeps its slot until the syscall returns. Later collections never share its +// stale result or start another probe for the same container in the meantime. +// The zero value is ready to use and must not be copied after first use. +type Observer struct { + mu sync.Mutex + inFlight map[string]struct{} + probe func(context.Context, ContainerRequest) ([]filesystem.Observation, error) + timeout time.Duration +} + +func (o *Observer) Observe(ctx context.Context, req ContainerRequest) ([]filesystem.Observation, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if err := validateRequest(req); err != nil { + return nil, err + } + // The asynchronous read must own its coordinates after its caller returns. + req.Mountpoints = append([]string(nil), req.Mountpoints...) + key := req.Runtime + ":" + req.ContainerID + o.mu.Lock() + if _, exists := o.inFlight[key]; exists { + o.mu.Unlock() + return nil, errors.New("previous filesystem observation is still running") + } + if len(o.inFlight) >= maxInFlight { + o.mu.Unlock() + return nil, errors.New("filesystem observation concurrency limit reached") + } + if o.inFlight == nil { + o.inFlight = make(map[string]struct{}) + } + o.inFlight[key] = struct{}{} + o.mu.Unlock() + + timeout := o.timeout + if timeout <= 0 { + timeout = probeTimeout + } + probeCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + type result struct { + observations []filesystem.Observation + err error + } + done := make(chan result, 1) + probe := o.probe + if probe == nil { + probe = observeContainer + } + go func() { + observations, err := probe(probeCtx, req) + o.mu.Lock() + delete(o.inFlight, key) + o.mu.Unlock() + done <- result{observations, err} + }() + select { + case r := <-done: + if err := probeCtx.Err(); err != nil { + return nil, err + } + return r.observations, r.err + case <-probeCtx.Done(): + return nil, probeCtx.Err() + } +} + +func validateRequest(req ContainerRequest) error { + if req.PID <= 0 { + return errors.New("container process is unavailable") + } + if len(req.ContainerID) != 64 || strings.IndexFunc(req.ContainerID, func(r rune) bool { + return !(r >= '0' && r <= '9' || r >= 'a' && r <= 'f') + }) >= 0 { + return errors.New("filesystem observation requires an exact container ID") + } + if req.Runtime != "docker" && req.Runtime != "podman" { + return errors.New("unsupported container runtime") + } + if len(req.Mountpoints) == 0 || len(req.Mountpoints) > maxMountpoints { + return errors.New("invalid filesystem mountpoint count") + } + seen := make(map[string]bool, len(req.Mountpoints)) + for _, mount := range req.Mountpoints { + if !path.IsAbs(mount) || path.Clean(mount) != mount || strings.ContainsRune(mount, 0) || len(mount) > 4096 || seen[mount] { + return fmt.Errorf("invalid or duplicate filesystem mountpoint %q", mount) + } + seen[mount] = true + } + return nil +} + +// cgroupMatches accepts an exact runtime-owned cgroup component, never a +// substring, container name, short ID, or unrelated process with the same PID. +func cgroupMatches(raw string, req ContainerRequest) bool { + scope := "docker-" + req.ContainerID + ".scope" + if req.Runtime == "podman" { + scope = "libpod-" + req.ContainerID + ".scope" + } + for _, line := range strings.Split(raw, "\n") { + parts := strings.SplitN(line, ":", 3) + if len(parts) != 3 { + continue + } + for _, component := range strings.Split(parts[2], "/") { + if component == scope || req.Runtime == "docker" && component == req.ContainerID { + return true + } + } + } + return false +} diff --git a/internal/filesystemprobe/observer_linux.go b/internal/filesystemprobe/observer_linux.go new file mode 100644 index 000000000..2e2b33ee4 --- /dev/null +++ b/internal/filesystemprobe/observer_linux.go @@ -0,0 +1,133 @@ +//go:build linux + +package filesystemprobe + +import ( + "context" + "errors" + "fmt" + "io" + "math" + "os" + "strconv" + "time" + + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" + "golang.org/x/sys/unix" +) + +func observeContainer(ctx context.Context, req ContainerRequest) ([]filesystem.Observation, error) { + // Hold this proc directory and root descriptor throughout the observation. + // Never re-resolve /proc/PID after the identity check, where PID reuse could + // otherwise change the process whose mount namespace is observed. + proc, err := unix.Open("/proc/"+strconv.Itoa(req.PID), unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + return nil, fmt.Errorf("open container process: %w", err) + } + defer unix.Close(proc) + if err := verifyCgroup(proc, req); err != nil { + return nil, err + } + root, err := unix.Openat(proc, "root", unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + return nil, fmt.Errorf("open container filesystem namespace: %w", err) + } + defer unix.Close(root) + if err := verifyCgroup(proc, req); err != nil { + return nil, err + } + observations := make([]filesystem.Observation, 0, len(req.Mountpoints)) + for _, mount := range req.Mountpoints { + if err := ctx.Err(); err != nil { + return nil, err + } + observation := filesystem.Observation{Mountpoint: mount, Source: Source} + observation.Type, observation.Usage, err = observePath(root, mount) + observation.ObservedAt = time.Now().UTC() + if err != nil { + observation.Error = err.Error() + } + observations = append(observations, observation) + } + if err := verifyCgroup(proc, req); err != nil { + return nil, err + } + return observations, nil +} + +func verifyCgroup(proc int, req ContainerRequest) error { + fd, err := unix.Openat(proc, "cgroup", unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if err != nil { + return fmt.Errorf("read container process identity: %w", err) + } + file := os.NewFile(uintptr(fd), "cgroup") + defer file.Close() + raw, err := io.ReadAll(io.LimitReader(file, 65537)) + if err != nil || len(raw) > 65536 { + return errors.New("container process identity is unavailable") + } + if !cgroupMatches(string(raw), req) { + return errors.New("process cgroup does not attest the exact container identity") + } + return nil +} + +func observePath(root int, mount string) (string, *filesystem.Usage, error) { + // Absolute symlinks stay within the resource root. Magic links cannot escape + // it. Unsupported kernels fail rather than falling back to unsafe resolution. + fd, err := unix.Openat2(root, mount, &unix.OpenHow{ + Flags: unix.O_PATH | unix.O_CLOEXEC, + Resolve: unix.RESOLVE_IN_ROOT | unix.RESOLVE_NO_MAGICLINKS, + }) + if err != nil { + return "", nil, fmt.Errorf("open mount within resource namespace: %w", err) + } + defer unix.Close(fd) + var stat unix.Statfs_t + if err := unix.Fstatfs(fd, &stat); err != nil { + return "", nil, fmt.Errorf("read filesystem counters: %w", err) + } + usage, err := filesystemUsage(stat) + if err != nil { + return "", nil, err + } + return filesystemType(int64(stat.Type)), usage, nil +} + +func filesystemUsage(stat unix.Statfs_t) (*filesystem.Usage, error) { + blockSize := stat.Frsize + if blockSize <= 0 { + blockSize = stat.Bsize + } + if blockSize <= 0 || stat.Blocks == 0 || stat.Blocks > math.MaxUint64/uint64(blockSize) || stat.Bfree > stat.Blocks || stat.Bavail > stat.Bfree { + return nil, errors.New("filesystem counters are unavailable or inconsistent") + } + usage := &filesystem.Usage{ + CapacityBytes: stat.Blocks * uint64(blockSize), + FreeBytes: stat.Bfree * uint64(blockSize), + AvailableBytes: stat.Bavail * uint64(blockSize), + } + if stat.Files > 0 && stat.Ffree <= stat.Files { + usage.Inodes = &filesystem.InodeUsage{Capacity: stat.Files, Free: stat.Ffree} + } + return usage, nil +} + +func filesystemType(kind int64) string { + switch kind { + case unix.TMPFS_MAGIC: + return "tmpfs" + case unix.OVERLAYFS_SUPER_MAGIC: + return "overlay" + case unix.EXT4_SUPER_MAGIC: + return "ext" + case unix.XFS_SUPER_MAGIC: + return "xfs" + case unix.BTRFS_SUPER_MAGIC: + return "btrfs" + case unix.NFS_SUPER_MAGIC: + return "nfs" + default: + return fmt.Sprintf("0x%x", uint64(kind)) + } +} diff --git a/internal/filesystemprobe/observer_linux_test.go b/internal/filesystemprobe/observer_linux_test.go new file mode 100644 index 000000000..2cfb97939 --- /dev/null +++ b/internal/filesystemprobe/observer_linux_test.go @@ -0,0 +1,83 @@ +//go:build linux + +package filesystemprobe + +import ( + "context" + "math" + "os" + "path/filepath" + "strconv" + "testing" + + "golang.org/x/sys/unix" +) + +func TestFilesystemCountersPreserveFullAndReservedSpace(t *testing.T) { + stat := unix.Statfs_t{Blocks: 2048, Bsize: 4096, Frsize: 4096, Bfree: 3, Bavail: 0, Files: 10, Ffree: 0} + u, err := filesystemUsage(stat) + if err != nil || u.CapacityBytes != 8<<20 || u.FreeBytes != 12288 || u.AvailableBytes != 0 || u.Inodes == nil || u.Inodes.Free != 0 { + t.Fatalf("counters: %+v %v", u, err) + } + stat.Files = 0 + stat.Ffree = math.MaxUint64 + u, err = filesystemUsage(stat) + if err != nil || u.Inodes != nil { + t.Fatalf("unsupported inode inventory invalidated capacity: %+v %v", u, err) + } + stat.Blocks = math.MaxUint64 + if _, err := filesystemUsage(stat); err == nil { + t.Fatal("overflow accepted") + } + stat.Blocks = 1 + stat.Bfree = 2 + if _, err := filesystemUsage(stat); err == nil { + t.Fatal("inconsistent free counters accepted") + } +} + +func TestMountResolutionStaysWithinResourceRoot(t *testing.T) { + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, "inside"), 0700); err != nil { + t.Fatal(err) + } + if err := os.Symlink("/inside", filepath.Join(dir, "absolute")); err != nil { + t.Fatal(err) + } + fd, err := unix.Open(dir, unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + defer unix.Close(fd) + for _, mount := range []string{"/", "/inside", "/absolute"} { + kind, u, err := observePath(fd, mount) + if err != nil || u == nil || kind == "" { + t.Fatalf("confined mount %s: %s %+v %v", mount, kind, u, err) + } + } + if err := os.Symlink("/etc", filepath.Join(dir, "outside")); err != nil { + t.Fatal(err) + } + if _, u, err := observePath(fd, "/outside"); err == nil || u != nil { + t.Fatal("absolute symlink escaped resource root") + } +} + +func TestMountResolutionRejectsProcMagicLinks(t *testing.T) { + root, err := unix.Open("/", unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + defer unix.Close(root) + if _, u, err := observePath(root, "/proc/self/fd/"+strconv.Itoa(root)); err == nil || u != nil { + t.Fatal("proc magic link accepted") + } +} + +func TestWrongProcessCannotSupplyContainerCapacity(t *testing.T) { + r := testRequest() + r.PID = os.Getpid() + if got, err := observeContainer(context.Background(), r); err == nil || got != nil { + t.Fatalf("unrelated process supplied capacity: %+v %v", got, err) + } +} diff --git a/internal/filesystemprobe/observer_other.go b/internal/filesystemprobe/observer_other.go new file mode 100644 index 000000000..23ce33bf4 --- /dev/null +++ b/internal/filesystemprobe/observer_other.go @@ -0,0 +1,14 @@ +//go:build !linux + +package filesystemprobe + +import ( + "context" + "errors" + + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" +) + +func observeContainer(context.Context, ContainerRequest) ([]filesystem.Observation, error) { + return nil, errors.New("container filesystem observations require access to the Linux process namespace") +} diff --git a/internal/filesystemprobe/observer_test.go b/internal/filesystemprobe/observer_test.go new file mode 100644 index 000000000..45b606e0f --- /dev/null +++ b/internal/filesystemprobe/observer_test.go @@ -0,0 +1,81 @@ +package filesystemprobe + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" +) + +func testRequest() ContainerRequest { + return ContainerRequest{PID: 42, ContainerID: strings.Repeat("a", 64), Runtime: "docker", Mountpoints: []string{"/", "/cache"}} +} + +func TestContainerCgroupRequiresExactIdentity(t *testing.T) { + r := testRequest() + for _, raw := range []string{"0::/docker/" + r.ContainerID, "1:cpu:/system.slice/docker-" + r.ContainerID + ".scope", "0::/docker/" + r.ContainerID + "/child"} { + if !cgroupMatches(raw, r) { + t.Fatalf("exact identity rejected: %q", raw) + } + } + for _, raw := range []string{"0::/", "0::/docker/" + r.ContainerID[:12], "0::/docker/" + r.ContainerID + "b", "0::/system.slice/docker-" + r.ContainerID + ".scope-extra", "0::/libpod-" + r.ContainerID + ".scope", "malformed"} { + if cgroupMatches(raw, r) { + t.Fatalf("unattested identity accepted: %q", raw) + } + } + r.Runtime = "podman" + if !cgroupMatches("0::/user.slice/libpod-"+r.ContainerID+".scope", r) || cgroupMatches("0::/docker/"+r.ContainerID, r) { + t.Fatal("runtime identity crossed") + } +} + +func TestObserverDoesNotAccumulateOrReuseTimedOutReads(t *testing.T) { + release := make(chan struct{}) + defer close(release) + var calls atomic.Int32 + o := &Observer{timeout: 20 * time.Millisecond, probe: func(context.Context, ContainerRequest) ([]filesystem.Observation, error) { + calls.Add(1) + <-release + return []filesystem.Observation{{Mountpoint: "/", Usage: &filesystem.Usage{CapacityBytes: 1}}}, nil + }} + if got, err := o.Observe(context.Background(), testRequest()); !errors.Is(err, context.DeadlineExceeded) || got != nil { + t.Fatalf("timeout: %+v %v", got, err) + } + for i := 0; i < 3; i++ { + if got, err := o.Observe(context.Background(), testRequest()); err == nil || got != nil { + t.Fatalf("stalled probe reused: %+v %v", got, err) + } + } + if calls.Load() != 1 { + t.Fatalf("started %d probes for one stuck container", calls.Load()) + } +} + +func TestObserverRejectsInvalidCoordinatesBeforeNativeRead(t *testing.T) { + cases := []func(*ContainerRequest){ + func(r *ContainerRequest) { r.PID = 0 }, func(r *ContainerRequest) { r.ContainerID = "name" }, + func(r *ContainerRequest) { r.Runtime = "remote" }, func(r *ContainerRequest) { r.Mountpoints = nil }, + func(r *ContainerRequest) { r.Mountpoints = []string{"relative"} }, func(r *ContainerRequest) { r.Mountpoints = []string{"/../host"} }, + func(r *ContainerRequest) { r.Mountpoints = []string{"/cache", "/cache"} }, func(r *ContainerRequest) { r.Mountpoints = []string{"/a\x00b"} }, + } + o := &Observer{probe: func(context.Context, ContainerRequest) ([]filesystem.Observation, error) { + t.Error("invalid coordinates reached native read") + return nil, nil + }} + for _, change := range cases { + r := testRequest() + change(&r) + if _, err := o.Observe(context.Background(), r); err == nil { + t.Fatalf("accepted %+v", r) + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := o.Observe(ctx, testRequest()); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} diff --git a/internal/models/converters.go b/internal/models/converters.go index 607863401..4b272130e 100644 --- a/internal/models/converters.go +++ b/internal/models/converters.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) @@ -673,6 +674,7 @@ func (c DockerContainer) ToFrontend() DockerContainerFrontend { } container.Mounts = mounts } + container.Filesystems = filesystem.Clone(c.Filesystems) if c.Podman != nil { container.Podman = &DockerPodmanContainerFrontend{ diff --git a/internal/models/deepcopy.go b/internal/models/deepcopy.go index 0cb215137..41f70ee63 100644 --- a/internal/models/deepcopy.go +++ b/internal/models/deepcopy.go @@ -4,6 +4,7 @@ import ( "reflect" "time" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) @@ -491,6 +492,7 @@ func cloneDockerContainer(src DockerContainer) DockerContainer { dest.Networks = append([]DockerContainerNetworkLink(nil), src.Networks...) dest.BlockIO = cloneDockerContainerBlockIO(src.BlockIO) dest.Mounts = append([]DockerContainerMount(nil), src.Mounts...) + dest.Filesystems = filesystem.Clone(src.Filesystems) dest.Podman = cloneDockerPodmanContainer(src.Podman) dest.UpdateStatus = cloneDockerContainerUpdateStatus(src.UpdateStatus) return dest.NormalizeCollections() diff --git a/internal/models/models.go b/internal/models/models.go index 082f8ca36..956e051b6 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -13,6 +13,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/proxmoxidentity" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) @@ -1165,6 +1166,7 @@ type DockerContainer struct { RootFilesystemBytes int64 `json:"rootFilesystemBytes,omitempty"` BlockIO *DockerContainerBlockIO `json:"blockIo,omitempty"` Mounts []DockerContainerMount `json:"mounts,omitempty"` + Filesystems []filesystem.Observation `json:"filesystems,omitempty"` Podman *DockerPodmanContainer `json:"podman,omitempty"` UpdateStatus *DockerContainerUpdateStatus `json:"updateStatus,omitempty"` // Image update detection status } diff --git a/internal/models/models_frontend.go b/internal/models/models_frontend.go index 36ed0a480..443c15301 100644 --- a/internal/models/models_frontend.go +++ b/internal/models/models_frontend.go @@ -4,6 +4,7 @@ import ( "encoding/json" "time" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) @@ -471,6 +472,7 @@ type DockerContainerFrontend struct { RootFilesystemBytes int64 `json:"rootFilesystemBytes,omitempty"` BlockIO *DockerContainerBlockIOFrontend `json:"blockIo,omitempty"` Mounts []DockerContainerMountFrontend `json:"mounts"` + Filesystems []filesystem.Observation `json:"filesystems,omitempty"` Podman *DockerPodmanContainerFrontend `json:"podman,omitempty"` UpdateStatus *DockerContainerUpdateStatusFrontend `json:"updateStatus,omitempty"` } diff --git a/internal/monitoring/docker_filesystem_evidence_test.go b/internal/monitoring/docker_filesystem_evidence_test.go new file mode 100644 index 000000000..d7fc602ad --- /dev/null +++ b/internal/monitoring/docker_filesystem_evidence_test.go @@ -0,0 +1,68 @@ +package monitoring + +import ( + "encoding/json" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/mock" + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" +) + +func TestDockerFilesystemEvidenceSurvivesIngestionAndReplacesStaleUsage(t *testing.T) { + previous := mock.IsMockEnabled() + mustSetMockEnabled(t, false) + t.Cleanup(func() { mustSetMockEnabled(t, previous) }) + m := newTestMonitor(t) + registry := unifiedresources.NewRegistry(nil) + now := time.Now().UTC() + report := agentsdocker.Report{Timestamp: now, Agent: agentsdocker.AgentInfo{ID: "fs-agent", IntervalSeconds: 30}, Host: agentsdocker.HostInfo{Hostname: "fs-host"}, Containers: []agentsdocker.Container{{ID: "fs-container", Name: "worker", State: "running", WritableLayerBytes: 999, RootFilesystemBytes: 9999}}} + observations := [][]filesystem.Observation{ + {{Mountpoint: "/cache", Source: "linux-proc-root-statfs", ObservedAt: now, Type: "tmpfs", Usage: &filesystem.Usage{CapacityBytes: 8 << 20, Inodes: &filesystem.InodeUsage{Capacity: 100, Free: 50}}}}, + {{Mountpoint: "/cache", Source: "linux-proc-root-statfs", ObservedAt: now.Add(time.Second), Error: "namespace unavailable"}}, + nil, + } + for i, rows := range observations { + report.Timestamp = now.Add(time.Duration(i) * time.Second) + report.Containers[0].Filesystems = rows + host, err := m.ApplyDockerReport(report, nil) + if err != nil { + t.Fatal(err) + } + registry.IngestSnapshot(models.StateSnapshot{DockerHosts: []models.DockerHost{host}}) + resources := registry.ListByType(unifiedresources.ResourceTypeAppContainer) + if len(resources) != 1 { + t.Fatalf("resources: %+v", resources) + } + got := resources[0].Docker.Filesystems + if len(got) != len(rows) { + t.Fatalf("old evidence retained at step %d: %+v", i, got) + } + if resources[0].Metrics != nil && resources[0].Metrics.Disk != nil { + t.Fatal("mount capacity was promoted into a container-wide disk percentage") + } + if i == 0 { + if got[0].Usage == nil || got[0].Usage.CapacityBytes != 8<<20 || got[0].Usage.AvailableBytes != 0 { + t.Fatalf("measured full filesystem lost: %+v", got) + } + got[0].Usage.CapacityBytes = 1 + got[0].Usage.Inodes.Free = 1 + report.Containers[0].Filesystems[0].Usage.CapacityBytes = 2 + fresh := registry.ListByType(unifiedresources.ResourceTypeAppContainer)[0].Docker.Filesystems[0] + if fresh.Usage.CapacityBytes != 8<<20 || fresh.Usage.Inodes.Free != 50 { + t.Fatal("resource snapshot aliases external observation storage") + } + } + if i == 1 && (got[0].Usage != nil || got[0].Error == "") { + t.Fatalf("failed read kept old numeric usage: %+v", got) + } + wire, err := json.Marshal(resources) + if err != nil { + t.Fatal(err) + } + t.Logf("FILESYSTEM_RESOURCE_STEP_%d %s", i, wire) + } +} diff --git a/internal/monitoring/monitor_agents.go b/internal/monitoring/monitor_agents.go index ae7f4e043..d455a8976 100644 --- a/internal/monitoring/monitor_agents.go +++ b/internal/monitoring/monitor_agents.go @@ -24,6 +24,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" unraidstatus "github.com/rcourtman/pulse-go-rewrite/internal/unraid" agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" "github.com/rcourtman/pulse-go-rewrite/pkg/fsfilters" pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing" @@ -2406,6 +2407,7 @@ func (m *Monitor) ApplyDockerReport(report agentsdocker.Report, tokenRecord *con } container.Mounts = mounts } + container.Filesystems = filesystem.Clone(payload.Filesystems) containers = append(containers, container) } diff --git a/internal/unifiedresources/adapters.go b/internal/unifiedresources/adapters.go index 5a8287413..dbfb71cab 100644 --- a/internal/unifiedresources/adapters.go +++ b/internal/unifiedresources/adapters.go @@ -12,6 +12,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/operationreceipt" "github.com/rcourtman/pulse-go-rewrite/internal/platformsupport" "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) @@ -2459,6 +2460,7 @@ func resourceFromDockerContainer(ct models.DockerContainer, host models.DockerHo Capabilities: dockerContainerLifecycleCapabilities(ct, host, runtime, now), } resource.Docker = docker + resource.Docker.Filesystems = filesystem.Clone(ct.Filesystems) identity := ResourceIdentity{ Hostnames: uniqueStrings([]string{ct.Name}), } diff --git a/internal/unifiedresources/clone.go b/internal/unifiedresources/clone.go index 8e0548419..bb6f90737 100644 --- a/internal/unifiedresources/clone.go +++ b/internal/unifiedresources/clone.go @@ -4,6 +4,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) @@ -314,6 +315,7 @@ func cloneDockerData(in *DockerData) *DockerData { out.EngineLabels = cloneStringMap(in.EngineLabels) out.Networks = cloneDockerNetworkMetaSlice(in.Networks) out.Mounts = cloneDockerMountMetaSlice(in.Mounts) + out.Filesystems = filesystem.Clone(in.Filesystems) out.UpdateStatus = cloneDockerUpdateStatusMeta(in.UpdateStatus) out.BlockIO = cloneDockerContainerBlockIOMeta(in.BlockIO) out.Podman = cloneDockerPodmanContainerMeta(in.Podman) diff --git a/internal/unifiedresources/store.go b/internal/unifiedresources/store.go index bbbf6cc65..f089adff7 100644 --- a/internal/unifiedresources/store.go +++ b/internal/unifiedresources/store.go @@ -1031,6 +1031,7 @@ func (s *SQLiteResourceStore) migrateResourceChangesSchema() error { func (s *SQLiteResourceStore) ensureResourceChangesIndexes() error { indexes := []string{ + `CREATE INDEX IF NOT EXISTS idx_resource_changes_time ON resource_changes(observed_at DESC)`, `CREATE INDEX IF NOT EXISTS idx_resource_changes_canonical_time ON resource_changes(canonical_id, observed_at DESC)`, `CREATE INDEX IF NOT EXISTS idx_resource_changes_kind_time ON resource_changes(kind, observed_at DESC)`, `CREATE INDEX IF NOT EXISTS idx_resource_changes_source_type_time ON resource_changes(source_type, observed_at DESC)`, diff --git a/internal/unifiedresources/store_history_index_test.go b/internal/unifiedresources/store_history_index_test.go new file mode 100644 index 000000000..9438e6340 --- /dev/null +++ b/internal/unifiedresources/store_history_index_test.go @@ -0,0 +1,61 @@ +package unifiedresources + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func TestExistingStoreIndexesGlobalRecentHistory(t *testing.T) { + dir := t.TempDir() + store, err := NewSQLiteResourceStore(dir, "default") + if err != nil { + t.Fatal(err) + } + // Reopen an existing canonical database without the global time index, + // matching installations that already have resource-specific indexes. + if _, err := store.db.Exec("DROP INDEX idx_resource_changes_time"); err != nil { + t.Fatal(err) + } + store.Close() + store, err = NewSQLiteResourceStore(dir, "default") + if err != nil { + t.Fatal(err) + } + defer store.Close() + rows, err := store.db.Query("EXPLAIN QUERY PLAN SELECT id FROM resource_changes WHERE observed_at >= ? ORDER BY observed_at DESC LIMIT 256", time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + indexed := false + defer rows.Close() + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + if strings.Contains(detail, "TEMP B-TREE") || strings.Contains(detail, "SCAN resource_changes") { + t.Fatalf("bounded global history scans or sorts retained history: %s", detail) + } + indexed = indexed || strings.Contains(detail, "idx_resource_changes_time") + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + rows.Close() + if !indexed { + t.Fatal("global history query did not use its chronological index") + } + at := time.Now().UTC() + for i, offset := range []time.Duration{3 * time.Minute, time.Minute, 2 * time.Minute} { + if err := store.RecordChange(ResourceChange{ID: fmt.Sprintf("history-%d", i), ResourceID: fmt.Sprintf("host-%d", i), ObservedAt: at.Add(-offset), Kind: ChangeKind("status_changed")}); err != nil { + t.Fatal(err) + } + } + changes, err := store.GetRecentChanges("", at.Add(-time.Hour), 2) + if err != nil || len(changes) != 2 || changes[0].ID != "history-1" || changes[1].ID != "history-2" { + t.Fatalf("bounded cross-resource chronology changed: %+v error=%v", changes, err) + } +} diff --git a/internal/unifiedresources/types.go b/internal/unifiedresources/types.go index f310a7bbe..7e4261e50 100644 --- a/internal/unifiedresources/types.go +++ b/internal/unifiedresources/types.go @@ -7,6 +7,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" "github.com/rcourtman/pulse-go-rewrite/pkg/tlsutil" ) @@ -1152,6 +1153,7 @@ type DockerData struct { Labels map[string]string `json:"labels,omitempty"` Networks []DockerNetworkMeta `json:"networks,omitempty"` Mounts []DockerMountMeta `json:"mounts,omitempty"` + Filesystems []filesystem.Observation `json:"filesystems,omitempty"` UpdateStatus *DockerUpdateStatusMeta `json:"updateStatus,omitempty"` // Service-specific fields (populated when Resource.Type == ResourceTypeDockerService) diff --git a/internal/unifiedresources/views.go b/internal/unifiedresources/views.go index 952fe4318..dce859df1 100644 --- a/internal/unifiedresources/views.go +++ b/internal/unifiedresources/views.go @@ -7,6 +7,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" ) @@ -4063,6 +4064,13 @@ func (v DockerContainerView) Mounts() []DockerMountMeta { return cloneDockerMountMetaSlice(v.r.Docker.Mounts) } +func (v DockerContainerView) Filesystems() []filesystem.Observation { + if v.r == nil || v.r.Docker == nil { + return nil + } + return filesystem.Clone(v.r.Docker.Filesystems) +} + func (v DockerContainerView) UpdateStatus() *DockerUpdateStatusMeta { if v.r == nil || v.r.Docker == nil { return nil diff --git a/pkg/agents/docker/report.go b/pkg/agents/docker/report.go index e23eb8697..3b83e8c6d 100644 --- a/pkg/agents/docker/report.go +++ b/pkg/agents/docker/report.go @@ -3,6 +3,7 @@ package dockeragent import ( "time" + "github.com/rcourtman/pulse-go-rewrite/pkg/agents/filesystem" hostagent "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" ) @@ -74,37 +75,38 @@ type HostSecurityInfo struct { // Container captures the runtime state for a Docker container at report time. type Container struct { - ID string `json:"id"` - Name string `json:"name"` - Image string `json:"image"` - ImageDigest string `json:"imageDigest,omitempty"` // Current image digest for update detection - CreatedAt time.Time `json:"createdAt"` - State string `json:"state"` - Status string `json:"status"` - Health string `json:"health,omitempty"` - HealthcheckTargets []string `json:"healthcheckTargets,omitempty"` - CPUPercent float64 `json:"cpuPercent"` - MemoryUsageBytes int64 `json:"memoryUsageBytes"` - MemoryLimitBytes int64 `json:"memoryLimitBytes"` - MemoryPercent float64 `json:"memoryPercent"` - UptimeSeconds int64 `json:"uptimeSeconds"` - RestartCount int `json:"restartCount"` - ExitCode int `json:"exitCode"` - OOMKilled *bool `json:"oomKilled,omitempty"` - StartedAt *time.Time `json:"startedAt,omitempty"` - FinishedAt *time.Time `json:"finishedAt,omitempty"` - Ports []ContainerPort `json:"ports,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - Env []string `json:"env,omitempty"` - Networks []ContainerNetwork `json:"networks,omitempty"` - NetworkRXBytes uint64 `json:"networkRxBytes,omitempty"` - NetworkTXBytes uint64 `json:"networkTxBytes,omitempty"` - WritableLayerBytes int64 `json:"writableLayerBytes,omitempty"` - RootFilesystemBytes int64 `json:"rootFilesystemBytes,omitempty"` - BlockIO *ContainerBlockIO `json:"blockIo,omitempty"` - Mounts []ContainerMount `json:"mounts,omitempty"` - Podman *PodmanContainer `json:"podman,omitempty"` - UpdateStatus *UpdateStatus `json:"updateStatus,omitempty"` // Image update detection status + ID string `json:"id"` + Name string `json:"name"` + Image string `json:"image"` + ImageDigest string `json:"imageDigest,omitempty"` // Current image digest for update detection + CreatedAt time.Time `json:"createdAt"` + State string `json:"state"` + Status string `json:"status"` + Health string `json:"health,omitempty"` + HealthcheckTargets []string `json:"healthcheckTargets,omitempty"` + CPUPercent float64 `json:"cpuPercent"` + MemoryUsageBytes int64 `json:"memoryUsageBytes"` + MemoryLimitBytes int64 `json:"memoryLimitBytes"` + MemoryPercent float64 `json:"memoryPercent"` + UptimeSeconds int64 `json:"uptimeSeconds"` + RestartCount int `json:"restartCount"` + ExitCode int `json:"exitCode"` + OOMKilled *bool `json:"oomKilled,omitempty"` + StartedAt *time.Time `json:"startedAt,omitempty"` + FinishedAt *time.Time `json:"finishedAt,omitempty"` + Ports []ContainerPort `json:"ports,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Env []string `json:"env,omitempty"` + Networks []ContainerNetwork `json:"networks,omitempty"` + NetworkRXBytes uint64 `json:"networkRxBytes,omitempty"` + NetworkTXBytes uint64 `json:"networkTxBytes,omitempty"` + WritableLayerBytes int64 `json:"writableLayerBytes,omitempty"` + RootFilesystemBytes int64 `json:"rootFilesystemBytes,omitempty"` + BlockIO *ContainerBlockIO `json:"blockIo,omitempty"` + Mounts []ContainerMount `json:"mounts,omitempty"` + Filesystems []filesystem.Observation `json:"filesystems,omitempty"` + Podman *PodmanContainer `json:"podman,omitempty"` + UpdateStatus *UpdateStatus `json:"updateStatus,omitempty"` // Image update detection status } // ContainerPort tracks an exposed container port mapping. diff --git a/pkg/agents/filesystem/report.go b/pkg/agents/filesystem/report.go new file mode 100644 index 000000000..5b337384c --- /dev/null +++ b/pkg/agents/filesystem/report.go @@ -0,0 +1,53 @@ +// Package filesystem defines source-authored filesystem observations shared by +// agent reports and canonical resource projections. +package filesystem + +import "time" + +// Observation describes the filesystem visible at one mountpoint in the +// observed resource's namespace. Shared filesystems are not resource quotas. +// A failed observation has no Usage. Missing evidence never implies zero usage. +type Observation struct { + Mountpoint string `json:"mountpoint"` + Source string `json:"source"` + ObservedAt time.Time `json:"observedAt"` + Type string `json:"type,omitempty"` + Usage *Usage `json:"usage,omitempty"` + Error string `json:"error,omitempty"` +} + +// Usage preserves the kernel counters, including observed zero. AvailableBytes +// excludes blocks reserved from ordinary users. FreeBytes includes them. +// Inodes is absent when the filesystem did not report a finite inode inventory. +type Usage struct { + CapacityBytes uint64 `json:"capacityBytes"` + FreeBytes uint64 `json:"freeBytes"` + AvailableBytes uint64 `json:"availableBytes"` + Inodes *InodeUsage `json:"inodes,omitempty"` +} + +type InodeUsage struct { + Capacity uint64 `json:"capacity"` + Free uint64 `json:"free"` +} + +// Clone prevents a projected or retained observation from sharing mutable +// measurement storage with its source report. +func Clone(in []Observation) []Observation { + if in == nil { + return nil + } + out := make([]Observation, len(in)) + copy(out, in) + for i := range out { + if in[i].Usage != nil { + usage := *in[i].Usage + if usage.Inodes != nil { + inodes := *usage.Inodes + usage.Inodes = &inodes + } + out[i].Usage = &usage + } + } + return out +} diff --git a/pkg/agents/filesystem/report_test.go b/pkg/agents/filesystem/report_test.go new file mode 100644 index 000000000..cdf78da56 --- /dev/null +++ b/pkg/agents/filesystem/report_test.go @@ -0,0 +1,40 @@ +package filesystem + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestObservationDistinguishesUnavailableFromExhausted(t *testing.T) { + unavailable, err := json.Marshal(Observation{Mountpoint: "/cache", Error: "namespace unavailable"}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(unavailable), "capacityBytes") || strings.Contains(string(unavailable), "usage") { + t.Fatal(string(unavailable)) + } + full, err := json.Marshal(Observation{Mountpoint: "/cache", Usage: &Usage{CapacityBytes: 8 << 20}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(full), `"availableBytes":0`) || !strings.Contains(string(full), `"freeBytes":0`) { + t.Fatal(string(full)) + } + if strings.Contains(string(full), `"inodes"`) { + t.Fatal("unknown inode inventory became observed zero") + } +} + +func TestCloneOwnsMeasurements(t *testing.T) { + in := []Observation{{Usage: &Usage{CapacityBytes: 123, Inodes: &InodeUsage{Capacity: 456}}}, {Error: "unavailable"}} + out := Clone(in) + out[0].Usage.CapacityBytes = 999 + out[0].Usage.Inodes.Capacity = 999 + if in[0].Usage.CapacityBytes != 123 || in[0].Usage.Inodes.Capacity != 456 || out[1].Usage != nil { + t.Fatal("cloned observations alias source") + } + if Clone(nil) != nil { + t.Fatal("nil evidence became present") + } +} diff --git a/scripts/release_control/canonical_completion_guard_test.py b/scripts/release_control/canonical_completion_guard_test.py index 53c41b475..95db2565f 100644 --- a/scripts/release_control/canonical_completion_guard_test.py +++ b/scripts/release_control/canonical_completion_guard_test.py @@ -225,6 +225,7 @@ class CanonicalCompletionGuardTest(unittest.TestCase): "internal/config/host_continuity_test.go", "internal/models/metrics_types_test.go", "internal/monitoring/availability_probe_agent_test.go", + "internal/monitoring/docker_filesystem_evidence_test.go", "internal/monitoring/docker_metric_presence_test.go", "internal/monitoring/monitor_host_agent_removal_lifecycle_test.go", "internal/monitoring/monitor_host_agents_test.go", @@ -283,6 +284,7 @@ class CanonicalCompletionGuardTest(unittest.TestCase): "diskinventory-collection-trust", "agent-fleet-diagnostics-runtime", "monitoring-runtime", + "filesystem-observations", ], ) self.assertEqual( @@ -356,7 +358,9 @@ class CanonicalCompletionGuardTest(unittest.TestCase): "internal/dockeragent/agent_cpu_test.go", "internal/dockeragent/agent_internal_test.go", "internal/dockeragent/blockio_presence_test.go", + "internal/dockeragent/collect_tmpfs_live_test.go", "internal/dockeragent/collect_tmpfs_test.go", + "internal/dockeragent/container_filesystems_test.go", "internal/dockeragent/swarm_coverage_test.go", ], } @@ -454,6 +458,7 @@ class CanonicalCompletionGuardTest(unittest.TestCase): "internal/config/host_continuity_test.go", "internal/models/metrics_types_test.go", "internal/monitoring/availability_probe_agent_test.go", + "internal/monitoring/docker_filesystem_evidence_test.go", "internal/monitoring/docker_metric_presence_test.go", "internal/monitoring/monitor_host_agent_removal_lifecycle_test.go", "internal/monitoring/monitor_host_agents_test.go", @@ -485,6 +490,7 @@ class CanonicalCompletionGuardTest(unittest.TestCase): "test_prefixes": [], "exact_files": [ "internal/config/host_continuity_test.go", + "internal/monitoring/docker_filesystem_evidence_test.go", "internal/monitoring/docker_metric_presence_test.go", "internal/monitoring/issue1485_unraid_lifecycle_test.go", "internal/monitoring/issue1595_collection_trust_test.go", diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.py index 36c799ae3..ffbce8444 100644 --- a/scripts/release_control/subsystem_lookup_test.py +++ b/scripts/release_control/subsystem_lookup_test.py @@ -4327,6 +4327,7 @@ class SubsystemLookupTest(unittest.TestCase): "internal/config/host_continuity_test.go", "internal/models/metrics_types_test.go", "internal/monitoring/availability_probe_agent_test.go", + "internal/monitoring/docker_filesystem_evidence_test.go", "internal/monitoring/docker_metric_presence_test.go", "internal/monitoring/monitor_host_agent_removal_lifecycle_test.go", "internal/monitoring/monitor_host_agents_test.go", @@ -4350,6 +4351,7 @@ class SubsystemLookupTest(unittest.TestCase): monitoring_match["verification_requirement"]["exact_files"], [ "internal/config/host_continuity_test.go", + "internal/monitoring/docker_filesystem_evidence_test.go", "internal/monitoring/docker_metric_presence_test.go", "internal/monitoring/issue1485_unraid_lifecycle_test.go", "internal/monitoring/issue1595_collection_trust_test.go", @@ -4473,6 +4475,7 @@ class SubsystemLookupTest(unittest.TestCase): "internal/unifiedresources/resolved_host_set_test.go", "internal/unifiedresources/resource_operator_state_policy_test.go", "internal/unifiedresources/snapshot_source_filter_test.go", + "internal/unifiedresources/store_history_index_test.go", "internal/unifiedresources/store_test.go", ], ) @@ -4504,6 +4507,7 @@ class SubsystemLookupTest(unittest.TestCase): "internal/unifiedresources/resolved_host_set_test.go", "internal/unifiedresources/resource_operator_state_policy_test.go", "internal/unifiedresources/snapshot_source_filter_test.go", + "internal/unifiedresources/store_history_index_test.go", "internal/unifiedresources/store_test.go", ], )