From a10c175f31d06e9c7c9db2870243200890093c8d Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 2 Sep 2026 07:02:51 +0100 Subject: [PATCH] Fold alert-mirroring findings, collapse flapping, surface lasting decisions Two users (discussions #1623, #1699) could not find Patrol's durable outcomes: the Needs attention detail offered only Acknowledge and Suppress, and Remember as expected, Dismiss: Not an issue, Dismiss: Later, and Create rule sat two levels below it. The same screenshot showed an alert with eleven open/resolved transitions in a day listed as eleven timeline rows, and Patrol findings restating an active alert appearing beside it. Telemetry shows most findings are seen and ignored. Backend: findings_alert_mirror.go is the single matcher for a finding that restates an active alert (explicit alert identifier, or same canonical resource plus the same condition class). Patrol stamps mirrors_alert_id/mirrors_alert_type after every real and demo cycle and clears them when the alert resolves; the finding is kept, not dropped. The storm throttler now also owns flap detection: four or more open/resolved transitions in 24h mark a finding flapping, and the store maintains one collapsed lifecycle row with the count instead of a row per transition, hydrating the window from persisted lifecycle after restart. The attention projection applies the same threshold to alert timelines and exposes a flapping summary while keeping the full timeline. Frontend: the attention detail gains a Lasting decisions section that offers the four Patrol decisions on the finding mirroring the alert, each with one line saying what it does and how long it lasts, an inline confirmation, and a remembered-decision state with Reopen. An alert with no mirrored finding says Patrol has nothing to remember and points at alert thresholds. Flapping items carry one label and a summarised timeline with every transition under an expansion. FindingsPanel demotes active alert-mirroring findings into a collapsed group. The Patrol surface loads dismissed findings so remembered decisions are visible. --- docs/release-control/v6/internal/status.json | 50 ++- .../v6/internal/subsystems/ai-runtime.md | 9 +- .../v6/internal/subsystems/api-contracts.md | 17 + .../subsystems/frontend-primitives.md | 6 + .../subsystems/patrol-intelligence.md | 39 ++ frontend-modern/browser-verification.json | 60 ++- .../src/api/__tests__/patrolAttention.test.ts | 16 + frontend-modern/src/api/patrol.ts | 21 + frontend-modern/src/api/patrolAttention.ts | 14 + .../src/components/AI/FindingsPanel.tsx | 61 ++- .../patrol/PatrolAttentionWorkbench.tsx | 425 +++++++++++++++--- .../patrol/PatrolIntelligenceSurface.tsx | 2 + .../PatrolAttentionWorkbench.test.tsx | 257 ++++++++++- .../usePatrolIntelligenceState.test.ts | 4 +- .../features/patrol/patrolHomePresentation.ts | 155 ++++++- .../patrol/usePatrolIntelligenceState.ts | 12 +- frontend-modern/src/stores/aiIntelligence.ts | 13 + .../__tests__/findingAlertIdentity.test.ts | 21 +- .../src/utils/findingAlertIdentity.ts | 17 + internal/ai/attention.go | 52 +++ internal/ai/attention_test.go | 72 +++ internal/ai/demo.go | 4 + internal/ai/findings.go | 117 ++++- internal/ai/findings_alert_mirror.go | 192 ++++++++ internal/ai/findings_alert_mirror_test.go | 154 +++++++ internal/ai/findings_flapping_test.go | 162 +++++++ internal/ai/findings_storm_throttler.go | 185 ++++++++ internal/ai/findings_storm_throttler_test.go | 101 +++++ internal/ai/patrol_run.go | 8 + 29 files changed, 2128 insertions(+), 118 deletions(-) create mode 100644 internal/ai/findings_alert_mirror.go create mode 100644 internal/ai/findings_alert_mirror_test.go create mode 100644 internal/ai/findings_flapping_test.go diff --git a/docs/release-control/v6/internal/status.json b/docs/release-control/v6/internal/status.json index 458c6fd77..044f12c44 100644 --- a/docs/release-control/v6/internal/status.json +++ b/docs/release-control/v6/internal/status.json @@ -9975,6 +9975,40 @@ "kind": "file" } ] + }, + { + "id": "patrol-findings-hygiene-attention-noise", + "summary": "Two users in a row (discussions #1623 and #1699) could not find Patrol's durable outcomes because Remember as expected, Dismiss: Not an issue, Dismiss: Later, and Create rule sat two levels below the Needs attention detail, which offered only Acknowledge and Suppress. The same screenshot showed an alert with eleven open/resolved transitions in a day rendered as eleven timeline rows, and Patrol findings that restate an active alert (same resource, same condition) listed as separate items. Telemetry on 2026-09-01 showed 4,709 findings across 736 installs but only 255 investigations across 27, so most findings are seen and ignored. Attention needs: alert-mirroring findings folded under the alert, flapping collapsed to one labelled item with a count, and the durable decisions on the detail with one-line explanations.", + "owner": "project-owner", + "status": "triaged", + "recorded_at": "2026-09-02", + "lane_ids": [ + "L6" + ], + "subsystem_ids": [ + "ai-runtime", + "api-contracts", + "patrol-intelligence" + ], + "proposed_resolution": "lane-expansion", + "coverage_impact": 3, + "evidence": [ + { + "repo": "pulse", + "path": "frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx", + "kind": "file" + }, + { + "repo": "pulse", + "path": "internal/ai/findings_alert_mirror.go", + "kind": "file" + }, + { + "repo": "pulse", + "path": "internal/ai/findings_storm_throttler.go", + "kind": "file" + } + ] } ], "candidate_lanes": [ @@ -10042,7 +10076,21 @@ ] } ], - "work_claims": [], + "work_claims": [ + { + "id": "claude-patrol-findings-hygiene-coverage-gap-patrol-findings-hygiene-attention-noise", + "agent_id": "claude-patrol-findings-hygiene", + "summary": "Fold alert-mirroring findings under the alert, collapse flapping, surface lasting Patrol decisions on the attention detail", + "target_id": "v6-product-lane-expansion", + "claimed_at": "2026-09-02T03:29:56Z", + "heartbeat_at": "2026-09-02T03:29:56Z", + "expires_at": "2026-09-02T09:29:56Z", + "work_item": { + "kind": "coverage-gap", + "id": "patrol-findings-hygiene-attention-noise" + } + } + ], "open_decisions": [], "source_of_truth_file": "docs/release-control/v6/internal/SOURCE_OF_TRUTH.md", "resolved_decisions": [ diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 3058e60ac..6d91c301b 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -4247,7 +4247,14 @@ titles: provider alarm messages yield their bounded alarm subject, generic resource/storage incident types become plain issue language, and known acronyms such as CPU, ZFS, and I/O retain their canonical casing. Clients must not receive machine-title-cased placeholders such as `Resource Incident` and -repair them independently. `internal/ai/patrol_metrics.go` exports only +repair them independently. The projection also collapses open/resolved churn: +when a record's timeline holds at least four open-to-resolved or +resolved-to-open transitions inside the last 24 hours, the item carries a +`flapping` summary (transition count, window, first and latest transition) and +clients present one item with that label; the full transition list stays in the +detail timeline as forensic detail. The window and threshold are the shared +constants in `internal/ai/findings_storm_throttler.go`, so alerts and Patrol +findings agree on what flapping means. `internal/ai/patrol_metrics.go` exports only low-cardinality lifecycle-state counts, queue age, acknowledgement time, and calm-evaluation age; raw resource IDs are forbidden as metric labels. diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 4226dd7f9..25e9f5e2e 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -4371,6 +4371,23 @@ auto-register mutation boundary. ## Current State +### Attention flapping summary and finding alert-mirror fields + +`GET /api/ai/patrol/attention` items and `GET /api/ai/patrol/attention/{id}` +details may carry `flapping` (`transitionCount`, `windowHours`, +`firstTransitionAt`, `lastTransitionAt`) when the record's timeline holds at +least four open/resolved transitions inside the last 24 hours; the timeline +itself is unchanged. `GET /api/ai/patrol/findings` findings may carry +`flapping` (`transition_count`, `window_hours`, `first_transition_at`, +`last_transition_at`) under the same rule, plus `mirrors_alert_id` and +`mirrors_alert_type` naming the active alert the finding restates (same +canonical resource, same condition class). All four fields are omitted when +not applicable and are derived read-model annotations, not lifecycle state; +clients must not treat their absence as evidence of stability. +`frontend-modern/src/api/patrolAttention.ts` (`AttentionFlapping`) and +`frontend-modern/src/api/patrol.ts` (`FindingFlapping`, `mirrors_alert_*`) +are the typed client boundary. + ### Resource-list facets preserve scoped navigation evidence `GET /api/resources` returns a compact `facets` object beside the paged row diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index a7ded1465..09523b1d7 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -3110,6 +3110,12 @@ viewport ordering, and reduced-motion-safe behavior. Detail deep links use The temporary-suppression reason and duration are labelled native controls owned by `FormTextarea` and `FormSelect`; Patrol must not recreate their label, focus, responsive touch-target, or controlled-value shells locally. +The Lasting decisions section reuses the same ownership: its four decision +triggers are shared `Button` primitives in a labelled list, the inline +confirmation note or rule reason is a `FormTextarea`, and the alert-only +guidance link is a `ButtonLink`. `PatrolIntelligenceSurface.tsx` passes the +Patrol findings accessor into the workbench; the workbench does not fetch or +poll findings itself. The objective brief and optional-context fields in `PatrolObjectivesPanel` share the same `FormTextarea` ownership contract. diff --git a/docs/release-control/v6/internal/subsystems/patrol-intelligence.md b/docs/release-control/v6/internal/subsystems/patrol-intelligence.md index 497270aad..5aa2a6db6 100644 --- a/docs/release-control/v6/internal/subsystems/patrol-intelligence.md +++ b/docs/release-control/v6/internal/subsystems/patrol-intelligence.md @@ -2498,3 +2498,42 @@ writes through `@/api/resourceOperatorState`; no alert-local mute document is created. `ResourceMonitoringPolicyAction.test.tsx` and the resource operator state section tests pin provider copy, expected-offline persistence, retirement, and compatibility preservation. + +### Lasting decisions live on the attention detail + +Two users in a row (discussions #1623 and #1699) could not find the durable +Patrol outcomes because they sat two levels below the Needs attention detail. +The selected detail in `PatrolAttentionWorkbench.tsx` now owns a `Lasting +decisions` section beside the alert lifecycle controls. When a Patrol finding +mirrors the selected alert (same resource, backend-stamped `mirrorsAlertType` +equal to the item kind, joined by `getLinkedPatrolFindings` in +`patrolHomePresentation.ts` over the surface-supplied `findings` accessor), +the section lists Remember as expected, Dismiss: Not an issue, Dismiss: Later, +and Create rule, each with one line stating what it does and how long it lasts +(`PATROL_LASTING_DECISIONS`). Choosing one opens an inline confirmation with an +optional note, or a required reason for a rule, and writes through the existing +finding actions (`aiIntelligenceStore.dismissFinding`, +`createSuppressionRuleFromFinding`); the alert lifecycle above is untouched. +A finding Patrol already remembers shows the remembered decision and a Reopen +path instead of re-offering the choice; to make that possible +`usePatrolIntelligenceState.ts` loads Patrol findings with dismissed and +resolved history included, because a dismissed finding is absent from the +active-only default set and the detail would otherwise claim Patrol has +nothing to remember. An alert with no mirrored finding says +plainly that Patrol has nothing to remember and points at alert thresholds, +because the only durable control for a bare alert is the alert configuration. +The temporary controls state in one visible line that acknowledgement and +suppression are short-term; the former `More ways to manage this issue` +disclosure is gone. + +Flapping is one item, not many. Queue rows and the detail show the backend +`flapping` label with the transition count; the detail timeline renders a +one-line flapping summary and keeps every transition under `Show all N +transitions`. `FindingsPanel.tsx` shows the same label on findings and demotes +active findings that mirror an active alert (`isAlertMirroredFinding` in +`frontend-modern/src/utils/findingAlertIdentity.ts`) into a collapsed `N +findings mirror an active alert` group below the list, where their finding +options remain available; dismissed and resolved views keep them in place. +`PatrolAttentionWorkbench.test.tsx` pins the four decisions and their copy, the +rule reason requirement, the remembered-decision path, the alert-only +guidance, and the flapping collapse. diff --git a/frontend-modern/browser-verification.json b/frontend-modern/browser-verification.json index 2d7ac6f24..8c1fbdf41 100644 --- a/frontend-modern/browser-verification.json +++ b/frontend-modern/browser-verification.json @@ -1,45 +1,57 @@ { "version": 1, - "base_sha": "9fba43ffed507f092f876acaf23bef013f0f6fab", - "verified_at": "2026-09-02T03:39:02Z", + "base_sha": "d7a4dcf8e7c59fc35348236d8aeb3f25dc6062fe", + "verified_at": "2026-09-02T06:02:51Z", "result": "passed", - "changed_paths": ["frontend-modern/src/components/shared/Dialog.tsx"], + "changed_paths": [ + "frontend-modern/src/api/patrol.ts", + "frontend-modern/src/api/patrolAttention.ts", + "frontend-modern/src/components/AI/FindingsPanel.tsx", + "frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx", + "frontend-modern/src/features/patrol/PatrolIntelligenceSurface.tsx", + "frontend-modern/src/features/patrol/patrolHomePresentation.ts", + "frontend-modern/src/features/patrol/usePatrolIntelligenceState.ts", + "frontend-modern/src/stores/aiIntelligence.ts", + "frontend-modern/src/utils/findingAlertIdentity.ts" + ], "content_sha256": { - "frontend-modern/src/components/shared/Dialog.tsx": "185f09e185a9e5ccddf73906a81552ea0cc8b07390fb5ab587fbe1b952697735" + "frontend-modern/src/api/patrol.ts": "962ddd9eccb31fcfa419cde956c90f319333de3c15f98118d6580b542b1f4825", + "frontend-modern/src/api/patrolAttention.ts": "6729696d1966d62a38ebcc52bcc1d0a62479c6e025c1a7d4f23f02ed77c4428f", + "frontend-modern/src/components/AI/FindingsPanel.tsx": "8a6f65c113e507e4d4016861559f5ca01c962b4173387a42a2140499ae1d3e30", + "frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx": "ababb1fd31897a9dd7ca518d574d30b21b8b4abeae597f6475937ad93f3ebf59", + "frontend-modern/src/features/patrol/PatrolIntelligenceSurface.tsx": "f7fbdc4d43c42e12f08671a78a4b903bb1ed7b2d25d1551c395691c4991f1a40", + "frontend-modern/src/features/patrol/patrolHomePresentation.ts": "78aa0072e9b6c8bed4bc5fa3a86010bd61c9aa0c2ed98b08accca2c42e5e68de", + "frontend-modern/src/features/patrol/usePatrolIntelligenceState.ts": "b9632efe1d21f9c125b52795fa950b5f40167c4e11ccff18652f32ac1b3faf49", + "frontend-modern/src/stores/aiIntelligence.ts": "177859f415e92bcd282844ececf6ef8edfd5d54bda612ae18f100aacc1b40921", + "frontend-modern/src/utils/findingAlertIdentity.ts": "7988efd6dbf80c60e4d66312abb5ff95bcae9fc98445af0f57940a7ed3ea358b" }, "routes": [ - "/settings/infrastructure", - "/actions", - "/alerts/overview", - "/settings/system-general", "/patrol", - "/" + "/patrol?attention=mock-cluster-2-pve10-local-zfs%3A%3Ametric-threshold%3Ausage" ], "viewports": [ { "width": 1280, - "height": 720 + "height": 1400 }, { "width": 390, - "height": 844 - }, - { - "width": 393, - "height": 851 + "height": 1600 } ], "states": [ - "Add infrastructure dialog open at desktop and narrow widths with reduced motion, a visible labelled heading, accessible description, focused close control, contained panel geometry, and no horizontal document overflow", - "Add infrastructure dialog dismissed at desktop and narrow widths with the underlying Infrastructure surface restored", - "authenticated Actions, Alerts, Infrastructure, General Settings, and Patrol surfaces with reduced motion and no automatically detectable WCAG A/AA violations", - "logged-out welcome surface with reduced motion and no automatically detectable WCAG A/AA violations" + "Patrol inbox queue with 47-50 mock decisions and the alert-only detail (no mirrored Patrol finding: Lasting decisions explains there is nothing for Patrol to remember and links alert thresholds)", + "Attention detail for east-d-service-pool storage usage with a mirrored Patrol finding: Lifecycle short-term note, Lasting decisions listing Remember as expected, Dismiss: Not an issue, Dismiss: Later, Create rule with one-line explanations", + "Inline confirmation form for Remember as expected with optional note", + "Remembered-decision state (Remembered as expected, note, Reopen finding) after the dismissal, and the four options again after Reopen", + "Activity tab findings list with the collapsed \"1 finding mirrors an active alert\" group, expanded to show the demoted finding with Finding options", + "Narrow 390px detail layout with stacked lasting decisions and no horizontal overflow" ], "interactions": [ - "opened Add infrastructure from its named trigger at desktop and narrow widths and verified the dialog accessible name and description", - "inspected final desktop and 390x844 screenshots for placement, clipping, stacking, scrolling, focus treatment, and responsive layout", - "verified the dialog bounds stay inside both viewports and the document has no horizontal overflow", - "dismissed the dialog with Escape at desktop and narrow widths and verified focus returned to Add infrastructure", - "scanned representative authenticated and logged-out surfaces for WCAG A/AA violations and unexpected reduced-motion effects" + "Opened the deep-linked attention detail and read the Lifecycle and Lasting decisions sections", + "Clicked Remember as expected, typed a note, confirmed; verified via /api/ai/patrol/findings?include_resolved=1 that the mirrored finding carried dismissed_reason=expected_behavior and the note", + "Reloaded the detail and confirmed the Remembered as expected state, then clicked Reopen finding and confirmed the finding returned to active with the four decisions offered again", + "Switched to the Activity tab, opened Finding options and history, expanded the mirrored-findings group", + "Triggered POST /api/ai/patrol/run and confirmed mirrors_alert_id/mirrors_alert_type stamped on the storage finding" ] } diff --git a/frontend-modern/src/api/__tests__/patrolAttention.test.ts b/frontend-modern/src/api/__tests__/patrolAttention.test.ts index 2cb6eb69a..f75c48497 100644 --- a/frontend-modern/src/api/__tests__/patrolAttention.test.ts +++ b/frontend-modern/src/api/__tests__/patrolAttention.test.ts @@ -26,6 +26,22 @@ describe('Patrol attention API', () => { fetchMock.mockResolvedValue({}); }); + it('returns the backend flapping summary on attention items unchanged', async () => { + const flapping = { + transitionCount: 11, + windowHours: 24, + firstTransitionAt: '2026-08-09T15:00:00Z', + lastTransitionAt: '2026-08-10T14:30:00Z', + }; + fetchMock.mockResolvedValue({ + data: [{ id: 'record-flap', flapping }], + summary: {}, + meta: { page: 1, limit: 50, total: 1, totalPages: 1 }, + }); + const response = await getPatrolAttention(); + expect(response.data[0].flapping).toEqual(flapping); + }); + it('uses one bounded typed list query', async () => { await getPatrolAttention('stale_unknown', 2, 40); expect(fetchMock).toHaveBeenCalledWith( diff --git a/frontend-modern/src/api/patrol.ts b/frontend-modern/src/api/patrol.ts index ce31d3ede..590b3fa0b 100644 --- a/frontend-modern/src/api/patrol.ts +++ b/frontend-modern/src/api/patrol.ts @@ -69,6 +69,27 @@ export interface Finding { // urgency signal. Present only for capacity-relevant findings whose resource // has enough utilization history to compute a trend. capacity_forecast?: CapacityForecast; + // flapping is present while the finding's open/resolved state has changed + // at least four times inside the last 24 hours. The store collapses those + // transitions into one lifecycle row; this summary carries the count. + flapping?: FindingFlapping; + // mirrors_alert_id / mirrors_alert_type name the active alert this finding + // restates (same resource, same condition). Patrol stamps them after every + // run so the UI can fold the finding under the alert instead of listing the + // same problem twice. + mirrors_alert_id?: string; + mirrors_alert_type?: string; +} + +/** + * FindingFlapping summarises open/resolved churn on one finding. Mirrors + * internal/ai FindingFlapping on the wire. + */ +export interface FindingFlapping { + transition_count: number; + window_hours: number; + first_transition_at: string; + last_transition_at: string; } /** diff --git a/frontend-modern/src/api/patrolAttention.ts b/frontend-modern/src/api/patrolAttention.ts index 88b7f4799..986548712 100644 --- a/frontend-modern/src/api/patrolAttention.ts +++ b/frontend-modern/src/api/patrolAttention.ts @@ -38,6 +38,19 @@ export interface AttentionResource { resourceId: string; } +/** + * AttentionFlapping summarises an item whose lifecycle keeps switching between + * open and resolved. Present only once the shared flapping threshold (four + * transitions inside 24 hours) is crossed; the full transition list stays in + * the detail timeline. + */ +export interface AttentionFlapping { + transitionCount: number; + windowHours: number; + firstTransitionAt: string; + lastTransitionAt: string; +} + export interface AttentionItem { id: string; operationalRecordId: string; @@ -59,6 +72,7 @@ export interface AttentionItem { recommendedNextStep?: string; availableActions: AttentionActionOffer[]; verificationState: AttentionVerificationState; + flapping?: AttentionFlapping; } export interface AttentionItemDetail { diff --git a/frontend-modern/src/components/AI/FindingsPanel.tsx b/frontend-modern/src/components/AI/FindingsPanel.tsx index 7bfa68e38..afeb23014 100644 --- a/frontend-modern/src/components/AI/FindingsPanel.tsx +++ b/frontend-modern/src/components/AI/FindingsPanel.tsx @@ -31,12 +31,17 @@ import { type PatrolAssistantApprovalBriefingInput, type PatrolAssistantProposedFixBriefingInput, } from '@/features/patrol/patrolInvestigationContextModel'; +import { getFlappingPresentation } from '@/features/patrol/patrolHomePresentation'; import { InvestigationSection, ApprovalSection } from '@/components/patrol'; import { AIAPI, type RemediationPlan } from '@/api/ai'; import { createSuppressionRuleFromFinding, reinvestigateFinding } from '@/api/patrol'; import type { PatrolRunRecord, PatrolRuntimeState, PatrolAutonomyLevel } from '@/api/patrol'; import { formatRelativeTime } from '@/utils/format'; -import { getFindingAlertIdentifier, hasTriggeringAlert } from '@/utils/findingAlertIdentity'; +import { + getFindingAlertIdentifier, + hasTriggeringAlert, + isAlertMirroredFinding, +} from '@/utils/findingAlertIdentity'; import { FilterSegmentedControl, type FilterSegmentOption, @@ -457,9 +462,16 @@ export const FindingsPanel: Component = (props) => { return Number.isFinite(due) && due <= nowMs; }).length; }); + // Active findings that restate an active alert (backend-stamped mirror). + // They are demoted into a collapsed group below the list rather than shown + // as separate items: the alert already owns that condition on the Patrol + // inbox and the Alerts page. + const alertMirroredFindings = createMemo(() => + isPatrolFindingsSource() ? filteredFindings().filter(isAlertMirroredFinding) : [], + ); const patrolFindings = createMemo(() => { if (isPatrolFindingsSource()) { - return filteredFindings(); + return filteredFindings().filter((finding) => !isAlertMirroredFinding(finding)); } return filteredFindings().filter( (f) => f.source !== 'threshold' && !f.isThreshold && !hasTriggeringAlert(f), @@ -1064,6 +1076,26 @@ export const FindingsPanel: Component = (props) => { > {severityPresentation.label} + {/* Flapping: the store collapsed repeated open/resolved + transitions into one row; say so instead of letting the + regression counter climb silently. */} + + {(flapping) => ( + + { + getFlappingPresentation(flapping().transition_count, flapping().window_hours) + .label + } + + )} + {/* Patrol actionable state — plain-language translation of investigation state that the contract allows on the collapsed Patrol row (approval required, investigating, @@ -2122,7 +2154,11 @@ export const FindingsPanel: Component = (props) => { {/* Content */}
- +
@@ -2201,6 +2237,25 @@ export const FindingsPanel: Component = (props) => { )} + 0}> +
+ + + {alertMirroredFindings().length}{' '} + {alertMirroredFindings().length === 1 ? 'finding mirrors' : 'findings mirror'}{' '} + an active alert + + + Already shown as alerts. Expand to manage the Patrol findings. + + +
+ + {(finding) => renderFindingItem(finding, false)} + +
+
+
diff --git a/frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx b/frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx index e32445688..ce0fdda47 100644 --- a/frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx +++ b/frontend-modern/src/features/patrol/PatrolAttentionWorkbench.tsx @@ -1,4 +1,4 @@ -import { A, useLocation } from '@solidjs/router'; +import { useLocation } from '@solidjs/router'; import { type Accessor, createEffect, @@ -20,6 +20,7 @@ import ClockIcon from 'lucide-solid/icons/clock'; import ExternalLinkIcon from 'lucide-solid/icons/external-link'; import RefreshIcon from 'lucide-solid/icons/refresh-cw'; import RotateCwIcon from 'lucide-solid/icons/rotate-cw'; +import RepeatIcon from 'lucide-solid/icons/repeat'; import ShieldOffIcon from 'lucide-solid/icons/shield-off'; import SparklesIcon from 'lucide-solid/icons/sparkles'; import XIcon from 'lucide-solid/icons/x'; @@ -35,6 +36,7 @@ import { unacknowledgePatrolAttention, unsuppressPatrolAttention, } from '@/api/patrolAttention'; +import { createSuppressionRuleFromFinding } from '@/api/patrol'; import { ResourceActionsAPI } from '@/api/resourceActions'; import { Button, ButtonLink, CopyValueButton } from '@/components/shared/Button'; import { FormSelect } from '@/components/shared/FormSelect'; @@ -48,6 +50,7 @@ import { getPatrolProtectionProviderLabels, } from '@/features/patrol/patrolControlPresentation'; import { aiChatStore } from '@/stores/aiChat'; +import { aiIntelligenceStore, type UnifiedFinding } from '@/stores/aiIntelligence'; import { patrolAttentionStore } from '@/stores/patrolAttention'; import { ALERT_THRESHOLDS_PATH, @@ -63,9 +66,14 @@ import { formatRelativeTime } from '@/utils/format'; import { copyToClipboard } from '@/utils/clipboard'; import type { PatrolAutonomyLevel } from '@/api/patrol'; import { + getAttentionFlappingPresentation, + getLinkedPatrolFindings, + getPatrolRememberedDecisionPresentation, partitionPatrolAttention, PATROL_AUTONOMY_EXPERIENCE, + PATROL_LASTING_DECISIONS, type PatrolAttentionDecision, + type PatrolLastingDecision, } from './patrolHomePresentation'; const PRIMARY_EVIDENCE_LIMIT = 3; @@ -138,6 +146,11 @@ export function PatrolAttentionWorkbench( autonomyLocked?: boolean; pendingActionCount?: number; onOpenFindings?: (item: AttentionItem) => void; + /** + * Current Patrol findings, so the selected detail can show the finding + * that mirrors this alert and offer Patrol's lasting decisions on it. + */ + findings?: () => UnifiedFinding[]; } = {}, ) { const location = useLocation(); @@ -557,6 +570,7 @@ export function PatrolAttentionWorkbench( }) } onOpenFindings={props.onOpenFindings} + findings={props.findings} />
@@ -676,6 +690,18 @@ function AttentionList(props: { {formatLabel(item.severity)} + + {(flapping) => ( + + {flapping().label} + + )} + {item.state === 'acknowledged' ? 'Reviewed' : 'Suppressed'} @@ -809,10 +835,17 @@ function AttentionDetail(props: { onSuppress: (itemId: string, reason: string, expiresAt: string) => Promise; onUnsuppress: (itemId: string) => Promise; onOpenFindings?: (item: AttentionItem) => void; + findings?: () => UnifiedFinding[]; }) { const [copiedResourceId, setCopiedResourceId] = createSignal(''); const detail = () => props.detail; const item = () => detail()?.item; + const linkedFindings = createMemo(() => { + const current = detail(); + const findings = props.findings?.(); + if (!current || !findings) return []; + return getLinkedPatrolFindings(current.item, findings); + }); const orderedEvidence = createMemo(() => [...(detail()?.evidence ?? [])].sort( (left, right) => new Date(right.observedAt).getTime() - new Date(left.observedAt).getTime(), @@ -1002,6 +1035,14 @@ function AttentionDetail(props: {

{loaded().item.plainLanguageSummary}

+ + {(flapping) => ( +

+

+ )} +
@@ -1246,7 +1299,6 @@ function AttentionLifecycleControls(props: { onUnacknowledge: (itemId: string) => Promise; onSuppress: (itemId: string, reason: string, expiresAt: string) => Promise; onUnsuppress: (itemId: string) => Promise; - onOpenFindings?: (item: AttentionItem) => void; }) { const [showSuppression, setShowSuppression] = createSignal(false); const [reason, setReason] = createSignal(''); @@ -1343,22 +1395,11 @@ function AttentionLifecycleControls(props: { - -
-

- Need a lasting outcome for a Patrol finding on this resource? Resolve it, dismiss it - as not an issue, remember expected behavior, or create a suppression rule. -

- -
- +

+ Both of these are short-term. Mark reviewed hides this occurrence until it resolves or you + return it to the inbox. Suppression hides it only until the time you pick. Lasting + decisions are below. +

-
- - More ways to manage this issue - -
-

- Mark reviewed removes this occurrence from the decision inbox until it resolves or you - return it to open. Suppression hides it only until the selected return time. Both - remain available under Reviewed and suppressed. -

-

- For a permanent change,{' '} - - adjust alert thresholds - {' '} - to change or turn off this alert for the affected resource. -

-
-
); } +function AttentionTimelineList(props: { timeline: AttentionItemDetail['timeline'] }) { + return ( +
    + + {(transition) => ( +
  1. +

    + {formatLabel(transition.from)} to {formatLabel(transition.to)} +

    +

    + {formatRelativeTime(transition.at, { compact: true })} ·{' '} + {formatLabel(transition.cause)} +

    + + {(reason) =>

    {reason()}

    } +     +
    2. + )} +     +
+ ); +} + +/** + * Patrol's durable decisions, offered where the operator already is. They act + * on the Patrol finding that mirrors this alert: the alert lifecycle above is + * reversible bookkeeping, while these teach Patrol what is normal here. An + * alert with no Patrol finding attached has nothing for Patrol to remember, + * so the section says that and points at alert thresholds instead of + * offering buttons that would do nothing. + */ +function AttentionLastingDecisions(props: { + item: AttentionItem; + linkedFindings: UnifiedFinding[]; + onOpenFindings?: (item: AttentionItem) => void; +}) { + const [pending, setPending] = createSignal(null); + const [note, setNote] = createSignal(''); + const [busy, setBusy] = createSignal(false); + const [error, setError] = createSignal(''); + const [notice, setNotice] = createSignal(''); + const activeFinding = createMemo(() => + props.linkedFindings.find((finding) => finding.status === 'active' && !finding.dismissedReason), + ); + const rememberedFinding = createMemo(() => + props.linkedFindings.find((finding) => Boolean(finding.dismissedReason)), + ); + const cancel = () => { + setPending(null); + setNote(''); + setError(''); + }; + const choose = (decision: PatrolLastingDecision) => { + setPending(decision); + setNote(''); + setError(''); + setNotice(''); + }; + const confirm = async (event: SubmitEvent) => { + event.preventDefault(); + const decision = pending(); + const finding = activeFinding(); + if (!decision || !finding || busy()) return; + const reason = note().trim(); + if (decision.requiresReason && !reason) return; + setBusy(true); + setError(''); + try { + if (decision.kind === 'create_rule') { + await createSuppressionRuleFromFinding({ + resourceId: finding.resourceId, + resourceName: finding.resourceName, + category: finding.category, + description: reason, + }); + await aiIntelligenceStore.loadPatrolFindings(); + setNotice( + `Rule created. Future ${finding.category} findings on ${finding.resourceName} are dismissed automatically.`, + ); + } else { + const saved = await aiIntelligenceStore.dismissFinding( + finding.id, + decision.kind, + reason || undefined, + ); + if (!saved) throw new Error('Patrol could not save this decision.'); + setNotice(`${decision.label} saved. ${decision.explanation}`); + } + cancel(); + } catch (cause) { + setError(cause instanceof Error ? cause.message : 'The decision could not be saved.'); + } finally { + setBusy(false); + } + }; + const reopen = async (finding: UnifiedFinding) => { + if (busy()) return; + setBusy(true); + setError(''); + try { + const reopened = await aiIntelligenceStore.reopenFinding(finding.id); + if (!reopened) throw new Error('Patrol could not reopen this finding.'); + setNotice('Finding reopened. Patrol will raise it again.'); + } catch (cause) { + setError(cause instanceof Error ? cause.message : 'The finding could not be reopened.'); + } finally { + setBusy(false); + } + }; + + return ( + +
+ + {(message) => ( +

+

+ )} + + + } + > + {(finding) => { + const remembered = () => getPatrolRememberedDecisionPresentation(finding()); + return ( +
+

+ {remembered()?.headline}.{' '} + {remembered()?.detail} +

+

+ Patrol finding: {finding().title} + {(value) => <> · Note: {value()}} +

+ +
+ ); + }} + + } + > + {(finding) => ( + <> +

+ Patrol also raised{' '} + {finding().title} for this + resource. Decide once here and Patrol remembers it. The alert above keeps its own + lifecycle. +

+
    + + {(decision) => ( +
  • + +

    {decision.explanation}

    +
    • + )} +     +
+ + {(decision) => ( + + setNote(event.currentTarget.value)} + /> +
+ + +
+ + )} + + + )} + + + + +
+ + ); +} + +function AttentionAlertOnlyGuidance(props: { + item: AttentionItem; + onOpenFindings?: (item: AttentionItem) => void; +}) { + return ( +
+

+ This is an alert and Patrol has not raised a finding about it, so there is nothing for + Patrol to remember here. To change it for good, adjust when this alert fires or turn it off + for {props.item.subjectResourceName} under alert thresholds. +

+
+ + Adjust alert thresholds + + + + +
+
+ ); +} + function ActionVerificationMessage(props: { state: AttentionItem['verificationState'] }) { const message = () => { switch (props.state) { diff --git a/frontend-modern/src/features/patrol/PatrolIntelligenceSurface.tsx b/frontend-modern/src/features/patrol/PatrolIntelligenceSurface.tsx index 613cd9e2f..b9587c060 100644 --- a/frontend-modern/src/features/patrol/PatrolIntelligenceSurface.tsx +++ b/frontend-modern/src/features/patrol/PatrolIntelligenceSurface.tsx @@ -5,6 +5,7 @@ import HistoryIcon from 'lucide-solid/icons/history'; import { ButtonLink } from '@/components/shared/Button'; import { MetadataBadge } from '@/components/shared/MetadataBadge'; import { actionInboxStore } from '@/stores/actionInbox'; +import { aiIntelligenceStore } from '@/stores/aiIntelligence'; import { usePatrolIntelligenceState } from './usePatrolIntelligenceState'; import { PatrolIntelligenceHeader } from './PatrolIntelligenceHeader'; import { PatrolIntelligenceBanners } from './PatrolIntelligenceBanners'; @@ -111,6 +112,7 @@ export function PatrolIntelligenceSurface() { autonomyLocked={state.autoFixLocked()} pendingActionCount={actionInboxStore.pendingActionCount} onOpenFindings={openFindings} + findings={() => aiIntelligenceStore.patrolFindings} />
diff --git a/frontend-modern/src/features/patrol/__tests__/PatrolAttentionWorkbench.test.tsx b/frontend-modern/src/features/patrol/__tests__/PatrolAttentionWorkbench.test.tsx index 66cdc6836..31e4ddd5d 100644 --- a/frontend-modern/src/features/patrol/__tests__/PatrolAttentionWorkbench.test.tsx +++ b/frontend-modern/src/features/patrol/__tests__/PatrolAttentionWorkbench.test.tsx @@ -18,8 +18,31 @@ const apiMocks = vi.hoisted(() => ({ unsuppress: vi.fn(), planAction: vi.fn(), getAction: vi.fn(), + dismissFinding: vi.fn(), + reopenFinding: vi.fn(), + loadPatrolFindings: vi.fn(), + createRule: vi.fn(), })); +vi.mock('@/stores/aiIntelligence', () => ({ + aiIntelligenceStore: { + dismissFinding: (...args: unknown[]) => apiMocks.dismissFinding(...args), + reopenFinding: (...args: unknown[]) => apiMocks.reopenFinding(...args), + loadPatrolFindings: (...args: unknown[]) => apiMocks.loadPatrolFindings(...args), + get patrolFindings() { + return []; + }, + }, +})); + +vi.mock('@/api/patrol', async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + createSuppressionRuleFromFinding: (...args: unknown[]) => apiMocks.createRule(...args), + }; +}); + vi.mock('@/api/patrolAttention', async (importOriginal) => { const original = await importOriginal(); return { @@ -73,6 +96,7 @@ import { sortPatrolAttentionDecisions, } from '../PatrolAttentionWorkbench'; import { patrolAttentionStore } from '@/stores/patrolAttention'; +import type { UnifiedFinding } from '@/stores/aiIntelligence'; const evaluatedAt = '2026-07-19T08:00:00Z'; @@ -124,6 +148,23 @@ const item = (overrides: Partial = {}): AttentionItem => ({ ...overrides, }); +const mirroredFinding = (overrides: Partial = {}): UnifiedFinding => ({ + id: 'finding-1', + source: 'ai-patrol', + resourceId: 'pve:vm:101', + resourceName: 'Database VM', + resourceType: 'vm', + category: 'capacity', + severity: 'warning', + title: 'Disk on Database VM is 95% full', + description: 'The root volume has been filling for a week.', + detectedAt: '2026-07-19T07:00:00Z', + status: 'active', + mirrorsAlertId: 'alert-record-1', + mirrorsAlertType: 'disk', + ...overrides, +}); + const listResponse = ( items: AttentionItem[], responseSummary: AttentionSummary, @@ -189,6 +230,10 @@ describe('PatrolAttentionWorkbench', () => { apiMocks.unsuppress.mockReset(); apiMocks.planAction.mockReset(); apiMocks.getAction.mockReset(); + apiMocks.dismissFinding.mockReset(); + apiMocks.reopenFinding.mockReset(); + apiMocks.loadPatrolFindings.mockReset(); + apiMocks.createRule.mockReset(); }); afterEach(() => { @@ -686,18 +731,18 @@ describe('PatrolAttentionWorkbench', () => { ).toBeInTheDocument(); }); - it('keeps durable Patrol finding outcomes visible beside alert lifecycle controls', async () => { + it('offers every lasting Patrol decision on the finding that mirrors the alert', async () => { const active = item(); apiMocks.getList.mockResolvedValue( listResponse([active], summary({ activeCount: 1, openCount: 1, calm: false })), ); apiMocks.getDetail.mockResolvedValue(detail(active)); - const onOpenFindings = vi.fn(); + apiMocks.dismissFinding.mockResolvedValue(true); render(() => ( } + component={() => [mirroredFinding()]} />} /> )); @@ -708,43 +753,211 @@ describe('PatrolAttentionWorkbench', () => { }), ); + expect(await screen.findByText('Lasting decisions')).toBeInTheDocument(); + expect(screen.getByText(/Patrol also raised/i)).toHaveTextContent( + 'Disk on Database VM is 95% full', + ); + const decisions = within(screen.getByRole('list', { name: 'Lasting decisions' })); + for (const label of [ + 'Remember as expected', + 'Dismiss: Not an issue', + 'Dismiss: Later', + 'Create rule', + ]) { + expect(decisions.getByRole('button', { name: label })).toBeInTheDocument(); + } + expect(decisions.getByText(/stops raising it unless it gets worse/i)).toHaveTextContent( + /Lasts until you reopen the finding/i, + ); + expect(decisions.getByText(/Hides this for 7 days/i)).toBeInTheDocument(); expect( - await screen.findByText(/Need a lasting outcome for a Patrol finding on this resource/i), - ).toHaveTextContent(/remember expected behavior/i); - fireEvent.click(screen.getByRole('button', { name: 'Find lasting options for this resource' })); - expect(onOpenFindings).toHaveBeenCalledWith(active); + decisions.getByText(/Permanent rule for this resource and category/i), + ).toBeInTheDocument(); + // The temporary controls say so in one line instead of hiding it behind a toggle. + expect(screen.getByText(/Both of these are short-term/i)).toBeInTheDocument(); + expect(screen.queryByText('More ways to manage this issue')).not.toBeInTheDocument(); - fireEvent.click(screen.getByText('More ways to manage this issue')); - expect(screen.getByText(/Mark reviewed removes this occurrence/i)).toHaveTextContent( - /until it resolves or you return it to open/i, + fireEvent.click(decisions.getByRole('button', { name: 'Remember as expected' })); + const form = screen.getByRole('form', { name: 'Confirm Remember as expected' }); + fireEvent.input(within(form).getByLabelText(/Note for Patrol/i), { + target: { value: 'This VM is a cold standby.' }, + }); + fireEvent.click(within(form).getByRole('button', { name: 'Confirm: Remember as expected' })); + + await waitFor(() => + expect(apiMocks.dismissFinding).toHaveBeenCalledWith( + 'finding-1', + 'expected_behavior', + 'This VM is a cold standby.', + ), ); - expect(screen.getByRole('link', { name: 'adjust alert thresholds' })).toHaveAttribute( - 'href', - '/alerts/thresholds', - ); - expect( - screen.queryByRole('button', { name: 'review Patrol findings' }), - ).not.toBeInTheDocument(); + expect(await screen.findByRole('status')).toHaveTextContent(/Remember as expected saved/i); }); - it('omits the findings pointer when no findings surface is wired', async () => { + it('requires a written reason before creating a permanent rule from the detail', async () => { const active = item(); apiMocks.getList.mockResolvedValue( listResponse([active], summary({ activeCount: 1, openCount: 1, calm: false })), ); apiMocks.getDetail.mockResolvedValue(detail(active)); - renderWorkbench(); + apiMocks.createRule.mockResolvedValue({ success: true, message: 'ok', rule: { id: 'r1' } }); + apiMocks.loadPatrolFindings.mockResolvedValue(undefined); + render(() => ( + + [mirroredFinding()]} />} + /> + + )); + fireEvent.click( + await screen.findByRole('button', { + name: 'Open Database VM · Disk pressure', + }), + ); + const decisions = within(await screen.findByRole('list', { name: 'Lasting decisions' })); + fireEvent.click(decisions.getByRole('button', { name: 'Create rule' })); + const form = screen.getByRole('form', { name: 'Confirm Create rule' }); + const confirm = within(form).getByRole('button', { name: 'Confirm: Create rule' }); + expect(confirm).toBeDisabled(); + fireEvent.input(within(form).getByLabelText(/Why this rule/i), { + target: { value: 'Archive volume is meant to run near full.' }, + }); + expect(confirm).not.toBeDisabled(); + fireEvent.click(confirm); + + await waitFor(() => + expect(apiMocks.createRule).toHaveBeenCalledWith({ + resourceId: 'pve:vm:101', + resourceName: 'Database VM', + category: 'capacity', + description: 'Archive volume is meant to run near full.', + }), + ); + expect(apiMocks.loadPatrolFindings).toHaveBeenCalled(); + expect(await screen.findByRole('status')).toHaveTextContent(/Rule created/i); + }); + + it('shows the remembered decision and a reopen path instead of re-offering it', async () => { + const active = item(); + apiMocks.getList.mockResolvedValue( + listResponse([active], summary({ activeCount: 1, openCount: 1, calm: false })), + ); + apiMocks.getDetail.mockResolvedValue(detail(active)); + apiMocks.reopenFinding.mockResolvedValue(true); + render(() => ( + + ( + [ + mirroredFinding({ + status: 'dismissed', + dismissedReason: 'expected_behavior', + userNote: 'Cold standby.', + }), + ]} + /> + )} + /> + + )); fireEvent.click( await screen.findByRole('button', { name: 'Open Database VM · Disk pressure', }), ); - expect(await screen.findByText(/Mark reviewed removes this occurrence/i)).toBeInTheDocument(); + expect(await screen.findByText(/Remembered as expected/i)).toBeInTheDocument(); + expect(screen.getByText(/Note: Cold standby/i)).toBeInTheDocument(); + expect(screen.queryByRole('list', { name: 'Lasting decisions' })).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: 'Reopen finding' })); + await waitFor(() => expect(apiMocks.reopenFinding).toHaveBeenCalledWith('finding-1')); + }); + + it('explains that an alert with no Patrol finding has nothing for Patrol to remember', async () => { + const active = item(); + apiMocks.getList.mockResolvedValue( + listResponse([active], summary({ activeCount: 1, openCount: 1, calm: false })), + ); + apiMocks.getDetail.mockResolvedValue(detail(active)); + const onOpenFindings = vi.fn(); + render(() => ( + + ( + []} onOpenFindings={onOpenFindings} /> + )} + /> + + )); + fireEvent.click( + await screen.findByRole('button', { + name: 'Open Database VM · Disk pressure', + }), + ); + + expect(await screen.findByText(/nothing for Patrol to remember here/i)).toHaveTextContent( + /turn it off for Database VM under alert thresholds/i, + ); + expect(screen.queryByRole('list', { name: 'Lasting decisions' })).not.toBeInTheDocument(); + expect(screen.getByRole('link', { name: 'Adjust alert thresholds' })).toHaveAttribute( + 'href', + '/alerts/thresholds', + ); + fireEvent.click( + screen.getByRole('button', { name: 'Browse Patrol findings for this resource' }), + ); + expect(onOpenFindings).toHaveBeenCalledWith(active); + }); + + it('collapses a flapping item into one label and a summarised timeline', async () => { + const flapping = item({ + id: 'record-flap', + operationalRecordId: 'record-flap', + kind: 'docker-container-state', + title: 'Docker container state on shlink-web-clientX', + subjectResourceName: 'shlink-web-clientX', + severity: 'warning', + flapping: { + transitionCount: 11, + windowHours: 24, + firstTransitionAt: '2026-07-18T09:00:00Z', + lastTransitionAt: '2026-07-19T07:30:00Z', + }, + }); + const flappingDetail = detail(flapping); + flappingDetail.timeline = Array.from({ length: 11 }, (_, index) => ({ + id: `flap-${index}`, + operationalRecordId: 'record-flap', + from: index % 2 === 0 ? 'open' : 'resolved', + to: index % 2 === 0 ? 'resolved' : 'open', + at: `2026-07-18T${String(9 + index).padStart(2, '0')}:00:00Z`, + cause: index % 2 === 0 ? 'recovery_evidence' : 'detector_decision', + causeKey: 'runtime-state', + evidenceIds: [], + })); + apiMocks.getList.mockResolvedValue( + listResponse([flapping], summary({ activeCount: 1, openCount: 1, calm: false })), + ); + apiMocks.getDetail.mockResolvedValue(flappingDetail); + renderWorkbench(); + + const row = await screen.findByRole('button', { + name: 'Open shlink-web-clientX · Docker container state', + }); + expect(within(row).getByText('Flapping · 11 changes in 24h')).toBeInTheDocument(); + fireEvent.click(row); + expect( - screen.queryByRole('button', { name: 'review Patrol findings' }), - ).not.toBeInTheDocument(); + await screen.findByText(/switched between open and resolved 11 times in the last day/i), + ).toBeInTheDocument(); + fireEvent.click(screen.getByText('Evidence and history')); + expect(screen.getByText(/11 open\/resolved changes in the last 24 hours/i)).toBeInTheDocument(); + expect(screen.getByText('Show all 11 transitions')).toBeInTheDocument(); }); it('opens the canonical governed action review from an eligible attention item', async () => { diff --git a/frontend-modern/src/features/patrol/__tests__/usePatrolIntelligenceState.test.ts b/frontend-modern/src/features/patrol/__tests__/usePatrolIntelligenceState.test.ts index f22f88961..cbd7bc760 100644 --- a/frontend-modern/src/features/patrol/__tests__/usePatrolIntelligenceState.test.ts +++ b/frontend-modern/src/features/patrol/__tests__/usePatrolIntelligenceState.test.ts @@ -117,7 +117,9 @@ describe('usePatrolIntelligenceState', () => { expect(patrolIntelligenceStateSource).toContain('async function loadVisiblePatrolData()'); expect(patrolIntelligenceStateSource).toContain('refetchPatrolStatus()'); - expect(patrolIntelligenceStateSource).toContain('aiIntelligenceStore.loadPatrolFindings()'); + expect(patrolIntelligenceStateSource).toContain( + 'aiIntelligenceStore.loadPatrolFindings({ includeResolved: true })', + ); expect(patrolIntelligenceStateSource).toContain('aiIntelligenceStore.loadPendingApprovals()'); expect(patrolIntelligenceStateSource).toContain( 'patrolRunHistory.refetch({ background: true })', diff --git a/frontend-modern/src/features/patrol/patrolHomePresentation.ts b/frontend-modern/src/features/patrol/patrolHomePresentation.ts index d9f3a5105..78d15e94c 100644 --- a/frontend-modern/src/features/patrol/patrolHomePresentation.ts +++ b/frontend-modern/src/features/patrol/patrolHomePresentation.ts @@ -1,5 +1,6 @@ import type { PatrolAutonomyLevel, PatrolObjective } from '@/api/patrol'; -import type { AttentionItem } from '@/api/patrolAttention'; +import type { AttentionFlapping, AttentionItem } from '@/api/patrolAttention'; +import type { UnifiedFinding } from '@/stores/aiIntelligence'; export interface PatrolAutonomyExperience { label: string; @@ -174,3 +175,155 @@ export function getPatrolObjectiveProtectionSummary( tone: 'warning', }; } + +export interface FlappingPresentation { + label: string; + detail: string; +} + +const formatFlappingWindow = (windowHours: number): string => + windowHours === 24 ? 'the last day' : `the last ${windowHours} hours`; + +/** + * One label for flapping across alerts and Patrol findings. The backend + * decides *whether* an item is flapping (four or more open/resolved + * transitions in 24 hours); this only phrases the count it reports. + */ +export function getFlappingPresentation( + transitionCount: number, + windowHours: number, +): FlappingPresentation { + const changes = transitionCount === 1 ? 'change' : 'changes'; + return { + label: `Flapping · ${transitionCount} ${changes} in ${windowHours}h`, + detail: `This issue has switched between open and resolved ${transitionCount} times in ${formatFlappingWindow(windowHours)}. It is shown once here. Every transition is listed under the timeline.`, + }; +} + +export function getAttentionFlappingPresentation( + flapping: AttentionFlapping | undefined, +): FlappingPresentation | null { + if (!flapping || flapping.transitionCount <= 0) return null; + return getFlappingPresentation(flapping.transitionCount, flapping.windowHours); +} + +const normalizeResourceKey = (value: string | undefined): string => + (value ?? '').trim().toLowerCase(); + +/** + * Patrol findings that restate this attention item's alert: same resource and + * the backend-stamped mirror type equals the alert kind. Resolved findings are + * history and are not linked. + */ +export function getLinkedPatrolFindings( + item: AttentionItem, + findings: readonly UnifiedFinding[], +): UnifiedFinding[] { + const resource = normalizeResourceKey(item.subjectResourceId); + const kind = normalizeResourceKey(item.kind); + if (!resource || !kind) return []; + return findings.filter( + (finding) => + finding.status !== 'resolved' && + Boolean(finding.mirrorsAlertId) && + normalizeResourceKey(finding.resourceId) === resource && + normalizeResourceKey(finding.mirrorsAlertType) === kind, + ); +} + +export type PatrolLastingDecisionKind = + 'expected_behavior' | 'not_an_issue' | 'will_fix_later' | 'create_rule'; + +export interface PatrolLastingDecision { + kind: PatrolLastingDecisionKind; + label: string; + /** One line: what it does and how long it lasts. */ + explanation: string; + /** Whether the inline confirmation requires a written reason. */ + requiresReason: boolean; +} + +/** + * The durable Patrol outcomes, in the order a first-time user should read + * them: the most common "this is normal here" first, the permanent rule last. + * Durations mirror internal/ai/findings.go: expected_behavior and not_an_issue + * hold until the finding is reopened (severity escalation still reactivates + * expected_behavior); will_fix_later reminds after seven days; a rule is + * permanent until deleted under Patrol suppression rules. + */ +export const PATROL_LASTING_DECISIONS: readonly PatrolLastingDecision[] = [ + { + kind: 'expected_behavior', + label: 'Remember as expected', + explanation: + 'Patrol treats this as normal for this resource and stops raising it unless it gets worse. Lasts until you reopen the finding.', + requiresReason: false, + }, + { + kind: 'not_an_issue', + label: 'Dismiss: Not an issue', + explanation: + 'Marks the detection wrong and silences similar findings on this resource. Lasts until you reopen the finding.', + requiresReason: false, + }, + { + kind: 'will_fix_later', + label: 'Dismiss: Later', + explanation: 'Hides this for 7 days. If it is still happening after that, Patrol reminds you.', + requiresReason: false, + }, + { + kind: 'create_rule', + label: 'Create rule', + explanation: + 'Permanent rule for this resource and category: matching findings are dismissed automatically until you delete the rule under Patrol suppression rules.', + requiresReason: true, + }, +]; + +export interface PatrolRememberedDecisionPresentation { + headline: string; + detail: string; +} + +/** + * Describes a finding Patrol already remembers a decision for, so the + * attention detail can say what was decided instead of offering it again. + */ +export function getPatrolRememberedDecisionPresentation( + finding: UnifiedFinding, + now: Date = new Date(), +): PatrolRememberedDecisionPresentation | null { + if (finding.status !== 'dismissed' && !finding.dismissedReason) return null; + switch (finding.dismissedReason) { + case 'expected_behavior': + return { + headline: 'Remembered as expected', + detail: + 'Patrol treats this as normal for this resource and will only raise it again if it gets worse.', + }; + case 'not_an_issue': + return { + headline: 'Dismissed as not an issue', + detail: 'Similar findings on this resource stay silent until you reopen this one.', + }; + case 'will_fix_later': { + const remindAt = finding.remindAt ? new Date(finding.remindAt) : undefined; + const overdue = + remindAt !== undefined && !Number.isNaN(remindAt.getTime()) && remindAt <= now; + return { + headline: overdue ? 'Reminder due' : 'Dismissed until later', + detail: overdue + ? 'You said you would fix this later and the reminder date has passed. It is still happening.' + : remindAt && !Number.isNaN(remindAt.getTime()) + ? `Patrol stays quiet and reminds you on ${remindAt.toLocaleDateString()} if it is still happening.` + : 'Patrol stays quiet and reminds you if it is still happening after the reminder date.', + }; + } + default: + return { + headline: 'Dismissed', + detail: 'Patrol remembers this decision until you reopen the finding.', + }; + } +} diff --git a/frontend-modern/src/features/patrol/usePatrolIntelligenceState.ts b/frontend-modern/src/features/patrol/usePatrolIntelligenceState.ts index da40b2b41..7572c50b5 100644 --- a/frontend-modern/src/features/patrol/usePatrolIntelligenceState.ts +++ b/frontend-modern/src/features/patrol/usePatrolIntelligenceState.ts @@ -807,6 +807,13 @@ export function usePatrolIntelligenceState() { try { await Promise.all([ + // The Patrol surface needs dismissed findings too: the attention + // detail shows the decision Patrol already remembers for a mirrored + // finding, and the store's default load is active-only. Listed first + // so it flips the store's sticky include-resolved preference before + // loadDashboardData issues its own findings read; otherwise the two + // responses race and the active-only one can win. + aiIntelligenceStore.loadPatrolFindings({ includeResolved: true }), aiIntelligenceStore.loadDashboardData(), refetchPatrolStatus(), loadAutonomySettings(), @@ -828,7 +835,10 @@ export function usePatrolIntelligenceState() { try { await Promise.all([ refetchPatrolStatus(), - aiIntelligenceStore.loadPatrolFindings(), + // Include dismissed and resolved findings: the attention detail shows + // the decision Patrol already remembers for a mirrored finding, and a + // dismissed finding is not in the active-only default set. + aiIntelligenceStore.loadPatrolFindings({ includeResolved: true }), aiIntelligenceStore.loadPendingApprovals(), patrolRunHistory.refetch({ background: true }), ]); diff --git a/frontend-modern/src/stores/aiIntelligence.ts b/frontend-modern/src/stores/aiIntelligence.ts index 93c030c2a..fb926c80b 100644 --- a/frontend-modern/src/stores/aiIntelligence.ts +++ b/frontend-modern/src/stores/aiIntelligence.ts @@ -20,6 +20,7 @@ import { getPatrolFindings, type Finding as PatrolFinding, type CapacityForecast, + type FindingFlapping, } from '@/api/patrol'; import type { ResourceCriticality } from '@/api/resourceOperatorState'; import type { @@ -215,6 +216,15 @@ export interface UnifiedFinding { // independently of the model prose. Surfaced as a first-class urgency // signal on capacity-relevant findings. capacityForecast?: CapacityForecast; + // flapping is set while the finding keeps switching between open and + // resolved (four or more transitions in 24 hours). The lifecycle log holds + // one collapsed "flapping" row instead of one row per transition. + flapping?: FindingFlapping; + // mirrorsAlertId / mirrorsAlertType identify the active alert this finding + // restates. Surfaces fold such findings under the alert rather than listing + // the same problem twice. + mirrorsAlertId?: string; + mirrorsAlertType?: string; } const [unifiedFindings, setUnifiedFindings] = createSignal([]); @@ -323,6 +333,9 @@ function normalizePatrolFindingRecord(item: PatrolFinding, now: number): Unified regressionCount: item.regression_count || 0, lastRegressionAt: item.last_regression_at || undefined, capacityForecast: item.capacity_forecast, + flapping: item.flapping, + mirrorsAlertId: item.mirrors_alert_id || undefined, + mirrorsAlertType: item.mirrors_alert_type || undefined, }; } diff --git a/frontend-modern/src/utils/__tests__/findingAlertIdentity.test.ts b/frontend-modern/src/utils/__tests__/findingAlertIdentity.test.ts index 62e25cc23..4c4969d04 100644 --- a/frontend-modern/src/utils/__tests__/findingAlertIdentity.test.ts +++ b/frontend-modern/src/utils/__tests__/findingAlertIdentity.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest'; -import { getFindingAlertIdentifier, hasTriggeringAlert } from '@/utils/findingAlertIdentity'; +import { + getFindingAlertIdentifier, + hasTriggeringAlert, + isAlertMirroredFinding, +} from '@/utils/findingAlertIdentity'; describe('findingAlertIdentity', () => { it('prefers canonical alertIdentifier when present', () => { @@ -18,4 +22,19 @@ describe('findingAlertIdentity', () => { it('detects when a finding was triggered by an alert', () => { expect(hasTriggeringAlert({ alertIdentifier: 'instance:node:100::metric/cpu' })).toBe(true); }); + + it('demotes only active findings that Patrol stamped as mirroring an alert', () => { + expect(isAlertMirroredFinding({ mirrorsAlertId: 'usage-storage-1', status: 'active' })).toBe( + true, + ); + expect(isAlertMirroredFinding({ mirrorsAlertId: ' ', status: 'active' })).toBe(false); + expect(isAlertMirroredFinding({ status: 'active' })).toBe(false); + // A remembered or resolved mirrored finding is Patrol history, not a duplicate item. + expect(isAlertMirroredFinding({ mirrorsAlertId: 'usage-storage-1', status: 'dismissed' })).toBe( + false, + ); + expect(isAlertMirroredFinding({ mirrorsAlertId: 'usage-storage-1', status: 'resolved' })).toBe( + false, + ); + }); }); diff --git a/frontend-modern/src/utils/findingAlertIdentity.ts b/frontend-modern/src/utils/findingAlertIdentity.ts index f603921ee..b599069de 100644 --- a/frontend-modern/src/utils/findingAlertIdentity.ts +++ b/frontend-modern/src/utils/findingAlertIdentity.ts @@ -10,3 +10,20 @@ export const getFindingAlertIdentifier = (finding: FindingAlertIdentity): string export const hasTriggeringAlert = (finding: FindingAlertIdentity): boolean => getFindingAlertIdentifier(finding) !== undefined; + +type FindingAlertMirror = { + mirrorsAlertId?: string; + status?: string; +}; + +/** + * True when Patrol stamped this finding as restating an active alert (same + * resource, same condition) and the finding is still active. Such findings are + * demoted under the alert instead of listed as separate items; dismissed and + * resolved findings keep their normal place because those views are about + * Patrol's own history. + */ +export const isAlertMirroredFinding = (finding: FindingAlertMirror): boolean => + typeof finding.mirrorsAlertId === 'string' && + finding.mirrorsAlertId.trim() !== '' && + finding.status === 'active'; diff --git a/internal/ai/attention.go b/internal/ai/attention.go index abd280051..3532bf518 100644 --- a/internal/ai/attention.go +++ b/internal/ai/attention.go @@ -75,6 +75,18 @@ type AttentionResource struct { ResourceID string `json:"resourceId"` } +// AttentionFlapping summarises an item whose lifecycle keeps moving between +// open and resolved. It uses the same window and threshold as finding +// flapping (findings_storm_throttler.go) so alerts and Patrol findings agree +// on the label. The full transition list stays in the detail timeline; this +// is the collapsed operator-facing summary. +type AttentionFlapping struct { + TransitionCount int `json:"transitionCount"` + WindowHours int `json:"windowHours"` + FirstTransitionAt time.Time `json:"firstTransitionAt"` + LastTransitionAt time.Time `json:"lastTransitionAt"` +} + type AttentionItem struct { ID string `json:"id"` OperationalRecordID string `json:"operationalRecordId"` @@ -96,6 +108,7 @@ type AttentionItem struct { RecommendedNextStep string `json:"recommendedNextStep,omitempty"` AvailableActions []AttentionActionOffer `json:"availableActions"` VerificationState AttentionVerificationState `json:"verificationState"` + Flapping *AttentionFlapping `json:"flapping,omitempty"` } type AttentionItemDetail struct { @@ -244,6 +257,7 @@ func projectAttentionAlert( RecommendedNextStep: record.RecommendedNextStep, AvailableActions: []AttentionActionOffer{}, VerificationState: AttentionVerificationNotAvailable, + Flapping: attentionFlapping(timeline, now), } return AttentionItemDetail{ Item: item, @@ -253,6 +267,44 @@ func projectAttentionAlert( }, true } +// attentionFlapping collapses open/resolved churn in the lifecycle timeline +// into one summary when it crosses the shared flapping threshold. Only +// transitions that open or resolve the record count; acknowledgement, +// suppression, and evidence refreshes are operator or detector bookkeeping, +// not the condition coming and going. +func attentionFlapping( + timeline []operationaltrust.LifecycleTransition, + now time.Time, +) *AttentionFlapping { + transitions := make([]time.Time, 0, len(timeline)) + for _, transition := range timeline { + if !attentionTransitionIsFlap(transition) { + continue + } + transitions = append(transitions, transition.At) + } + summary := summarizeFlapTransitions(transitions, now) + if summary == nil { + return nil + } + return &AttentionFlapping{ + TransitionCount: summary.TransitionCount, + WindowHours: summary.WindowHours, + FirstTransitionAt: summary.FirstTransitionAt, + LastTransitionAt: summary.LastTransitionAt, + } +} + +func attentionTransitionIsFlap(transition operationaltrust.LifecycleTransition) bool { + if transition.To == operationaltrust.OperationalResolved { + return transition.From != operationaltrust.OperationalResolved + } + if transition.To == operationaltrust.OperationalOpen { + return transition.From == operationaltrust.OperationalResolved + } + return false +} + func summarizeAttentionEvidence( state operationaltrust.OperationalState, evidence []operationaltrust.EvidenceEnvelope, diff --git a/internal/ai/attention_test.go b/internal/ai/attention_test.go index c84b43e40..cc8556b0a 100644 --- a/internal/ai/attention_test.go +++ b/internal/ai/attention_test.go @@ -378,3 +378,75 @@ func attentionTestPosture( EvaluatedAt: evaluatedAt, } } + +func TestProjectAttentionItemsCollapsesOpenResolvedChurnIntoFlapping(t *testing.T) { + now := time.Date(2026, 8, 10, 15, 0, 0, 0, time.UTC) + alert := attentionTestAlert("flapper", operationaltrust.OperationalOpen, operationaltrust.SeverityWarning, now.Add(-2*24*time.Hour), now) + from := operationaltrust.OperationalOpen + to := operationaltrust.OperationalResolved + // Eleven open/resolved transitions in the last day, plus older churn that + // must not count, plus acknowledgement bookkeeping that never counts. + for i := 0; i < 11; i++ { + alert.Transitions = append(alert.Transitions, operationaltrust.LifecycleTransition{ + ID: "t-" + string(rune('a'+i)), + OperationalRecordID: "flapper", + From: from, + To: to, + At: now.Add(-time.Duration(11-i) * time.Hour), + Cause: operationaltrust.TransitionRecoveryEvidence, + }) + from, to = to, from + } + alert.Transitions = append(alert.Transitions, + operationaltrust.LifecycleTransition{ + ID: "old", OperationalRecordID: "flapper", + From: operationaltrust.OperationalOpen, To: operationaltrust.OperationalResolved, + At: now.Add(-30 * time.Hour), Cause: operationaltrust.TransitionRecoveryEvidence, + }, + operationaltrust.LifecycleTransition{ + ID: "ack", OperationalRecordID: "flapper", + From: operationaltrust.OperationalOpen, To: operationaltrust.OperationalAcknowledged, + At: now.Add(-20 * time.Minute), Cause: operationaltrust.TransitionAcknowledgement, + }, + ) + + projected := ProjectAttentionItems([]alerts.Alert{alert}, nil, nil, now) + if len(projected.Details) != 1 { + t.Fatalf("details = %d, want 1", len(projected.Details)) + } + item := projected.Details[0].Item + if item.Flapping == nil { + t.Fatal("eleven open/resolved transitions in a day were not labelled flapping") + } + if item.Flapping.TransitionCount != 11 { + t.Fatalf("TransitionCount = %d, want 11 (older and acknowledgement transitions excluded)", item.Flapping.TransitionCount) + } + if item.Flapping.WindowHours != 24 { + t.Fatalf("WindowHours = %d, want 24", item.Flapping.WindowHours) + } + if len(projected.Details[0].Timeline) != 13 { + t.Fatalf("timeline = %d, want the full forensic list of 13 transitions retained", len(projected.Details[0].Timeline)) + } +} + +func TestProjectAttentionItemsLeavesSingleRecurrenceUnlabelled(t *testing.T) { + now := time.Date(2026, 8, 10, 15, 0, 0, 0, time.UTC) + alert := attentionTestAlert("recurred", operationaltrust.OperationalOpen, operationaltrust.SeverityWarning, now.Add(-24*time.Hour), now) + alert.Transitions = append(alert.Transitions, + operationaltrust.LifecycleTransition{ + ID: "r1", OperationalRecordID: "recurred", + From: operationaltrust.OperationalOpen, To: operationaltrust.OperationalResolved, + At: now.Add(-3 * time.Hour), Cause: operationaltrust.TransitionRecoveryEvidence, + }, + operationaltrust.LifecycleTransition{ + ID: "r2", OperationalRecordID: "recurred", + From: operationaltrust.OperationalResolved, To: operationaltrust.OperationalOpen, + At: now.Add(-time.Hour), Cause: operationaltrust.TransitionDetectorDecision, + }, + ) + + projected := ProjectAttentionItems([]alerts.Alert{alert}, nil, nil, now) + if projected.Details[0].Item.Flapping != nil { + t.Fatalf("one recurrence was labelled flapping: %+v", projected.Details[0].Item.Flapping) + } +} diff --git a/internal/ai/demo.go b/internal/ai/demo.go index a7c25a96c..c2e562af0 100644 --- a/internal/ai/demo.go +++ b/internal/ai/demo.go @@ -169,6 +169,10 @@ func (p *PatrolService) runDemoPatrolCycleWithStart(trigger TriggerReason, accep } } + // Demo findings restate mock alerts on purpose (a storage pool near full + // beside its usage alert), so fold them the same way a real run does. + p.reconcileAlertMirrors() + // A simulated pass stands in for a healthy provider-backed run: clear any // stale "Provider analysis error" finding left by real runs that failed // before mock mode enabled. diff --git a/internal/ai/findings.go b/internal/ai/findings.go index e7828afc0..7bd30671b 100644 --- a/internal/ai/findings.go +++ b/internal/ai/findings.go @@ -4,6 +4,7 @@ package ai import ( "encoding/json" "fmt" + "strconv" "strings" "sync" "time" @@ -247,6 +248,21 @@ type Finding struct { // local evidence caused this finding to be assessed. It is durable context // for investigation, not action or approval authority. ObjectiveContext *aicontracts.PatrolObjectiveContext `json:"objective_context,omitempty"` + // Flapping is set while the finding's open/resolved state has changed + // at least findingFlapThreshold times inside findingFlapWindow. The + // store collapses the individual regressed/resolved lifecycle rows into + // one flapping row while it is set. Derived: rebuilt from the lifecycle + // on the first transition after a restart rather than persisted. + Flapping *FindingFlapping `json:"flapping,omitempty"` + // MirrorsAlertID names the active alert whose resource and condition + // this finding restates (for example a Proxmox guest stopped finding + // beside a powered-off alert). Stamped by Patrol after every run and + // cleared when the alert resolves; the attention surface folds such + // findings under the alert instead of listing them twice. + MirrorsAlertID string `json:"mirrors_alert_id,omitempty"` + // MirrorsAlertType is the alert type of MirrorsAlertID, so a client + // holding the alert projection can join on resource plus kind. + MirrorsAlertType string `json:"mirrors_alert_type,omitempty"` } // findingJSON is the marshal mirror of Finding: identical fields, but @@ -299,6 +315,9 @@ type findingJSON struct { LastRegressionAt *time.Time `json:"last_regression_at,omitempty"` CapacityForecast *CapacityForecast `json:"capacity_forecast,omitempty"` ObjectiveContext *aicontracts.PatrolObjectiveContext `json:"objective_context,omitempty"` + Flapping *FindingFlapping `json:"flapping,omitempty"` + MirrorsAlertID string `json:"mirrors_alert_id,omitempty"` + MirrorsAlertType string `json:"mirrors_alert_type,omitempty"` } func (f Finding) MarshalJSON() ([]byte, error) { @@ -347,6 +366,9 @@ func (f Finding) MarshalJSON() ([]byte, error) { LastRegressionAt: f.LastRegressionAt, CapacityForecast: f.CapacityForecast, ObjectiveContext: clonePatrolObjectiveContext(f.ObjectiveContext), + Flapping: cloneFindingFlapping(f.Flapping), + MirrorsAlertID: f.MirrorsAlertID, + MirrorsAlertType: f.MirrorsAlertType, }) } @@ -400,6 +422,9 @@ func (f *Finding) UnmarshalJSON(data []byte) error { LastRegressionAt: payload.LastRegressionAt, CapacityForecast: payload.CapacityForecast, ObjectiveContext: clonePatrolObjectiveContext(payload.ObjectiveContext), + Flapping: cloneFindingFlapping(payload.Flapping), + MirrorsAlertID: payload.MirrorsAlertID, + MirrorsAlertType: payload.MirrorsAlertType, } return nil } @@ -1337,6 +1362,92 @@ func (s *FindingsStore) appendLifecycleLocked(f *Finding, typ, msg, from, to str } } +// recordTransitionLifecycleLocked records an open/resolved transition for f. +// Below the flapping threshold it appends the ordinary lifecycle row. Once +// the storm throttler reports the finding is flapping, it stamps f.Flapping +// and folds the churn into one "flapping" row that carries the transition +// count, updating that row in place on every further transition instead of +// appending another regressed/resolved entry. The individual rows recorded +// before the label applied stay in the log as forensic detail. +func (s *FindingsStore) recordTransitionLifecycleLocked(f *Finding, typ, msg, from, to string, meta map[string]string) { + if f == nil { + return + } + var flapping *FindingFlapping + if t := s.stormThrottler; t != nil { + flapping = t.observeFlapLocked(f, time.Now()) + } + f.Flapping = flapping + if flapping == nil { + s.appendLifecycleLocked(f, typ, msg, from, to, meta) + return + } + collapsed := map[string]string{ + "transition_count": strconv.Itoa(flapping.TransitionCount), + "window_hours": strconv.Itoa(flapping.WindowHours), + "first_transition_at": flapping.FirstTransitionAt.UTC().Format(time.RFC3339), + "last_transition": typ, + } + message := fmt.Sprintf( + "State changed %d times in the last %d hours; individual transitions are collapsed while the finding is flapping", + flapping.TransitionCount, + flapping.WindowHours, + ) + if n := len(f.Lifecycle); n > 0 && f.Lifecycle[n-1].Type == FindingLifecycleFlapping { + last := &f.Lifecycle[n-1] + last.At = time.Now() + last.Message = message + last.From = from + last.To = to + last.Metadata = collapsed + return + } + s.appendLifecycleLocked(f, FindingLifecycleFlapping, message, from, to, collapsed) +} + +func cloneFindingFlapping(value *FindingFlapping) *FindingFlapping { + if value == nil { + return nil + } + clone := *value + return &clone +} + +// findingAlertMirror names the active alert a finding restates. +type findingAlertMirror struct { + AlertID string + AlertType string +} + +// StampAlertMirrors records, for every unresolved finding, which active +// alert it mirrors (same resource, same condition). Findings absent from +// the map lose any previous stamp, so a resolved alert releases its +// finding back to the normal list on the next reconcile. The stamp is a +// derived read-model annotation and does not schedule a save. +func (s *FindingsStore) StampAlertMirrors(mirrors map[string]findingAlertMirror) int { + s.mu.Lock() + defer s.mu.Unlock() + + changed := 0 + for id, f := range s.findings { + mirror, ok := mirrors[id] + if !ok || f.ResolvedAt != nil { + if f.MirrorsAlertID != "" || f.MirrorsAlertType != "" { + f.MirrorsAlertID = "" + f.MirrorsAlertType = "" + changed++ + } + continue + } + if f.MirrorsAlertID != mirror.AlertID || f.MirrorsAlertType != mirror.AlertType { + f.MirrorsAlertID = mirror.AlertID + f.MirrorsAlertType = mirror.AlertType + changed++ + } + } + return changed +} + func isKnownLoopState(v string) bool { switch FindingLoopState(v) { case FindingLoopStateDetected, FindingLoopStateInvestigating, FindingLoopStateRemediationPlanned, FindingLoopStateRemediating, FindingLoopStateRemediationFailed, FindingLoopStateNeedsAttention, FindingLoopStateTimedOut, FindingLoopStateResolved, FindingLoopStateDismissed, FindingLoopStateSnoozed, FindingLoopStateSuppressed: @@ -1954,7 +2065,7 @@ func (s *FindingsStore) Add(f *Finding) bool { if hadAcknowledgement { meta["previous_acknowledged"] = "true" } - s.appendLifecycleLocked(existing, "regressed", "Finding re-detected after resolution", string(FindingLoopStateResolved), string(FindingLoopStateDetected), meta) + s.recordTransitionLifecycleLocked(existing, "regressed", "Finding re-detected after resolution", string(FindingLoopStateResolved), string(FindingLoopStateDetected), meta) if existing.IsActive() { s.activeCounts[existing.Severity]++ } @@ -2099,7 +2210,7 @@ func (s *FindingsStore) Resolve(id string, auto bool) bool { if auto { etype = "auto_resolved" } - s.appendLifecycleLocked(f, etype, f.ResolveReason, f.LoopState, string(FindingLoopStateResolved), nil) + s.recordTransitionLifecycleLocked(f, etype, f.ResolveReason, f.LoopState, string(FindingLoopStateResolved), nil) s.syncLoopStateLocked(f) s.activeCounts[f.Severity]-- s.mu.Unlock() @@ -2123,7 +2234,7 @@ func (s *FindingsStore) ResolveWithReason(id string, reason string) bool { f.ResolvedAt = &now f.AutoResolved = true f.ResolveReason = reason - s.appendLifecycleLocked(f, "auto_resolved", reason, f.LoopState, string(FindingLoopStateResolved), nil) + s.recordTransitionLifecycleLocked(f, "auto_resolved", reason, f.LoopState, string(FindingLoopStateResolved), nil) s.syncLoopStateLocked(f) s.activeCounts[f.Severity]-- s.mu.Unlock() diff --git a/internal/ai/findings_alert_mirror.go b/internal/ai/findings_alert_mirror.go new file mode 100644 index 000000000..27e931744 --- /dev/null +++ b/internal/ai/findings_alert_mirror.go @@ -0,0 +1,192 @@ +// findings_alert_mirror.go decides when a Patrol finding restates an active +// alert. Real-time alerts own down, threshold, and age conditions; Patrol's +// deterministic watchers (Proxmox guest lifecycle, PDM bridge, signal +// detection) and the model can still emit a finding for the same resource +// and condition. Rather than dropping those findings, Patrol stamps them +// with the alert they mirror after every run so the attention surface can +// fold them under the alert and the findings list can demote them into an +// expansion. The matcher is deliberately conservative: it requires the same +// canonical resource and a recognised condition class on both sides, or an +// explicit AlertIdentifier link. +package ai + +import ( + "strings" + + unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +// alertMirrorCandidate is the minimal alert shape the matcher needs. +type alertMirrorCandidate struct { + ID string + ResourceID string + Type string +} + +// Condition classes shared by alerts and findings. An empty class never +// matches, so unknown alert types and free-form findings are left alone. +const ( + mirrorConditionDown = "down" + mirrorConditionRestartLoop = "restart-loop" + mirrorConditionContainerHealth = "container-health" + mirrorConditionDiskCapacity = "disk-capacity" + mirrorConditionMemory = "memory" + mirrorConditionCPU = "cpu" + mirrorConditionBackupAge = "backup-age" + mirrorConditionSnapshotAge = "snapshot-age" + mirrorConditionTemperature = "temperature" +) + +// alertConditionClass maps an alert type onto the shared condition class. +func alertConditionClass(alertType string) string { + normalized := strings.NewReplacer("_", "-", " ", "-").Replace(strings.ToLower(strings.TrimSpace(alertType))) + switch normalized { + case "offline", "connectivity", "powered-off", "docker-container-state", "docker-host-offline", "node-offline": + return mirrorConditionDown + case "docker-container-restart-loop": + return mirrorConditionRestartLoop + case "docker-container-health", "docker-service-health": + return mirrorConditionContainerHealth + case "disk", "usage", "storage-usage": + return mirrorConditionDiskCapacity + case "memory", "docker-container-memory-limit", "docker-container-oom-kill": + return mirrorConditionMemory + case "cpu": + return mirrorConditionCPU + case "backup-age": + return mirrorConditionBackupAge + case "snapshot-age": + return mirrorConditionSnapshotAge + case "temperature", "disk-temperature": + return mirrorConditionTemperature + } + return "" +} + +// findingConditionClass maps a finding onto the shared condition class using +// its stable key or dedup prefix first and its category plus title words +// second. Findings that do not describe an alert-owned condition return "". +func findingConditionClass(f *Finding) string { + if f == nil { + return "" + } + id := strings.ToLower(strings.TrimSpace(f.ID)) + key := strings.ToLower(strings.TrimSpace(f.Key)) + switch { + case strings.HasPrefix(id, proxmoxGuestStoppedFindingIDPrefix), key == proxmoxGuestStoppedFindingKey: + return mirrorConditionDown + case strings.HasPrefix(id, strings.ToLower(PDMAlertFindingPrefix)): + return mirrorConditionDown + } + + text := strings.ToLower(f.Title) + contains := func(words ...string) bool { + for _, word := range words { + if strings.Contains(text, word) { + return true + } + } + return false + } + switch f.Category { + case FindingCategoryCapacity: + switch { + case contains("memory", "swap"): + return mirrorConditionMemory + case contains("disk", "storage", "pool", "datastore", "volume", "full", "%"): + return mirrorConditionDiskCapacity + } + case FindingCategoryPerformance: + switch { + case contains("memory", "swap", "oom"): + return mirrorConditionMemory + case contains("cpu", "load"): + return mirrorConditionCPU + case contains("temperature", "thermal"): + return mirrorConditionTemperature + } + case FindingCategoryReliability: + switch { + case contains("restarting repeatedly", "restart loop", "crash loop", "crashlooping"): + return mirrorConditionRestartLoop + case contains("unhealthy", "health check"): + return mirrorConditionContainerHealth + case contains("stopped", "exited", "offline", "powered off", "is down", "unreachable", "not running"): + return mirrorConditionDown + } + case FindingCategoryBackup: + if contains("snapshot") { + return mirrorConditionSnapshotAge + } + return mirrorConditionBackupAge + } + return "" +} + +// findingMirrorsAlert reports whether f restates alert: an explicit +// AlertIdentifier link, or the same canonical resource with the same +// condition class. +func findingMirrorsAlert(f *Finding, alert alertMirrorCandidate) bool { + if f == nil { + return false + } + alertID := strings.TrimSpace(alert.ID) + if alertID != "" && strings.TrimSpace(f.AlertIdentifier) == alertID { + return true + } + findingResource := unified.CanonicalResourceID(f.ResourceID) + alertResource := unified.CanonicalResourceID(alert.ResourceID) + if findingResource == "" || alertResource == "" || findingResource != alertResource { + return false + } + class := findingConditionClass(f) + return class != "" && class == alertConditionClass(alert.Type) +} + +// matchFindingsToAlerts returns, for every unresolved finding that mirrors +// an active alert, the alert it mirrors. Storm meta-findings are never +// mirrors; they summarise several findings rather than one condition. When +// several alerts match, the first in the caller's order wins so the result +// is deterministic for a stable alert list. +func matchFindingsToAlerts(findings []*Finding, alerts []alertMirrorCandidate) map[string]findingAlertMirror { + mirrors := make(map[string]findingAlertMirror) + if len(findings) == 0 || len(alerts) == 0 { + return mirrors + } + for _, f := range findings { + if f == nil || f.ResolvedAt != nil || f.Source == stormFindingSource { + continue + } + for _, alert := range alerts { + if findingMirrorsAlert(f, alert) { + mirrors[f.ID] = findingAlertMirror{ + AlertID: strings.TrimSpace(alert.ID), + AlertType: strings.TrimSpace(alert.Type), + } + break + } + } + } + return mirrors +} + +// reconcileAlertMirrors re-stamps every unresolved finding against the +// current unscoped active-alert set. It runs at the end of every patrol +// cycle so a finding that mirrors an alert is folded while the alert is +// active and released once the alert resolves. +func (p *PatrolService) reconcileAlertMirrors() int { + if p == nil || p.findings == nil || p.stateProvider == nil { + return 0 + } + snapshot := p.stateProvider.ReadSnapshot() + candidates := make([]alertMirrorCandidate, 0, len(snapshot.ActiveAlerts)) + for _, alert := range snapshot.ActiveAlerts { + candidates = append(candidates, alertMirrorCandidate{ + ID: alert.ID, + ResourceID: alert.ResourceID, + Type: alert.Type, + }) + } + findings := p.findings.GetAll(nil) + return p.findings.StampAlertMirrors(matchFindingsToAlerts(findings, candidates)) +} diff --git a/internal/ai/findings_alert_mirror_test.go b/internal/ai/findings_alert_mirror_test.go new file mode 100644 index 000000000..0a3d743ed --- /dev/null +++ b/internal/ai/findings_alert_mirror_test.go @@ -0,0 +1,154 @@ +package ai + +import ( + "testing" + "time" +) + +func TestFindingMirrorsAlert_SameResourceSameCondition(t *testing.T) { + stopped := &Finding{ + ID: proxmoxGuestStoppedFindingIDPrefix + "pve1/qemu/101", + Key: proxmoxGuestStoppedFindingKey, + Category: FindingCategoryReliability, + ResourceID: "pve:vm:101", + Title: "Proxmox VM database stopped", + } + poweredOff := alertMirrorCandidate{ID: "powered-off-pve:vm:101", ResourceID: "pve:vm:101", Type: "powered-off"} + if !findingMirrorsAlert(stopped, poweredOff) { + t.Fatal("guest stopped finding did not mirror the powered-off alert on the same resource") + } + + otherResource := alertMirrorCandidate{ID: "powered-off-pve:vm:102", ResourceID: "pve:vm:102", Type: "powered-off"} + if findingMirrorsAlert(stopped, otherResource) { + t.Fatal("matched an alert on a different resource") + } + + memoryAlert := alertMirrorCandidate{ID: "memory-pve:vm:101", ResourceID: "pve:vm:101", Type: "memory"} + if findingMirrorsAlert(stopped, memoryAlert) { + t.Fatal("matched an alert with a different condition on the same resource") + } +} + +func TestFindingMirrorsAlert_ExplicitAlertIdentifierWins(t *testing.T) { + f := &Finding{ + ID: "llm-finding", + Category: FindingCategoryGeneral, + ResourceID: "docker:host/web", + Title: "Something Patrol noticed", + AlertIdentifier: "docker-container-state-docker:host/web", + } + alert := alertMirrorCandidate{ID: "docker-container-state-docker:host/web", ResourceID: "docker:other", Type: "docker-container-state"} + if !findingMirrorsAlert(f, alert) { + t.Fatal("explicit AlertIdentifier link was not honoured") + } +} + +func TestFindingConditionClass_CoversAlertOwnedConditions(t *testing.T) { + cases := []struct { + name string + f *Finding + want string + }{ + {"container exited", &Finding{Category: FindingCategoryReliability, Title: `Container "vaultwarden" exited unexpectedly`}, mirrorConditionDown}, + {"restart loop", &Finding{Category: FindingCategoryReliability, Title: `Container "sftp-ingest" is restarting repeatedly`}, mirrorConditionRestartLoop}, + {"unhealthy container", &Finding{Category: FindingCategoryReliability, Title: `Container "jellyfin" is unhealthy`}, mirrorConditionContainerHealth}, + {"storage full", &Finding{Category: FindingCategoryCapacity, Title: `Storage "core-b-service-pool" is 88% full`}, mirrorConditionDiskCapacity}, + {"memory capacity", &Finding{Category: FindingCategoryCapacity, Title: `Memory headroom on node pve1 is nearly exhausted`}, mirrorConditionMemory}, + {"guest memory", &Finding{Category: FindingCategoryPerformance, Title: `Container "artifact-cache" memory usage at 92%`}, mirrorConditionMemory}, + {"cpu", &Finding{Category: FindingCategoryPerformance, Title: `Sustained CPU saturation on pve3`}, mirrorConditionCPU}, + {"snapshot age", &Finding{Category: FindingCategoryBackup, Title: `Snapshot "base" is 349 days old`}, mirrorConditionSnapshotAge}, + {"backup age", &Finding{Category: FindingCategoryBackup, Title: `No recent backup for VM 104`}, mirrorConditionBackupAge}, + {"pdm offline", &Finding{ID: PDMAlertFindingPrefix + ":dc/node/pve1", Category: FindingCategoryReliability, Title: `PDM: node "pve1" is offline`}, mirrorConditionDown}, + {"mail gateway degraded is Patrol-owned", &Finding{Category: FindingCategoryReliability, Title: `Mail gateway "mail-gateway-us" is degraded`}, ""}, + {"security finding never mirrors", &Finding{Category: FindingCategorySecurity, Title: `Container runs as root and is offline`}, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := findingConditionClass(tc.f); got != tc.want { + t.Fatalf("findingConditionClass() = %q, want %q", got, tc.want) + } + }) + } +} + +func TestAlertConditionClass_MapsAlertTypes(t *testing.T) { + cases := map[string]string{ + "docker-container-state": mirrorConditionDown, + "powered-off": mirrorConditionDown, + "offline": mirrorConditionDown, + "connectivity": mirrorConditionDown, + "docker-container-restart-loop": mirrorConditionRestartLoop, + "docker-container-health": mirrorConditionContainerHealth, + "usage": mirrorConditionDiskCapacity, + "disk": mirrorConditionDiskCapacity, + "memory": mirrorConditionMemory, + "cpu": mirrorConditionCPU, + "snapshot-age": mirrorConditionSnapshotAge, + "backup-age": mirrorConditionBackupAge, + "temperature": mirrorConditionTemperature, + "zfs-pool-state": "", + "": "", + } + for alertType, want := range cases { + if got := alertConditionClass(alertType); got != want { + t.Errorf("alertConditionClass(%q) = %q, want %q", alertType, got, want) + } + } +} + +func TestMatchFindingsToAlerts_SkipsResolvedAndStormFindings(t *testing.T) { + resolvedAt := time.Now() + alerts := []alertMirrorCandidate{ + {ID: "docker-container-state-docker:h/a", ResourceID: "docker:h/a", Type: "docker-container-state"}, + {ID: "docker-container-state-docker:h/b", ResourceID: "docker:h/b", Type: "docker-container-state"}, + } + findings := []*Finding{ + {ID: "live", Category: FindingCategoryReliability, ResourceID: "docker:h/a", Title: `Container "a" exited`}, + {ID: "gone", Category: FindingCategoryReliability, ResourceID: "docker:h/b", Title: `Container "b" exited`, ResolvedAt: &resolvedAt}, + {ID: stormFindingIDPrefix + "docker:h/a", Source: stormFindingSource, Category: FindingCategoryReliability, ResourceID: "docker:h/a", Title: "Multiple findings emitted"}, + nil, + } + got := matchFindingsToAlerts(findings, alerts) + if len(got) != 1 { + t.Fatalf("mirrors = %d, want 1: %+v", len(got), got) + } + mirror, ok := got["live"] + if !ok { + t.Fatalf("live finding was not matched: %+v", got) + } + if mirror.AlertID != "docker-container-state-docker:h/a" || mirror.AlertType != "docker-container-state" { + t.Fatalf("mirror = %+v", mirror) + } +} + +func TestStampAlertMirrors_SetsAndClears(t *testing.T) { + store := NewFindingsStore() + store.Add(&Finding{ID: "f1", Severity: FindingSeverityWarning, Category: FindingCategoryReliability, ResourceID: "docker:h/a", Title: `Container "a" exited`}) + store.Add(&Finding{ID: "f2", Severity: FindingSeverityWarning, Category: FindingCategoryReliability, ResourceID: "docker:h/b", Title: `Container "b" exited`}) + + changed := store.StampAlertMirrors(map[string]findingAlertMirror{ + "f1": {AlertID: "alert-a", AlertType: "docker-container-state"}, + }) + if changed != 1 { + t.Fatalf("changed = %d, want 1", changed) + } + if f := store.Get("f1"); f.MirrorsAlertID != "alert-a" || f.MirrorsAlertType != "docker-container-state" { + t.Fatalf("f1 was not stamped: %+v", f) + } + if f := store.Get("f2"); f.MirrorsAlertID != "" { + t.Fatalf("f2 was stamped without a match: %+v", f) + } + + // Same stamp again is a no-op. + if changed := store.StampAlertMirrors(map[string]findingAlertMirror{"f1": {AlertID: "alert-a", AlertType: "docker-container-state"}}); changed != 0 { + t.Fatalf("idempotent restamp changed = %d, want 0", changed) + } + + // Alert resolved: the finding is released. + if changed := store.StampAlertMirrors(map[string]findingAlertMirror{}); changed != 1 { + t.Fatalf("clear changed = %d, want 1", changed) + } + if f := store.Get("f1"); f.MirrorsAlertID != "" || f.MirrorsAlertType != "" { + t.Fatalf("f1 stamp was not cleared: %+v", f) + } +} diff --git a/internal/ai/findings_flapping_test.go b/internal/ai/findings_flapping_test.go new file mode 100644 index 000000000..bc8242f45 --- /dev/null +++ b/internal/ai/findings_flapping_test.go @@ -0,0 +1,162 @@ +package ai + +import ( + "testing" + "time" +) + +// flapFinding builds a fresh warning finding for the flap tests. +func flapFinding() *Finding { + return &Finding{ + ID: "docker:h/web:exited", + Key: "container-exited", + Severity: FindingSeverityWarning, + Category: FindingCategoryReliability, + ResourceID: "docker:h/web", + ResourceName: "web", + Title: `Container "web" exited`, + } +} + +func lifecycleTypes(f *Finding) []string { + types := make([]string, 0, len(f.Lifecycle)) + for _, event := range f.Lifecycle { + types = append(types, event.Type) + } + return types +} + +func TestFindingsStore_CollapsesFlappingIntoOneLifecycleRow(t *testing.T) { + store := NewFindingsStore() + store.SetStormThrottler(newFindingStormThrottler()) + + store.Add(flapFinding()) + // Resolve and re-detect repeatedly. Each resolve and each regression is + // one open/resolved transition. + for i := 0; i < 6; i++ { + if !store.ResolveWithReason("docker:h/web:exited", "container running again") { + t.Fatalf("resolve %d failed", i) + } + if store.Add(flapFinding()) { + t.Fatalf("re-detection %d was treated as a new finding", i) + } + } + + f := store.Get("docker:h/web:exited") + if f == nil { + t.Fatal("finding disappeared") + } + if f.Flapping == nil { + t.Fatalf("twelve transitions in a minute were not labelled flapping: %v", lifecycleTypes(f)) + } + if f.Flapping.TransitionCount != 12 { + t.Fatalf("TransitionCount = %d, want 12", f.Flapping.TransitionCount) + } + if f.RegressionCount != 6 { + t.Fatalf("RegressionCount = %d, want 6 (regressions still counted while collapsed)", f.RegressionCount) + } + + // Transitions 1-3 were recorded individually; from the fourth onward the + // store maintains a single flapping row instead of appending nine more. + types := lifecycleTypes(f) + flappingRows := 0 + individualTransitions := 0 + for _, typ := range types { + switch typ { + case FindingLifecycleFlapping: + flappingRows++ + case "regressed", "auto_resolved", "resolved": + individualTransitions++ + } + } + if flappingRows != 1 { + t.Fatalf("flapping rows = %d, want exactly one collapsed row: %v", flappingRows, types) + } + if individualTransitions != findingFlapThreshold-1 { + t.Fatalf("individual transition rows = %d, want %d pre-label rows: %v", individualTransitions, findingFlapThreshold-1, types) + } + last := f.Lifecycle[len(f.Lifecycle)-1] + if last.Type != FindingLifecycleFlapping { + t.Fatalf("last lifecycle row = %q, want the collapsed flapping row", last.Type) + } + if last.Metadata["transition_count"] != "12" { + t.Fatalf("collapsed row transition_count = %q, want 12", last.Metadata["transition_count"]) + } + if last.Metadata["last_transition"] != "regressed" { + t.Fatalf("collapsed row last_transition = %q, want regressed", last.Metadata["last_transition"]) + } +} + +func TestFindingsStore_WithoutThrottlerKeepsIndividualRows(t *testing.T) { + store := NewFindingsStore() + store.Add(flapFinding()) + for i := 0; i < 3; i++ { + store.ResolveWithReason("docker:h/web:exited", "running again") + store.Add(flapFinding()) + } + f := store.Get("docker:h/web:exited") + if f.Flapping != nil { + t.Fatal("no throttler installed but the finding was labelled flapping") + } + for _, typ := range lifecycleTypes(f) { + if typ == FindingLifecycleFlapping { + t.Fatalf("collapsed row appeared without a throttler: %v", lifecycleTypes(f)) + } + } +} + +func TestFindingsStore_FlappingLabelClearsWhenChurnStops(t *testing.T) { + store := NewFindingsStore() + throttler := newFindingStormThrottler() + store.SetStormThrottler(throttler) + store.Add(flapFinding()) + for i := 0; i < 3; i++ { + store.ResolveWithReason("docker:h/web:exited", "running again") + store.Add(flapFinding()) + } + if store.Get("docker:h/web:exited").Flapping == nil { + t.Fatal("expected flapping after six transitions") + } + + // Age the tracked window out, then one more transition: no longer flapping. + throttler.mu.Lock() + tracker := throttler.flaps["docker:h/web:exited"] + for i := range tracker.transitions { + tracker.transitions[i] = tracker.transitions[i].Add(-findingFlapWindow - time.Hour) + } + throttler.mu.Unlock() + store.ResolveWithReason("docker:h/web:exited", "running again") + + f := store.Get("docker:h/web:exited") + if f.Flapping != nil { + t.Fatalf("label persisted after the window emptied: %+v", f.Flapping) + } + if last := f.Lifecycle[len(f.Lifecycle)-1]; last.Type != "auto_resolved" { + t.Fatalf("post-flap transition row = %q, want an ordinary auto_resolved row", last.Type) + } +} + +func TestFindingJSONRoundTripsFlappingAndMirror(t *testing.T) { + now := time.Date(2026, 8, 10, 12, 0, 0, 0, time.UTC) + f := flapFinding() + f.DetectedAt = now + f.LastSeenAt = now + f.Flapping = &FindingFlapping{TransitionCount: 7, WindowHours: 24, FirstTransitionAt: now.Add(-20 * time.Hour), LastTransitionAt: now} + f.MirrorsAlertID = "docker-container-state-docker:h/web" + f.MirrorsAlertType = "docker-container-state" + + data, err := f.MarshalJSON() + if err != nil { + t.Fatal(err) + } + var decoded Finding + if err := decoded.UnmarshalJSON(data); err != nil { + t.Fatal(err) + } + if decoded.Flapping == nil || decoded.Flapping.TransitionCount != 7 || decoded.Flapping.WindowHours != 24 { + t.Fatalf("flapping did not round-trip: %+v", decoded.Flapping) + } + if decoded.MirrorsAlertID != f.MirrorsAlertID || decoded.MirrorsAlertType != f.MirrorsAlertType { + t.Fatalf("mirror fields did not round-trip: %q %q", decoded.MirrorsAlertID, decoded.MirrorsAlertType) + } +} diff --git a/internal/ai/findings_storm_throttler.go b/internal/ai/findings_storm_throttler.go index ec5927714..d9f9e4628 100644 --- a/internal/ai/findings_storm_throttler.go +++ b/internal/ai/findings_storm_throttler.go @@ -10,12 +10,21 @@ // store mutex held; it owns its own internal mutex so the throttler does // not couple to the store's lock-ordering invariants. The observer must // NOT call back into FindingsStore. +// +// The same throttler owns flap detection (observeFlapLocked): a finding +// whose open/resolved state changes at least findingFlapThreshold times +// inside findingFlapWindow is "flapping". The store then collapses those +// transitions into one lifecycle entry carrying the count instead of +// appending one regressed/resolved row per change, and the attention +// projection reads the same constants so alert and finding surfaces agree +// on what "flapping" means. package ai import ( "fmt" "sort" + "strconv" "strings" "sync" "time" @@ -28,8 +37,48 @@ const ( stormFindingIDPrefix = "finding-storm:" stormClusterCap = 1024 stormResolveReason = "finding-storm:rate_dropped_below_threshold" + + // findingFlapWindow is the sliding window over which open/resolved + // transitions count toward flapping. Twenty-four hours matches the + // operator-visible complaint (eleven transitions in a day) rather than + // the alert engine's five-minute notification-suppression window. + findingFlapWindow = 24 * time.Hour + // findingFlapThreshold is the number of open/resolved transitions inside + // findingFlapWindow at which an item is labelled flapping. Four means + // the condition came back at least twice; a single recurrence is a + // regression, not a flap. + findingFlapThreshold = 4 + // findingFlapTrackerCap bounds per-finding transition tracking. + findingFlapTrackerCap = 4096 + // FindingLifecycleFlapping is the collapsed lifecycle event type that + // stands in for the individual regressed/resolved rows while a finding + // is flapping. Its metadata carries transition_count and window_hours. + FindingLifecycleFlapping = "flapping" ) +// FindingFlapping summarises a finding whose open/resolved state keeps +// changing. It is stamped by the store while the transition count inside +// the window is at or above findingFlapThreshold and cleared otherwise. +type FindingFlapping struct { + // TransitionCount is the number of open/resolved transitions observed + // inside the window ending at LastTransitionAt. + TransitionCount int `json:"transition_count"` + // WindowHours is the length of the sliding window in hours. + WindowHours int `json:"window_hours"` + // FirstTransitionAt is the oldest transition still inside the window. + FirstTransitionAt time.Time `json:"first_transition_at"` + // LastTransitionAt is the most recent transition. + LastTransitionAt time.Time `json:"last_transition_at"` +} + +// findingFlapTracker holds the sliding window of open/resolved transition +// timestamps for one finding. +type findingFlapTracker struct { + transitions []time.Time + hydrated bool + lastTouchedAt time.Time +} + // findingStormCluster carries per-clusterKey state: the sliding window of // emission timestamps, identity of the storm finding currently emitted for // the cluster (if any), and the freshest contributor metadata so the storm @@ -55,12 +104,17 @@ type findingStormThrottler struct { mu sync.Mutex clusters map[string]*findingStormCluster cap int + + flaps map[string]*findingFlapTracker + flapCap int } func newFindingStormThrottler() *findingStormThrottler { return &findingStormThrottler{ clusters: make(map[string]*findingStormCluster), cap: stormClusterCap, + flaps: make(map[string]*findingFlapTracker), + flapCap: findingFlapTrackerCap, } } @@ -240,3 +294,134 @@ func (t *findingStormThrottler) buildStormFindingLocked(cluster *findingStormClu LastSeenAt: now, } } + +// observeFlapLocked records one open/resolved transition for f at now and +// returns the resulting FindingFlapping summary, or nil while the finding +// is below the flapping threshold. Like observeLocked it is called with +// the store mutex held and uses the throttler's own mutex; it never calls +// back into the store. +// +// The first observation of a finding hydrates the window from the +// finding's persisted lifecycle so a restart does not forget that an item +// was flapping until the next transition lands. +func (t *findingStormThrottler) observeFlapLocked(f *Finding, now time.Time) *FindingFlapping { + if t == nil || f == nil { + return nil + } + key := strings.TrimSpace(f.ID) + if key == "" { + return nil + } + + t.mu.Lock() + defer t.mu.Unlock() + + if t.flaps == nil { + t.flaps = make(map[string]*findingFlapTracker) + } + tracker, ok := t.flaps[key] + if !ok { + tracker = &findingFlapTracker{} + t.flaps[key] = tracker + } + if !tracker.hydrated { + tracker.transitions = flapTransitionsFromLifecycle(f.Lifecycle, now) + tracker.hydrated = true + } + tracker.lastTouchedAt = now + tracker.transitions = append(trimFlapWindow(tracker.transitions, now), now) + t.pruneFlapLRULocked(key) + + return summarizeFlapTransitions(tracker.transitions, now) +} + +// pruneFlapLRULocked evicts the least-recently-touched trackers (excluding +// the one just touched) until the map is at or under flapCap. +func (t *findingStormThrottler) pruneFlapLRULocked(protectKey string) { + if t.flapCap <= 0 || len(t.flaps) <= t.flapCap { + return + } + type entry struct { + key string + at time.Time + } + candidates := make([]entry, 0, len(t.flaps)-1) + for k, tracker := range t.flaps { + if k == protectKey { + continue + } + candidates = append(candidates, entry{key: k, at: tracker.lastTouchedAt}) + } + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].at.Before(candidates[j].at) + }) + excess := len(t.flaps) - t.flapCap + for i := 0; i < excess && i < len(candidates); i++ { + delete(t.flaps, candidates[i].key) + } +} + +// trimFlapWindow drops transitions older than findingFlapWindow before now. +func trimFlapWindow(transitions []time.Time, now time.Time) []time.Time { + cutoff := now.Add(-findingFlapWindow) + keep := transitions[:0] + for _, ts := range transitions { + if !ts.Before(cutoff) { + keep = append(keep, ts) + } + } + return keep +} + +// summarizeFlapTransitions returns the flapping summary for a window of +// transition timestamps, or nil below the threshold. Shared by the finding +// store and the attention projection so both surfaces agree. +func summarizeFlapTransitions(transitions []time.Time, now time.Time) *FindingFlapping { + inWindow := make([]time.Time, 0, len(transitions)) + cutoff := now.Add(-findingFlapWindow) + for _, ts := range transitions { + if !ts.Before(cutoff) { + inWindow = append(inWindow, ts) + } + } + if len(inWindow) < findingFlapThreshold { + return nil + } + sort.Slice(inWindow, func(i, j int) bool { return inWindow[i].Before(inWindow[j]) }) + return &FindingFlapping{ + TransitionCount: len(inWindow), + WindowHours: int(findingFlapWindow / time.Hour), + FirstTransitionAt: inWindow[0], + LastTransitionAt: inWindow[len(inWindow)-1], + } +} + +// flapTransitionsFromLifecycle rebuilds the in-window transition +// timestamps from a persisted lifecycle log. Individual regressed and +// resolved rows count once each; a collapsed flapping row contributes its +// recorded transition_count, spread as repeats of its timestamp so the +// count survives without inventing intermediate times. +func flapTransitionsFromLifecycle(lifecycle []FindingLifecycleEvent, now time.Time) []time.Time { + cutoff := now.Add(-findingFlapWindow) + var transitions []time.Time + for _, event := range lifecycle { + if event.At.Before(cutoff) || event.At.After(now) { + continue + } + switch event.Type { + case "regressed", "resolved", "auto_resolved": + transitions = append(transitions, event.At) + case FindingLifecycleFlapping: + count := 1 + if raw, ok := event.Metadata["transition_count"]; ok { + if parsed, err := strconv.Atoi(raw); err == nil && parsed > 0 { + count = parsed + } + } + for i := 0; i < count; i++ { + transitions = append(transitions, event.At) + } + } + } + return transitions +} diff --git a/internal/ai/findings_storm_throttler_test.go b/internal/ai/findings_storm_throttler_test.go index 9cc253894..68d525036 100644 --- a/internal/ai/findings_storm_throttler_test.go +++ b/internal/ai/findings_storm_throttler_test.go @@ -232,3 +232,104 @@ func clusterKeys(tr *findingStormThrottler) []string { } return keys } + +func TestStormThrottler_FlapBelowThresholdReturnsNil(t *testing.T) { + tr := newFindingStormThrottler() + base := time.Now() + f := newStormFinding("f1", "vm/100", "container exited") + + for i := 0; i < findingFlapThreshold-1; i++ { + if got := tr.observeFlapLocked(f, base.Add(time.Duration(i)*time.Hour)); got != nil { + t.Fatalf("transition %d below threshold: want nil, got %+v", i+1, got) + } + } +} + +func TestStormThrottler_FlapAtThresholdSummarisesWindow(t *testing.T) { + tr := newFindingStormThrottler() + base := time.Now() + f := newStormFinding("f1", "vm/100", "container exited") + + var got *FindingFlapping + for i := 0; i < findingFlapThreshold; i++ { + got = tr.observeFlapLocked(f, base.Add(time.Duration(i)*time.Hour)) + } + if got == nil { + t.Fatalf("expected flapping summary at threshold %d", findingFlapThreshold) + } + if got.TransitionCount != findingFlapThreshold { + t.Errorf("TransitionCount = %d, want %d", got.TransitionCount, findingFlapThreshold) + } + if got.WindowHours != int(findingFlapWindow/time.Hour) { + t.Errorf("WindowHours = %d, want %d", got.WindowHours, int(findingFlapWindow/time.Hour)) + } + if !got.FirstTransitionAt.Equal(base) { + t.Errorf("FirstTransitionAt = %v, want %v", got.FirstTransitionAt, base) + } + last := base.Add(time.Duration(findingFlapThreshold-1) * time.Hour) + if !got.LastTransitionAt.Equal(last) { + t.Errorf("LastTransitionAt = %v, want %v", got.LastTransitionAt, last) + } +} + +func TestStormThrottler_FlapWindowEvictionClearsLabel(t *testing.T) { + tr := newFindingStormThrottler() + base := time.Now() + f := newStormFinding("f1", "vm/100", "container exited") + + for i := 0; i < findingFlapThreshold; i++ { + tr.observeFlapLocked(f, base.Add(time.Duration(i)*time.Minute)) + } + // One more transition a day and an hour later: every earlier transition + // has aged out, so the finding is a single recurrence again. + if got := tr.observeFlapLocked(f, base.Add(findingFlapWindow+time.Hour)); got != nil { + t.Fatalf("aged-out window still reported flapping: %+v", got) + } +} + +func TestStormThrottler_FlapHydratesFromPersistedLifecycle(t *testing.T) { + tr := newFindingStormThrottler() + base := time.Now() + f := newStormFinding("f1", "vm/100", "container exited") + f.Lifecycle = []FindingLifecycleEvent{ + {At: base.Add(-3 * time.Hour), Type: "regressed"}, + {At: base.Add(-2 * time.Hour), Type: "auto_resolved"}, + {At: base.Add(-30 * time.Hour), Type: "regressed"}, // outside window + {At: base.Add(-time.Hour), Type: FindingLifecycleFlapping, Metadata: map[string]string{"transition_count": "5"}}, + } + + got := tr.observeFlapLocked(f, base) + if got == nil { + t.Fatal("expected persisted lifecycle to hydrate the flap window") + } + // 1 + 1 + 5 hydrated transitions plus the observed one; the 30h-old row is dropped. + if got.TransitionCount != 8 { + t.Errorf("TransitionCount = %d, want 8", got.TransitionCount) + } +} + +func TestStormThrottler_FlapTrackerLRUEvictsAtCap(t *testing.T) { + tr := newFindingStormThrottler() + tr.flapCap = 2 + base := time.Now() + + tr.observeFlapLocked(newStormFinding("a", "vm/1", "x"), base) + tr.observeFlapLocked(newStormFinding("b", "vm/2", "x"), base.Add(time.Second)) + tr.observeFlapLocked(newStormFinding("c", "vm/3", "x"), base.Add(2*time.Second)) + if _, ok := tr.flaps["a"]; ok { + t.Fatal("least-recently-touched flap tracker survived the cap") + } + if len(tr.flaps) != 2 { + t.Fatalf("tracker count = %d, want 2", len(tr.flaps)) + } +} + +func TestStormThrottler_FlapIgnoresEmptyID(t *testing.T) { + tr := newFindingStormThrottler() + if got := tr.observeFlapLocked(&Finding{}, time.Now()); got != nil { + t.Fatalf("empty ID must not be tracked, got %+v", got) + } + if len(tr.flaps) != 0 { + t.Fatalf("empty ID created a tracker") + } +} diff --git a/internal/ai/patrol_run.go b/internal/ai/patrol_run.go index 5bbf0ffa1..17d8c2e48 100644 --- a/internal/ai/patrol_run.go +++ b/internal/ai/patrol_run.go @@ -654,6 +654,14 @@ func (p *PatrolService) runPatrolWithTriggerStart(ctx context.Context, trigger T } } + // Fold findings that restate an active alert under that alert. Runs after + // both the deterministic watchers and the model have written this cycle's + // findings, against the unscoped alert set, so scoped runs cannot release + // mirrors outside their scope. + if mirrored := p.reconcileAlertMirrors(); mirrored > 0 { + log.Debug().Int("changed", mirrored).Msg("AI Patrol: Updated alert-mirror stamps on findings") + } + // Count resolved findings: LLM-resolved (via tool) + auto-reconciled stale findings. var resolvedCount int if runStats.aiAnalysis != nil {