mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
Canonicalize Patrol activity semantics
This commit is contained in:
@@ -118,6 +118,7 @@ management, and fleet control surfaces.
|
||||
1. Add or change install-command generation, canonical /api/auto-register behavior, or installer download behavior through the owned `internal/api/` files above.
|
||||
2. Add or change update continuity and persisted-version handoff through `internal/agentupdate/`.
|
||||
3. Add or change runtime-side Unified Agent startup, first-report assembly, and enroll/runtime continuity through `internal/hostagent/`.
|
||||
4. Keep shared `internal/api/` helper edits isolated from agent lifecycle semantics: Patrol-specific status transport or alert-trigger wiring changes in shared handlers must not bleed into auto-register, installer, or fleet-control behavior unless this contract moves in the same slice.
|
||||
4. Keep legacy Unified Agent compatibility names explicitly secondary when touching shared `internal/api/` runtime helpers: the legacy host-route family and `host-agent:*` scope names may remain as ingress or migration aliases, but they must not retake primary ownership in router state, live runtime scope checks, handler commentary, or operator-facing guidance.
|
||||
5. Add or change installer flags, persisted service arguments, or upgrade-safe re-entry behavior through `scripts/install.sh` and `scripts/install.ps1`.
|
||||
6. Add or change profile management, the extracted agent profiles runtime owner, the pure unified-agent inventory/install model, the direct Proxmox workspace shell, route model, reporting summary owner, shared install/inventory/dialog section owners, the split infrastructure install/reporting state owners, the split direct-node/discovery infrastructure settings owners plus their shared model, shared frontend install-command assembly, Proxmox setup/install API transport, setup-completion install handoff transport, deploy-fallback manual install transport, and fleet-control presentation through `frontend-modern/src/api/agentProfiles.ts`, `frontend-modern/src/api/nodes.ts`, `frontend-modern/src/components/Settings/AgentProfilesPanel.tsx`, `frontend-modern/src/components/Settings/useAgentProfilesPanelState.ts`, `frontend-modern/src/components/Settings/InfrastructureOperationsController.tsx`, `frontend-modern/src/components/Settings/infrastructureOperationsModel.tsx`, `frontend-modern/src/components/Settings/InfrastructureInstallPanel.tsx`, `frontend-modern/src/components/Settings/InfrastructureInstallerSection.tsx`, `frontend-modern/src/components/Settings/InfrastructureReportingPanel.tsx`, `frontend-modern/src/components/Settings/InfrastructureInventorySection.tsx`, `frontend-modern/src/components/Settings/InfrastructureActiveRowDetails.tsx`, `frontend-modern/src/components/Settings/InfrastructureIgnoredRowDetails.tsx`, `frontend-modern/src/components/Settings/InfrastructureStopMonitoringDialog.tsx`, `frontend-modern/src/components/Settings/InfrastructureDirectConnectionsSummaryCard.tsx`, `frontend-modern/src/components/Settings/InfrastructureWorkspace.tsx`, `frontend-modern/src/components/Settings/infrastructureWorkspaceModel.ts`, `frontend-modern/src/components/Settings/ProxmoxSettingsPanel.tsx`, `frontend-modern/src/components/Settings/proxmoxSettingsModel.ts`, `frontend-modern/src/components/Settings/ProxmoxDirectWorkspace.tsx`, `frontend-modern/src/components/Settings/ProxmoxConfiguredNodesTable.tsx`, `frontend-modern/src/components/Settings/ProxmoxDirectConnectionsCard.tsx`, `frontend-modern/src/components/Settings/ProxmoxDiscoveryResultsCard.tsx`, `frontend-modern/src/components/Settings/ProxmoxDeleteNodeDialog.tsx`, `frontend-modern/src/components/Settings/ProxmoxNodeModalStack.tsx`, `frontend-modern/src/components/Settings/ConfiguredNodeTables.tsx`, `frontend-modern/src/components/Settings/SettingsSectionNav.tsx`, `frontend-modern/src/components/Settings/infrastructureSettingsModel.ts`, `frontend-modern/src/components/Settings/useInfrastructureConfiguredNodesState.ts`, `frontend-modern/src/components/Settings/useInfrastructureDiscoveryRuntimeState.ts`, `frontend-modern/src/components/Settings/useInfrastructureInstallState.tsx`, `frontend-modern/src/components/Settings/useInfrastructureOperationsState.tsx`, `frontend-modern/src/components/Settings/useInfrastructureReportingState.tsx`, `frontend-modern/src/components/Settings/useInfrastructureSettingsState.ts`, `frontend-modern/src/components/Settings/useProxmoxDirectWorkspaceState.ts`, `frontend-modern/src/components/Settings/NodeModal.tsx`, `frontend-modern/src/components/Settings/nodeModalModel.ts`, `frontend-modern/src/components/Settings/useNodeModalState.ts`, `frontend-modern/src/components/SetupWizard/SetupCompletionPanel.tsx`, `frontend-modern/src/components/Infrastructure/deploy/ResultsStep.tsx`, and `frontend-modern/src/utils/agentInstallCommand.ts`.
|
||||
|
||||
@@ -531,6 +531,11 @@ The Patrol status payload must keep that same scope distinction explicit in its
|
||||
own recency fields. `last_patrol_at` is reserved for the most recent completed
|
||||
full Patrol run, while scoped runs and fix-verification checks advance
|
||||
`last_activity_at` without pretending a full verification sweep just happened.
|
||||
That same runtime boundary also owns which Patrol work counts toward
|
||||
full-patrol cadence gates. Community-tier or other full-run limits must key
|
||||
off completed full sweeps only; recent scoped or verification activity may
|
||||
advance `last_activity_at`, but it must not block a manual full Patrol request
|
||||
as if a scheduled estate-wide sweep already happened.
|
||||
The Patrol startup scheduler must preserve that coverage guarantee as well:
|
||||
`internal/ai/patrol_run.go` may skip the startup full patrol only when recent
|
||||
run history already includes a successful full Patrol run, not merely because
|
||||
|
||||
@@ -87,6 +87,12 @@ The alert webhook editor now mirrors that canonical Pushover field rule through
|
||||
`frontend-modern/src/utils/alertWebhookPresentation.ts`, so the UI shares the
|
||||
same alias, preset, and custom-field input mapping instead of carrying its own
|
||||
local webhook-field normalization fork.
|
||||
The alert manager callback layer now also has to stay fan-out-safe. Monitor
|
||||
delivery, the unified alert bridge, and Patrol-adjacent AI listeners must
|
||||
compose through additive fired/resolved subscriptions instead of overwriting a
|
||||
single callback slot, and alert-triggered Patrol enqueueing must stay on the
|
||||
canonical unified alert bridge plus trigger-manager path rather than reviving
|
||||
duplicate callback-side Patrol shortcuts.
|
||||
That shared alert presentation boundary now also has explicit alerts ownership.
|
||||
`frontend-modern/src/utils/alertWebhookPresentation.ts` is the canonical owner
|
||||
for webhook setup copy, service labels, mention-help phrasing, custom-field
|
||||
|
||||
@@ -135,6 +135,8 @@ Own canonical runtime payload shapes between backend and frontend.
|
||||
and the Patrol findings empty-state behavior, so `0 active findings` only renders as a healthy frontend conclusion when the same governed AI summary contract still reports healthy overall health; degraded or not-fully-verified health predictions must flow through to the Patrol findings surface instead of being replaced by page-local "looks healthy" copy
|
||||
and the Patrol assessment headline plus compact summary-strip behavior, so the same governed AI summary contract decides whether the page leads with verified health, issues detected, coverage incomplete, or another attention state instead of letting count-only page fragments emit a stale `No issues found` conclusion
|
||||
and the Patrol verification summary derived from run history, so the page also states whether recent Patrol evidence came from a successful full patrol or only from scoped/erroring runs instead of leaving verification scope implicit
|
||||
and the Patrol status recency split, so `last_patrol_at` remains reserved for completed full Patrol sweeps while scoped runs and verification checks advance `last_activity_at` without claiming a fresh full-estate verification pass
|
||||
and the canonical alert-triggered Patrol enqueue path in `internal/api/router.go`, so alert-fired Patrol work flows through the unified alert bridge and trigger manager instead of being duplicated by monitor callback wiring
|
||||
and the shared `frontend-modern/src/components/Infrastructure/ResourceChangeSummary.tsx` card, so canonical recent-change timelines stay rendered through one governed frontend card instead of separate page-local list loops
|
||||
and the shared `frontend-modern/src/utils/resourceChangePresentation.ts` formatter used by the summary page and resource drawer, so canonical change wording does not drift across surfaces
|
||||
and the `/api/ai/intelligence/changes` route plus `internal/api/contract_test.go`, so the canonical recent-changes endpoint stays on the same intelligence facade and contract snapshot instead of bypassing the shared timeline source
|
||||
|
||||
@@ -1066,6 +1066,10 @@ That same summary shell should also surface verification scope from the
|
||||
owning run-history contract. Operators should be able to see, inside the same
|
||||
summary surface, whether Patrol recently completed a full verification pass or
|
||||
whether recent activity was limited to scoped/erroring patrol runs.
|
||||
That same shell rule also owns Patrol recency labels. Shared Patrol header and
|
||||
status-shell surfaces must keep `Last full patrol` tied only to the full-sweep
|
||||
transport fact and use `Last activity` for scoped or verification work instead
|
||||
of collapsing both timestamps back into a generic `Last run` label.
|
||||
|
||||
Shared primitive consumers that split status-dot tone and status-text tone
|
||||
must now keep both values routed through the same exported presentation helper.
|
||||
|
||||
@@ -201,6 +201,9 @@ The summary recency chip must follow the same governed scope distinction. When
|
||||
the latest completed activity was only a scoped run, the summary should label
|
||||
that timestamp as `Last activity` instead of `Last patrol`; `Last full patrol`
|
||||
belongs only to the most recent completed full Patrol run.
|
||||
That same distinction is transport-backed. `last_patrol_at` names the last
|
||||
completed full Patrol sweep, while `last_activity_at` may advance on scoped
|
||||
work or fix-verification checks without claiming a new full verification pass.
|
||||
That same recency contract also applies to the header metadata row. The top
|
||||
header must not revert to a generic `Last:` timestamp when the rest of Patrol
|
||||
is explicitly distinguishing activity from full verification recency.
|
||||
@@ -224,6 +227,8 @@ operator whether Patrol recently completed a successful full patrol, only ran
|
||||
scoped alert-triggered checks, or ended its most recent full patrol with
|
||||
errors, so the page does not leave trust and coverage as implicit background
|
||||
knowledge.
|
||||
Fix-verification checks belong to that same explanation layer as targeted
|
||||
activity, not as evidence of a fresh full-estate sweep.
|
||||
The same hierarchy applies to investigation context. Correlations, recent
|
||||
changes, and policy posture are secondary evidence for deeper investigation, so
|
||||
the `Investigation context` section belongs beneath the primary findings/history
|
||||
|
||||
@@ -78,6 +78,7 @@ querying, and the operator-facing storage health presentation layer.
|
||||
17. Preserve backend-owned Pulse Mobile relay runtime credential minting in those same shared `internal/api/` auth/security helpers so storage- and recovery-adjacent transport surfaces do not inherit browser-authored wildcard token bundles when they depend on the canonical security helper layer.
|
||||
18. Preserve the dedicated backend-owned `relay:mobile:access` capability and its governed backward-compatible route inventory plus the shared helper call sites around it, so storage- and recovery-adjacent transport surfaces do not treat the mobile relay credential as a general AI scope bundle.
|
||||
19. Preserve shipped local security-doc guidance in shared `internal/api/` config/setup helpers so storage- and recovery-adjacent transport surfaces do not reintroduce GitHub `main` security links when the running build already serves its own local security documentation route.
|
||||
20. Keep shared `internal/api/` Patrol transport and alert-trigger edits feature-isolated: Patrol-specific recency fields, callback fan-out, or alert-bridge wiring changes must not leak into recovery queries, storage links, or recovery-adjacent install/setup flows unless this contract changes in the same slice.
|
||||
|
||||
## Forbidden Paths
|
||||
|
||||
|
||||
@@ -62,17 +62,35 @@ describe('patrol api', () => {
|
||||
it('preserves the canonical patrol runtime state payload', async () => {
|
||||
apiFetchJSONMock.mockResolvedValueOnce({
|
||||
runtime_state: 'blocked',
|
||||
blocked_reason: 'Quickstart credits exhausted. Connect your API key to continue using AI Patrol.',
|
||||
blocked_reason:
|
||||
'Quickstart credits exhausted. Connect your API key to continue using AI Patrol.',
|
||||
healthy: false,
|
||||
} as any);
|
||||
|
||||
await expect(getPatrolStatus()).resolves.toMatchObject({
|
||||
runtime_state: 'blocked',
|
||||
blocked_reason: 'Quickstart credits exhausted. Connect your API key to continue using AI Patrol.',
|
||||
blocked_reason:
|
||||
'Quickstart credits exhausted. Connect your API key to continue using AI Patrol.',
|
||||
healthy: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('preserves split patrol recency transport fields', async () => {
|
||||
apiFetchJSONMock.mockResolvedValueOnce({
|
||||
runtime_state: 'active',
|
||||
healthy: true,
|
||||
last_patrol_at: '2026-03-12T09:30:00Z',
|
||||
last_activity_at: '2026-03-12T09:59:00Z',
|
||||
} as any);
|
||||
|
||||
await expect(getPatrolStatus()).resolves.toMatchObject({
|
||||
runtime_state: 'active',
|
||||
healthy: true,
|
||||
last_patrol_at: '2026-03-12T09:30:00Z',
|
||||
last_activity_at: '2026-03-12T09:59:00Z',
|
||||
});
|
||||
});
|
||||
|
||||
it('normalizes patrol run alert identifiers', async () => {
|
||||
apiFetchJSONMock.mockResolvedValueOnce([
|
||||
{
|
||||
|
||||
@@ -145,6 +145,7 @@ export interface PatrolStatus {
|
||||
running: boolean;
|
||||
enabled: boolean;
|
||||
last_patrol_at?: string;
|
||||
last_activity_at?: string;
|
||||
next_patrol_at?: string;
|
||||
last_duration_ms: number;
|
||||
resources_checked: number;
|
||||
|
||||
@@ -16,7 +16,6 @@ import {
|
||||
} from '@/utils/patrolRunPresentation';
|
||||
import { getPatrolRuntimePresentation } from '@/utils/patrolRuntimePresentation';
|
||||
import ActivityIcon from 'lucide-solid/icons/activity';
|
||||
import CheckCircleIcon from 'lucide-solid/icons/check-circle';
|
||||
import AlertCircleIcon from 'lucide-solid/icons/alert-circle';
|
||||
import AlertTriangleIcon from 'lucide-solid/icons/alert-triangle';
|
||||
|
||||
@@ -27,6 +26,13 @@ interface PatrolStatusBarProps {
|
||||
blockedReason?: string;
|
||||
}
|
||||
|
||||
function normalizePatrolRunType(type: string | undefined): string {
|
||||
return String(type || '')
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/\s+/g, '_');
|
||||
}
|
||||
|
||||
export const PatrolStatusBar: Component<PatrolStatusBarProps> = (props) => {
|
||||
const [runs] = createResource(
|
||||
() => props.refreshTrigger ?? 0,
|
||||
@@ -48,7 +54,8 @@ export const PatrolStatusBar: Component<PatrolStatusBarProps> = (props) => {
|
||||
const todayRuns = allRuns.filter((r) => new Date(r.started_at) >= todayStart);
|
||||
|
||||
const lastRun = allRuns[0];
|
||||
const lastRunTime = lastRun ? new Date(lastRun.started_at) : null;
|
||||
const lastRunTime = lastRun ? new Date(lastRun.completed_at || lastRun.started_at) : null;
|
||||
const lastRunType = normalizePatrolRunType(lastRun?.type);
|
||||
const lastRunStatus = getPatrolRunStatusPresentation(
|
||||
lastRun?.status ?? 'unknown',
|
||||
lastRun?.error_count ?? 0,
|
||||
@@ -59,6 +66,10 @@ export const PatrolStatusBar: Component<PatrolStatusBarProps> = (props) => {
|
||||
runsToday: todayRuns.length,
|
||||
newFindingsToday: todayRuns.reduce((sum, r) => sum + (r.new_findings || 0), 0),
|
||||
lastRunTime: lastRunTime ? formatRelativeTime(lastRunTime, { compact: true }) : null,
|
||||
lastRunTimeLabel:
|
||||
lastRunType === '' || lastRunType === 'full' || lastRunType === 'patrol'
|
||||
? 'Last full patrol'
|
||||
: 'Last activity',
|
||||
lastRunTrigger: formatTriggerReason(lastRun?.trigger_reason),
|
||||
lastRunTypeLabel: getPatrolRunKindLabel(lastRun?.type),
|
||||
lastRunStatus,
|
||||
@@ -73,9 +84,7 @@ export const PatrolStatusBar: Component<PatrolStatusBarProps> = (props) => {
|
||||
const showRuntimeState = createMemo(() => {
|
||||
const runtimeState = props.runtimeState;
|
||||
return (
|
||||
runtimeState === 'blocked' ||
|
||||
runtimeState === 'disabled' ||
|
||||
runtimeState === 'unavailable'
|
||||
runtimeState === 'blocked' || runtimeState === 'disabled' || runtimeState === 'unavailable'
|
||||
);
|
||||
});
|
||||
const showRunInProgress = createMemo(() => props.runtimeState === 'running');
|
||||
@@ -153,7 +162,7 @@ export const PatrolStatusBar: Component<PatrolStatusBarProps> = (props) => {
|
||||
{/* Last run */}
|
||||
<Show when={s().lastRunTime}>
|
||||
<span class="text-xs text-muted">
|
||||
Last run: {s().lastRunTime}
|
||||
{s().lastRunTimeLabel}: {s().lastRunTime}
|
||||
<Show when={s().lastRunTrigger}>
|
||||
<span class=" "> ({s().lastRunTrigger})</span>
|
||||
</Show>
|
||||
|
||||
@@ -32,6 +32,7 @@ export function PatrolIntelligenceHeader(props: { state: PatrolIntelligenceState
|
||||
getPatrolRecencyPresentation({
|
||||
runs: state.patrolRunHistory() ?? [],
|
||||
lastPatrolAt: state.patrolStatus()?.last_patrol_at,
|
||||
lastActivityAt: state.patrolStatus()?.last_activity_at,
|
||||
}),
|
||||
);
|
||||
const showQuickstartStatus = createMemo(() => {
|
||||
|
||||
@@ -59,10 +59,7 @@ export function PatrolIntelligenceSummary(props: { state: PatrolIntelligenceStat
|
||||
getSemanticTonePresentation(assessment().tone),
|
||||
);
|
||||
const activeFindingsSummaryPresentation = createMemo(() =>
|
||||
getPatrolSummaryPresentation(
|
||||
metricState().primarySeverity,
|
||||
metricState().primaryValue > 0,
|
||||
),
|
||||
getPatrolSummaryPresentation(metricState().primarySeverity, metricState().primaryValue > 0),
|
||||
);
|
||||
const verification = createMemo(() =>
|
||||
getPatrolVerificationPresentation({
|
||||
@@ -75,6 +72,7 @@ export function PatrolIntelligenceSummary(props: { state: PatrolIntelligenceStat
|
||||
getPatrolRecencyPresentation({
|
||||
runs: state.patrolRunHistory() ?? [],
|
||||
lastPatrolAt: state.patrolStatus()?.last_patrol_at,
|
||||
lastActivityAt: state.patrolStatus()?.last_activity_at,
|
||||
}),
|
||||
);
|
||||
const fixedSummaryPresentation = createMemo(() =>
|
||||
@@ -156,7 +154,9 @@ export function PatrolIntelligenceSummary(props: { state: PatrolIntelligenceStat
|
||||
</Show>
|
||||
}
|
||||
>
|
||||
<CheckCircleIcon class={`w-5 h-5 ${assessmentTonePresentation().iconClass}`} />
|
||||
<CheckCircleIcon
|
||||
class={`w-5 h-5 ${assessmentTonePresentation().iconClass}`}
|
||||
/>
|
||||
</Show>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -610,7 +610,9 @@ describe('AIIntelligence entitlement gating', () => {
|
||||
const findingsPanel = screen.getByTestId('findings-panel');
|
||||
const contextHeading = screen.getByText('Investigation context');
|
||||
expect(
|
||||
Boolean(findingsPanel.compareDocumentPosition(contextHeading) & Node.DOCUMENT_POSITION_FOLLOWING),
|
||||
Boolean(
|
||||
findingsPanel.compareDocumentPosition(contextHeading) & Node.DOCUMENT_POSITION_FOLLOWING,
|
||||
),
|
||||
).toBe(true);
|
||||
|
||||
expect(screen.getByText(/Health A · 91\/100/)).toBeInTheDocument();
|
||||
@@ -862,6 +864,54 @@ describe('AIIntelligence entitlement gating', () => {
|
||||
expect(findingsPanelState.latestProps?.patrolIntervalMs).toBeUndefined();
|
||||
});
|
||||
|
||||
it('prefers last activity transport over last full patrol transport when no run history is loaded', async () => {
|
||||
hasFeatureMock.mockReturnValue(true);
|
||||
licenseStatusMock.mockReturnValue({ subscription_state: 'active' });
|
||||
getPatrolStatusMock.mockResolvedValue(
|
||||
defaultPatrolStatus({
|
||||
runtime_state: 'active',
|
||||
last_patrol_at: '2026-03-12T09:30:00Z',
|
||||
last_activity_at: '2026-03-12T09:59:00Z',
|
||||
}),
|
||||
);
|
||||
getPatrolRunHistoryMock.mockResolvedValue([]);
|
||||
intelligenceState.summary = {
|
||||
timestamp: '2026-03-12T10:00:00Z',
|
||||
overall_health: {
|
||||
score: 100,
|
||||
grade: 'A',
|
||||
trend: 'stable',
|
||||
factors: [],
|
||||
prediction: 'Infrastructure is healthy with no significant issues detected.',
|
||||
},
|
||||
findings_count: {
|
||||
critical: 0,
|
||||
warning: 0,
|
||||
watch: 0,
|
||||
info: 0,
|
||||
total: 0,
|
||||
},
|
||||
predictions_count: 0,
|
||||
recent_changes_count: 0,
|
||||
learning: {
|
||||
resources_with_knowledge: 0,
|
||||
total_notes: 0,
|
||||
resources_with_baselines: 0,
|
||||
patterns_detected: 0,
|
||||
correlations_learned: 0,
|
||||
incidents_tracked: 0,
|
||||
},
|
||||
};
|
||||
|
||||
render(() => <AIIntelligence />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/Last activity:/i)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
expect(screen.queryByText(/Last full patrol:/i)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('describes both findings and incomplete coverage when active issues exist', async () => {
|
||||
hasFeatureMock.mockReturnValue(true);
|
||||
licenseStatusMock.mockReturnValue({ subscription_state: 'active' });
|
||||
|
||||
@@ -83,14 +83,14 @@ describe('patrolRunPresentation', () => {
|
||||
|
||||
it('returns canonical patrol run kind labels', () => {
|
||||
expect(getPatrolRunKindLabel('scoped')).toBe('Scoped run');
|
||||
expect(getPatrolRunKindLabel('verification')).toBe('Verification check');
|
||||
expect(getPatrolRunKindLabel('patrol')).toBe('Full patrol');
|
||||
expect(getPatrolRunKindLabel('')).toBe('Full patrol');
|
||||
expect(getPatrolRunKindLabel('unexpected')).toBe('Patrol run');
|
||||
});
|
||||
|
||||
it('expresses scoped run coverage against the effective scope', () => {
|
||||
expect(getPatrolRunCoverageSummary(scopedCoverageRun)).toBe(
|
||||
'Checked 1 of 2 scoped resources',
|
||||
);
|
||||
expect(getPatrolRunCoverageSummary(scopedCoverageRun)).toBe('Checked 1 of 2 scoped resources');
|
||||
expect(getPatrolRunResourcesHeading(scopedCoverageRun)).toBe(
|
||||
'Resources checked (1 of 2 scoped)',
|
||||
);
|
||||
@@ -118,20 +118,13 @@ describe('patrolRunPresentation', () => {
|
||||
});
|
||||
|
||||
it('warns when visible runs include legacy findings snapshots', () => {
|
||||
expect(
|
||||
getRunHistorySelectionHint([
|
||||
{ finding_ids: [] },
|
||||
{ finding_ids: undefined },
|
||||
]),
|
||||
).toBe(
|
||||
expect(getRunHistorySelectionHint([{ finding_ids: [] }, { finding_ids: undefined }])).toBe(
|
||||
'Select a run to filter findings when available. Some older runs do not include findings snapshots.',
|
||||
);
|
||||
});
|
||||
|
||||
it('explains selected legacy runs in the run-history shell', () => {
|
||||
expect(
|
||||
getRunHistorySelectionHint([{ finding_ids: [] }], { finding_ids: undefined }),
|
||||
).toBe(
|
||||
expect(getRunHistorySelectionHint([{ finding_ids: [] }], { finding_ids: undefined })).toBe(
|
||||
'Selected run predates findings snapshots; run-scoped findings cannot be fully verified.',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -387,6 +387,49 @@ describe('getPatrolSummaryPresentation', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('reports limited verification when only verification checks are recent', () => {
|
||||
expect(
|
||||
getPatrolVerificationPresentation({
|
||||
runs: [
|
||||
{
|
||||
id: 'run-1',
|
||||
started_at: '2026-03-12T09:58:00Z',
|
||||
completed_at: '2026-03-12T09:59:00Z',
|
||||
duration_ms: 60000,
|
||||
type: 'verification',
|
||||
trigger_reason: 'verification',
|
||||
resources_checked: 1,
|
||||
nodes_checked: 1,
|
||||
guests_checked: 0,
|
||||
docker_checked: 0,
|
||||
storage_checked: 0,
|
||||
hosts_checked: 0,
|
||||
pbs_checked: 0,
|
||||
pmg_checked: 0,
|
||||
kubernetes_checked: 0,
|
||||
new_findings: 0,
|
||||
existing_findings: 0,
|
||||
rejected_findings: 0,
|
||||
resolved_findings: 1,
|
||||
auto_fix_count: 0,
|
||||
findings_summary: 'Verification: issue resolved',
|
||||
finding_ids: ['finding-1'],
|
||||
error_count: 0,
|
||||
status: 'healthy',
|
||||
triage_flags: 0,
|
||||
tool_call_count: 0,
|
||||
},
|
||||
] as never,
|
||||
}),
|
||||
).toEqual({
|
||||
title: 'No recent full patrol',
|
||||
description:
|
||||
'Recent activity was limited to verification checks over 1 resource, so Patrol has not recently re-verified your full infrastructure.',
|
||||
compactLabel: 'Partial verification',
|
||||
tone: 'warning',
|
||||
});
|
||||
});
|
||||
|
||||
it('labels scoped recency as activity rather than patrol', () => {
|
||||
expect(
|
||||
getPatrolRecencyPresentation({
|
||||
@@ -465,4 +508,16 @@ describe('getPatrolSummaryPresentation', () => {
|
||||
timestamp: '2026-03-12T09:57:00Z',
|
||||
});
|
||||
});
|
||||
|
||||
it('prefers explicit last activity transport over last full patrol transport when no run history is loaded', () => {
|
||||
expect(
|
||||
getPatrolRecencyPresentation({
|
||||
lastPatrolAt: '2026-03-12T09:57:00Z',
|
||||
lastActivityAt: '2026-03-12T09:59:00Z',
|
||||
}),
|
||||
).toEqual({
|
||||
label: 'Last activity',
|
||||
timestamp: '2026-03-12T09:59:00Z',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -94,10 +94,26 @@ export function getToolCallResultTextClass(success: boolean): string {
|
||||
}
|
||||
|
||||
export function getPatrolRunKindLabel(type: string | undefined): string {
|
||||
return normalizePatrolRunType(type) === 'scoped' ? 'Scoped run' : 'Full patrol';
|
||||
switch (normalizePatrolRunType(type)) {
|
||||
case 'scoped':
|
||||
return 'Scoped run';
|
||||
case 'verification':
|
||||
return 'Verification check';
|
||||
case '':
|
||||
case 'full':
|
||||
case 'patrol':
|
||||
return 'Full patrol';
|
||||
default:
|
||||
return 'Patrol run';
|
||||
}
|
||||
}
|
||||
|
||||
export function getPatrolRunCoverageSummary(run: Pick<PatrolRunRecord, 'resources_checked' | 'scope_resource_ids' | 'effective_scope_resource_ids'>): string {
|
||||
export function getPatrolRunCoverageSummary(
|
||||
run: Pick<
|
||||
PatrolRunRecord,
|
||||
'resources_checked' | 'scope_resource_ids' | 'effective_scope_resource_ids'
|
||||
>,
|
||||
): string {
|
||||
const resourcesChecked = Math.max(0, run.resources_checked || 0);
|
||||
const scopedResourceCount = getCanonicalScopeResourceIds(run)?.length ?? 0;
|
||||
|
||||
@@ -117,7 +133,12 @@ export function getPatrolRunCoverageSummary(run: Pick<PatrolRunRecord, 'resource
|
||||
return '';
|
||||
}
|
||||
|
||||
export function getPatrolRunResourcesHeading(run: Pick<PatrolRunRecord, 'resources_checked' | 'scope_resource_ids' | 'effective_scope_resource_ids'>): string {
|
||||
export function getPatrolRunResourcesHeading(
|
||||
run: Pick<
|
||||
PatrolRunRecord,
|
||||
'resources_checked' | 'scope_resource_ids' | 'effective_scope_resource_ids'
|
||||
>,
|
||||
): string {
|
||||
const resourcesChecked = Math.max(0, run.resources_checked || 0);
|
||||
const scopedResourceCount = getCanonicalScopeResourceIds(run)?.length ?? 0;
|
||||
|
||||
|
||||
@@ -300,7 +300,16 @@ function normalizeRunType(type: string | undefined): string {
|
||||
}
|
||||
|
||||
function isFullPatrolRun(run: PatrolRunRecord): boolean {
|
||||
return normalizeRunType(run.type) !== 'scoped';
|
||||
const normalized = normalizeRunType(run.type);
|
||||
return normalized === '' || normalized === 'full' || normalized === 'patrol';
|
||||
}
|
||||
|
||||
function isScopedPatrolRun(run: PatrolRunRecord): boolean {
|
||||
return normalizeRunType(run.type) === 'scoped';
|
||||
}
|
||||
|
||||
function isVerificationPatrolRun(run: PatrolRunRecord): boolean {
|
||||
return normalizeRunType(run.type) === 'verification';
|
||||
}
|
||||
|
||||
function isCompletedPatrolRun(run: PatrolRunRecord): boolean {
|
||||
@@ -308,7 +317,12 @@ function isCompletedPatrolRun(run: PatrolRunRecord): boolean {
|
||||
}
|
||||
|
||||
function hasRunErrors(run: PatrolRunRecord): boolean {
|
||||
return run.error_count > 0 || String(run.status || '').trim().toLowerCase() === 'error';
|
||||
return (
|
||||
run.error_count > 0 ||
|
||||
String(run.status || '')
|
||||
.trim()
|
||||
.toLowerCase() === 'error'
|
||||
);
|
||||
}
|
||||
|
||||
export function getPatrolAssessmentPresentation(args: {
|
||||
@@ -390,8 +404,7 @@ export function getPatrolAssessmentPresentation(args: {
|
||||
return {
|
||||
title: 'Health requires attention',
|
||||
description:
|
||||
args.overallHealth.prediction?.trim() ||
|
||||
'Patrol assessment still needs attention.',
|
||||
args.overallHealth.prediction?.trim() || 'Patrol assessment still needs attention.',
|
||||
eyebrow: 'Patrol assessment',
|
||||
compactLabel: 'Health requires attention',
|
||||
tone: getHealthSummaryTone(args.overallHealth),
|
||||
@@ -458,15 +471,29 @@ export function getPatrolVerificationPresentation(args: {
|
||||
};
|
||||
}
|
||||
|
||||
const recentScopedRun = completedRuns.find((run) => !isFullPatrolRun(run));
|
||||
if (recentScopedRun) {
|
||||
const resourcesChecked = recentScopedRun.resources_checked || 0;
|
||||
const recentLimitedRun = completedRuns.find((run) => !isFullPatrolRun(run));
|
||||
if (recentLimitedRun) {
|
||||
const resourcesChecked = recentLimitedRun.resources_checked || 0;
|
||||
let description =
|
||||
'Recent activity was limited to targeted Patrol checks, so Patrol has not recently re-verified your full infrastructure.';
|
||||
|
||||
if (isVerificationPatrolRun(recentLimitedRun)) {
|
||||
description =
|
||||
resourcesChecked > 0
|
||||
? `Recent activity was limited to verification checks over ${resourcesChecked} resource${resourcesChecked === 1 ? '' : 's'}, so Patrol has not recently re-verified your full infrastructure.`
|
||||
: 'Recent activity was limited to verification checks, so Patrol has not recently re-verified your full infrastructure.';
|
||||
} else if (isScopedPatrolRun(recentLimitedRun)) {
|
||||
description =
|
||||
resourcesChecked > 0
|
||||
? `Recent activity was limited to scoped ${recentLimitedRun.trigger_reason ? String(recentLimitedRun.trigger_reason).replace(/_/g, ' ') : 'patrol'} runs over ${resourcesChecked} resource${resourcesChecked === 1 ? '' : 's'}, so Patrol has not recently re-verified your full infrastructure.`
|
||||
: 'Recent activity was limited to scoped patrol runs, so Patrol has not recently re-verified your full infrastructure.';
|
||||
} else if (resourcesChecked > 0) {
|
||||
description = `Recent activity was limited to targeted Patrol checks over ${resourcesChecked} resource${resourcesChecked === 1 ? '' : 's'}, so Patrol has not recently re-verified your full infrastructure.`;
|
||||
}
|
||||
|
||||
return {
|
||||
title: 'No recent full patrol',
|
||||
description:
|
||||
resourcesChecked > 0
|
||||
? `Recent activity was limited to scoped ${recentScopedRun.trigger_reason ? String(recentScopedRun.trigger_reason).replace(/_/g, ' ') : 'patrol'} runs over ${resourcesChecked} resource${resourcesChecked === 1 ? '' : 's'}, so Patrol has not recently re-verified your full infrastructure.`
|
||||
: 'Recent activity was limited to scoped patrol runs, so Patrol has not recently re-verified your full infrastructure.',
|
||||
description,
|
||||
compactLabel: 'Partial verification',
|
||||
tone: 'warning',
|
||||
};
|
||||
@@ -483,6 +510,7 @@ export function getPatrolVerificationPresentation(args: {
|
||||
export function getPatrolRecencyPresentation(args: {
|
||||
runs?: PatrolRunRecord[];
|
||||
lastPatrolAt?: string;
|
||||
lastActivityAt?: string;
|
||||
}): PatrolRecencyPresentation {
|
||||
const latestCompletedRun = (args.runs ?? []).find((run) => isCompletedPatrolRun(run));
|
||||
if (latestCompletedRun?.completed_at) {
|
||||
@@ -492,10 +520,41 @@ export function getPatrolRecencyPresentation(args: {
|
||||
};
|
||||
}
|
||||
|
||||
if (args.lastPatrolAt?.trim()) {
|
||||
const lastPatrolAt = args.lastPatrolAt?.trim();
|
||||
const lastActivityAt = args.lastActivityAt?.trim();
|
||||
|
||||
if (lastActivityAt && lastPatrolAt) {
|
||||
const activityMs = Date.parse(lastActivityAt);
|
||||
const patrolMs = Date.parse(lastPatrolAt);
|
||||
if (Number.isNaN(activityMs) && !Number.isNaN(patrolMs)) {
|
||||
return {
|
||||
label: 'Last full patrol',
|
||||
timestamp: lastPatrolAt,
|
||||
};
|
||||
}
|
||||
if (!Number.isNaN(activityMs) && !Number.isNaN(patrolMs) && patrolMs >= activityMs) {
|
||||
return {
|
||||
label: 'Last full patrol',
|
||||
timestamp: lastPatrolAt,
|
||||
};
|
||||
}
|
||||
return {
|
||||
label: 'Last activity',
|
||||
timestamp: args.lastPatrolAt,
|
||||
timestamp: lastActivityAt,
|
||||
};
|
||||
}
|
||||
|
||||
if (lastActivityAt) {
|
||||
return {
|
||||
label: 'Last activity',
|
||||
timestamp: lastActivityAt,
|
||||
};
|
||||
}
|
||||
|
||||
if (lastPatrolAt) {
|
||||
return {
|
||||
label: 'Last full patrol',
|
||||
timestamp: lastPatrolAt,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+59
-10
@@ -1074,22 +1074,23 @@ func summarizeRecentPatrolCoverage(
|
||||
var recentErrors int
|
||||
var hasSuccessfulFullRun bool
|
||||
var hasRecentFullRun bool
|
||||
var scopedRuns int
|
||||
var limitedActivityRuns int
|
||||
for _, run := range relevant {
|
||||
if run.ErrorCount > 0 || strings.EqualFold(strings.TrimSpace(run.Status), "error") {
|
||||
recentErrors++
|
||||
}
|
||||
if !isScopedPatrolRun(run) {
|
||||
if isFullPatrolRun(run) {
|
||||
hasRecentFullRun = true
|
||||
} else {
|
||||
limitedActivityRuns++
|
||||
}
|
||||
if isSuccessfulFullPatrolRun(run) {
|
||||
hasSuccessfulFullRun = true
|
||||
}
|
||||
if isScopedPatrolRun(run) {
|
||||
scopedRuns++
|
||||
}
|
||||
}
|
||||
|
||||
limitedActivityLabel := describeLimitedPatrolActivity(relevant)
|
||||
|
||||
switch {
|
||||
case !hasSuccessfulFullRun && hasRecentFullRun && recentErrors > 0:
|
||||
return patrolCoverageFactor{
|
||||
@@ -1100,13 +1101,13 @@ func summarizeRecentPatrolCoverage(
|
||||
case !hasSuccessfulFullRun && recentErrors > 0:
|
||||
return patrolCoverageFactor{
|
||||
name: "Patrol coverage incomplete",
|
||||
description: "Patrol coverage is incomplete: recent activity was limited to scoped runs and ended with errors, so overall health is not fully verified.",
|
||||
description: fmt.Sprintf("Patrol coverage is incomplete: recent activity was limited to %s and ended with errors, so overall health is not fully verified.", limitedActivityLabel),
|
||||
impact: 35,
|
||||
}, true
|
||||
case !hasSuccessfulFullRun && scopedRuns == len(relevant):
|
||||
case !hasSuccessfulFullRun && limitedActivityRuns == len(relevant):
|
||||
return patrolCoverageFactor{
|
||||
name: "Patrol coverage incomplete",
|
||||
description: "Patrol coverage is incomplete: recent activity was limited to scoped runs, so overall infrastructure health is not fully verified.",
|
||||
description: fmt.Sprintf("Patrol coverage is incomplete: recent activity was limited to %s, so overall infrastructure health is not fully verified.", limitedActivityLabel),
|
||||
impact: 20,
|
||||
}, true
|
||||
case recentErrors > 0:
|
||||
@@ -1120,12 +1121,60 @@ func summarizeRecentPatrolCoverage(
|
||||
}
|
||||
}
|
||||
|
||||
func normalizePatrolRunType(run PatrolRunRecord) string {
|
||||
return strings.ToLower(strings.TrimSpace(run.Type))
|
||||
}
|
||||
|
||||
func isFullPatrolRun(run PatrolRunRecord) bool {
|
||||
switch normalizePatrolRunType(run) {
|
||||
case "", "full", "patrol":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isScopedPatrolRun(run PatrolRunRecord) bool {
|
||||
return strings.EqualFold(strings.TrimSpace(run.Type), "scoped")
|
||||
return normalizePatrolRunType(run) == "scoped"
|
||||
}
|
||||
|
||||
func isVerificationPatrolRun(run PatrolRunRecord) bool {
|
||||
return normalizePatrolRunType(run) == "verification"
|
||||
}
|
||||
|
||||
func describeLimitedPatrolActivity(runs []PatrolRunRecord) string {
|
||||
hasScoped := false
|
||||
hasVerification := false
|
||||
hasOther := false
|
||||
|
||||
for _, run := range runs {
|
||||
if isFullPatrolRun(run) {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case isScopedPatrolRun(run):
|
||||
hasScoped = true
|
||||
case isVerificationPatrolRun(run):
|
||||
hasVerification = true
|
||||
default:
|
||||
hasOther = true
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case hasScoped && !hasVerification && !hasOther:
|
||||
return "scoped runs"
|
||||
case hasVerification && !hasScoped && !hasOther:
|
||||
return "verification checks"
|
||||
case hasScoped && hasVerification && !hasOther:
|
||||
return "scoped runs and verification checks"
|
||||
default:
|
||||
return "targeted Patrol activity"
|
||||
}
|
||||
}
|
||||
|
||||
func isSuccessfulFullPatrolRun(run PatrolRunRecord) bool {
|
||||
return !isScopedPatrolRun(run) &&
|
||||
return isFullPatrolRun(run) &&
|
||||
run.ErrorCount == 0 &&
|
||||
!strings.EqualFold(strings.TrimSpace(run.Status), "error")
|
||||
}
|
||||
|
||||
@@ -199,6 +199,34 @@ func TestIntelligence_GetSummary_DegradesWhenRecentPatrolCoverageIsScopedAndErro
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntelligence_GetSummary_DegradesWhenRecentPatrolCoverageIsVerificationOnly(t *testing.T) {
|
||||
intel := NewIntelligence(IntelligenceConfig{})
|
||||
runHistory := NewPatrolRunHistoryStore(10)
|
||||
now := time.Now()
|
||||
runHistory.Add(PatrolRunRecord{
|
||||
ID: "verification-1",
|
||||
Type: "verification",
|
||||
TriggerReason: "verification",
|
||||
CompletedAt: now.Add(-2 * time.Minute),
|
||||
ErrorCount: 0,
|
||||
Status: "healthy",
|
||||
ResourcesChecked: 1,
|
||||
})
|
||||
intel.SetRunHistoryStore(runHistory)
|
||||
|
||||
summary := intel.GetSummary()
|
||||
if summary.OverallHealth.Score >= 100 {
|
||||
t.Fatalf("expected reduced health score, got %f", summary.OverallHealth.Score)
|
||||
}
|
||||
if summary.OverallHealth.Grade == HealthGradeA {
|
||||
t.Fatalf("expected non-A grade, got %s", summary.OverallHealth.Grade)
|
||||
}
|
||||
want := "Patrol coverage is incomplete: recent activity was limited to verification checks, so overall infrastructure health is not fully verified."
|
||||
if summary.OverallHealth.Prediction != want {
|
||||
t.Fatalf("expected verification-only coverage warning, got %q", summary.OverallHealth.Prediction)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntelligence_GetSummary_DegradesWhenRecentFullPatrolErrored(t *testing.T) {
|
||||
intel := NewIntelligence(IntelligenceConfig{})
|
||||
runHistory := NewPatrolRunHistoryStore(10)
|
||||
|
||||
@@ -94,7 +94,8 @@ type PatrolStatus struct {
|
||||
RuntimeState PatrolRuntimeState `json:"runtime_state"`
|
||||
Running bool `json:"running"`
|
||||
Enabled bool `json:"enabled"`
|
||||
LastPatrolAt *time.Time `json:"last_patrol_at,omitempty"`
|
||||
LastPatrolAt *time.Time `json:"last_patrol_at,omitempty"` // Last completed full patrol
|
||||
LastActivityAt *time.Time `json:"last_activity_at,omitempty"` // Last completed Patrol activity of any kind
|
||||
NextPatrolAt *time.Time `json:"next_patrol_at,omitempty"`
|
||||
LastDuration time.Duration `json:"last_duration_ms"`
|
||||
ResourcesChecked int `json:"resources_checked"`
|
||||
@@ -443,7 +444,8 @@ type PatrolService struct {
|
||||
runStartedAt time.Time
|
||||
stopCh chan struct{}
|
||||
configChanged chan struct{} // Signal when config changes to reset ticker
|
||||
lastPatrol time.Time
|
||||
lastFullPatrol time.Time
|
||||
lastActivity time.Time
|
||||
lastDuration time.Duration
|
||||
resourcesChecked int
|
||||
errorCount int
|
||||
|
||||
@@ -1341,25 +1341,45 @@ func (p *PatrolService) VerifyFixResolved(ctx context.Context, resourceID, resou
|
||||
}
|
||||
|
||||
verifyRecord := PatrolRunRecord{
|
||||
ID: fmt.Sprintf("%d", startTime.UnixNano()),
|
||||
StartedAt: startTime,
|
||||
CompletedAt: endTime,
|
||||
Duration: duration,
|
||||
DurationMs: duration.Milliseconds(),
|
||||
Type: "verification",
|
||||
TriggerReason: string(TriggerReasonVerification),
|
||||
ScopeResourceIDs: []string{resourceID},
|
||||
ScopeResourceTypes: []string{resourceType},
|
||||
ScopeContext: fmt.Sprintf("Verifying fix for finding: %s", findingID),
|
||||
FindingID: findingID,
|
||||
NewFindings: 0,
|
||||
FindingsSummary: summary,
|
||||
Status: status,
|
||||
ID: fmt.Sprintf("%d", startTime.UnixNano()),
|
||||
StartedAt: startTime,
|
||||
CompletedAt: endTime,
|
||||
Duration: duration,
|
||||
DurationMs: duration.Milliseconds(),
|
||||
Type: "verification",
|
||||
TriggerReason: string(TriggerReasonVerification),
|
||||
ScopeResourceIDs: []string{resourceID},
|
||||
EffectiveScopeResourceIDs: []string{resourceID},
|
||||
ScopeResourceTypes: []string{resourceType},
|
||||
ScopeContext: fmt.Sprintf("Verifying fix for finding: %s", findingID),
|
||||
FindingID: findingID,
|
||||
ResourcesChecked: 1,
|
||||
NewFindings: 0,
|
||||
FindingsSummary: summary,
|
||||
Status: status,
|
||||
}
|
||||
if strings.TrimSpace(resourceID) == "" {
|
||||
verifyRecord.ScopeResourceIDs = nil
|
||||
verifyRecord.EffectiveScopeResourceIDs = nil
|
||||
verifyRecord.ResourcesChecked = 0
|
||||
}
|
||||
if strings.TrimSpace(resourceType) == "" {
|
||||
verifyRecord.ScopeResourceTypes = nil
|
||||
}
|
||||
if verifyErr != nil {
|
||||
verifyRecord.ErrorCount = 1
|
||||
}
|
||||
if p.runHistoryStore != nil {
|
||||
p.runHistoryStore.Add(verifyRecord)
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
p.lastActivity = endTime
|
||||
p.lastDuration = duration
|
||||
p.resourcesChecked = verifyRecord.ResourcesChecked
|
||||
p.errorCount = verifyRecord.ErrorCount
|
||||
p.mu.Unlock()
|
||||
|
||||
return verified, verifyErr
|
||||
}
|
||||
|
||||
|
||||
@@ -268,6 +268,66 @@ func TestVerifyFixResolved_UsesReadStateWithoutSnapshotProvider(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyFixResolved_RecordsVerificationAsActivityWithoutReplacingLastFullPatrol(t *testing.T) {
|
||||
ps := NewPatrolService(nil, nil)
|
||||
ps.thresholds = PatrolThresholds{NodeCPUWarning: 90}
|
||||
|
||||
nodeView := unifiedresources.NewNodeView(&unifiedresources.Resource{
|
||||
ID: "node-1",
|
||||
Name: "node-1",
|
||||
Type: unifiedresources.ResourceTypeAgent,
|
||||
Status: unifiedresources.StatusOnline,
|
||||
Proxmox: &unifiedresources.ProxmoxData{
|
||||
NodeName: "node-1",
|
||||
},
|
||||
Metrics: &unifiedresources.ResourceMetrics{
|
||||
CPU: &unifiedresources.MetricValue{Percent: 20},
|
||||
},
|
||||
})
|
||||
ps.SetReadState(&mockReadState{nodes: []*unifiedresources.NodeView{&nodeView}})
|
||||
|
||||
lastFullPatrol := time.Now().Add(-1 * time.Hour).UTC().Truncate(time.Second)
|
||||
ps.mu.Lock()
|
||||
ps.lastFullPatrol = lastFullPatrol
|
||||
ps.mu.Unlock()
|
||||
|
||||
verified, err := ps.VerifyFixResolved(context.Background(), "node-1", "node", "cpu-high", "finding-1")
|
||||
if err != nil {
|
||||
t.Fatalf("expected verification to succeed, got %v", err)
|
||||
}
|
||||
if !verified {
|
||||
t.Fatal("expected verification to resolve the issue")
|
||||
}
|
||||
|
||||
status := ps.GetStatus()
|
||||
if status.LastPatrolAt == nil || !status.LastPatrolAt.Equal(lastFullPatrol) {
|
||||
t.Fatalf("last full patrol = %v, want %v", status.LastPatrolAt, lastFullPatrol)
|
||||
}
|
||||
if status.LastActivityAt == nil {
|
||||
t.Fatal("expected verification run to update last activity")
|
||||
}
|
||||
if !status.LastActivityAt.After(lastFullPatrol) {
|
||||
t.Fatalf("expected last activity %v to be after last full patrol %v", *status.LastActivityAt, lastFullPatrol)
|
||||
}
|
||||
if status.LastDuration <= 0 {
|
||||
t.Fatalf("expected verification run to update last duration, got %v", status.LastDuration)
|
||||
}
|
||||
if status.ResourcesChecked != 1 {
|
||||
t.Fatalf("resources checked = %d, want 1", status.ResourcesChecked)
|
||||
}
|
||||
|
||||
runs := ps.GetRunHistory(1)
|
||||
if len(runs) != 1 {
|
||||
t.Fatalf("expected one verification run, got %d", len(runs))
|
||||
}
|
||||
if runs[0].Type != "verification" {
|
||||
t.Fatalf("run type = %q, want verification", runs[0].Type)
|
||||
}
|
||||
if runs[0].ResourcesChecked != 1 {
|
||||
t.Fatalf("verification run resources_checked = %d, want 1", runs[0].ResourcesChecked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyFixResolved_WithoutRuntimeStateFailsClosed(t *testing.T) {
|
||||
ps := NewPatrolService(nil, nil)
|
||||
|
||||
|
||||
@@ -101,11 +101,13 @@ func (p *PatrolService) Stop() {
|
||||
|
||||
// patrolLoop is the main background loop
|
||||
func (p *PatrolService) patrolLoop(ctx context.Context) {
|
||||
// Seed lastPatrol from persisted run history so the API can return
|
||||
// last_patrol_at immediately (before the first in-process patrol completes).
|
||||
if history := p.GetRunHistory(1); len(history) > 0 && !history[0].CompletedAt.IsZero() {
|
||||
// Seed recency from persisted run history so the API can return Patrol timing
|
||||
// metadata immediately (before the first in-process patrol completes).
|
||||
if history := p.GetRunHistory(10); len(history) > 0 {
|
||||
lastActivity, lastFullPatrol := patrolRecencyFromHistory(history)
|
||||
p.mu.Lock()
|
||||
p.lastPatrol = history[0].CompletedAt
|
||||
p.lastActivity = lastActivity
|
||||
p.lastFullPatrol = lastFullPatrol
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
@@ -216,6 +218,23 @@ func shouldSkipInitialFullPatrol(runHistory []PatrolRunRecord, now time.Time) bo
|
||||
return false
|
||||
}
|
||||
|
||||
func patrolRecencyFromHistory(runHistory []PatrolRunRecord) (time.Time, time.Time) {
|
||||
var lastActivity time.Time
|
||||
var lastFullPatrol time.Time
|
||||
for _, run := range runHistory {
|
||||
if run.CompletedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
if lastActivity.IsZero() || run.CompletedAt.After(lastActivity) {
|
||||
lastActivity = run.CompletedAt
|
||||
}
|
||||
if isFullPatrolRun(run) && (lastFullPatrol.IsZero() || run.CompletedAt.After(lastFullPatrol)) {
|
||||
lastFullPatrol = run.CompletedAt
|
||||
}
|
||||
}
|
||||
return lastActivity, lastFullPatrol
|
||||
}
|
||||
|
||||
// runPatrol executes a scheduled patrol run
|
||||
func (p *PatrolService) runPatrol(ctx context.Context) {
|
||||
p.runPatrolWithTrigger(ctx, TriggerReasonScheduled, nil)
|
||||
@@ -600,7 +619,8 @@ func (p *PatrolService) runPatrolWithTrigger(ctx context.Context, trigger Trigge
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
p.lastPatrol = completedAt
|
||||
p.lastActivity = completedAt
|
||||
p.lastFullPatrol = completedAt
|
||||
p.lastDuration = duration
|
||||
p.resourcesChecked = runStats.resourceCount
|
||||
p.errorCount = runStats.errors
|
||||
@@ -941,7 +961,7 @@ func (p *PatrolService) runScopedPatrol(ctx context.Context, scope PatrolScope)
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
p.lastPatrol = completedAt
|
||||
p.lastActivity = completedAt
|
||||
p.lastDuration = duration
|
||||
p.resourcesChecked = runStats.resourceCount
|
||||
p.errorCount = runStats.errors
|
||||
@@ -1555,8 +1575,11 @@ func (p *PatrolService) GetStatus() PatrolStatus {
|
||||
status.RuntimeState = PatrolRuntimeStateActive
|
||||
}
|
||||
|
||||
if !p.lastPatrol.IsZero() {
|
||||
status.LastPatrolAt = &p.lastPatrol
|
||||
if !p.lastFullPatrol.IsZero() {
|
||||
status.LastPatrolAt = &p.lastFullPatrol
|
||||
}
|
||||
if !p.lastActivity.IsZero() {
|
||||
status.LastActivityAt = &p.lastActivity
|
||||
}
|
||||
if strings.TrimSpace(status.BlockedReason) != "" && !p.lastBlockedAt.IsZero() {
|
||||
status.BlockedAt = &p.lastBlockedAt
|
||||
|
||||
@@ -1391,14 +1391,14 @@ func TestGetStatus_NextPatrolAt(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetStatus_NextPatrolAt_IndependentOfLastPatrol(t *testing.T) {
|
||||
// nextScheduledAt should drive NextPatrolAt, not lastPatrol + interval.
|
||||
// nextScheduledAt should drive NextPatrolAt, not lastFullPatrol + interval.
|
||||
// This is the scenario that was previously broken: user changes interval
|
||||
// mid-cycle, lastPatrol is old, so lastPatrol+newInterval could be in the past.
|
||||
// mid-cycle, lastFullPatrol is old, so lastFullPatrol+newInterval could be in the past.
|
||||
ps := NewPatrolService(nil, nil)
|
||||
|
||||
// Simulate: patrol ran 45 min ago
|
||||
ps.mu.Lock()
|
||||
ps.lastPatrol = time.Now().Add(-45 * time.Minute)
|
||||
ps.lastFullPatrol = time.Now().Add(-45 * time.Minute)
|
||||
// But the ticker was just reset with a 15-min interval, so next fire is ~15 min from now
|
||||
expectedNext := time.Now().Add(15 * time.Minute)
|
||||
ps.nextScheduledAt = expectedNext
|
||||
@@ -1409,7 +1409,7 @@ func TestGetStatus_NextPatrolAt_IndependentOfLastPatrol(t *testing.T) {
|
||||
t.Fatal("expected NextPatrolAt to be set")
|
||||
}
|
||||
// NextPatrolAt must be the tracked nextScheduledAt (in the future),
|
||||
// NOT lastPatrol + interval (which would be 30 min in the past).
|
||||
// NOT lastFullPatrol + interval (which would be 30 min in the past).
|
||||
if !status.NextPatrolAt.Equal(expectedNext) {
|
||||
t.Errorf("expected NextPatrolAt = %v, got %v", expectedNext, *status.NextPatrolAt)
|
||||
}
|
||||
|
||||
@@ -598,6 +598,7 @@ func TestPatrolStatus_Fields(t *testing.T) {
|
||||
Running: true,
|
||||
Enabled: true,
|
||||
LastPatrolAt: &now,
|
||||
LastActivityAt: &now,
|
||||
NextPatrolAt: &next,
|
||||
LastDuration: 5 * time.Second,
|
||||
ResourcesChecked: 25,
|
||||
@@ -619,6 +620,9 @@ func TestPatrolStatus_Fields(t *testing.T) {
|
||||
if status.LastPatrolAt == nil {
|
||||
t.Error("Expected LastPatrolAt to be set")
|
||||
}
|
||||
if status.LastActivityAt == nil {
|
||||
t.Error("Expected LastActivityAt to be set")
|
||||
}
|
||||
if *status.NextPatrolAt != next {
|
||||
t.Error("NextPatrolAt value mismatch")
|
||||
}
|
||||
|
||||
@@ -53,7 +53,7 @@ func (a *AlertManagerAdapter) SetAlertCallback(cb func(AlertAdapter)) {
|
||||
return
|
||||
}
|
||||
|
||||
a.manager.SetAlertCallback(func(alert *alerts.Alert) {
|
||||
a.manager.SubscribeAlertCallback(func(alert *alerts.Alert) {
|
||||
if cb != nil && alert != nil {
|
||||
cb(&alertWrapper{alert: alert})
|
||||
}
|
||||
@@ -66,7 +66,7 @@ func (a *AlertManagerAdapter) SetResolvedCallback(cb func(alertID string)) {
|
||||
return
|
||||
}
|
||||
|
||||
a.manager.SetResolvedCallback(cb)
|
||||
a.manager.SubscribeResolvedCallback(cb)
|
||||
}
|
||||
|
||||
// alertWrapper wraps an alerts.Alert to implement AlertAdapter
|
||||
|
||||
@@ -98,8 +98,14 @@ func TestAlertManagerAdapter_WithManagerAndCallbacks(t *testing.T) {
|
||||
adapter.SetAlertCallback(func(ad AlertAdapter) {
|
||||
alertCh <- ad.GetAlertIdentifier()
|
||||
})
|
||||
onAlert := getUnexportedField(t, manager, "onAlert").Interface().(func(alert *alerts.Alert))
|
||||
onAlert(alert)
|
||||
alertSubs := getUnexportedField(t, manager, "alertSubs")
|
||||
if alertSubs.Len() == 0 {
|
||||
t.Fatal("expected subscribed alert callback")
|
||||
}
|
||||
for _, key := range alertSubs.MapKeys() {
|
||||
alertSubs.MapIndex(key).Interface().(func(alert *alerts.Alert))(alert)
|
||||
break
|
||||
}
|
||||
select {
|
||||
case got := <-alertCh:
|
||||
if got != alert.ID {
|
||||
@@ -113,8 +119,14 @@ func TestAlertManagerAdapter_WithManagerAndCallbacks(t *testing.T) {
|
||||
adapter.SetResolvedCallback(func(alertID string) {
|
||||
resolvedCh <- alertID
|
||||
})
|
||||
onResolved := getUnexportedField(t, manager, "onResolved").Interface().(func(alertID string))
|
||||
onResolved(alert.ID)
|
||||
resolvedSubs := getUnexportedField(t, manager, "resolvedSubs")
|
||||
if resolvedSubs.Len() == 0 {
|
||||
t.Fatal("expected subscribed resolved callback")
|
||||
}
|
||||
for _, key := range resolvedSubs.MapKeys() {
|
||||
resolvedSubs.MapIndex(key).Interface().(func(alertID string))(alert.ID)
|
||||
break
|
||||
}
|
||||
select {
|
||||
case got := <-resolvedCh:
|
||||
if got != alert.ID {
|
||||
|
||||
+149
-21
@@ -534,11 +534,15 @@ type Manager struct {
|
||||
activeAlertAlias map[string]string
|
||||
historyManager *HistoryManager
|
||||
onAlert func(alert *Alert)
|
||||
alertSubs map[int]func(alert *Alert)
|
||||
onResolved func(alertID string)
|
||||
resolvedSubs map[int]func(alertID string)
|
||||
onAcknowledged func(alert *Alert, user string)
|
||||
onUnacknowledged func(alert *Alert, user string)
|
||||
onEscalate func(alert *Alert, level int)
|
||||
onAlertForAI func(alert *Alert) // AI analysis callback - bypasses notification suppression
|
||||
alertForAISubs map[int]func(alert *Alert)
|
||||
nextCallbackID int
|
||||
escalationStop chan struct{}
|
||||
alertRateLimit map[string][]time.Time // Track alert times for rate limiting
|
||||
// New fields for deduplication and suppression
|
||||
@@ -625,6 +629,9 @@ func NewManagerWithDataDir(dataDir string) *Manager {
|
||||
activeAlertAlias: make(map[string]string),
|
||||
historyManager: NewHistoryManager(alertsDir),
|
||||
escalationStop: make(chan struct{}),
|
||||
alertSubs: make(map[int]func(*Alert)),
|
||||
resolvedSubs: make(map[int]func(string)),
|
||||
alertForAISubs: make(map[int]func(*Alert)),
|
||||
alertRateLimit: make(map[string][]time.Time),
|
||||
recentAlerts: make(map[string]*Alert),
|
||||
suppressedUntil: make(map[string]time.Time),
|
||||
@@ -834,6 +841,27 @@ func (m *Manager) SetAlertCallback(cb func(alert *Alert)) {
|
||||
m.onAlert = cb
|
||||
}
|
||||
|
||||
// SubscribeAlertCallback registers an additional alert callback without
|
||||
// replacing the legacy single callback slot. The returned function removes the
|
||||
// subscription when called.
|
||||
func (m *Manager) SubscribeAlertCallback(cb func(alert *Alert)) func() {
|
||||
if cb == nil {
|
||||
return func() {}
|
||||
}
|
||||
|
||||
m.callbackMu.Lock()
|
||||
m.nextCallbackID++
|
||||
id := m.nextCallbackID
|
||||
m.alertSubs[id] = cb
|
||||
m.callbackMu.Unlock()
|
||||
|
||||
return func() {
|
||||
m.callbackMu.Lock()
|
||||
delete(m.alertSubs, id)
|
||||
m.callbackMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// SetAlertForAICallback sets a callback for AI analysis when alerts are created.
|
||||
// Unlike SetAlertCallback, this callback is invoked unconditionally - it bypasses
|
||||
// activation state, quiet hours, and other notification suppression checks.
|
||||
@@ -845,6 +873,27 @@ func (m *Manager) SetAlertForAICallback(cb func(alert *Alert)) {
|
||||
log.Info().Msg("alert-for-AI callback registered (bypasses notification suppression)")
|
||||
}
|
||||
|
||||
// SubscribeAlertForAICallback registers an additional AI alert callback without
|
||||
// replacing the legacy single callback slot. The returned function removes the
|
||||
// subscription when called.
|
||||
func (m *Manager) SubscribeAlertForAICallback(cb func(alert *Alert)) func() {
|
||||
if cb == nil {
|
||||
return func() {}
|
||||
}
|
||||
|
||||
m.callbackMu.Lock()
|
||||
m.nextCallbackID++
|
||||
id := m.nextCallbackID
|
||||
m.alertForAISubs[id] = cb
|
||||
m.callbackMu.Unlock()
|
||||
|
||||
return func() {
|
||||
m.callbackMu.Lock()
|
||||
delete(m.alertForAISubs, id)
|
||||
m.callbackMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// SetResolvedCallback sets the callback for resolved alerts
|
||||
func (m *Manager) SetResolvedCallback(cb func(alertID string)) {
|
||||
m.callbackMu.Lock()
|
||||
@@ -852,6 +901,27 @@ func (m *Manager) SetResolvedCallback(cb func(alertID string)) {
|
||||
m.onResolved = cb
|
||||
}
|
||||
|
||||
// SubscribeResolvedCallback registers an additional resolved-alert callback
|
||||
// without replacing the legacy single callback slot. The returned function
|
||||
// removes the subscription when called.
|
||||
func (m *Manager) SubscribeResolvedCallback(cb func(alertID string)) func() {
|
||||
if cb == nil {
|
||||
return func() {}
|
||||
}
|
||||
|
||||
m.callbackMu.Lock()
|
||||
m.nextCallbackID++
|
||||
id := m.nextCallbackID
|
||||
m.resolvedSubs[id] = cb
|
||||
m.callbackMu.Unlock()
|
||||
|
||||
return func() {
|
||||
m.callbackMu.Lock()
|
||||
delete(m.resolvedSubs, id)
|
||||
m.callbackMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// SetAcknowledgedCallback sets the callback for acknowledged alerts.
|
||||
func (m *Manager) SetAcknowledgedCallback(cb func(alert *Alert, user string)) {
|
||||
m.callbackMu.Lock()
|
||||
@@ -880,6 +950,22 @@ func (m *Manager) getAlertCallback() func(alert *Alert) {
|
||||
return cb
|
||||
}
|
||||
|
||||
func (m *Manager) getAlertCallbacks() []func(alert *Alert) {
|
||||
m.callbackMu.RLock()
|
||||
defer m.callbackMu.RUnlock()
|
||||
|
||||
callbacks := make([]func(alert *Alert), 0, len(m.alertSubs)+1)
|
||||
if m.onAlert != nil {
|
||||
callbacks = append(callbacks, m.onAlert)
|
||||
}
|
||||
for _, cb := range m.alertSubs {
|
||||
if cb != nil {
|
||||
callbacks = append(callbacks, cb)
|
||||
}
|
||||
}
|
||||
return callbacks
|
||||
}
|
||||
|
||||
func (m *Manager) getAlertForAICallback() func(alert *Alert) {
|
||||
m.callbackMu.RLock()
|
||||
cb := m.onAlertForAI
|
||||
@@ -887,6 +973,22 @@ func (m *Manager) getAlertForAICallback() func(alert *Alert) {
|
||||
return cb
|
||||
}
|
||||
|
||||
func (m *Manager) getAlertForAICallbacks() []func(alert *Alert) {
|
||||
m.callbackMu.RLock()
|
||||
defer m.callbackMu.RUnlock()
|
||||
|
||||
callbacks := make([]func(alert *Alert), 0, len(m.alertForAISubs)+1)
|
||||
if m.onAlertForAI != nil {
|
||||
callbacks = append(callbacks, m.onAlertForAI)
|
||||
}
|
||||
for _, cb := range m.alertForAISubs {
|
||||
if cb != nil {
|
||||
callbacks = append(callbacks, cb)
|
||||
}
|
||||
}
|
||||
return callbacks
|
||||
}
|
||||
|
||||
func (m *Manager) getResolvedCallback() func(alertID string) {
|
||||
m.callbackMu.RLock()
|
||||
cb := m.onResolved
|
||||
@@ -894,6 +996,22 @@ func (m *Manager) getResolvedCallback() func(alertID string) {
|
||||
return cb
|
||||
}
|
||||
|
||||
func (m *Manager) getResolvedCallbacks() []func(alertID string) {
|
||||
m.callbackMu.RLock()
|
||||
defer m.callbackMu.RUnlock()
|
||||
|
||||
callbacks := make([]func(alertID string), 0, len(m.resolvedSubs)+1)
|
||||
if m.onResolved != nil {
|
||||
callbacks = append(callbacks, m.onResolved)
|
||||
}
|
||||
for _, cb := range m.resolvedSubs {
|
||||
if cb != nil {
|
||||
callbacks = append(callbacks, cb)
|
||||
}
|
||||
}
|
||||
return callbacks
|
||||
}
|
||||
|
||||
func (m *Manager) getAcknowledgedCallback() func(alert *Alert, user string) {
|
||||
m.callbackMu.RLock()
|
||||
cb := m.onAcknowledged
|
||||
@@ -919,8 +1037,8 @@ func (m *Manager) getEscalateCallback() func(alert *Alert, level int) {
|
||||
// preserving canonical state as the internal identity and emitting the public
|
||||
// alert ID to external callbacks for compatibility.
|
||||
func (m *Manager) safeCallResolvedAlertCallback(alert *Alert, fallbackID string, async bool) {
|
||||
callback := m.getResolvedCallback()
|
||||
if callback == nil {
|
||||
callbacks := m.getResolvedCallbacks()
|
||||
if len(callbacks) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -937,7 +1055,9 @@ func (m *Manager) safeCallResolvedAlertCallback(alert *Alert, fallbackID string,
|
||||
Msg("Panic in onResolved callback")
|
||||
}
|
||||
}()
|
||||
callback(publicID)
|
||||
for _, callback := range callbacks {
|
||||
callback(publicID)
|
||||
}
|
||||
}
|
||||
|
||||
if async {
|
||||
@@ -1070,8 +1190,8 @@ func (m *Manager) checkFlappingLocked(trackingKey string) bool {
|
||||
}
|
||||
|
||||
func (m *Manager) dispatchAlert(alert *Alert, async bool) bool {
|
||||
callback := m.getAlertCallback()
|
||||
if callback == nil || alert == nil {
|
||||
callbacks := m.getAlertCallbacks()
|
||||
if len(callbacks) == 0 || alert == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1130,7 +1250,7 @@ func (m *Manager) dispatchAlert(alert *Alert, async bool) bool {
|
||||
|
||||
alertCopy := cloneAlertForOutput(alert)
|
||||
if async {
|
||||
go func(a *Alert) {
|
||||
go func(a *Alert, fns []func(*Alert)) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Error().
|
||||
@@ -1140,11 +1260,13 @@ func (m *Manager) dispatchAlert(alert *Alert, async bool) bool {
|
||||
Msg("Panic in onAlert callback")
|
||||
}
|
||||
}()
|
||||
callback(a)
|
||||
}(alertCopy)
|
||||
for _, callback := range fns {
|
||||
callback(a)
|
||||
}
|
||||
}(alertCopy, callbacks)
|
||||
} else {
|
||||
// Synchronous calls also need panic recovery to prevent service crash
|
||||
func() {
|
||||
func(fns []func(*Alert)) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Error().
|
||||
@@ -1154,8 +1276,10 @@ func (m *Manager) dispatchAlert(alert *Alert, async bool) bool {
|
||||
Msg("Panic in onAlert callback (synchronous)")
|
||||
}
|
||||
}()
|
||||
callback(alertCopy)
|
||||
}()
|
||||
for _, callback := range fns {
|
||||
callback(alertCopy)
|
||||
}
|
||||
}(callbacks)
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -4873,16 +4997,18 @@ func (m *Manager) HandleDockerHostOffline(host models.DockerHost) {
|
||||
alert, _ := m.getActiveAlertNoLock(alertID)
|
||||
m.mu.RUnlock()
|
||||
if alert != nil {
|
||||
if alertForAICallback := m.getAlertForAICallback(); alertForAICallback != nil {
|
||||
if callbacks := m.getAlertForAICallbacks(); len(callbacks) > 0 {
|
||||
alertCopy := cloneAlertForOutput(alert)
|
||||
go func(a *Alert) {
|
||||
go func(a *Alert, fns []func(*Alert)) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Error().Interface("panic", r).Str("alertID", a.ID).Msg("panic in AI alert callback")
|
||||
}
|
||||
}()
|
||||
alertForAICallback(a)
|
||||
}(alertCopy)
|
||||
for _, callback := range fns {
|
||||
callback(a)
|
||||
}
|
||||
}(alertCopy, callbacks)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6875,16 +7001,18 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||
Msg("Alert triggered")
|
||||
|
||||
// Trigger AI analysis callback unconditionally (bypasses notification suppression)
|
||||
if alertForAICallback := m.getAlertForAICallback(); alertForAICallback != nil {
|
||||
if callbacks := m.getAlertForAICallbacks(); len(callbacks) > 0 {
|
||||
alertCopy := cloneAlertForOutput(alert)
|
||||
go func(a *Alert) {
|
||||
go func(a *Alert, fns []func(*Alert)) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Error().Interface("panic", r).Str("alertID", a.ID).Msg("panic in AI alert callback")
|
||||
}
|
||||
}()
|
||||
alertForAICallback(a)
|
||||
}(alertCopy)
|
||||
for _, callback := range fns {
|
||||
callback(a)
|
||||
}
|
||||
}(alertCopy, callbacks)
|
||||
}
|
||||
|
||||
// Check rate limit (but don't remove alert from tracking)
|
||||
@@ -6899,7 +7027,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||
}
|
||||
|
||||
// Notify callback (may be suppressed by quiet hours)
|
||||
if m.getAlertCallback() != nil {
|
||||
if len(m.getAlertCallbacks()) > 0 {
|
||||
now := time.Now()
|
||||
alert.LastNotified = &now
|
||||
if m.dispatchAlert(alert, true) {
|
||||
@@ -6967,7 +7095,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||
}
|
||||
|
||||
// Send re-notification if appropriate (may be suppressed by quiet hours)
|
||||
if shouldRenotify && m.getAlertCallback() != nil {
|
||||
if shouldRenotify && len(m.getAlertCallbacks()) > 0 {
|
||||
now := time.Now()
|
||||
existingAlert.LastNotified = &now
|
||||
// Dispatch asynchronously so callback I/O cannot block alert evaluation.
|
||||
|
||||
@@ -213,6 +213,56 @@ func TestCheckMetricInvokesAICallbackWhenNotificationsSuppressed(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatchAlertInvokesSubscribedAlertCallbacks(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
alert := &Alert{
|
||||
ID: "subscribed-alert",
|
||||
Type: "cpu",
|
||||
Level: AlertLevelWarning,
|
||||
ResourceID: "vm-100",
|
||||
ResourceName: "web-1",
|
||||
StartTime: time.Now().Add(-time.Minute),
|
||||
LastSeen: time.Now(),
|
||||
}
|
||||
|
||||
firstDone := make(chan string, 1)
|
||||
secondDone := make(chan string, 1)
|
||||
m.SubscribeAlertCallback(func(alert *Alert) {
|
||||
firstDone <- alert.ID
|
||||
})
|
||||
m.SubscribeAlertCallback(func(alert *Alert) {
|
||||
secondDone <- alert.ID
|
||||
})
|
||||
|
||||
m.mu.Lock()
|
||||
m.config.ActivationState = ActivationActive
|
||||
dispatched := m.dispatchAlert(alert, false)
|
||||
m.mu.Unlock()
|
||||
|
||||
if !dispatched {
|
||||
t.Fatal("expected dispatchAlert to report a subscribed callback dispatch")
|
||||
}
|
||||
|
||||
select {
|
||||
case got := <-firstDone:
|
||||
if got != alert.ID {
|
||||
t.Fatalf("first callback ID = %q, want %q", got, alert.ID)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("expected first subscribed callback to fire")
|
||||
}
|
||||
|
||||
select {
|
||||
case got := <-secondDone:
|
||||
if got != alert.ID {
|
||||
t.Fatalf("second callback ID = %q, want %q", got, alert.ID)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("expected second subscribed callback to fire")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnAlertHistoryRegistersCallback(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
|
||||
|
||||
@@ -4711,6 +4711,7 @@ type PatrolStatusResponse struct {
|
||||
Running bool `json:"running"`
|
||||
Enabled bool `json:"enabled"`
|
||||
LastPatrolAt *time.Time `json:"last_patrol_at,omitempty"`
|
||||
LastActivityAt *time.Time `json:"last_activity_at,omitempty"`
|
||||
NextPatrolAt *time.Time `json:"next_patrol_at,omitempty"`
|
||||
LastDurationMs int64 `json:"last_duration_ms"`
|
||||
ResourcesChecked int `json:"resources_checked"`
|
||||
@@ -4816,6 +4817,7 @@ func (h *AISettingsHandler) HandleGetPatrolStatus(w http.ResponseWriter, r *http
|
||||
Running: status.Running,
|
||||
Enabled: status.Enabled,
|
||||
LastPatrolAt: status.LastPatrolAt,
|
||||
LastActivityAt: status.LastActivityAt,
|
||||
NextPatrolAt: status.NextPatrolAt,
|
||||
LastDurationMs: status.LastDuration.Milliseconds(),
|
||||
ResourcesChecked: status.ResourcesChecked,
|
||||
|
||||
@@ -186,6 +186,36 @@ func TestHandleGetPatrolStatus_DerivesBlockedRuntimeStateForExhaustedQuickstartC
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetPatrolStatus_DistinguishesLastFullPatrolFromLastActivity(t *testing.T) {
|
||||
handler, patrol, _, _ := setupAIHandlerWithPatrol(t)
|
||||
|
||||
lastPatrolAt := time.Date(2026, 3, 12, 9, 30, 0, 0, time.UTC)
|
||||
lastActivityAt := lastPatrolAt.Add(8 * time.Minute)
|
||||
setUnexportedField(t, patrol, "lastFullPatrol", lastPatrolAt)
|
||||
setUnexportedField(t, patrol, "lastActivity", lastActivityAt)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/ai/patrol/status", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetPatrolStatus(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
|
||||
var resp PatrolStatusResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
|
||||
if resp.LastPatrolAt == nil || !resp.LastPatrolAt.Equal(lastPatrolAt) {
|
||||
t.Fatalf("last_patrol_at = %v, want %v", resp.LastPatrolAt, lastPatrolAt)
|
||||
}
|
||||
if resp.LastActivityAt == nil || !resp.LastActivityAt.Equal(lastActivityAt) {
|
||||
t.Fatalf("last_activity_at = %v, want %v", resp.LastActivityAt, lastActivityAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatrolActionHandlers_NoAIService_ReturnStructuredServiceUnavailable(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
cfg := &config.Config{DataPath: tmp}
|
||||
@@ -672,3 +702,22 @@ func TestHandleForcePatrol_ConfigDisabled(t *testing.T) {
|
||||
t.Fatalf("expected success message")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleForcePatrol_CommunityTierIgnoresRecentScopedActivityForFullPatrolRateLimit(t *testing.T) {
|
||||
handler, patrol, _, _ := setupAIHandlerWithPatrol(t)
|
||||
handler.defaultAIService.SetLicenseChecker(communityLicenseChecker{})
|
||||
|
||||
setUnexportedField(t, patrol, "lastActivity", time.Now().Add(-10*time.Minute))
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/ai/patrol/run", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleForcePatrol(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), "patrol_rate_limited") {
|
||||
t.Fatalf("expected community force patrol to ignore scoped-only activity, got %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4364,6 +4364,7 @@ func TestContract_MetricsHistoryLiveFallbackJSONSnapshot(t *testing.T) {
|
||||
|
||||
func TestContract_PatrolStatusResponseJSONSnapshot(t *testing.T) {
|
||||
lastPatrolAt := time.Date(2026, 3, 12, 9, 30, 0, 0, time.UTC)
|
||||
lastActivityAt := lastPatrolAt.Add(5 * time.Minute)
|
||||
nextPatrolAt := lastPatrolAt.Add(6 * time.Hour)
|
||||
blockedAt := lastPatrolAt.Add(15 * time.Minute)
|
||||
|
||||
@@ -4372,6 +4373,7 @@ func TestContract_PatrolStatusResponseJSONSnapshot(t *testing.T) {
|
||||
Running: false,
|
||||
Enabled: true,
|
||||
LastPatrolAt: &lastPatrolAt,
|
||||
LastActivityAt: &lastActivityAt,
|
||||
NextPatrolAt: &nextPatrolAt,
|
||||
LastDurationMs: 12345,
|
||||
ResourcesChecked: 18,
|
||||
@@ -4404,6 +4406,7 @@ func TestContract_PatrolStatusResponseJSONSnapshot(t *testing.T) {
|
||||
"running":false,
|
||||
"enabled":true,
|
||||
"last_patrol_at":"2026-03-12T09:30:00Z",
|
||||
"last_activity_at":"2026-03-12T09:35:00Z",
|
||||
"next_patrol_at":"2026-03-12T15:30:00Z",
|
||||
"last_duration_ms":12345,
|
||||
"resources_checked":18,
|
||||
|
||||
+6
-15
@@ -3125,32 +3125,23 @@ func (r *Router) WireAlertTriggeredAI() {
|
||||
return
|
||||
}
|
||||
|
||||
// 2. Get the Patrol Service (The Watchdog)
|
||||
patrol := aiService.GetPatrolService()
|
||||
if patrol == nil {
|
||||
log.Debug().Msg("Patrol service not available for wiring")
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Get the Monitor (The Trigger)
|
||||
// 2. Get the Monitor (The Trigger)
|
||||
if r.monitor == nil {
|
||||
log.Debug().Msg("Monitor not available for AI alert callback")
|
||||
return
|
||||
}
|
||||
|
||||
// 4. Connect Trigger -> Watchdog
|
||||
// When an alert fires, we immediately trigger the Patrol Agent to investigate
|
||||
// 3. Connect alert-fired events to the dedicated alert-triggered analyzer.
|
||||
// Patrol's event-triggered runs are owned by the canonical alert bridge /
|
||||
// trigger-manager path, so this callback should not enqueue Patrol directly.
|
||||
r.monitor.SetAlertTriggeredAICallback(func(alert *alerts.Alert) {
|
||||
log.Info().Str("alert_identifier", alert.ID).Msg("Alert fired leading to Patrol Trigger")
|
||||
patrol.TriggerPatrolForAlert(alert)
|
||||
|
||||
// We also trigger the specific analyzer if enabled, as it tracks specific stats
|
||||
if analyzer := r.GetAlertTriggeredAnalyzer(); analyzer != nil {
|
||||
log.Info().Str("alert_identifier", alert.ID).Msg("Alert fired leading to alert-triggered analysis")
|
||||
analyzer.OnAlertFired(alert)
|
||||
}
|
||||
})
|
||||
|
||||
log.Info().Msg("Alert-triggered AI Watchdog wired to monitor")
|
||||
log.Info().Msg("Alert-triggered AI analyzer wired to monitor")
|
||||
}
|
||||
|
||||
// Deprecated: deriveResourceTypeFromAlert uses heuristic string matching.
|
||||
|
||||
Reference in New Issue
Block a user