Canonicalize Patrol activity semantics

This commit is contained in:
rcourtman
2026-03-29 13:38:06 +01:00
parent 82b24f5d90
commit 3c0707751b
33 changed files with 774 additions and 122 deletions
@@ -118,6 +118,7 @@ management, and fleet control surfaces.
1. Add or change install-command generation, canonical /api/auto-register behavior, or installer download behavior through the owned `internal/api/` files above.
2. Add or change update continuity and persisted-version handoff through `internal/agentupdate/`.
3. Add or change runtime-side Unified Agent startup, first-report assembly, and enroll/runtime continuity through `internal/hostagent/`.
4. Keep shared `internal/api/` helper edits isolated from agent lifecycle semantics: Patrol-specific status transport or alert-trigger wiring changes in shared handlers must not bleed into auto-register, installer, or fleet-control behavior unless this contract moves in the same slice.
4. Keep legacy Unified Agent compatibility names explicitly secondary when touching shared `internal/api/` runtime helpers: the legacy host-route family and `host-agent:*` scope names may remain as ingress or migration aliases, but they must not retake primary ownership in router state, live runtime scope checks, handler commentary, or operator-facing guidance.
5. Add or change installer flags, persisted service arguments, or upgrade-safe re-entry behavior through `scripts/install.sh` and `scripts/install.ps1`.
6. Add or change profile management, the extracted agent profiles runtime owner, the pure unified-agent inventory/install model, the direct Proxmox workspace shell, route model, reporting summary owner, shared install/inventory/dialog section owners, the split infrastructure install/reporting state owners, the split direct-node/discovery infrastructure settings owners plus their shared model, shared frontend install-command assembly, Proxmox setup/install API transport, setup-completion install handoff transport, deploy-fallback manual install transport, and fleet-control presentation through `frontend-modern/src/api/agentProfiles.ts`, `frontend-modern/src/api/nodes.ts`, `frontend-modern/src/components/Settings/AgentProfilesPanel.tsx`, `frontend-modern/src/components/Settings/useAgentProfilesPanelState.ts`, `frontend-modern/src/components/Settings/InfrastructureOperationsController.tsx`, `frontend-modern/src/components/Settings/infrastructureOperationsModel.tsx`, `frontend-modern/src/components/Settings/InfrastructureInstallPanel.tsx`, `frontend-modern/src/components/Settings/InfrastructureInstallerSection.tsx`, `frontend-modern/src/components/Settings/InfrastructureReportingPanel.tsx`, `frontend-modern/src/components/Settings/InfrastructureInventorySection.tsx`, `frontend-modern/src/components/Settings/InfrastructureActiveRowDetails.tsx`, `frontend-modern/src/components/Settings/InfrastructureIgnoredRowDetails.tsx`, `frontend-modern/src/components/Settings/InfrastructureStopMonitoringDialog.tsx`, `frontend-modern/src/components/Settings/InfrastructureDirectConnectionsSummaryCard.tsx`, `frontend-modern/src/components/Settings/InfrastructureWorkspace.tsx`, `frontend-modern/src/components/Settings/infrastructureWorkspaceModel.ts`, `frontend-modern/src/components/Settings/ProxmoxSettingsPanel.tsx`, `frontend-modern/src/components/Settings/proxmoxSettingsModel.ts`, `frontend-modern/src/components/Settings/ProxmoxDirectWorkspace.tsx`, `frontend-modern/src/components/Settings/ProxmoxConfiguredNodesTable.tsx`, `frontend-modern/src/components/Settings/ProxmoxDirectConnectionsCard.tsx`, `frontend-modern/src/components/Settings/ProxmoxDiscoveryResultsCard.tsx`, `frontend-modern/src/components/Settings/ProxmoxDeleteNodeDialog.tsx`, `frontend-modern/src/components/Settings/ProxmoxNodeModalStack.tsx`, `frontend-modern/src/components/Settings/ConfiguredNodeTables.tsx`, `frontend-modern/src/components/Settings/SettingsSectionNav.tsx`, `frontend-modern/src/components/Settings/infrastructureSettingsModel.ts`, `frontend-modern/src/components/Settings/useInfrastructureConfiguredNodesState.ts`, `frontend-modern/src/components/Settings/useInfrastructureDiscoveryRuntimeState.ts`, `frontend-modern/src/components/Settings/useInfrastructureInstallState.tsx`, `frontend-modern/src/components/Settings/useInfrastructureOperationsState.tsx`, `frontend-modern/src/components/Settings/useInfrastructureReportingState.tsx`, `frontend-modern/src/components/Settings/useInfrastructureSettingsState.ts`, `frontend-modern/src/components/Settings/useProxmoxDirectWorkspaceState.ts`, `frontend-modern/src/components/Settings/NodeModal.tsx`, `frontend-modern/src/components/Settings/nodeModalModel.ts`, `frontend-modern/src/components/Settings/useNodeModalState.ts`, `frontend-modern/src/components/SetupWizard/SetupCompletionPanel.tsx`, `frontend-modern/src/components/Infrastructure/deploy/ResultsStep.tsx`, and `frontend-modern/src/utils/agentInstallCommand.ts`.
@@ -531,6 +531,11 @@ The Patrol status payload must keep that same scope distinction explicit in its
own recency fields. `last_patrol_at` is reserved for the most recent completed
full Patrol run, while scoped runs and fix-verification checks advance
`last_activity_at` without pretending a full verification sweep just happened.
That same runtime boundary also owns which Patrol work counts toward
full-patrol cadence gates. Community-tier or other full-run limits must key
off completed full sweeps only; recent scoped or verification activity may
advance `last_activity_at`, but it must not block a manual full Patrol request
as if a scheduled estate-wide sweep already happened.
The Patrol startup scheduler must preserve that coverage guarantee as well:
`internal/ai/patrol_run.go` may skip the startup full patrol only when recent
run history already includes a successful full Patrol run, not merely because
@@ -87,6 +87,12 @@ The alert webhook editor now mirrors that canonical Pushover field rule through
`frontend-modern/src/utils/alertWebhookPresentation.ts`, so the UI shares the
same alias, preset, and custom-field input mapping instead of carrying its own
local webhook-field normalization fork.
The alert manager callback layer now also has to stay fan-out-safe. Monitor
delivery, the unified alert bridge, and Patrol-adjacent AI listeners must
compose through additive fired/resolved subscriptions instead of overwriting a
single callback slot, and alert-triggered Patrol enqueueing must stay on the
canonical unified alert bridge plus trigger-manager path rather than reviving
duplicate callback-side Patrol shortcuts.
That shared alert presentation boundary now also has explicit alerts ownership.
`frontend-modern/src/utils/alertWebhookPresentation.ts` is the canonical owner
for webhook setup copy, service labels, mention-help phrasing, custom-field
@@ -135,6 +135,8 @@ Own canonical runtime payload shapes between backend and frontend.
and the Patrol findings empty-state behavior, so `0 active findings` only renders as a healthy frontend conclusion when the same governed AI summary contract still reports healthy overall health; degraded or not-fully-verified health predictions must flow through to the Patrol findings surface instead of being replaced by page-local "looks healthy" copy
and the Patrol assessment headline plus compact summary-strip behavior, so the same governed AI summary contract decides whether the page leads with verified health, issues detected, coverage incomplete, or another attention state instead of letting count-only page fragments emit a stale `No issues found` conclusion
and the Patrol verification summary derived from run history, so the page also states whether recent Patrol evidence came from a successful full patrol or only from scoped/erroring runs instead of leaving verification scope implicit
and the Patrol status recency split, so `last_patrol_at` remains reserved for completed full Patrol sweeps while scoped runs and verification checks advance `last_activity_at` without claiming a fresh full-estate verification pass
and the canonical alert-triggered Patrol enqueue path in `internal/api/router.go`, so alert-fired Patrol work flows through the unified alert bridge and trigger manager instead of being duplicated by monitor callback wiring
and the shared `frontend-modern/src/components/Infrastructure/ResourceChangeSummary.tsx` card, so canonical recent-change timelines stay rendered through one governed frontend card instead of separate page-local list loops
and the shared `frontend-modern/src/utils/resourceChangePresentation.ts` formatter used by the summary page and resource drawer, so canonical change wording does not drift across surfaces
and the `/api/ai/intelligence/changes` route plus `internal/api/contract_test.go`, so the canonical recent-changes endpoint stays on the same intelligence facade and contract snapshot instead of bypassing the shared timeline source
@@ -1066,6 +1066,10 @@ That same summary shell should also surface verification scope from the
owning run-history contract. Operators should be able to see, inside the same
summary surface, whether Patrol recently completed a full verification pass or
whether recent activity was limited to scoped/erroring patrol runs.
That same shell rule also owns Patrol recency labels. Shared Patrol header and
status-shell surfaces must keep `Last full patrol` tied only to the full-sweep
transport fact and use `Last activity` for scoped or verification work instead
of collapsing both timestamps back into a generic `Last run` label.
Shared primitive consumers that split status-dot tone and status-text tone
must now keep both values routed through the same exported presentation helper.
@@ -201,6 +201,9 @@ The summary recency chip must follow the same governed scope distinction. When
the latest completed activity was only a scoped run, the summary should label
that timestamp as `Last activity` instead of `Last patrol`; `Last full patrol`
belongs only to the most recent completed full Patrol run.
That same distinction is transport-backed. `last_patrol_at` names the last
completed full Patrol sweep, while `last_activity_at` may advance on scoped
work or fix-verification checks without claiming a new full verification pass.
That same recency contract also applies to the header metadata row. The top
header must not revert to a generic `Last:` timestamp when the rest of Patrol
is explicitly distinguishing activity from full verification recency.
@@ -224,6 +227,8 @@ operator whether Patrol recently completed a successful full patrol, only ran
scoped alert-triggered checks, or ended its most recent full patrol with
errors, so the page does not leave trust and coverage as implicit background
knowledge.
Fix-verification checks belong to that same explanation layer as targeted
activity, not as evidence of a fresh full-estate sweep.
The same hierarchy applies to investigation context. Correlations, recent
changes, and policy posture are secondary evidence for deeper investigation, so
the `Investigation context` section belongs beneath the primary findings/history
@@ -78,6 +78,7 @@ querying, and the operator-facing storage health presentation layer.
17. Preserve backend-owned Pulse Mobile relay runtime credential minting in those same shared `internal/api/` auth/security helpers so storage- and recovery-adjacent transport surfaces do not inherit browser-authored wildcard token bundles when they depend on the canonical security helper layer.
18. Preserve the dedicated backend-owned `relay:mobile:access` capability and its governed backward-compatible route inventory plus the shared helper call sites around it, so storage- and recovery-adjacent transport surfaces do not treat the mobile relay credential as a general AI scope bundle.
19. Preserve shipped local security-doc guidance in shared `internal/api/` config/setup helpers so storage- and recovery-adjacent transport surfaces do not reintroduce GitHub `main` security links when the running build already serves its own local security documentation route.
20. Keep shared `internal/api/` Patrol transport and alert-trigger edits feature-isolated: Patrol-specific recency fields, callback fan-out, or alert-bridge wiring changes must not leak into recovery queries, storage links, or recovery-adjacent install/setup flows unless this contract changes in the same slice.
## Forbidden Paths
@@ -62,17 +62,35 @@ describe('patrol api', () => {
it('preserves the canonical patrol runtime state payload', async () => {
apiFetchJSONMock.mockResolvedValueOnce({
runtime_state: 'blocked',
blocked_reason: 'Quickstart credits exhausted. Connect your API key to continue using AI Patrol.',
blocked_reason:
'Quickstart credits exhausted. Connect your API key to continue using AI Patrol.',
healthy: false,
} as any);
await expect(getPatrolStatus()).resolves.toMatchObject({
runtime_state: 'blocked',
blocked_reason: 'Quickstart credits exhausted. Connect your API key to continue using AI Patrol.',
blocked_reason:
'Quickstart credits exhausted. Connect your API key to continue using AI Patrol.',
healthy: false,
});
});
it('preserves split patrol recency transport fields', async () => {
apiFetchJSONMock.mockResolvedValueOnce({
runtime_state: 'active',
healthy: true,
last_patrol_at: '2026-03-12T09:30:00Z',
last_activity_at: '2026-03-12T09:59:00Z',
} as any);
await expect(getPatrolStatus()).resolves.toMatchObject({
runtime_state: 'active',
healthy: true,
last_patrol_at: '2026-03-12T09:30:00Z',
last_activity_at: '2026-03-12T09:59:00Z',
});
});
it('normalizes patrol run alert identifiers', async () => {
apiFetchJSONMock.mockResolvedValueOnce([
{
+1
View File
@@ -145,6 +145,7 @@ export interface PatrolStatus {
running: boolean;
enabled: boolean;
last_patrol_at?: string;
last_activity_at?: string;
next_patrol_at?: string;
last_duration_ms: number;
resources_checked: number;
@@ -16,7 +16,6 @@ import {
} from '@/utils/patrolRunPresentation';
import { getPatrolRuntimePresentation } from '@/utils/patrolRuntimePresentation';
import ActivityIcon from 'lucide-solid/icons/activity';
import CheckCircleIcon from 'lucide-solid/icons/check-circle';
import AlertCircleIcon from 'lucide-solid/icons/alert-circle';
import AlertTriangleIcon from 'lucide-solid/icons/alert-triangle';
@@ -27,6 +26,13 @@ interface PatrolStatusBarProps {
blockedReason?: string;
}
function normalizePatrolRunType(type: string | undefined): string {
return String(type || '')
.trim()
.toLowerCase()
.replace(/\s+/g, '_');
}
export const PatrolStatusBar: Component<PatrolStatusBarProps> = (props) => {
const [runs] = createResource(
() => props.refreshTrigger ?? 0,
@@ -48,7 +54,8 @@ export const PatrolStatusBar: Component<PatrolStatusBarProps> = (props) => {
const todayRuns = allRuns.filter((r) => new Date(r.started_at) >= todayStart);
const lastRun = allRuns[0];
const lastRunTime = lastRun ? new Date(lastRun.started_at) : null;
const lastRunTime = lastRun ? new Date(lastRun.completed_at || lastRun.started_at) : null;
const lastRunType = normalizePatrolRunType(lastRun?.type);
const lastRunStatus = getPatrolRunStatusPresentation(
lastRun?.status ?? 'unknown',
lastRun?.error_count ?? 0,
@@ -59,6 +66,10 @@ export const PatrolStatusBar: Component<PatrolStatusBarProps> = (props) => {
runsToday: todayRuns.length,
newFindingsToday: todayRuns.reduce((sum, r) => sum + (r.new_findings || 0), 0),
lastRunTime: lastRunTime ? formatRelativeTime(lastRunTime, { compact: true }) : null,
lastRunTimeLabel:
lastRunType === '' || lastRunType === 'full' || lastRunType === 'patrol'
? 'Last full patrol'
: 'Last activity',
lastRunTrigger: formatTriggerReason(lastRun?.trigger_reason),
lastRunTypeLabel: getPatrolRunKindLabel(lastRun?.type),
lastRunStatus,
@@ -73,9 +84,7 @@ export const PatrolStatusBar: Component<PatrolStatusBarProps> = (props) => {
const showRuntimeState = createMemo(() => {
const runtimeState = props.runtimeState;
return (
runtimeState === 'blocked' ||
runtimeState === 'disabled' ||
runtimeState === 'unavailable'
runtimeState === 'blocked' || runtimeState === 'disabled' || runtimeState === 'unavailable'
);
});
const showRunInProgress = createMemo(() => props.runtimeState === 'running');
@@ -153,7 +162,7 @@ export const PatrolStatusBar: Component<PatrolStatusBarProps> = (props) => {
{/* Last run */}
<Show when={s().lastRunTime}>
<span class="text-xs text-muted">
Last run: {s().lastRunTime}
{s().lastRunTimeLabel}: {s().lastRunTime}
<Show when={s().lastRunTrigger}>
<span class=" "> ({s().lastRunTrigger})</span>
</Show>
@@ -32,6 +32,7 @@ export function PatrolIntelligenceHeader(props: { state: PatrolIntelligenceState
getPatrolRecencyPresentation({
runs: state.patrolRunHistory() ?? [],
lastPatrolAt: state.patrolStatus()?.last_patrol_at,
lastActivityAt: state.patrolStatus()?.last_activity_at,
}),
);
const showQuickstartStatus = createMemo(() => {
@@ -59,10 +59,7 @@ export function PatrolIntelligenceSummary(props: { state: PatrolIntelligenceStat
getSemanticTonePresentation(assessment().tone),
);
const activeFindingsSummaryPresentation = createMemo(() =>
getPatrolSummaryPresentation(
metricState().primarySeverity,
metricState().primaryValue > 0,
),
getPatrolSummaryPresentation(metricState().primarySeverity, metricState().primaryValue > 0),
);
const verification = createMemo(() =>
getPatrolVerificationPresentation({
@@ -75,6 +72,7 @@ export function PatrolIntelligenceSummary(props: { state: PatrolIntelligenceStat
getPatrolRecencyPresentation({
runs: state.patrolRunHistory() ?? [],
lastPatrolAt: state.patrolStatus()?.last_patrol_at,
lastActivityAt: state.patrolStatus()?.last_activity_at,
}),
);
const fixedSummaryPresentation = createMemo(() =>
@@ -156,7 +154,9 @@ export function PatrolIntelligenceSummary(props: { state: PatrolIntelligenceStat
</Show>
}
>
<CheckCircleIcon class={`w-5 h-5 ${assessmentTonePresentation().iconClass}`} />
<CheckCircleIcon
class={`w-5 h-5 ${assessmentTonePresentation().iconClass}`}
/>
</Show>
</div>
@@ -610,7 +610,9 @@ describe('AIIntelligence entitlement gating', () => {
const findingsPanel = screen.getByTestId('findings-panel');
const contextHeading = screen.getByText('Investigation context');
expect(
Boolean(findingsPanel.compareDocumentPosition(contextHeading) & Node.DOCUMENT_POSITION_FOLLOWING),
Boolean(
findingsPanel.compareDocumentPosition(contextHeading) & Node.DOCUMENT_POSITION_FOLLOWING,
),
).toBe(true);
expect(screen.getByText(/Health A · 91\/100/)).toBeInTheDocument();
@@ -862,6 +864,54 @@ describe('AIIntelligence entitlement gating', () => {
expect(findingsPanelState.latestProps?.patrolIntervalMs).toBeUndefined();
});
it('prefers last activity transport over last full patrol transport when no run history is loaded', async () => {
hasFeatureMock.mockReturnValue(true);
licenseStatusMock.mockReturnValue({ subscription_state: 'active' });
getPatrolStatusMock.mockResolvedValue(
defaultPatrolStatus({
runtime_state: 'active',
last_patrol_at: '2026-03-12T09:30:00Z',
last_activity_at: '2026-03-12T09:59:00Z',
}),
);
getPatrolRunHistoryMock.mockResolvedValue([]);
intelligenceState.summary = {
timestamp: '2026-03-12T10:00:00Z',
overall_health: {
score: 100,
grade: 'A',
trend: 'stable',
factors: [],
prediction: 'Infrastructure is healthy with no significant issues detected.',
},
findings_count: {
critical: 0,
warning: 0,
watch: 0,
info: 0,
total: 0,
},
predictions_count: 0,
recent_changes_count: 0,
learning: {
resources_with_knowledge: 0,
total_notes: 0,
resources_with_baselines: 0,
patterns_detected: 0,
correlations_learned: 0,
incidents_tracked: 0,
},
};
render(() => <AIIntelligence />);
await waitFor(() => {
expect(screen.getByText(/Last activity:/i)).toBeInTheDocument();
});
expect(screen.queryByText(/Last full patrol:/i)).not.toBeInTheDocument();
});
it('describes both findings and incomplete coverage when active issues exist', async () => {
hasFeatureMock.mockReturnValue(true);
licenseStatusMock.mockReturnValue({ subscription_state: 'active' });
@@ -83,14 +83,14 @@ describe('patrolRunPresentation', () => {
it('returns canonical patrol run kind labels', () => {
expect(getPatrolRunKindLabel('scoped')).toBe('Scoped run');
expect(getPatrolRunKindLabel('verification')).toBe('Verification check');
expect(getPatrolRunKindLabel('patrol')).toBe('Full patrol');
expect(getPatrolRunKindLabel('')).toBe('Full patrol');
expect(getPatrolRunKindLabel('unexpected')).toBe('Patrol run');
});
it('expresses scoped run coverage against the effective scope', () => {
expect(getPatrolRunCoverageSummary(scopedCoverageRun)).toBe(
'Checked 1 of 2 scoped resources',
);
expect(getPatrolRunCoverageSummary(scopedCoverageRun)).toBe('Checked 1 of 2 scoped resources');
expect(getPatrolRunResourcesHeading(scopedCoverageRun)).toBe(
'Resources checked (1 of 2 scoped)',
);
@@ -118,20 +118,13 @@ describe('patrolRunPresentation', () => {
});
it('warns when visible runs include legacy findings snapshots', () => {
expect(
getRunHistorySelectionHint([
{ finding_ids: [] },
{ finding_ids: undefined },
]),
).toBe(
expect(getRunHistorySelectionHint([{ finding_ids: [] }, { finding_ids: undefined }])).toBe(
'Select a run to filter findings when available. Some older runs do not include findings snapshots.',
);
});
it('explains selected legacy runs in the run-history shell', () => {
expect(
getRunHistorySelectionHint([{ finding_ids: [] }], { finding_ids: undefined }),
).toBe(
expect(getRunHistorySelectionHint([{ finding_ids: [] }], { finding_ids: undefined })).toBe(
'Selected run predates findings snapshots; run-scoped findings cannot be fully verified.',
);
});
@@ -387,6 +387,49 @@ describe('getPatrolSummaryPresentation', () => {
});
});
it('reports limited verification when only verification checks are recent', () => {
expect(
getPatrolVerificationPresentation({
runs: [
{
id: 'run-1',
started_at: '2026-03-12T09:58:00Z',
completed_at: '2026-03-12T09:59:00Z',
duration_ms: 60000,
type: 'verification',
trigger_reason: 'verification',
resources_checked: 1,
nodes_checked: 1,
guests_checked: 0,
docker_checked: 0,
storage_checked: 0,
hosts_checked: 0,
pbs_checked: 0,
pmg_checked: 0,
kubernetes_checked: 0,
new_findings: 0,
existing_findings: 0,
rejected_findings: 0,
resolved_findings: 1,
auto_fix_count: 0,
findings_summary: 'Verification: issue resolved',
finding_ids: ['finding-1'],
error_count: 0,
status: 'healthy',
triage_flags: 0,
tool_call_count: 0,
},
] as never,
}),
).toEqual({
title: 'No recent full patrol',
description:
'Recent activity was limited to verification checks over 1 resource, so Patrol has not recently re-verified your full infrastructure.',
compactLabel: 'Partial verification',
tone: 'warning',
});
});
it('labels scoped recency as activity rather than patrol', () => {
expect(
getPatrolRecencyPresentation({
@@ -465,4 +508,16 @@ describe('getPatrolSummaryPresentation', () => {
timestamp: '2026-03-12T09:57:00Z',
});
});
it('prefers explicit last activity transport over last full patrol transport when no run history is loaded', () => {
expect(
getPatrolRecencyPresentation({
lastPatrolAt: '2026-03-12T09:57:00Z',
lastActivityAt: '2026-03-12T09:59:00Z',
}),
).toEqual({
label: 'Last activity',
timestamp: '2026-03-12T09:59:00Z',
});
});
});
@@ -94,10 +94,26 @@ export function getToolCallResultTextClass(success: boolean): string {
}
export function getPatrolRunKindLabel(type: string | undefined): string {
return normalizePatrolRunType(type) === 'scoped' ? 'Scoped run' : 'Full patrol';
switch (normalizePatrolRunType(type)) {
case 'scoped':
return 'Scoped run';
case 'verification':
return 'Verification check';
case '':
case 'full':
case 'patrol':
return 'Full patrol';
default:
return 'Patrol run';
}
}
export function getPatrolRunCoverageSummary(run: Pick<PatrolRunRecord, 'resources_checked' | 'scope_resource_ids' | 'effective_scope_resource_ids'>): string {
export function getPatrolRunCoverageSummary(
run: Pick<
PatrolRunRecord,
'resources_checked' | 'scope_resource_ids' | 'effective_scope_resource_ids'
>,
): string {
const resourcesChecked = Math.max(0, run.resources_checked || 0);
const scopedResourceCount = getCanonicalScopeResourceIds(run)?.length ?? 0;
@@ -117,7 +133,12 @@ export function getPatrolRunCoverageSummary(run: Pick<PatrolRunRecord, 'resource
return '';
}
export function getPatrolRunResourcesHeading(run: Pick<PatrolRunRecord, 'resources_checked' | 'scope_resource_ids' | 'effective_scope_resource_ids'>): string {
export function getPatrolRunResourcesHeading(
run: Pick<
PatrolRunRecord,
'resources_checked' | 'scope_resource_ids' | 'effective_scope_resource_ids'
>,
): string {
const resourcesChecked = Math.max(0, run.resources_checked || 0);
const scopedResourceCount = getCanonicalScopeResourceIds(run)?.length ?? 0;
@@ -300,7 +300,16 @@ function normalizeRunType(type: string | undefined): string {
}
function isFullPatrolRun(run: PatrolRunRecord): boolean {
return normalizeRunType(run.type) !== 'scoped';
const normalized = normalizeRunType(run.type);
return normalized === '' || normalized === 'full' || normalized === 'patrol';
}
function isScopedPatrolRun(run: PatrolRunRecord): boolean {
return normalizeRunType(run.type) === 'scoped';
}
function isVerificationPatrolRun(run: PatrolRunRecord): boolean {
return normalizeRunType(run.type) === 'verification';
}
function isCompletedPatrolRun(run: PatrolRunRecord): boolean {
@@ -308,7 +317,12 @@ function isCompletedPatrolRun(run: PatrolRunRecord): boolean {
}
function hasRunErrors(run: PatrolRunRecord): boolean {
return run.error_count > 0 || String(run.status || '').trim().toLowerCase() === 'error';
return (
run.error_count > 0 ||
String(run.status || '')
.trim()
.toLowerCase() === 'error'
);
}
export function getPatrolAssessmentPresentation(args: {
@@ -390,8 +404,7 @@ export function getPatrolAssessmentPresentation(args: {
return {
title: 'Health requires attention',
description:
args.overallHealth.prediction?.trim() ||
'Patrol assessment still needs attention.',
args.overallHealth.prediction?.trim() || 'Patrol assessment still needs attention.',
eyebrow: 'Patrol assessment',
compactLabel: 'Health requires attention',
tone: getHealthSummaryTone(args.overallHealth),
@@ -458,15 +471,29 @@ export function getPatrolVerificationPresentation(args: {
};
}
const recentScopedRun = completedRuns.find((run) => !isFullPatrolRun(run));
if (recentScopedRun) {
const resourcesChecked = recentScopedRun.resources_checked || 0;
const recentLimitedRun = completedRuns.find((run) => !isFullPatrolRun(run));
if (recentLimitedRun) {
const resourcesChecked = recentLimitedRun.resources_checked || 0;
let description =
'Recent activity was limited to targeted Patrol checks, so Patrol has not recently re-verified your full infrastructure.';
if (isVerificationPatrolRun(recentLimitedRun)) {
description =
resourcesChecked > 0
? `Recent activity was limited to verification checks over ${resourcesChecked} resource${resourcesChecked === 1 ? '' : 's'}, so Patrol has not recently re-verified your full infrastructure.`
: 'Recent activity was limited to verification checks, so Patrol has not recently re-verified your full infrastructure.';
} else if (isScopedPatrolRun(recentLimitedRun)) {
description =
resourcesChecked > 0
? `Recent activity was limited to scoped ${recentLimitedRun.trigger_reason ? String(recentLimitedRun.trigger_reason).replace(/_/g, ' ') : 'patrol'} runs over ${resourcesChecked} resource${resourcesChecked === 1 ? '' : 's'}, so Patrol has not recently re-verified your full infrastructure.`
: 'Recent activity was limited to scoped patrol runs, so Patrol has not recently re-verified your full infrastructure.';
} else if (resourcesChecked > 0) {
description = `Recent activity was limited to targeted Patrol checks over ${resourcesChecked} resource${resourcesChecked === 1 ? '' : 's'}, so Patrol has not recently re-verified your full infrastructure.`;
}
return {
title: 'No recent full patrol',
description:
resourcesChecked > 0
? `Recent activity was limited to scoped ${recentScopedRun.trigger_reason ? String(recentScopedRun.trigger_reason).replace(/_/g, ' ') : 'patrol'} runs over ${resourcesChecked} resource${resourcesChecked === 1 ? '' : 's'}, so Patrol has not recently re-verified your full infrastructure.`
: 'Recent activity was limited to scoped patrol runs, so Patrol has not recently re-verified your full infrastructure.',
description,
compactLabel: 'Partial verification',
tone: 'warning',
};
@@ -483,6 +510,7 @@ export function getPatrolVerificationPresentation(args: {
export function getPatrolRecencyPresentation(args: {
runs?: PatrolRunRecord[];
lastPatrolAt?: string;
lastActivityAt?: string;
}): PatrolRecencyPresentation {
const latestCompletedRun = (args.runs ?? []).find((run) => isCompletedPatrolRun(run));
if (latestCompletedRun?.completed_at) {
@@ -492,10 +520,41 @@ export function getPatrolRecencyPresentation(args: {
};
}
if (args.lastPatrolAt?.trim()) {
const lastPatrolAt = args.lastPatrolAt?.trim();
const lastActivityAt = args.lastActivityAt?.trim();
if (lastActivityAt && lastPatrolAt) {
const activityMs = Date.parse(lastActivityAt);
const patrolMs = Date.parse(lastPatrolAt);
if (Number.isNaN(activityMs) && !Number.isNaN(patrolMs)) {
return {
label: 'Last full patrol',
timestamp: lastPatrolAt,
};
}
if (!Number.isNaN(activityMs) && !Number.isNaN(patrolMs) && patrolMs >= activityMs) {
return {
label: 'Last full patrol',
timestamp: lastPatrolAt,
};
}
return {
label: 'Last activity',
timestamp: args.lastPatrolAt,
timestamp: lastActivityAt,
};
}
if (lastActivityAt) {
return {
label: 'Last activity',
timestamp: lastActivityAt,
};
}
if (lastPatrolAt) {
return {
label: 'Last full patrol',
timestamp: lastPatrolAt,
};
}
+59 -10
View File
@@ -1074,22 +1074,23 @@ func summarizeRecentPatrolCoverage(
var recentErrors int
var hasSuccessfulFullRun bool
var hasRecentFullRun bool
var scopedRuns int
var limitedActivityRuns int
for _, run := range relevant {
if run.ErrorCount > 0 || strings.EqualFold(strings.TrimSpace(run.Status), "error") {
recentErrors++
}
if !isScopedPatrolRun(run) {
if isFullPatrolRun(run) {
hasRecentFullRun = true
} else {
limitedActivityRuns++
}
if isSuccessfulFullPatrolRun(run) {
hasSuccessfulFullRun = true
}
if isScopedPatrolRun(run) {
scopedRuns++
}
}
limitedActivityLabel := describeLimitedPatrolActivity(relevant)
switch {
case !hasSuccessfulFullRun && hasRecentFullRun && recentErrors > 0:
return patrolCoverageFactor{
@@ -1100,13 +1101,13 @@ func summarizeRecentPatrolCoverage(
case !hasSuccessfulFullRun && recentErrors > 0:
return patrolCoverageFactor{
name: "Patrol coverage incomplete",
description: "Patrol coverage is incomplete: recent activity was limited to scoped runs and ended with errors, so overall health is not fully verified.",
description: fmt.Sprintf("Patrol coverage is incomplete: recent activity was limited to %s and ended with errors, so overall health is not fully verified.", limitedActivityLabel),
impact: 35,
}, true
case !hasSuccessfulFullRun && scopedRuns == len(relevant):
case !hasSuccessfulFullRun && limitedActivityRuns == len(relevant):
return patrolCoverageFactor{
name: "Patrol coverage incomplete",
description: "Patrol coverage is incomplete: recent activity was limited to scoped runs, so overall infrastructure health is not fully verified.",
description: fmt.Sprintf("Patrol coverage is incomplete: recent activity was limited to %s, so overall infrastructure health is not fully verified.", limitedActivityLabel),
impact: 20,
}, true
case recentErrors > 0:
@@ -1120,12 +1121,60 @@ func summarizeRecentPatrolCoverage(
}
}
func normalizePatrolRunType(run PatrolRunRecord) string {
return strings.ToLower(strings.TrimSpace(run.Type))
}
func isFullPatrolRun(run PatrolRunRecord) bool {
switch normalizePatrolRunType(run) {
case "", "full", "patrol":
return true
default:
return false
}
}
func isScopedPatrolRun(run PatrolRunRecord) bool {
return strings.EqualFold(strings.TrimSpace(run.Type), "scoped")
return normalizePatrolRunType(run) == "scoped"
}
func isVerificationPatrolRun(run PatrolRunRecord) bool {
return normalizePatrolRunType(run) == "verification"
}
func describeLimitedPatrolActivity(runs []PatrolRunRecord) string {
hasScoped := false
hasVerification := false
hasOther := false
for _, run := range runs {
if isFullPatrolRun(run) {
continue
}
switch {
case isScopedPatrolRun(run):
hasScoped = true
case isVerificationPatrolRun(run):
hasVerification = true
default:
hasOther = true
}
}
switch {
case hasScoped && !hasVerification && !hasOther:
return "scoped runs"
case hasVerification && !hasScoped && !hasOther:
return "verification checks"
case hasScoped && hasVerification && !hasOther:
return "scoped runs and verification checks"
default:
return "targeted Patrol activity"
}
}
func isSuccessfulFullPatrolRun(run PatrolRunRecord) bool {
return !isScopedPatrolRun(run) &&
return isFullPatrolRun(run) &&
run.ErrorCount == 0 &&
!strings.EqualFold(strings.TrimSpace(run.Status), "error")
}
+28
View File
@@ -199,6 +199,34 @@ func TestIntelligence_GetSummary_DegradesWhenRecentPatrolCoverageIsScopedAndErro
}
}
func TestIntelligence_GetSummary_DegradesWhenRecentPatrolCoverageIsVerificationOnly(t *testing.T) {
intel := NewIntelligence(IntelligenceConfig{})
runHistory := NewPatrolRunHistoryStore(10)
now := time.Now()
runHistory.Add(PatrolRunRecord{
ID: "verification-1",
Type: "verification",
TriggerReason: "verification",
CompletedAt: now.Add(-2 * time.Minute),
ErrorCount: 0,
Status: "healthy",
ResourcesChecked: 1,
})
intel.SetRunHistoryStore(runHistory)
summary := intel.GetSummary()
if summary.OverallHealth.Score >= 100 {
t.Fatalf("expected reduced health score, got %f", summary.OverallHealth.Score)
}
if summary.OverallHealth.Grade == HealthGradeA {
t.Fatalf("expected non-A grade, got %s", summary.OverallHealth.Grade)
}
want := "Patrol coverage is incomplete: recent activity was limited to verification checks, so overall infrastructure health is not fully verified."
if summary.OverallHealth.Prediction != want {
t.Fatalf("expected verification-only coverage warning, got %q", summary.OverallHealth.Prediction)
}
}
func TestIntelligence_GetSummary_DegradesWhenRecentFullPatrolErrored(t *testing.T) {
intel := NewIntelligence(IntelligenceConfig{})
runHistory := NewPatrolRunHistoryStore(10)
+4 -2
View File
@@ -94,7 +94,8 @@ type PatrolStatus struct {
RuntimeState PatrolRuntimeState `json:"runtime_state"`
Running bool `json:"running"`
Enabled bool `json:"enabled"`
LastPatrolAt *time.Time `json:"last_patrol_at,omitempty"`
LastPatrolAt *time.Time `json:"last_patrol_at,omitempty"` // Last completed full patrol
LastActivityAt *time.Time `json:"last_activity_at,omitempty"` // Last completed Patrol activity of any kind
NextPatrolAt *time.Time `json:"next_patrol_at,omitempty"`
LastDuration time.Duration `json:"last_duration_ms"`
ResourcesChecked int `json:"resources_checked"`
@@ -443,7 +444,8 @@ type PatrolService struct {
runStartedAt time.Time
stopCh chan struct{}
configChanged chan struct{} // Signal when config changes to reset ticker
lastPatrol time.Time
lastFullPatrol time.Time
lastActivity time.Time
lastDuration time.Duration
resourcesChecked int
errorCount int
+34 -14
View File
@@ -1341,25 +1341,45 @@ func (p *PatrolService) VerifyFixResolved(ctx context.Context, resourceID, resou
}
verifyRecord := PatrolRunRecord{
ID: fmt.Sprintf("%d", startTime.UnixNano()),
StartedAt: startTime,
CompletedAt: endTime,
Duration: duration,
DurationMs: duration.Milliseconds(),
Type: "verification",
TriggerReason: string(TriggerReasonVerification),
ScopeResourceIDs: []string{resourceID},
ScopeResourceTypes: []string{resourceType},
ScopeContext: fmt.Sprintf("Verifying fix for finding: %s", findingID),
FindingID: findingID,
NewFindings: 0,
FindingsSummary: summary,
Status: status,
ID: fmt.Sprintf("%d", startTime.UnixNano()),
StartedAt: startTime,
CompletedAt: endTime,
Duration: duration,
DurationMs: duration.Milliseconds(),
Type: "verification",
TriggerReason: string(TriggerReasonVerification),
ScopeResourceIDs: []string{resourceID},
EffectiveScopeResourceIDs: []string{resourceID},
ScopeResourceTypes: []string{resourceType},
ScopeContext: fmt.Sprintf("Verifying fix for finding: %s", findingID),
FindingID: findingID,
ResourcesChecked: 1,
NewFindings: 0,
FindingsSummary: summary,
Status: status,
}
if strings.TrimSpace(resourceID) == "" {
verifyRecord.ScopeResourceIDs = nil
verifyRecord.EffectiveScopeResourceIDs = nil
verifyRecord.ResourcesChecked = 0
}
if strings.TrimSpace(resourceType) == "" {
verifyRecord.ScopeResourceTypes = nil
}
if verifyErr != nil {
verifyRecord.ErrorCount = 1
}
if p.runHistoryStore != nil {
p.runHistoryStore.Add(verifyRecord)
}
p.mu.Lock()
p.lastActivity = endTime
p.lastDuration = duration
p.resourcesChecked = verifyRecord.ResourcesChecked
p.errorCount = verifyRecord.ErrorCount
p.mu.Unlock()
return verified, verifyErr
}
@@ -268,6 +268,66 @@ func TestVerifyFixResolved_UsesReadStateWithoutSnapshotProvider(t *testing.T) {
}
}
func TestVerifyFixResolved_RecordsVerificationAsActivityWithoutReplacingLastFullPatrol(t *testing.T) {
ps := NewPatrolService(nil, nil)
ps.thresholds = PatrolThresholds{NodeCPUWarning: 90}
nodeView := unifiedresources.NewNodeView(&unifiedresources.Resource{
ID: "node-1",
Name: "node-1",
Type: unifiedresources.ResourceTypeAgent,
Status: unifiedresources.StatusOnline,
Proxmox: &unifiedresources.ProxmoxData{
NodeName: "node-1",
},
Metrics: &unifiedresources.ResourceMetrics{
CPU: &unifiedresources.MetricValue{Percent: 20},
},
})
ps.SetReadState(&mockReadState{nodes: []*unifiedresources.NodeView{&nodeView}})
lastFullPatrol := time.Now().Add(-1 * time.Hour).UTC().Truncate(time.Second)
ps.mu.Lock()
ps.lastFullPatrol = lastFullPatrol
ps.mu.Unlock()
verified, err := ps.VerifyFixResolved(context.Background(), "node-1", "node", "cpu-high", "finding-1")
if err != nil {
t.Fatalf("expected verification to succeed, got %v", err)
}
if !verified {
t.Fatal("expected verification to resolve the issue")
}
status := ps.GetStatus()
if status.LastPatrolAt == nil || !status.LastPatrolAt.Equal(lastFullPatrol) {
t.Fatalf("last full patrol = %v, want %v", status.LastPatrolAt, lastFullPatrol)
}
if status.LastActivityAt == nil {
t.Fatal("expected verification run to update last activity")
}
if !status.LastActivityAt.After(lastFullPatrol) {
t.Fatalf("expected last activity %v to be after last full patrol %v", *status.LastActivityAt, lastFullPatrol)
}
if status.LastDuration <= 0 {
t.Fatalf("expected verification run to update last duration, got %v", status.LastDuration)
}
if status.ResourcesChecked != 1 {
t.Fatalf("resources checked = %d, want 1", status.ResourcesChecked)
}
runs := ps.GetRunHistory(1)
if len(runs) != 1 {
t.Fatalf("expected one verification run, got %d", len(runs))
}
if runs[0].Type != "verification" {
t.Fatalf("run type = %q, want verification", runs[0].Type)
}
if runs[0].ResourcesChecked != 1 {
t.Fatalf("verification run resources_checked = %d, want 1", runs[0].ResourcesChecked)
}
}
func TestVerifyFixResolved_WithoutRuntimeStateFailsClosed(t *testing.T) {
ps := NewPatrolService(nil, nil)
+31 -8
View File
@@ -101,11 +101,13 @@ func (p *PatrolService) Stop() {
// patrolLoop is the main background loop
func (p *PatrolService) patrolLoop(ctx context.Context) {
// Seed lastPatrol from persisted run history so the API can return
// last_patrol_at immediately (before the first in-process patrol completes).
if history := p.GetRunHistory(1); len(history) > 0 && !history[0].CompletedAt.IsZero() {
// Seed recency from persisted run history so the API can return Patrol timing
// metadata immediately (before the first in-process patrol completes).
if history := p.GetRunHistory(10); len(history) > 0 {
lastActivity, lastFullPatrol := patrolRecencyFromHistory(history)
p.mu.Lock()
p.lastPatrol = history[0].CompletedAt
p.lastActivity = lastActivity
p.lastFullPatrol = lastFullPatrol
p.mu.Unlock()
}
@@ -216,6 +218,23 @@ func shouldSkipInitialFullPatrol(runHistory []PatrolRunRecord, now time.Time) bo
return false
}
func patrolRecencyFromHistory(runHistory []PatrolRunRecord) (time.Time, time.Time) {
var lastActivity time.Time
var lastFullPatrol time.Time
for _, run := range runHistory {
if run.CompletedAt.IsZero() {
continue
}
if lastActivity.IsZero() || run.CompletedAt.After(lastActivity) {
lastActivity = run.CompletedAt
}
if isFullPatrolRun(run) && (lastFullPatrol.IsZero() || run.CompletedAt.After(lastFullPatrol)) {
lastFullPatrol = run.CompletedAt
}
}
return lastActivity, lastFullPatrol
}
// runPatrol executes a scheduled patrol run
func (p *PatrolService) runPatrol(ctx context.Context) {
p.runPatrolWithTrigger(ctx, TriggerReasonScheduled, nil)
@@ -600,7 +619,8 @@ func (p *PatrolService) runPatrolWithTrigger(ctx context.Context, trigger Trigge
}
p.mu.Lock()
p.lastPatrol = completedAt
p.lastActivity = completedAt
p.lastFullPatrol = completedAt
p.lastDuration = duration
p.resourcesChecked = runStats.resourceCount
p.errorCount = runStats.errors
@@ -941,7 +961,7 @@ func (p *PatrolService) runScopedPatrol(ctx context.Context, scope PatrolScope)
}
p.mu.Lock()
p.lastPatrol = completedAt
p.lastActivity = completedAt
p.lastDuration = duration
p.resourcesChecked = runStats.resourceCount
p.errorCount = runStats.errors
@@ -1555,8 +1575,11 @@ func (p *PatrolService) GetStatus() PatrolStatus {
status.RuntimeState = PatrolRuntimeStateActive
}
if !p.lastPatrol.IsZero() {
status.LastPatrolAt = &p.lastPatrol
if !p.lastFullPatrol.IsZero() {
status.LastPatrolAt = &p.lastFullPatrol
}
if !p.lastActivity.IsZero() {
status.LastActivityAt = &p.lastActivity
}
if strings.TrimSpace(status.BlockedReason) != "" && !p.lastBlockedAt.IsZero() {
status.BlockedAt = &p.lastBlockedAt
+4 -4
View File
@@ -1391,14 +1391,14 @@ func TestGetStatus_NextPatrolAt(t *testing.T) {
}
func TestGetStatus_NextPatrolAt_IndependentOfLastPatrol(t *testing.T) {
// nextScheduledAt should drive NextPatrolAt, not lastPatrol + interval.
// nextScheduledAt should drive NextPatrolAt, not lastFullPatrol + interval.
// This is the scenario that was previously broken: user changes interval
// mid-cycle, lastPatrol is old, so lastPatrol+newInterval could be in the past.
// mid-cycle, lastFullPatrol is old, so lastFullPatrol+newInterval could be in the past.
ps := NewPatrolService(nil, nil)
// Simulate: patrol ran 45 min ago
ps.mu.Lock()
ps.lastPatrol = time.Now().Add(-45 * time.Minute)
ps.lastFullPatrol = time.Now().Add(-45 * time.Minute)
// But the ticker was just reset with a 15-min interval, so next fire is ~15 min from now
expectedNext := time.Now().Add(15 * time.Minute)
ps.nextScheduledAt = expectedNext
@@ -1409,7 +1409,7 @@ func TestGetStatus_NextPatrolAt_IndependentOfLastPatrol(t *testing.T) {
t.Fatal("expected NextPatrolAt to be set")
}
// NextPatrolAt must be the tracked nextScheduledAt (in the future),
// NOT lastPatrol + interval (which would be 30 min in the past).
// NOT lastFullPatrol + interval (which would be 30 min in the past).
if !status.NextPatrolAt.Equal(expectedNext) {
t.Errorf("expected NextPatrolAt = %v, got %v", expectedNext, *status.NextPatrolAt)
}
+4
View File
@@ -598,6 +598,7 @@ func TestPatrolStatus_Fields(t *testing.T) {
Running: true,
Enabled: true,
LastPatrolAt: &now,
LastActivityAt: &now,
NextPatrolAt: &next,
LastDuration: 5 * time.Second,
ResourcesChecked: 25,
@@ -619,6 +620,9 @@ func TestPatrolStatus_Fields(t *testing.T) {
if status.LastPatrolAt == nil {
t.Error("Expected LastPatrolAt to be set")
}
if status.LastActivityAt == nil {
t.Error("Expected LastActivityAt to be set")
}
if *status.NextPatrolAt != next {
t.Error("NextPatrolAt value mismatch")
}
+2 -2
View File
@@ -53,7 +53,7 @@ func (a *AlertManagerAdapter) SetAlertCallback(cb func(AlertAdapter)) {
return
}
a.manager.SetAlertCallback(func(alert *alerts.Alert) {
a.manager.SubscribeAlertCallback(func(alert *alerts.Alert) {
if cb != nil && alert != nil {
cb(&alertWrapper{alert: alert})
}
@@ -66,7 +66,7 @@ func (a *AlertManagerAdapter) SetResolvedCallback(cb func(alertID string)) {
return
}
a.manager.SetResolvedCallback(cb)
a.manager.SubscribeResolvedCallback(cb)
}
// alertWrapper wraps an alerts.Alert to implement AlertAdapter
+16 -4
View File
@@ -98,8 +98,14 @@ func TestAlertManagerAdapter_WithManagerAndCallbacks(t *testing.T) {
adapter.SetAlertCallback(func(ad AlertAdapter) {
alertCh <- ad.GetAlertIdentifier()
})
onAlert := getUnexportedField(t, manager, "onAlert").Interface().(func(alert *alerts.Alert))
onAlert(alert)
alertSubs := getUnexportedField(t, manager, "alertSubs")
if alertSubs.Len() == 0 {
t.Fatal("expected subscribed alert callback")
}
for _, key := range alertSubs.MapKeys() {
alertSubs.MapIndex(key).Interface().(func(alert *alerts.Alert))(alert)
break
}
select {
case got := <-alertCh:
if got != alert.ID {
@@ -113,8 +119,14 @@ func TestAlertManagerAdapter_WithManagerAndCallbacks(t *testing.T) {
adapter.SetResolvedCallback(func(alertID string) {
resolvedCh <- alertID
})
onResolved := getUnexportedField(t, manager, "onResolved").Interface().(func(alertID string))
onResolved(alert.ID)
resolvedSubs := getUnexportedField(t, manager, "resolvedSubs")
if resolvedSubs.Len() == 0 {
t.Fatal("expected subscribed resolved callback")
}
for _, key := range resolvedSubs.MapKeys() {
resolvedSubs.MapIndex(key).Interface().(func(alertID string))(alert.ID)
break
}
select {
case got := <-resolvedCh:
if got != alert.ID {
+149 -21
View File
@@ -534,11 +534,15 @@ type Manager struct {
activeAlertAlias map[string]string
historyManager *HistoryManager
onAlert func(alert *Alert)
alertSubs map[int]func(alert *Alert)
onResolved func(alertID string)
resolvedSubs map[int]func(alertID string)
onAcknowledged func(alert *Alert, user string)
onUnacknowledged func(alert *Alert, user string)
onEscalate func(alert *Alert, level int)
onAlertForAI func(alert *Alert) // AI analysis callback - bypasses notification suppression
alertForAISubs map[int]func(alert *Alert)
nextCallbackID int
escalationStop chan struct{}
alertRateLimit map[string][]time.Time // Track alert times for rate limiting
// New fields for deduplication and suppression
@@ -625,6 +629,9 @@ func NewManagerWithDataDir(dataDir string) *Manager {
activeAlertAlias: make(map[string]string),
historyManager: NewHistoryManager(alertsDir),
escalationStop: make(chan struct{}),
alertSubs: make(map[int]func(*Alert)),
resolvedSubs: make(map[int]func(string)),
alertForAISubs: make(map[int]func(*Alert)),
alertRateLimit: make(map[string][]time.Time),
recentAlerts: make(map[string]*Alert),
suppressedUntil: make(map[string]time.Time),
@@ -834,6 +841,27 @@ func (m *Manager) SetAlertCallback(cb func(alert *Alert)) {
m.onAlert = cb
}
// SubscribeAlertCallback registers an additional alert callback without
// replacing the legacy single callback slot. The returned function removes the
// subscription when called.
func (m *Manager) SubscribeAlertCallback(cb func(alert *Alert)) func() {
if cb == nil {
return func() {}
}
m.callbackMu.Lock()
m.nextCallbackID++
id := m.nextCallbackID
m.alertSubs[id] = cb
m.callbackMu.Unlock()
return func() {
m.callbackMu.Lock()
delete(m.alertSubs, id)
m.callbackMu.Unlock()
}
}
// SetAlertForAICallback sets a callback for AI analysis when alerts are created.
// Unlike SetAlertCallback, this callback is invoked unconditionally - it bypasses
// activation state, quiet hours, and other notification suppression checks.
@@ -845,6 +873,27 @@ func (m *Manager) SetAlertForAICallback(cb func(alert *Alert)) {
log.Info().Msg("alert-for-AI callback registered (bypasses notification suppression)")
}
// SubscribeAlertForAICallback registers an additional AI alert callback without
// replacing the legacy single callback slot. The returned function removes the
// subscription when called.
func (m *Manager) SubscribeAlertForAICallback(cb func(alert *Alert)) func() {
if cb == nil {
return func() {}
}
m.callbackMu.Lock()
m.nextCallbackID++
id := m.nextCallbackID
m.alertForAISubs[id] = cb
m.callbackMu.Unlock()
return func() {
m.callbackMu.Lock()
delete(m.alertForAISubs, id)
m.callbackMu.Unlock()
}
}
// SetResolvedCallback sets the callback for resolved alerts
func (m *Manager) SetResolvedCallback(cb func(alertID string)) {
m.callbackMu.Lock()
@@ -852,6 +901,27 @@ func (m *Manager) SetResolvedCallback(cb func(alertID string)) {
m.onResolved = cb
}
// SubscribeResolvedCallback registers an additional resolved-alert callback
// without replacing the legacy single callback slot. The returned function
// removes the subscription when called.
func (m *Manager) SubscribeResolvedCallback(cb func(alertID string)) func() {
if cb == nil {
return func() {}
}
m.callbackMu.Lock()
m.nextCallbackID++
id := m.nextCallbackID
m.resolvedSubs[id] = cb
m.callbackMu.Unlock()
return func() {
m.callbackMu.Lock()
delete(m.resolvedSubs, id)
m.callbackMu.Unlock()
}
}
// SetAcknowledgedCallback sets the callback for acknowledged alerts.
func (m *Manager) SetAcknowledgedCallback(cb func(alert *Alert, user string)) {
m.callbackMu.Lock()
@@ -880,6 +950,22 @@ func (m *Manager) getAlertCallback() func(alert *Alert) {
return cb
}
func (m *Manager) getAlertCallbacks() []func(alert *Alert) {
m.callbackMu.RLock()
defer m.callbackMu.RUnlock()
callbacks := make([]func(alert *Alert), 0, len(m.alertSubs)+1)
if m.onAlert != nil {
callbacks = append(callbacks, m.onAlert)
}
for _, cb := range m.alertSubs {
if cb != nil {
callbacks = append(callbacks, cb)
}
}
return callbacks
}
func (m *Manager) getAlertForAICallback() func(alert *Alert) {
m.callbackMu.RLock()
cb := m.onAlertForAI
@@ -887,6 +973,22 @@ func (m *Manager) getAlertForAICallback() func(alert *Alert) {
return cb
}
func (m *Manager) getAlertForAICallbacks() []func(alert *Alert) {
m.callbackMu.RLock()
defer m.callbackMu.RUnlock()
callbacks := make([]func(alert *Alert), 0, len(m.alertForAISubs)+1)
if m.onAlertForAI != nil {
callbacks = append(callbacks, m.onAlertForAI)
}
for _, cb := range m.alertForAISubs {
if cb != nil {
callbacks = append(callbacks, cb)
}
}
return callbacks
}
func (m *Manager) getResolvedCallback() func(alertID string) {
m.callbackMu.RLock()
cb := m.onResolved
@@ -894,6 +996,22 @@ func (m *Manager) getResolvedCallback() func(alertID string) {
return cb
}
func (m *Manager) getResolvedCallbacks() []func(alertID string) {
m.callbackMu.RLock()
defer m.callbackMu.RUnlock()
callbacks := make([]func(alertID string), 0, len(m.resolvedSubs)+1)
if m.onResolved != nil {
callbacks = append(callbacks, m.onResolved)
}
for _, cb := range m.resolvedSubs {
if cb != nil {
callbacks = append(callbacks, cb)
}
}
return callbacks
}
func (m *Manager) getAcknowledgedCallback() func(alert *Alert, user string) {
m.callbackMu.RLock()
cb := m.onAcknowledged
@@ -919,8 +1037,8 @@ func (m *Manager) getEscalateCallback() func(alert *Alert, level int) {
// preserving canonical state as the internal identity and emitting the public
// alert ID to external callbacks for compatibility.
func (m *Manager) safeCallResolvedAlertCallback(alert *Alert, fallbackID string, async bool) {
callback := m.getResolvedCallback()
if callback == nil {
callbacks := m.getResolvedCallbacks()
if len(callbacks) == 0 {
return
}
@@ -937,7 +1055,9 @@ func (m *Manager) safeCallResolvedAlertCallback(alert *Alert, fallbackID string,
Msg("Panic in onResolved callback")
}
}()
callback(publicID)
for _, callback := range callbacks {
callback(publicID)
}
}
if async {
@@ -1070,8 +1190,8 @@ func (m *Manager) checkFlappingLocked(trackingKey string) bool {
}
func (m *Manager) dispatchAlert(alert *Alert, async bool) bool {
callback := m.getAlertCallback()
if callback == nil || alert == nil {
callbacks := m.getAlertCallbacks()
if len(callbacks) == 0 || alert == nil {
return false
}
@@ -1130,7 +1250,7 @@ func (m *Manager) dispatchAlert(alert *Alert, async bool) bool {
alertCopy := cloneAlertForOutput(alert)
if async {
go func(a *Alert) {
go func(a *Alert, fns []func(*Alert)) {
defer func() {
if r := recover(); r != nil {
log.Error().
@@ -1140,11 +1260,13 @@ func (m *Manager) dispatchAlert(alert *Alert, async bool) bool {
Msg("Panic in onAlert callback")
}
}()
callback(a)
}(alertCopy)
for _, callback := range fns {
callback(a)
}
}(alertCopy, callbacks)
} else {
// Synchronous calls also need panic recovery to prevent service crash
func() {
func(fns []func(*Alert)) {
defer func() {
if r := recover(); r != nil {
log.Error().
@@ -1154,8 +1276,10 @@ func (m *Manager) dispatchAlert(alert *Alert, async bool) bool {
Msg("Panic in onAlert callback (synchronous)")
}
}()
callback(alertCopy)
}()
for _, callback := range fns {
callback(alertCopy)
}
}(callbacks)
}
return true
}
@@ -4873,16 +4997,18 @@ func (m *Manager) HandleDockerHostOffline(host models.DockerHost) {
alert, _ := m.getActiveAlertNoLock(alertID)
m.mu.RUnlock()
if alert != nil {
if alertForAICallback := m.getAlertForAICallback(); alertForAICallback != nil {
if callbacks := m.getAlertForAICallbacks(); len(callbacks) > 0 {
alertCopy := cloneAlertForOutput(alert)
go func(a *Alert) {
go func(a *Alert, fns []func(*Alert)) {
defer func() {
if r := recover(); r != nil {
log.Error().Interface("panic", r).Str("alertID", a.ID).Msg("panic in AI alert callback")
}
}()
alertForAICallback(a)
}(alertCopy)
for _, callback := range fns {
callback(a)
}
}(alertCopy, callbacks)
}
}
@@ -6875,16 +7001,18 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
Msg("Alert triggered")
// Trigger AI analysis callback unconditionally (bypasses notification suppression)
if alertForAICallback := m.getAlertForAICallback(); alertForAICallback != nil {
if callbacks := m.getAlertForAICallbacks(); len(callbacks) > 0 {
alertCopy := cloneAlertForOutput(alert)
go func(a *Alert) {
go func(a *Alert, fns []func(*Alert)) {
defer func() {
if r := recover(); r != nil {
log.Error().Interface("panic", r).Str("alertID", a.ID).Msg("panic in AI alert callback")
}
}()
alertForAICallback(a)
}(alertCopy)
for _, callback := range fns {
callback(a)
}
}(alertCopy, callbacks)
}
// Check rate limit (but don't remove alert from tracking)
@@ -6899,7 +7027,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
}
// Notify callback (may be suppressed by quiet hours)
if m.getAlertCallback() != nil {
if len(m.getAlertCallbacks()) > 0 {
now := time.Now()
alert.LastNotified = &now
if m.dispatchAlert(alert, true) {
@@ -6967,7 +7095,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
}
// Send re-notification if appropriate (may be suppressed by quiet hours)
if shouldRenotify && m.getAlertCallback() != nil {
if shouldRenotify && len(m.getAlertCallbacks()) > 0 {
now := time.Now()
existingAlert.LastNotified = &now
// Dispatch asynchronously so callback I/O cannot block alert evaluation.
@@ -213,6 +213,56 @@ func TestCheckMetricInvokesAICallbackWhenNotificationsSuppressed(t *testing.T) {
}
}
func TestDispatchAlertInvokesSubscribedAlertCallbacks(t *testing.T) {
m := newTestManager(t)
alert := &Alert{
ID: "subscribed-alert",
Type: "cpu",
Level: AlertLevelWarning,
ResourceID: "vm-100",
ResourceName: "web-1",
StartTime: time.Now().Add(-time.Minute),
LastSeen: time.Now(),
}
firstDone := make(chan string, 1)
secondDone := make(chan string, 1)
m.SubscribeAlertCallback(func(alert *Alert) {
firstDone <- alert.ID
})
m.SubscribeAlertCallback(func(alert *Alert) {
secondDone <- alert.ID
})
m.mu.Lock()
m.config.ActivationState = ActivationActive
dispatched := m.dispatchAlert(alert, false)
m.mu.Unlock()
if !dispatched {
t.Fatal("expected dispatchAlert to report a subscribed callback dispatch")
}
select {
case got := <-firstDone:
if got != alert.ID {
t.Fatalf("first callback ID = %q, want %q", got, alert.ID)
}
case <-time.After(time.Second):
t.Fatal("expected first subscribed callback to fire")
}
select {
case got := <-secondDone:
if got != alert.ID {
t.Fatalf("second callback ID = %q, want %q", got, alert.ID)
}
case <-time.After(time.Second):
t.Fatal("expected second subscribed callback to fire")
}
}
func TestOnAlertHistoryRegistersCallback(t *testing.T) {
m := newTestManager(t)
+2
View File
@@ -4711,6 +4711,7 @@ type PatrolStatusResponse struct {
Running bool `json:"running"`
Enabled bool `json:"enabled"`
LastPatrolAt *time.Time `json:"last_patrol_at,omitempty"`
LastActivityAt *time.Time `json:"last_activity_at,omitempty"`
NextPatrolAt *time.Time `json:"next_patrol_at,omitempty"`
LastDurationMs int64 `json:"last_duration_ms"`
ResourcesChecked int `json:"resources_checked"`
@@ -4816,6 +4817,7 @@ func (h *AISettingsHandler) HandleGetPatrolStatus(w http.ResponseWriter, r *http
Running: status.Running,
Enabled: status.Enabled,
LastPatrolAt: status.LastPatrolAt,
LastActivityAt: status.LastActivityAt,
NextPatrolAt: status.NextPatrolAt,
LastDurationMs: status.LastDuration.Milliseconds(),
ResourcesChecked: status.ResourcesChecked,
@@ -186,6 +186,36 @@ func TestHandleGetPatrolStatus_DerivesBlockedRuntimeStateForExhaustedQuickstartC
}
}
func TestHandleGetPatrolStatus_DistinguishesLastFullPatrolFromLastActivity(t *testing.T) {
handler, patrol, _, _ := setupAIHandlerWithPatrol(t)
lastPatrolAt := time.Date(2026, 3, 12, 9, 30, 0, 0, time.UTC)
lastActivityAt := lastPatrolAt.Add(8 * time.Minute)
setUnexportedField(t, patrol, "lastFullPatrol", lastPatrolAt)
setUnexportedField(t, patrol, "lastActivity", lastActivityAt)
req := httptest.NewRequest(http.MethodGet, "/api/ai/patrol/status", nil)
rec := httptest.NewRecorder()
handler.HandleGetPatrolStatus(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rec.Code)
}
var resp PatrolStatusResponse
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode response: %v", err)
}
if resp.LastPatrolAt == nil || !resp.LastPatrolAt.Equal(lastPatrolAt) {
t.Fatalf("last_patrol_at = %v, want %v", resp.LastPatrolAt, lastPatrolAt)
}
if resp.LastActivityAt == nil || !resp.LastActivityAt.Equal(lastActivityAt) {
t.Fatalf("last_activity_at = %v, want %v", resp.LastActivityAt, lastActivityAt)
}
}
func TestPatrolActionHandlers_NoAIService_ReturnStructuredServiceUnavailable(t *testing.T) {
tmp := t.TempDir()
cfg := &config.Config{DataPath: tmp}
@@ -672,3 +702,22 @@ func TestHandleForcePatrol_ConfigDisabled(t *testing.T) {
t.Fatalf("expected success message")
}
}
func TestHandleForcePatrol_CommunityTierIgnoresRecentScopedActivityForFullPatrolRateLimit(t *testing.T) {
handler, patrol, _, _ := setupAIHandlerWithPatrol(t)
handler.defaultAIService.SetLicenseChecker(communityLicenseChecker{})
setUnexportedField(t, patrol, "lastActivity", time.Now().Add(-10*time.Minute))
req := httptest.NewRequest(http.MethodPost, "/api/ai/patrol/run", nil)
rec := httptest.NewRecorder()
handler.HandleForcePatrol(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200: %s", rec.Code, rec.Body.String())
}
if strings.Contains(rec.Body.String(), "patrol_rate_limited") {
t.Fatalf("expected community force patrol to ignore scoped-only activity, got %s", rec.Body.String())
}
}
+3
View File
@@ -4364,6 +4364,7 @@ func TestContract_MetricsHistoryLiveFallbackJSONSnapshot(t *testing.T) {
func TestContract_PatrolStatusResponseJSONSnapshot(t *testing.T) {
lastPatrolAt := time.Date(2026, 3, 12, 9, 30, 0, 0, time.UTC)
lastActivityAt := lastPatrolAt.Add(5 * time.Minute)
nextPatrolAt := lastPatrolAt.Add(6 * time.Hour)
blockedAt := lastPatrolAt.Add(15 * time.Minute)
@@ -4372,6 +4373,7 @@ func TestContract_PatrolStatusResponseJSONSnapshot(t *testing.T) {
Running: false,
Enabled: true,
LastPatrolAt: &lastPatrolAt,
LastActivityAt: &lastActivityAt,
NextPatrolAt: &nextPatrolAt,
LastDurationMs: 12345,
ResourcesChecked: 18,
@@ -4404,6 +4406,7 @@ func TestContract_PatrolStatusResponseJSONSnapshot(t *testing.T) {
"running":false,
"enabled":true,
"last_patrol_at":"2026-03-12T09:30:00Z",
"last_activity_at":"2026-03-12T09:35:00Z",
"next_patrol_at":"2026-03-12T15:30:00Z",
"last_duration_ms":12345,
"resources_checked":18,
+6 -15
View File
@@ -3125,32 +3125,23 @@ func (r *Router) WireAlertTriggeredAI() {
return
}
// 2. Get the Patrol Service (The Watchdog)
patrol := aiService.GetPatrolService()
if patrol == nil {
log.Debug().Msg("Patrol service not available for wiring")
return
}
// 3. Get the Monitor (The Trigger)
// 2. Get the Monitor (The Trigger)
if r.monitor == nil {
log.Debug().Msg("Monitor not available for AI alert callback")
return
}
// 4. Connect Trigger -> Watchdog
// When an alert fires, we immediately trigger the Patrol Agent to investigate
// 3. Connect alert-fired events to the dedicated alert-triggered analyzer.
// Patrol's event-triggered runs are owned by the canonical alert bridge /
// trigger-manager path, so this callback should not enqueue Patrol directly.
r.monitor.SetAlertTriggeredAICallback(func(alert *alerts.Alert) {
log.Info().Str("alert_identifier", alert.ID).Msg("Alert fired leading to Patrol Trigger")
patrol.TriggerPatrolForAlert(alert)
// We also trigger the specific analyzer if enabled, as it tracks specific stats
if analyzer := r.GetAlertTriggeredAnalyzer(); analyzer != nil {
log.Info().Str("alert_identifier", alert.ID).Msg("Alert fired leading to alert-triggered analysis")
analyzer.OnAlertFired(alert)
}
})
log.Info().Msg("Alert-triggered AI Watchdog wired to monitor")
log.Info().Msg("Alert-triggered AI analyzer wired to monitor")
}
// Deprecated: deriveResourceTypeFromAlert uses heuristic string matching.