From 015e7f6555f845ff5430eec44e5ee210957f7651 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Tue, 12 May 2026 18:02:16 +0100 Subject: [PATCH] Add maintenance verification reports When a maintenance window ends on a resource, the sentinel runs deterministic checks (active alerts, Patrol findings, failed actions since window start, basic post-window metric recovery) and writes a durable LoopReport. Operators can list reports per resource, mark them reviewed, or rerun verification immediately. UI surfaces the section in the resource detail drawer; scoped Patrol runs and Assistant deep-link are deferred until those entry points stabilise. --- .../src/api/maintenanceVerification.ts | 133 +++++ .../MaintenanceVerificationSection.tsx | 288 ++++++++++ .../ResourceDetailDrawerOverviewTab.tsx | 10 + .../MaintenanceVerificationSection.test.ts | 89 +++ internal/api/maintenance_verification.go | 372 +++++++++++++ internal/api/maintenance_verification_test.go | 184 +++++++ .../api/maintenance_verification_wiring.go | 289 ++++++++++ internal/api/route_inventory_test.go | 4 + internal/api/router.go | 8 + internal/api/router_routes_monitoring.go | 11 + internal/maintenancesentinel/sentinel.go | 288 ++++++++++ internal/maintenancesentinel/sentinel_test.go | 191 +++++++ internal/maintenancesentinel/verification.go | 400 ++++++++++++++ .../maintenancesentinel/verification_test.go | 194 +++++++ internal/unifiedresources/loop_reports.go | 289 ++++++++++ .../unifiedresources/loop_reports_store.go | 510 ++++++++++++++++++ .../loop_reports_store_test.go | 185 +++++++ internal/unifiedresources/store.go | 50 ++ 18 files changed, 3495 insertions(+) create mode 100644 frontend-modern/src/api/maintenanceVerification.ts create mode 100644 frontend-modern/src/components/Infrastructure/MaintenanceVerificationSection.tsx create mode 100644 frontend-modern/src/components/Infrastructure/__tests__/MaintenanceVerificationSection.test.ts create mode 100644 internal/api/maintenance_verification.go create mode 100644 internal/api/maintenance_verification_test.go create mode 100644 internal/api/maintenance_verification_wiring.go create mode 100644 internal/maintenancesentinel/sentinel.go create mode 100644 internal/maintenancesentinel/sentinel_test.go create mode 100644 internal/maintenancesentinel/verification.go create mode 100644 internal/maintenancesentinel/verification_test.go create mode 100644 internal/unifiedresources/loop_reports.go create mode 100644 internal/unifiedresources/loop_reports_store.go create mode 100644 internal/unifiedresources/loop_reports_store_test.go diff --git a/frontend-modern/src/api/maintenanceVerification.ts b/frontend-modern/src/api/maintenanceVerification.ts new file mode 100644 index 000000000..b2c4050bd --- /dev/null +++ b/frontend-modern/src/api/maintenanceVerification.ts @@ -0,0 +1,133 @@ +import { apiFetchJSON } from '@/utils/apiClient'; + +/** + * Maintenance Verification Report — the durable summary Pulse writes + * when a maintenance window ends for a resource. Mirrors the Go + * `unified.LoopReport` projected through + * `internal/api/maintenance_verification.go`. + * + * Surfaced as "Maintenance Verification Report" in the product copy. + * The status enum is small on purpose so the UI can render each value + * with a stable visual treatment: + * - healthy → green check / quiet + * - needs_review → amber prompt for an operator look + * - failed_verification → red surface, operator action expected + * - pending → in-flight; the sentinel hasn't decided + */ +export type MaintenanceVerificationStatus = + | 'pending' + | 'healthy' + | 'needs_review' + | 'failed_verification'; + +export type MaintenanceVerificationUserOutcome = 'reviewed' | ''; + +export interface MaintenanceVerificationMetricRecovery { + metricsObserved?: string[]; + samplesAfterEnd: number; + trend?: 'improving' | 'stable' | 'degrading' | 'unknown' | ''; + note?: string; +} + +export interface MaintenanceVerificationEvidence { + operatorStateSummary?: string; + activeCriticalAlerts: number; + activeWarningAlerts: number; + activeCriticalFindings: number; + activeWarningFindings: number; + failedActionsSinceWindowStart: number; + metricRecovery?: MaintenanceVerificationMetricRecovery; + /** + * Breadcrumb set by the sentinel when deterministic evidence was + * ambiguous and a scoped Patrol run would have helped, but + * triggering one was not safe in this build. Empty when the report + * was unambiguous. + */ + patrolRunTodo?: string; +} + +export interface MaintenanceVerificationReport { + id: string; + resourceId: string; + trigger: string; + goal?: string; + status: MaintenanceVerificationStatus; + startedAt: string; + completedAt: string; + windowStartedAt?: string; + windowEndedAt?: string; + evidence: MaintenanceVerificationEvidence; + linkedFindingIds: string[]; + linkedAlertIds: string[]; + linkedActionIds: string[]; + linkedPatrolRunId?: string; + recommendation?: string; + userOutcome?: MaintenanceVerificationUserOutcome; + reviewedAt?: string; + reviewedBy?: string; + reviewNote?: string; +} + +export interface MaintenanceVerificationListResponse { + data: MaintenanceVerificationReport[]; + meta: { + resourceId: string; + limit: number; + total: number; + }; +} + +/** + * Fetch recent Maintenance Verification Reports for a resource, + * newest first. + */ +export async function listMaintenanceVerificationsForResource( + resourceId: string, + limit = 25, +): Promise { + const params = new URLSearchParams(); + if (limit && limit > 0) { + params.set('limit', String(limit)); + } + const query = params.toString(); + const path = `/api/resources/${encodeURIComponent(resourceId)}/maintenance-verifications${ + query ? `?${query}` : '' + }`; + return apiFetchJSON(path, { cache: 'no-store' }); +} + +/** + * Mark a report as reviewed by the operator. The report's status and + * evidence stay immutable — only the user verdict and review fields + * update. + */ +export async function reviewMaintenanceVerification( + reportId: string, + note?: string, +): Promise { + return apiFetchJSON( + `/api/maintenance-verifications/${encodeURIComponent(reportId)}/review`, + { + method: 'POST', + body: JSON.stringify({ note: note ?? '' }), + headers: { 'Content-Type': 'application/json' }, + }, + ); +} + +/** + * Re-run the deterministic verification immediately. The new report + * is persisted with a `-rerun-N` suffix on its id so the review + * history is preserved. + */ +export async function rerunMaintenanceVerification( + resourceId: string, +): Promise { + return apiFetchJSON( + `/api/resources/${encodeURIComponent(resourceId)}/maintenance-verifications/rerun`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + ); +} diff --git a/frontend-modern/src/components/Infrastructure/MaintenanceVerificationSection.tsx b/frontend-modern/src/components/Infrastructure/MaintenanceVerificationSection.tsx new file mode 100644 index 000000000..e71eb1452 --- /dev/null +++ b/frontend-modern/src/components/Infrastructure/MaintenanceVerificationSection.tsx @@ -0,0 +1,288 @@ +import { Component, For, Show, createMemo, createSignal } from 'solid-js'; +import { createNonSuspendingQuery } from '@/hooks/createNonSuspendingQuery'; +import { notificationStore } from '@/stores/notifications'; +import { + listMaintenanceVerificationsForResource, + rerunMaintenanceVerification, + reviewMaintenanceVerification, + type MaintenanceVerificationReport, + type MaintenanceVerificationStatus, +} from '@/api/maintenanceVerification'; +import { formatRelativeTime } from '@/utils/format'; + +/** + * MaintenanceVerificationSection surfaces Maintenance Verification + * Reports in the resource detail drawer. Reports are durable records + * the sentinel writes when a maintenance window ends for a resource, + * summarizing whether the resource recovered cleanly. + * + * The section is empty (and hidden) until the first report is + * written. Operators can: + * - Read the deterministic evidence (alerts, findings, failed + * actions, basic metric recovery summary) + * - Mark a report as reviewed (without changing the underlying + * status / evidence — those stay immutable) + * - Rerun verification now (writes a new -rerun-N report) + * + * The "open Assistant with this report context" action is intentionally + * omitted from this first pass — see Status: missing-actions note at + * the bottom. The other two actions (mark reviewed, rerun verification) + * are sufficient for the MVP review loop. + */ +interface MaintenanceVerificationSectionProps { + resourceId: string; +} + +const STATUS_LABELS: Record = { + pending: 'Pending', + healthy: 'Healthy', + needs_review: 'Needs review', + failed_verification: 'Failed verification', +}; + +const STATUS_CLASSES: Record = { + pending: 'bg-surface-hover text-base-content', + healthy: + 'bg-emerald-100 text-emerald-700 dark:bg-emerald-900 dark:text-emerald-300', + needs_review: + 'bg-amber-100 text-amber-700 dark:bg-amber-900 dark:text-amber-300', + failed_verification: + 'bg-rose-100 text-rose-700 dark:bg-rose-900 dark:text-rose-300', +}; + +export const MaintenanceVerificationSection: Component = ( + props, +) => { + const [refreshTick, setRefreshTick] = createSignal(0); + + const query = createNonSuspendingQuery({ + source: () => (props.resourceId ? `${props.resourceId}:${refreshTick()}` : null), + fetcher: async () => { + try { + const res = await listMaintenanceVerificationsForResource(props.resourceId); + return res.data ?? []; + } catch (err) { + notificationStore.error( + err instanceof Error + ? err.message + : 'Failed to load Maintenance Verification Reports', + ); + return []; + } + }, + initialValue: [] as MaintenanceVerificationReport[], + cacheKey: (key: string) => `maintenance-verifications:${key}`, + }); + + const reports = createMemo(() => query.value() ?? []); + const hasReports = createMemo(() => reports().length > 0); + + const [rerunning, setRerunning] = createSignal(false); + const [reviewingId, setReviewingId] = createSignal(null); + + const refresh = () => setRefreshTick((n) => n + 1); + + const handleRerun = async () => { + if (rerunning() || !props.resourceId) return; + setRerunning(true); + try { + await rerunMaintenanceVerification(props.resourceId); + notificationStore.success('Maintenance verification rerun complete'); + refresh(); + } catch (err) { + notificationStore.error( + err instanceof Error + ? err.message + : 'Failed to rerun maintenance verification', + ); + } finally { + setRerunning(false); + } + }; + + const handleReview = async (report: MaintenanceVerificationReport) => { + if (reviewingId() || !report.id) return; + setReviewingId(report.id); + try { + await reviewMaintenanceVerification(report.id); + notificationStore.success('Marked report reviewed'); + refresh(); + } catch (err) { + notificationStore.error( + err instanceof Error + ? err.message + : 'Failed to mark report reviewed', + ); + } finally { + setReviewingId(null); + } + }; + + return ( +
+
+
+

+ Maintenance verification +

+

+ Pulse runs deterministic checks each time a maintenance window + ends and writes a Maintenance Verification Report. The result + sticks here for review. +

+
+ +
+ + + No verification reports yet. A report is written automatically the + next time this resource exits a maintenance window. + + } + > +
    + + {(report) => ( +
  • +
    +
    + + {STATUS_LABELS[report.status]} + + + Window ended{' '} + {report.windowEndedAt + ? formatRelativeTime(report.windowEndedAt) + : 'unknown'} + +
    + + + + + + Reviewed{' '} + {report.reviewedAt + ? formatRelativeTime(report.reviewedAt) + : ''} + {report.reviewedBy ? ` by ${report.reviewedBy}` : ''} + + +
    + + +

    {report.recommendation}

    +     + +
    +
    +
    Critical alerts
    +
    {report.evidence.activeCriticalAlerts}
    +
    +
    +
    Warning alerts
    +
    {report.evidence.activeWarningAlerts}
    +
    +
    +
    + Critical findings +
    +
    {report.evidence.activeCriticalFindings}
    +
    +
    +
    + Warning findings +
    +
    {report.evidence.activeWarningFindings}
    +
    +
    +
    Failed actions
    +
    {report.evidence.failedActionsSinceWindowStart}
    +
    + + {(rec) => ( + <> +
    +
    + Metric samples +
    +
    {rec().samplesAfterEnd}
    +
    +
    +
    + Metric trend +
    +
    {rec().trend || 'unknown'}
    +
    + + )} +     +
    + + +

    + {report.evidence.operatorStateSummary} +

    +     + + +

    + {report.evidence.patrolRunTodo} +

    +     + + +

    + Reviewer note: {report.reviewNote} +

    +     +
    • + )} +     +
+ + + {/* + MVP omission: "Open Assistant with this report context" is the + third action called out in the product brief. Wiring it requires + a stable Assistant deep-link contract for report payloads which + is not yet implemented — left as a follow-up so the surface + does not ship a button that does nothing. + */} +
+ ); +}; + +export default MaintenanceVerificationSection; diff --git a/frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx b/frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx index 6f22db263..0ac1aafbf 100644 --- a/frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx +++ b/frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx @@ -29,6 +29,7 @@ import { ResourceChangeSummary } from './ResourceChangeSummary'; import { ResourceFacetSummary } from './ResourceFacetSummary'; import { ResourceActionHistory } from './ResourceActionHistory'; import { ResourceOperatorStateSection } from './ResourceOperatorStateSection'; +import { MaintenanceVerificationSection } from './MaintenanceVerificationSection'; import { RESOURCE_CHANGE_KIND_ORDER, RESOURCE_CHANGE_SOURCE_ADAPTER_ORDER, @@ -571,6 +572,15 @@ export const ResourceDetailDrawerOverviewTab: Component + {/* Maintenance Verification Reports sit directly under the + operator-state section because the same operator who set + the maintenance window above is the one who needs to read + the verification result here. Self-fetching component; + empty state hides gracefully when no reports exist. */} + + + + { + it('routes list/rerun/review through the canonical maintenanceVerification API client', () => { + expect(sectionSource).toContain("from '@/api/maintenanceVerification'"); + expect(sectionSource).toContain('listMaintenanceVerificationsForResource'); + expect(sectionSource).toContain('rerunMaintenanceVerification'); + expect(sectionSource).toContain('reviewMaintenanceVerification'); + }); + + it('keeps the section out of the parent Suspense fallback by using createNonSuspendingQuery', () => { + expect(sectionSource).toContain('createNonSuspendingQuery'); + expect(sectionSource).not.toContain('createResource<'); + }); + + it('renders an empty state for resources with no reports yet', () => { + expect(sectionSource).toContain('No verification reports yet.'); + }); + + it('surfaces the deterministic evidence counts the report exposes', () => { + expect(sectionSource).toContain('Critical alerts'); + expect(sectionSource).toContain('Warning alerts'); + expect(sectionSource).toContain('Critical findings'); + expect(sectionSource).toContain('Warning findings'); + expect(sectionSource).toContain('Failed actions'); + }); + + it('exposes both operator actions (rerun + mark reviewed) the MVP committed to', () => { + expect(sectionSource).toContain('data-testid="maintenance-verification-rerun"'); + expect(sectionSource).toContain('data-testid="maintenance-verification-review"'); + }); + + it('hides the review button once the report has been reviewed', () => { + expect(sectionSource).toContain("when={!report.userOutcome}"); + expect(sectionSource).toContain("when={report.userOutcome === 'reviewed'}"); + }); + + it('renders the patrolRunTodo breadcrumb when the sentinel surfaced one', () => { + expect(sectionSource).toContain('report.evidence.patrolRunTodo'); + }); +}); + +describe('maintenanceVerification API client', () => { + it('encodes the resource id segment so canonical ids with colons survive', () => { + expect(apiClientSource).toContain('encodeURIComponent(resourceId)'); + expect(apiClientSource).toContain('encodeURIComponent(reportId)'); + }); + + it('exposes the four operations the section needs', () => { + expect(apiClientSource).toContain('export async function listMaintenanceVerificationsForResource'); + expect(apiClientSource).toContain('export async function reviewMaintenanceVerification'); + expect(apiClientSource).toContain('export async function rerunMaintenanceVerification'); + }); + + it('pins the four-state status enum the UI branches on', () => { + expect(apiClientSource).toContain("'pending'"); + expect(apiClientSource).toContain("'healthy'"); + expect(apiClientSource).toContain("'needs_review'"); + expect(apiClientSource).toContain("'failed_verification'"); + }); +}); + +describe('ResourceDetailDrawerOverviewTab integration', () => { + it('renders MaintenanceVerificationSection directly under the operator-state section', () => { + expect(overviewTabSource).toContain("from './MaintenanceVerificationSection'"); + expect(overviewTabSource).toContain(''); + const operatorIndex = overviewTabSource.indexOf(' max { + return max + } + return v +} + +// extractMaintenanceVerificationsResourceID parses the canonical +// resource id out of `/api/resources//maintenance-verifications`. +func extractMaintenanceVerificationsResourceID(path string) string { + trimmed := strings.TrimPrefix(path, "/api/resources/") + trimmed = strings.TrimSuffix(trimmed, "/") + trimmed = strings.TrimSuffix(trimmed, "/maintenance-verifications") + trimmed = strings.TrimSuffix(trimmed, "/") + return unified.CanonicalResourceID(trimmed) +} + +// extractMaintenanceVerificationRerunResourceID parses the canonical +// resource id out of +// `/api/resources//maintenance-verifications/rerun`. +func extractMaintenanceVerificationRerunResourceID(path string) string { + trimmed := strings.TrimPrefix(path, "/api/resources/") + trimmed = strings.TrimSuffix(trimmed, "/") + trimmed = strings.TrimSuffix(trimmed, "/maintenance-verifications/rerun") + trimmed = strings.TrimSuffix(trimmed, "/") + return unified.CanonicalResourceID(trimmed) +} + +// extractMaintenanceVerificationReportID parses the report id out of +// `/api/maintenance-verifications/`. +func extractMaintenanceVerificationReportID(path string) string { + trimmed := strings.TrimPrefix(path, "/api/maintenance-verifications/") + trimmed = strings.TrimSuffix(trimmed, "/") + return strings.TrimSpace(trimmed) +} + +// extractMaintenanceVerificationReviewReportID parses the report id +// out of `/api/maintenance-verifications//review`. +func extractMaintenanceVerificationReviewReportID(path string) string { + trimmed := strings.TrimPrefix(path, "/api/maintenance-verifications/") + trimmed = strings.TrimSuffix(trimmed, "/") + trimmed = strings.TrimSuffix(trimmed, "/review") + trimmed = strings.TrimSuffix(trimmed, "/") + return strings.TrimSpace(trimmed) +} + +func isMaintenanceWindowMissing(err error) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), "no operator state for") || + strings.Contains(err.Error(), "no maintenance window to verify") +} + +// Compile-time interface satisfaction check so the router never wires +// a misshapen sentinel pointer. +var _ context.Context = context.Background() diff --git a/internal/api/maintenance_verification_test.go b/internal/api/maintenance_verification_test.go new file mode 100644 index 000000000..b6b992e66 --- /dev/null +++ b/internal/api/maintenance_verification_test.go @@ -0,0 +1,184 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/config" + "github.com/rcourtman/pulse-go-rewrite/internal/maintenancesentinel" + unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +func newMaintenanceVerificationFixture(t *testing.T) (*MaintenanceVerificationHandlers, unified.ResourceStore) { + t.Helper() + rh := NewResourceHandlers(&config.Config{DataPath: t.TempDir()}) + store, err := rh.getStore("default") + if err != nil { + t.Fatalf("get store: %v", err) + } + providers := maintenancesentinel.Providers{ + Stores: func(orgID string) (unified.ResourceStore, error) { + return rh.getStore(orgID) + }, + Now: func() time.Time { return time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) }, + } + sentinel, err := maintenancesentinel.New(maintenancesentinel.Config{OrgID: "default"}, providers) + if err != nil { + t.Fatalf("new sentinel: %v", err) + } + return NewMaintenanceVerificationHandlers(rh, sentinel), store +} + +func withDefaultOrg(req *http.Request) *http.Request { + return req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "default")) +} + +func seedMaintenanceWindow(t *testing.T, store unified.ResourceStore, canonicalID string, start, end time.Time) { + t.Helper() + if err := store.SetResourceOperatorState(unified.ResourceOperatorState{ + CanonicalID: canonicalID, + MaintenanceStartAt: &start, + MaintenanceEndAt: &end, + SetAt: start, + SetBy: "operator", + }); err != nil { + t.Fatalf("seed operator state: %v", err) + } +} + +func TestMaintenanceVerification_HandleListForResource_Empty(t *testing.T) { + h, _ := newMaintenanceVerificationFixture(t) + rec := httptest.NewRecorder() + req := withDefaultOrg(httptest.NewRequest(http.MethodGet, "/api/resources/vm:101/maintenance-verifications", nil)) + h.HandleListForResource(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String()) + } + var resp struct { + Data []maintenanceVerificationReportAPI `json:"data"` + Meta struct { + ResourceID string `json:"resourceId"` + Limit int `json:"limit"` + Total int `json:"total"` + } `json:"meta"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("parse body: %v", err) + } + if resp.Meta.ResourceID != "vm:101" || resp.Meta.Total != 0 || resp.Meta.Limit != 25 { + t.Fatalf("meta = %+v", resp.Meta) + } +} + +func TestMaintenanceVerification_RerunWritesReport(t *testing.T) { + h, store := newMaintenanceVerificationFixture(t) + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + seedMaintenanceWindow(t, store, "vm:101", now.Add(-time.Hour), now.Add(-15*time.Minute)) + + rec := httptest.NewRecorder() + req := withDefaultOrg(httptest.NewRequest(http.MethodPost, "/api/resources/vm:101/maintenance-verifications/rerun", nil)) + h.HandleRerun(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("rerun status = %d body=%s", rec.Code, rec.Body.String()) + } + var report maintenanceVerificationReportAPI + if err := json.Unmarshal(rec.Body.Bytes(), &report); err != nil { + t.Fatalf("parse: %v", err) + } + if report.ResourceID != "vm:101" { + t.Fatalf("resource id = %q", report.ResourceID) + } + if report.Status == "" { + t.Fatalf("status must be set") + } + + // Confirm the report landed in the store. + reports, err := store.ListLoopReportsForResource(unified.LoopReportTypeMaintenanceVerification, "vm:101", 0) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(reports) != 1 { + t.Fatalf("expected 1 report after rerun, got %d", len(reports)) + } +} + +func TestMaintenanceVerification_RerunWithoutWindowReturns400(t *testing.T) { + h, _ := newMaintenanceVerificationFixture(t) + rec := httptest.NewRecorder() + req := withDefaultOrg(httptest.NewRequest(http.MethodPost, "/api/resources/vm:101/maintenance-verifications/rerun", nil)) + h.HandleRerun(rec, req) + if rec.Code != http.StatusBadRequest { + t.Fatalf("expected 400 on missing window, got %d body=%s", rec.Code, rec.Body.String()) + } + var body map[string]string + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("body must be JSON: %v", err) + } + if body["error"] != "maintenance_window_missing" { + t.Fatalf("error code = %q want maintenance_window_missing", body["error"]) + } +} + +func TestMaintenanceVerification_ReviewMarksReportReviewed(t *testing.T) { + h, store := newMaintenanceVerificationFixture(t) + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + seedMaintenanceWindow(t, store, "vm:101", now.Add(-time.Hour), now.Add(-15*time.Minute)) + + rerunRec := httptest.NewRecorder() + rerunReq := withDefaultOrg(httptest.NewRequest(http.MethodPost, "/api/resources/vm:101/maintenance-verifications/rerun", nil)) + h.HandleRerun(rerunRec, rerunReq) + if rerunRec.Code != http.StatusOK { + t.Fatalf("rerun status = %d body=%s", rerunRec.Code, rerunRec.Body.String()) + } + var seeded maintenanceVerificationReportAPI + if err := json.Unmarshal(rerunRec.Body.Bytes(), &seeded); err != nil { + t.Fatalf("parse rerun: %v", err) + } + + rec := httptest.NewRecorder() + req := withDefaultOrg(httptest.NewRequest( + http.MethodPost, + "/api/maintenance-verifications/"+seeded.ID+"/review", + nil, + )) + h.HandleReview(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("review status = %d body=%s", rec.Code, rec.Body.String()) + } + var reviewed maintenanceVerificationReportAPI + if err := json.Unmarshal(rec.Body.Bytes(), &reviewed); err != nil { + t.Fatalf("parse review response: %v", err) + } + if reviewed.UserOutcome != string(unified.LoopReportUserOutcomeReviewed) { + t.Fatalf("userOutcome = %q want reviewed", reviewed.UserOutcome) + } + if reviewed.Status != seeded.Status { + t.Fatalf("status mutated: was %q, now %q", seeded.Status, reviewed.Status) + } +} + +func TestMaintenanceVerification_ReviewMissingReportReturns404(t *testing.T) { + h, _ := newMaintenanceVerificationFixture(t) + rec := httptest.NewRecorder() + req := withDefaultOrg(httptest.NewRequest(http.MethodPost, "/api/maintenance-verifications/no-such-id/review", nil)) + h.HandleReview(rec, req) + if rec.Code != http.StatusNotFound { + t.Fatalf("expected 404 on missing report, got %d body=%s", rec.Code, rec.Body.String()) + } +} + +func TestMaintenanceVerification_RerunReturns503WhenSentinelNil(t *testing.T) { + rh := NewResourceHandlers(&config.Config{DataPath: t.TempDir()}) + h := NewMaintenanceVerificationHandlers(rh, nil) + + rec := httptest.NewRecorder() + req := withDefaultOrg(httptest.NewRequest(http.MethodPost, "/api/resources/vm:101/maintenance-verifications/rerun", nil)) + h.HandleRerun(rec, req) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("expected 503 when sentinel disabled, got %d body=%s", rec.Code, rec.Body.String()) + } +} diff --git a/internal/api/maintenance_verification_wiring.go b/internal/api/maintenance_verification_wiring.go new file mode 100644 index 000000000..e083e58a7 --- /dev/null +++ b/internal/api/maintenance_verification_wiring.go @@ -0,0 +1,289 @@ +package api + +import ( + "context" + "strings" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/ai" + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + "github.com/rcourtman/pulse-go-rewrite/internal/maintenancesentinel" + "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" + unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +// buildMaintenanceVerificationSentinel constructs the sentinel with +// adapters that pull alerts, findings, action audits, and recent +// metric samples out of the live runtime. Closures capture `r` so the +// providers always read the current monitor / aiSettings / store +// state (multi-tenant rebuilds replace those in place; latching to a +// snapshot would go stale). +// +// Returns nil when the resource handlers are not initialized — the +// caller treats nil as "sentinel disabled" and the API rerun endpoint +// returns 503. +func (r *Router) buildMaintenanceVerificationSentinel() *maintenancesentinel.Sentinel { + if r == nil || r.resourceHandlers == nil { + return nil + } + providers := maintenancesentinel.Providers{ + Stores: func(orgID string) (unified.ResourceStore, error) { + return r.resourceHandlers.getStore(orgID) + }, + ActiveAlerts: r.maintenanceVerificationActiveAlerts, + ActiveFindings: r.maintenanceVerificationActiveFindings, + RecentActions: r.maintenanceVerificationRecentActions, + PostWindowMetricSamples: r.maintenanceVerificationMetricSamples, + Now: func() time.Time { return time.Now().UTC() }, + } + sentinel, err := maintenancesentinel.New(maintenancesentinel.Config{ + OrgID: "default", + }, providers) + if err != nil { + return nil + } + return sentinel +} + +func (r *Router) maintenanceVerificationActiveAlerts(orgID, canonicalID string) []maintenancesentinel.AlertSummary { + mgr := r.resolveAlertManagerForOrg(orgID) + if mgr == nil { + return nil + } + canonicalID = unified.CanonicalResourceID(canonicalID) + if canonicalID == "" { + return nil + } + out := []maintenancesentinel.AlertSummary{} + for _, a := range mgr.GetActiveAlerts() { + if !alertMatchesResource(a, canonicalID) { + continue + } + out = append(out, maintenancesentinel.AlertSummary{ + ID: a.ID, + Severity: mapAlertLevel(a.Level), + Type: a.Type, + Acknowledged: a.Acknowledged, + }) + } + return out +} + +func (r *Router) maintenanceVerificationActiveFindings(orgID, canonicalID string) []maintenancesentinel.FindingSummary { + patrol := r.resolvePatrolServiceForOrg(orgID) + if patrol == nil { + return nil + } + canonicalID = unified.CanonicalResourceID(canonicalID) + if canonicalID == "" { + return nil + } + findings := patrol.GetFindingsForResource(canonicalID) + out := make([]maintenancesentinel.FindingSummary, 0, len(findings)) + for _, f := range findings { + if f == nil { + continue + } + out = append(out, maintenancesentinel.FindingSummary{ + ID: f.ID, + Severity: mapFindingSeverity(f.Severity), + Category: string(f.Category), + Resolved: f.ResolvedAt != nil, + Acknowledged: f.AcknowledgedAt != nil, + }) + } + return out +} + +func (r *Router) maintenanceVerificationRecentActions(orgID, canonicalID string, since time.Time) []maintenancesentinel.ActionSummary { + if r.resourceHandlers == nil { + return nil + } + store, err := r.resourceHandlers.getStore(orgID) + if err != nil || store == nil { + return nil + } + audits, err := store.GetActionAudits(canonicalID, since, 50) + if err != nil { + return nil + } + out := make([]maintenancesentinel.ActionSummary, 0, len(audits)) + for _, a := range audits { + out = append(out, maintenancesentinel.ActionSummary{ + ID: a.ID, + State: string(a.State), + UpdatedAt: a.UpdatedAt, + }) + } + return out +} + +func (r *Router) maintenanceVerificationMetricSamples(orgID, canonicalID string, windowEnd, now time.Time) ([]maintenancesentinel.MetricSample, bool) { + history := r.resolveMetricsHistoryForOrg(orgID) + if history == nil { + return nil, false + } + canonicalID = unified.CanonicalResourceID(canonicalID) + if canonicalID == "" { + return nil, false + } + // The metrics history is keyed by source IDs (e.g. "qemu/101", + // "node/pve") not canonical resource IDs. Strip the canonical + // `kind:` prefix so the lookup works for the common case (vm, + // container, node). Resources whose source-id form does not + // match this convention (storage, agents, docker hosts) won't + // have metric history available — the sentinel reports + // MetricSourceAvailable=false in that case, which is honest. + sourceID := canonicalToSourceID(canonicalID) + if sourceID == "" { + return nil, false + } + since := now.Sub(windowEnd) + if since <= 0 { + since = time.Hour + } + // Inspect cpu + memory only for the MVP. These are the metrics + // every guest/node reports; storage / disk / network are not + // universally available across resource kinds. + samples := []maintenancesentinel.MetricSample{} + anyData := false + for _, metric := range []string{"cpu", "memory"} { + // Try both guest and node lookups; whichever has data wins. + points := history.GetGuestMetrics(sourceID, metric, since) + if len(points) == 0 { + points = history.GetNodeMetrics(sourceID, metric, since) + } + if len(points) > 0 { + anyData = true + for _, p := range points { + if !p.Timestamp.After(windowEnd) { + continue + } + samples = append(samples, maintenancesentinel.MetricSample{ + Metric: metric, + Value: p.Value, + Timestamp: p.Timestamp, + }) + } + } + } + // available=true even when samples is empty if we know there's + // a history bucket for the resource — that's still useful + // evidence ("the resource exists in history but has not reported + // since the window closed"). + return samples, anyData +} + +// resolveAlertManagerForOrg returns the AlertManager for the supplied +// org. MVP runs the default org only; multi-tenant resolution is +// supported as a future change. +func (r *Router) resolveAlertManagerForOrg(orgID string) *alerts.Manager { + monitor := r.resolveMonitorForOrg(orgID) + if monitor == nil { + return nil + } + return monitor.GetAlertManager() +} + +func (r *Router) resolvePatrolServiceForOrg(orgID string) *ai.PatrolService { + if r.aiSettingsHandler == nil { + return nil + } + aiService := r.aiSettingsHandler.GetAIService(maintenanceVerificationOrgContext(orgID)) + if aiService == nil { + return nil + } + return aiService.GetPatrolService() +} + +func (r *Router) resolveMetricsHistoryForOrg(orgID string) *monitoring.MetricsHistory { + monitor := r.resolveMonitorForOrg(orgID) + if monitor == nil { + return nil + } + return monitor.GetMetricsHistory() +} + +func (r *Router) resolveMonitorForOrg(orgID string) *monitoring.Monitor { + orgID = strings.TrimSpace(orgID) + if orgID == "" || orgID == "default" { + return r.monitor + } + if r.mtMonitor == nil { + return nil + } + monitor, err := r.mtMonitor.GetMonitor(orgID) + if err != nil { + return nil + } + return monitor +} + +func maintenanceVerificationOrgContext(orgID string) context.Context { + if orgID == "" { + orgID = "default" + } + return context.WithValue(context.Background(), OrgIDContextKey, orgID) +} + +// alertMatchesResource reports whether the alert targets the supplied +// canonical resource id. Alert IDs are not canonical — the alert +// model carries `ResourceID` (legacy form) and `CanonicalSpecID` / +// `CanonicalState` from the canonical engine. We try the canonical +// form first, then fall back to canonicalizing the legacy form so we +// catch both providers. +func alertMatchesResource(a alerts.Alert, canonicalID string) bool { + if unified.CanonicalResourceID(a.CanonicalState) == canonicalID { + return true + } + if unified.CanonicalResourceID(a.CanonicalSpecID) == canonicalID { + return true + } + return unified.CanonicalResourceID(a.ResourceID) == canonicalID +} + +func mapAlertLevel(level alerts.AlertLevel) maintenancesentinel.Severity { + switch level { + case alerts.AlertLevelCritical: + return maintenancesentinel.SeverityCritical + case alerts.AlertLevelWarning: + return maintenancesentinel.SeverityWarning + default: + return "" + } +} + +func mapFindingSeverity(s ai.FindingSeverity) maintenancesentinel.Severity { + switch s { + case ai.FindingSeverityCritical: + return maintenancesentinel.SeverityCritical + case ai.FindingSeverityWarning: + return maintenancesentinel.SeverityWarning + default: + return "" + } +} + +// canonicalToSourceID best-effort maps a canonical resource ID into +// the legacy source-id form used by `monitoring.MetricsHistory`. +// Canonical ids look like `vm:101`, `ct:200`, `node:pve`, +// `docker-container:abc`, etc. The metrics history keys are +// `qemu/101`, `lxc/200`, `node/pve`. Resources not covered here will +// not surface metric history evidence on their report — the sentinel +// honestly reports MetricSourceAvailable=false. +func canonicalToSourceID(canonicalID string) string { + parts := strings.SplitN(canonicalID, ":", 2) + if len(parts) != 2 { + return "" + } + kind, id := parts[0], parts[1] + switch kind { + case "vm": + return "qemu/" + id + case "ct": + return "lxc/" + id + case "node": + return "node/" + id + } + return "" +} diff --git a/internal/api/route_inventory_test.go b/internal/api/route_inventory_test.go index 442b07558..939dcb446 100644 --- a/internal/api/route_inventory_test.go +++ b/internal/api/route_inventory_test.go @@ -412,6 +412,10 @@ var allRouteAllowlist = []string{ "/api/resources/{id}/facets", "/api/resources/{id}/timeline", "/api/resources/{id}/operator-state", + "/api/resources/{id}/maintenance-verifications", + "POST /api/resources/{id}/maintenance-verifications/rerun", + "/api/maintenance-verifications/{reportId}", + "POST /api/maintenance-verifications/{reportId}/review", "/api/agent/resource-context/{id}", "/api/agent/fleet-context", "/api/agent/capabilities", diff --git a/internal/api/router.go b/internal/api/router.go index d1594e287..0e2157a12 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -42,6 +42,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/alerts" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/deploy" + "github.com/rcourtman/pulse-go-rewrite/internal/maintenancesentinel" "github.com/rcourtman/pulse-go-rewrite/internal/metrics" "github.com/rcourtman/pulse-go-rewrite/internal/mock" "github.com/rcourtman/pulse-go-rewrite/internal/models" @@ -88,6 +89,8 @@ type Router struct { aiHandler *AIHandler // AI chat handler discoveryHandlers *DiscoveryHandlers resourceHandlers *ResourceHandlers + maintenanceVerificationHandlers *MaintenanceVerificationHandlers + maintenanceSentinel *maintenancesentinel.Sentinel agentContextHandler *AgentContextHandler agentEventBroadcaster *AgentEventBroadcaster resourceRegistry *unifiedresources.ResourceRegistry @@ -436,6 +439,11 @@ func (r *Router) setupRoutes() { r.unifiedAgentHandlers = NewUnifiedAgentHandlers(r.mtMonitor, r.monitor, r.wsHub) r.kubernetesAgentHandlers.SetRecoveryIngestor(r.recoveryHandlers) r.resourceHandlers = NewResourceHandlers(r.config) + r.maintenanceSentinel = r.buildMaintenanceVerificationSentinel() + r.maintenanceVerificationHandlers = NewMaintenanceVerificationHandlers(r.resourceHandlers, r.maintenanceSentinel) + if r.maintenanceSentinel != nil { + r.maintenanceSentinel.Start(r.lifecycleCtx) + } r.agentContextHandler = NewAgentContextHandler(r.resourceHandlers) // Wire pending-approvals into the bundle. The provider resolves // the approval store at request time so multi-tenant rebuilds diff --git a/internal/api/router_routes_monitoring.go b/internal/api/router_routes_monitoring.go index 4c8b6830f..1293502e4 100644 --- a/internal/api/router_routes_monitoring.go +++ b/internal/api/router_routes_monitoring.go @@ -57,6 +57,17 @@ func (r *Router) registerMonitoringResourceRoutes( } r.resourceHandlers.HandleResourceOperatorState(w, req) })) + // Per-resource Maintenance Verification Reports. Listing and reading + // are monitoring-read; rerun is monitoring-write because it produces + // a new persisted record. Review (mark-reviewed) is monitoring-write + // for the same reason. Wired before the broad `/api/resources/` + // catch-all so the more specific paths route here. + if r.maintenanceVerificationHandlers != nil { + r.mux.HandleFunc("/api/resources/{id}/maintenance-verifications", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.maintenanceVerificationHandlers.HandleListForResource))) + r.mux.HandleFunc("POST /api/resources/{id}/maintenance-verifications/rerun", RequireAuth(r.config, RequireScope(config.ScopeMonitoringWrite, r.maintenanceVerificationHandlers.HandleRerun))) + r.mux.HandleFunc("/api/maintenance-verifications/{reportId}", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.maintenanceVerificationHandlers.HandleGet))) + r.mux.HandleFunc("POST /api/maintenance-verifications/{reportId}/review", RequireAuth(r.config, RequireScope(config.ScopeMonitoringWrite, r.maintenanceVerificationHandlers.HandleReview))) + } r.mux.HandleFunc("/api/resources/", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.resourceHandlers.HandleResourceRoutes))) // Agent-consumable bundled context — substrate for any LLM agent // (in-process Patrol/Assistant or external) that needs the full diff --git a/internal/maintenancesentinel/sentinel.go b/internal/maintenancesentinel/sentinel.go new file mode 100644 index 000000000..3855a9c1e --- /dev/null +++ b/internal/maintenancesentinel/sentinel.go @@ -0,0 +1,288 @@ +package maintenancesentinel + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rs/zerolog/log" +) + +// Providers bundles the slim, package-defined interfaces the sentinel +// needs from the rest of the system. The caller (router wiring) is +// responsible for adapting alerts/findings/actions/metrics into these +// shapes. Keeps this package free of `internal/alerts`, `internal/ai`, +// and `internal/monitoring` imports so there are no cycles. +type Providers struct { + // Stores resolves the ResourceStore for the given org id. The + // sentinel runs against the default org for MVP; tenant-scoped + // scheduling is left to a future change. + Stores func(orgID string) (unified.ResourceStore, error) + // ActiveAlerts returns active alerts for a resource at this moment. + ActiveAlerts func(orgID, canonicalID string) []AlertSummary + // ActiveFindings returns active Patrol findings for a resource. + ActiveFindings func(orgID, canonicalID string) []FindingSummary + // RecentActions returns action audit records for the resource + // since the supplied lower bound. The sentinel itself filters for + // failed state after the call returns. + RecentActions func(orgID, canonicalID string, since time.Time) []ActionSummary + // PostWindowMetricSamples returns metric samples observed after + // the maintenance window closed for the resource. The boolean + // reports whether a metric source was available for the + // resource — distinct from "available but empty". + PostWindowMetricSamples func(orgID, canonicalID string, windowEnd, now time.Time) ([]MetricSample, bool) + // Now is the clock; tests inject a fixed clock. Defaults to + // time.Now. + Now func() time.Time +} + +// Sentinel watches operator state for maintenance-window-end events +// and writes Maintenance Verification Reports. +type Sentinel struct { + orgID string + providers Providers + tick time.Duration + lookbackLimit time.Duration + + mu sync.Mutex +} + +// Config configures a Sentinel instance. +type Config struct { + // OrgID is the tenant the sentinel scans. MVP runs the default + // org only — multi-tenant scheduling is deferred. + OrgID string + // Tick controls the sweep cadence. Defaults to one minute. + Tick time.Duration + // LookbackLimit prevents the sentinel from generating reports + // for ancient windows on first start (e.g. after a long + // downtime). Window-end events older than `now - LookbackLimit` + // are ignored. Default: 7 days. + LookbackLimit time.Duration +} + +// New returns a Sentinel ready to start. Providers is required. +func New(cfg Config, providers Providers) (*Sentinel, error) { + if providers.Stores == nil { + return nil, errors.New("maintenancesentinel: Providers.Stores is required") + } + tick := cfg.Tick + if tick <= 0 { + tick = time.Minute + } + lookback := cfg.LookbackLimit + if lookback <= 0 { + lookback = 7 * 24 * time.Hour + } + org := cfg.OrgID + if org == "" { + org = "default" + } + return &Sentinel{ + orgID: org, + providers: providers, + tick: tick, + lookbackLimit: lookback, + }, nil +} + +// Start runs the sentinel loop in a goroutine and returns. The loop +// stops when ctx is canceled. Safe to call once per Sentinel. +func (s *Sentinel) Start(ctx context.Context) { + go s.run(ctx) +} + +func (s *Sentinel) run(ctx context.Context) { + ticker := time.NewTicker(s.tick) + defer ticker.Stop() + // Run one sweep immediately so a newly-ended window doesn't have + // to wait a full tick for a report. + s.tickOnce(ctx) + for { + select { + case <-ctx.Done(): + log.Info().Msg("maintenance-verification sentinel stopped") + return + case <-ticker.C: + s.tickOnce(ctx) + } + } +} + +func (s *Sentinel) tickOnce(ctx context.Context) { + s.mu.Lock() + defer s.mu.Unlock() + + store, err := s.providers.Stores(s.orgID) + if err != nil { + log.Debug().Err(err).Msg("maintenance-verification sentinel: get store") + return + } + states, err := store.ListResourceOperatorStates() + if err != nil { + log.Debug().Err(err).Msg("maintenance-verification sentinel: list operator states") + return + } + + now := s.now() + cutoff := now.Add(-s.lookbackLimit) + + for _, state := range states { + if ctx.Err() != nil { + return + } + if state.MaintenanceStartAt == nil || state.MaintenanceEndAt == nil { + continue + } + if !state.MaintenanceEndAt.Before(now) && !state.MaintenanceEndAt.Equal(now) { + // Window hasn't ended yet. + continue + } + if state.MaintenanceEndAt.Before(cutoff) { + // Window ended too long ago — don't backfill ancient events. + continue + } + s.evaluateForState(state, store, now) + } +} + +func (s *Sentinel) evaluateForState(state unified.ResourceOperatorState, store unified.ResourceStore, now time.Time) { + canonicalID := unified.CanonicalResourceID(state.CanonicalID) + if canonicalID == "" || state.MaintenanceEndAt == nil { + return + } + if _, exists, err := store.FindLoopReportByWindow(unified.LoopReportTypeMaintenanceVerification, canonicalID, *state.MaintenanceEndAt); err != nil { + log.Debug().Err(err).Str("resource", canonicalID).Msg("maintenance-verification sentinel: dedupe lookup") + return + } else if exists { + // Already wrote a report for this (resource, window-end). + return + } + + inputs := s.buildInputs(state, now) + report := EvaluateVerification(inputs) + if err := store.RecordLoopReport(report); err != nil { + // A unique-constraint conflict (race against a parallel + // tick) is benign — another writer wrote the same window. + log.Debug().Err(err).Str("resource", canonicalID).Msg("maintenance-verification sentinel: record report") + return + } + log.Info(). + Str("resource", canonicalID). + Str("status", string(report.Status)). + Str("report_id", report.ID). + Msg("maintenance verification report written") +} + +// buildInputs gathers the deterministic input bundle for the +// evaluator. Provider closures may be nil — the inputs simply carry +// zero values in that case. +func (s *Sentinel) buildInputs(state unified.ResourceOperatorState, now time.Time) VerificationInputs { + canonicalID := unified.CanonicalResourceID(state.CanonicalID) + inputs := VerificationInputs{ + ResourceID: canonicalID, + OperatorState: cloneOperatorState(state), + Now: now, + } + if state.MaintenanceStartAt != nil { + inputs.WindowStartedAt = state.MaintenanceStartAt.UTC() + } + if state.MaintenanceEndAt != nil { + inputs.WindowEndedAt = state.MaintenanceEndAt.UTC() + } + if s.providers.ActiveAlerts != nil { + inputs.ActiveAlerts = s.providers.ActiveAlerts(s.orgID, canonicalID) + } + if s.providers.ActiveFindings != nil { + inputs.ActiveFindings = s.providers.ActiveFindings(s.orgID, canonicalID) + } + if s.providers.RecentActions != nil { + windowStart := inputs.WindowStartedAt + if windowStart.IsZero() { + windowStart = now.Add(-24 * time.Hour) + } + inputs.RecentActions = s.providers.RecentActions(s.orgID, canonicalID, windowStart) + } + if s.providers.PostWindowMetricSamples != nil { + samples, available := s.providers.PostWindowMetricSamples(s.orgID, canonicalID, inputs.WindowEndedAt, now) + inputs.PostWindowMetricSamples = samples + inputs.MetricSourceAvailable = available + } + return inputs +} + +func cloneOperatorState(s unified.ResourceOperatorState) *unified.ResourceOperatorState { + clone := s + if s.MaintenanceStartAt != nil { + t := *s.MaintenanceStartAt + clone.MaintenanceStartAt = &t + } + if s.MaintenanceEndAt != nil { + t := *s.MaintenanceEndAt + clone.MaintenanceEndAt = &t + } + return &clone +} + +func (s *Sentinel) now() time.Time { + if s.providers.Now != nil { + return s.providers.Now().UTC() + } + return time.Now().UTC() +} + +// EvaluateOnce runs the deterministic verification for the supplied +// resource exactly once, immediately, writing a fresh report. Used by +// the "rerun verification" handler so an operator can re-evaluate +// without waiting for the next tick. +// +// If the existing report for the (resource, window-end) triple is +// found, this writes a *new* report with a -rerun-N suffix so the +// review history is preserved. +func (s *Sentinel) EvaluateOnce(ctx context.Context, canonicalID string) (unified.LoopReport, error) { + if ctx.Err() != nil { + return unified.LoopReport{}, ctx.Err() + } + canonicalID = unified.CanonicalResourceID(canonicalID) + if canonicalID == "" { + return unified.LoopReport{}, fmt.Errorf("maintenancesentinel: canonical id is required") + } + store, err := s.providers.Stores(s.orgID) + if err != nil { + return unified.LoopReport{}, err + } + state, found, err := store.GetResourceOperatorState(canonicalID) + if err != nil { + return unified.LoopReport{}, err + } + if !found { + return unified.LoopReport{}, fmt.Errorf("maintenancesentinel: no operator state for %q", canonicalID) + } + if state.MaintenanceStartAt == nil || state.MaintenanceEndAt == nil { + return unified.LoopReport{}, fmt.Errorf("maintenancesentinel: resource %q has no maintenance window to verify", canonicalID) + } + now := s.now() + inputs := s.buildInputs(state, now) + report := EvaluateVerification(inputs) + report.ID = uniqueRerunID(store, report.ID) + if err := store.RecordLoopReport(report); err != nil { + return unified.LoopReport{}, err + } + return report, nil +} + +func uniqueRerunID(store unified.ResourceStore, base string) string { + if _, exists, err := store.GetLoopReport(base); err == nil && !exists { + return base + } + for i := 1; i < 1000; i++ { + candidate := fmt.Sprintf("%s-rerun-%d", base, i) + if _, exists, err := store.GetLoopReport(candidate); err == nil && !exists { + return candidate + } + } + return fmt.Sprintf("%s-rerun-%d", base, time.Now().UnixNano()) +} diff --git a/internal/maintenancesentinel/sentinel_test.go b/internal/maintenancesentinel/sentinel_test.go new file mode 100644 index 000000000..4ce313e28 --- /dev/null +++ b/internal/maintenancesentinel/sentinel_test.go @@ -0,0 +1,191 @@ +package maintenancesentinel + +import ( + "context" + "strings" + "testing" + "time" + + unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +func TestSentinelTickOnceWritesReportAndDedupes(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + windowStart := now.Add(-time.Hour) + windowEnd := now.Add(-15 * time.Minute) + + store := unified.NewMemoryStore() + if err := store.SetResourceOperatorState(unified.ResourceOperatorState{ + CanonicalID: "vm:101", + MaintenanceStartAt: &windowStart, + MaintenanceEndAt: &windowEnd, + SetAt: windowStart, + SetBy: "operator", + }); err != nil { + t.Fatalf("seed operator state: %v", err) + } + + providers := Providers{ + Stores: func(orgID string) (unified.ResourceStore, error) { return store, nil }, + Now: func() time.Time { return now }, + } + sentinel, err := New(Config{OrgID: "default", Tick: time.Minute}, providers) + if err != nil { + t.Fatalf("new sentinel: %v", err) + } + + sentinel.tickOnce(context.Background()) + + reports, err := store.ListLoopReportsForResource(unified.LoopReportTypeMaintenanceVerification, "vm:101", 0) + if err != nil { + t.Fatalf("list reports: %v", err) + } + if len(reports) != 1 { + t.Fatalf("expected 1 report after first tick, got %d", len(reports)) + } + + sentinel.tickOnce(context.Background()) + + reports, err = store.ListLoopReportsForResource(unified.LoopReportTypeMaintenanceVerification, "vm:101", 0) + if err != nil { + t.Fatalf("list reports: %v", err) + } + if len(reports) != 1 { + t.Fatalf("expected dedupe; got %d reports after second tick", len(reports)) + } +} + +func TestSentinelSkipsAncientWindows(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + windowStart := now.Add(-30 * 24 * time.Hour) + windowEnd := now.Add(-29 * 24 * time.Hour) + + store := unified.NewMemoryStore() + if err := store.SetResourceOperatorState(unified.ResourceOperatorState{ + CanonicalID: "vm:202", + MaintenanceStartAt: &windowStart, + MaintenanceEndAt: &windowEnd, + SetAt: windowStart, + SetBy: "operator", + }); err != nil { + t.Fatalf("seed operator state: %v", err) + } + + providers := Providers{ + Stores: func(orgID string) (unified.ResourceStore, error) { return store, nil }, + Now: func() time.Time { return now }, + } + sentinel, err := New(Config{OrgID: "default", LookbackLimit: 7 * 24 * time.Hour}, providers) + if err != nil { + t.Fatalf("new sentinel: %v", err) + } + sentinel.tickOnce(context.Background()) + + reports, err := store.ListLoopReportsForResource(unified.LoopReportTypeMaintenanceVerification, "vm:202", 0) + if err != nil { + t.Fatalf("list reports: %v", err) + } + if len(reports) != 0 { + t.Fatalf("expected no report for ancient window, got %d", len(reports)) + } +} + +func TestSentinelSkipsOpenWindow(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + windowStart := now.Add(-30 * time.Minute) + windowEnd := now.Add(30 * time.Minute) + + store := unified.NewMemoryStore() + if err := store.SetResourceOperatorState(unified.ResourceOperatorState{ + CanonicalID: "vm:303", + MaintenanceStartAt: &windowStart, + MaintenanceEndAt: &windowEnd, + SetAt: windowStart, + SetBy: "operator", + }); err != nil { + t.Fatalf("seed operator state: %v", err) + } + + providers := Providers{ + Stores: func(orgID string) (unified.ResourceStore, error) { return store, nil }, + Now: func() time.Time { return now }, + } + sentinel, err := New(Config{OrgID: "default"}, providers) + if err != nil { + t.Fatalf("new sentinel: %v", err) + } + sentinel.tickOnce(context.Background()) + + reports, err := store.ListLoopReportsForResource(unified.LoopReportTypeMaintenanceVerification, "vm:303", 0) + if err != nil { + t.Fatalf("list reports: %v", err) + } + if len(reports) != 0 { + t.Fatalf("expected no report while window still open, got %d", len(reports)) + } +} + +func TestSentinelEvaluateOncePersistsRerunReport(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + windowStart := now.Add(-time.Hour) + windowEnd := now.Add(-15 * time.Minute) + + store := unified.NewMemoryStore() + if err := store.SetResourceOperatorState(unified.ResourceOperatorState{ + CanonicalID: "vm:101", + MaintenanceStartAt: &windowStart, + MaintenanceEndAt: &windowEnd, + SetAt: windowStart, + SetBy: "operator", + }); err != nil { + t.Fatalf("seed operator state: %v", err) + } + + providers := Providers{ + Stores: func(orgID string) (unified.ResourceStore, error) { return store, nil }, + Now: func() time.Time { return now }, + } + sentinel, err := New(Config{OrgID: "default"}, providers) + if err != nil { + t.Fatalf("new sentinel: %v", err) + } + sentinel.tickOnce(context.Background()) + + rerun, err := sentinel.EvaluateOnce(context.Background(), "vm:101") + if err != nil { + t.Fatalf("evaluate once: %v", err) + } + if !strings.Contains(rerun.ID, "rerun") { + t.Fatalf("expected rerun id suffix, got %q", rerun.ID) + } + reports, err := store.ListLoopReportsForResource(unified.LoopReportTypeMaintenanceVerification, "vm:101", 0) + if err != nil { + t.Fatalf("list reports: %v", err) + } + if len(reports) != 2 { + t.Fatalf("expected 2 reports after rerun, got %d", len(reports)) + } +} + +func TestSentinelEvaluateOnceErrorsWhenNoWindow(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + store := unified.NewMemoryStore() + if err := store.SetResourceOperatorState(unified.ResourceOperatorState{ + CanonicalID: "vm:404", + SetAt: now, + SetBy: "operator", + }); err != nil { + t.Fatalf("seed operator state: %v", err) + } + providers := Providers{ + Stores: func(orgID string) (unified.ResourceStore, error) { return store, nil }, + Now: func() time.Time { return now }, + } + sentinel, err := New(Config{OrgID: "default"}, providers) + if err != nil { + t.Fatalf("new sentinel: %v", err) + } + if _, err := sentinel.EvaluateOnce(context.Background(), "vm:404"); err == nil { + t.Fatalf("expected error when no maintenance window present") + } +} diff --git a/internal/maintenancesentinel/verification.go b/internal/maintenancesentinel/verification.go new file mode 100644 index 000000000..890abddce --- /dev/null +++ b/internal/maintenancesentinel/verification.go @@ -0,0 +1,400 @@ +// Package maintenancesentinel implements the Maintenance Verification +// Report loop. When a maintenance window ends for a resource, the +// sentinel runs deterministic checks against the resource's current +// state and writes a durable Maintenance Verification Report. +// +// The package is intentionally small and shaped so future loops +// (post-incident verification, post-deployment verification) can reuse +// the same persistence (LoopReport) and review actions. This is +// substrate, not a generic agent framework. +package maintenancesentinel + +import ( + "fmt" + "strings" + "time" + + unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +// Severity classifies an alert or finding for the verification +// decision tree. Maintenance Verification only distinguishes critical +// vs. warning — the deeper severity grades (watch/info) are not +// load-bearing for the decision. +type Severity string + +const ( + SeverityCritical Severity = "critical" + SeverityWarning Severity = "warning" +) + +// AlertSummary is the slim projection of an alert the sentinel needs. +// The router adapts the canonical `alerts.Alert` into this shape so +// this package does not import `internal/alerts`. +type AlertSummary struct { + ID string + Severity Severity + Type string + Acknowledged bool +} + +// FindingSummary is the slim projection of a Patrol finding the +// sentinel needs. The router adapts `ai.Finding` into this shape so +// this package does not import `internal/ai`. +type FindingSummary struct { + ID string + Severity Severity + Category string + Resolved bool + Acknowledged bool +} + +// ActionSummary is the slim projection of an action audit record the +// sentinel needs. +type ActionSummary struct { + ID string + State string + UpdatedAt time.Time +} + +// MetricSample is a single recent metric data point used to decide +// whether a resource is reporting metrics after maintenance closed. +type MetricSample struct { + Metric string + Value float64 + Timestamp time.Time +} + +// VerificationInputs is the full deterministic input bundle the +// evaluator consumes. The sentinel collects this once before calling +// EvaluateVerification. +type VerificationInputs struct { + ResourceID string + ResourceName string + OperatorState *unified.ResourceOperatorState + WindowStartedAt time.Time + WindowEndedAt time.Time + Now time.Time + ActiveAlerts []AlertSummary + ActiveFindings []FindingSummary + // RecentActions should be the action audits with UpdatedAt >= + // WindowStartedAt. The evaluator counts how many of them ended in + // "failed" state. + RecentActions []ActionSummary + // PostWindowMetricSamples are the metric samples observed after + // WindowEndedAt. Empty slice means no metrics have been reported + // since the window closed. + PostWindowMetricSamples []MetricSample + // MetricSourceAvailable indicates whether the sentinel had access + // to a metric source for this resource at all. Distinguishes + // "we looked and saw nothing" from "we have no way to look". + MetricSourceAvailable bool +} + +// EvaluateVerification applies the deterministic decision tree to the +// inputs and returns a fully-populated LoopReport. The function is +// pure: no I/O, no time.Now() (always uses inputs.Now), no goroutines. +func EvaluateVerification(inputs VerificationInputs) unified.LoopReport { + now := inputs.Now + if now.IsZero() { + now = time.Now().UTC() + } + now = now.UTC() + + canonicalID := unified.CanonicalResourceID(inputs.ResourceID) + report := unified.LoopReport{ + ID: newReportID(canonicalID, inputs.WindowEndedAt), + Type: unified.LoopReportTypeMaintenanceVerification, + Scope: canonicalID, + Trigger: "maintenance_window_end", + Goal: "Confirm the resource recovered after the maintenance window closed.", + Status: unified.LoopReportStatusPending, + StartedAt: now, + CompletedAt: now, + } + if !inputs.WindowStartedAt.IsZero() { + t := inputs.WindowStartedAt.UTC() + report.WindowStartedAt = &t + } + if !inputs.WindowEndedAt.IsZero() { + t := inputs.WindowEndedAt.UTC() + report.WindowEndedAt = &t + } + + evidence := unified.LoopReportEvidence{ + OperatorStateSummary: summarizeOperatorState(inputs.OperatorState, inputs.WindowEndedAt, now), + } + + criticalAlerts, warningAlerts, alertIDs := classifyAlerts(inputs.ActiveAlerts) + evidence.ActiveCriticalAlerts = criticalAlerts + evidence.ActiveWarningAlerts = warningAlerts + report.LinkedAlertIDs = alertIDs + + criticalFindings, warningFindings, findingIDs := classifyFindings(inputs.ActiveFindings) + evidence.ActiveCriticalFindings = criticalFindings + evidence.ActiveWarningFindings = warningFindings + report.LinkedFindingIDs = findingIDs + + failedActions, failedActionIDs := classifyFailedActions(inputs.RecentActions, inputs.WindowStartedAt) + evidence.FailedActionsSinceWindowStart = failedActions + report.LinkedActionIDs = failedActionIDs + + if inputs.MetricSourceAvailable { + evidence.MetricRecovery = summarizeMetricRecovery(inputs.PostWindowMetricSamples, inputs.WindowEndedAt, now) + } + + // Patrol-run trigger is deferred — see PatrolRunTODO contract. + evidence.PatrolRunTODO = patrolRunTODO(inputs, evidence) + + report.Evidence = evidence + report.Status, report.Recommendation = decideStatus(inputs, evidence) + return report +} + +func summarizeOperatorState(state *unified.ResourceOperatorState, windowEnd, now time.Time) string { + if state == nil { + return "no operator state recorded for this resource" + } + parts := []string{} + if state.MaintenanceStartAt != nil && state.MaintenanceEndAt != nil { + if state.MaintenanceEndAt.After(now) { + parts = append(parts, "maintenance window still open") + } else { + parts = append(parts, "maintenance window ended") + } + } else { + parts = append(parts, "no maintenance window") + } + if state.IntentionallyOffline { + parts = append(parts, "intentionally offline") + } + if state.NeverAutoRemediate { + parts = append(parts, "never auto-remediate") + } + _ = windowEnd + return strings.Join(parts, "; ") +} + +func classifyAlerts(alerts []AlertSummary) (criticalCount, warningCount int, ids []string) { + for _, a := range alerts { + switch a.Severity { + case SeverityCritical: + criticalCount++ + case SeverityWarning: + warningCount++ + } + if a.ID != "" { + ids = append(ids, a.ID) + } + } + return criticalCount, warningCount, ids +} + +func classifyFindings(findings []FindingSummary) (criticalCount, warningCount int, ids []string) { + for _, f := range findings { + if f.Resolved { + continue + } + switch f.Severity { + case SeverityCritical: + criticalCount++ + case SeverityWarning: + warningCount++ + } + if f.ID != "" { + ids = append(ids, f.ID) + } + } + return criticalCount, warningCount, ids +} + +func classifyFailedActions(actions []ActionSummary, windowStart time.Time) (count int, ids []string) { + for _, a := range actions { + if a.State != "failed" { + continue + } + if !windowStart.IsZero() && a.UpdatedAt.Before(windowStart) { + continue + } + count++ + if a.ID != "" { + ids = append(ids, a.ID) + } + } + return count, ids +} + +func summarizeMetricRecovery(samples []MetricSample, windowEnd, now time.Time) *unified.MetricRecoveryEvidence { + rec := &unified.MetricRecoveryEvidence{} + metrics := map[string]struct{}{} + relevant := make([]MetricSample, 0, len(samples)) + for _, s := range samples { + if !s.Timestamp.After(windowEnd) { + continue + } + relevant = append(relevant, s) + if s.Metric != "" { + metrics[s.Metric] = struct{}{} + } + } + rec.SamplesAfterEnd = len(relevant) + if len(metrics) > 0 { + rec.MetricsObserved = make([]string, 0, len(metrics)) + for m := range metrics { + rec.MetricsObserved = append(rec.MetricsObserved, m) + } + } + if len(relevant) == 0 { + rec.Trend = "unknown" + rec.Note = "no metric samples have been recorded since the window closed" + return rec + } + rec.Trend = trendFromSamples(relevant) + if rec.Trend == "stable" { + rec.Note = fmt.Sprintf("%d post-window samples observed; values within expected band", len(relevant)) + } + return rec +} + +// trendFromSamples reports a simple plain-English trend label by +// comparing the first half of samples to the second half. The MVP is +// deterministic and crude on purpose — anomaly detection is the +// baseline store's job, not the sentinel's. +func trendFromSamples(samples []MetricSample) string { + if len(samples) < 2 { + return "unknown" + } + byMetric := map[string][]MetricSample{} + for _, s := range samples { + byMetric[s.Metric] = append(byMetric[s.Metric], s) + } + var trends []string + for _, ms := range byMetric { + if len(ms) < 2 { + trends = append(trends, "unknown") + continue + } + mid := len(ms) / 2 + firstAvg := averageValue(ms[:mid]) + secondAvg := averageValue(ms[mid:]) + // Threshold is intentionally wide — the trend label only + // changes the report's note, not its overall status. + switch { + case firstAvg == 0 && secondAvg == 0: + trends = append(trends, "stable") + case secondAvg <= firstAvg*1.05+0.01: + trends = append(trends, "stable") + case secondAvg > firstAvg*1.5: + trends = append(trends, "degrading") + default: + trends = append(trends, "stable") + } + } + // Combine: any degrading wins, else stable. + for _, t := range trends { + if t == "degrading" { + return "degrading" + } + } + return "stable" +} + +func averageValue(samples []MetricSample) float64 { + if len(samples) == 0 { + return 0 + } + var sum float64 + for _, s := range samples { + sum += s.Value + } + return sum / float64(len(samples)) +} + +func patrolRunTODO(inputs VerificationInputs, evidence unified.LoopReportEvidence) string { + // If the deterministic evidence is unambiguous (clear failure or + // clear pass), there's nothing for Patrol to add. + if evidence.ActiveCriticalAlerts > 0 || evidence.ActiveCriticalFindings > 0 || evidence.FailedActionsSinceWindowStart > 0 { + return "" + } + if evidence.ActiveWarningAlerts == 0 && evidence.ActiveWarningFindings == 0 { + if evidence.MetricRecovery != nil && evidence.MetricRecovery.SamplesAfterEnd > 0 { + return "" + } + } + _ = inputs + // Ambiguous case: a scoped Patrol run would help. + // + // MVP: deterministic checks only. The Patrol API surfaces a global + // scheduled run today; a scoped per-resource trigger that does + // not race the global scheduler is not yet available. Until that + // lands, we leave a TODO breadcrumb on the report so the operator + // knows verification was conservative, and so future work can + // pick this up cleanly. + return "scoped Patrol run not triggered — per-resource scoped run entrypoint is not yet implemented; review evidence manually" +} + +func decideStatus(inputs VerificationInputs, evidence unified.LoopReportEvidence) (unified.LoopReportStatus, string) { + // Strong negative signals → failed verification. + if evidence.ActiveCriticalAlerts > 0 { + return unified.LoopReportStatusFailedVerification, + "Critical alert(s) are active after maintenance closed. Investigate before clearing operator state." + } + if evidence.ActiveCriticalFindings > 0 { + return unified.LoopReportStatusFailedVerification, + "Critical Patrol finding(s) are active. Resource has not recovered cleanly." + } + if evidence.FailedActionsSinceWindowStart > 0 { + return unified.LoopReportStatusFailedVerification, + "One or more recovery actions failed during or after the maintenance window. Review the action audit timeline." + } + + // Ambiguous signals → needs review. + if inputs.OperatorState != nil && inputs.OperatorState.IntentionallyOffline { + return unified.LoopReportStatusNeedsReview, + "Resource is marked intentionally offline. Verification cannot confirm recovery automatically." + } + if evidence.ActiveWarningAlerts > 0 || evidence.ActiveWarningFindings > 0 { + return unified.LoopReportStatusNeedsReview, + "Warning-level signals are active after maintenance closed. Operator review recommended." + } + if evidence.MetricRecovery != nil { + if evidence.MetricRecovery.SamplesAfterEnd == 0 { + return unified.LoopReportStatusNeedsReview, + "No metric samples have been recorded since the window closed. Confirm the resource is reporting again." + } + if evidence.MetricRecovery.Trend == "degrading" { + return unified.LoopReportStatusNeedsReview, + "Post-window metric trend is degrading. Operator review recommended." + } + } else { + return unified.LoopReportStatusNeedsReview, + "No metric source was available for this resource. Confirm recovery manually." + } + + return unified.LoopReportStatusHealthy, + "All deterministic checks passed. No active alerts, findings, or failed actions since the window started." +} + +func newReportID(canonicalID string, windowEndedAt time.Time) string { + ts := windowEndedAt.UTC().Format("20060102T150405Z") + if windowEndedAt.IsZero() { + ts = time.Now().UTC().Format("20060102T150405Z") + } + return fmt.Sprintf("mv-%s-%s", sanitizeIDComponent(canonicalID), ts) +} + +func sanitizeIDComponent(s string) string { + var b strings.Builder + b.Grow(len(s)) + for _, r := range s { + switch { + case (r >= 'a' && r <= 'z'), (r >= 'A' && r <= 'Z'), (r >= '0' && r <= '9'): + b.WriteRune(r) + case r == '-' || r == '_' || r == '.': + b.WriteRune(r) + default: + b.WriteRune('_') + } + } + return b.String() +} diff --git a/internal/maintenancesentinel/verification_test.go b/internal/maintenancesentinel/verification_test.go new file mode 100644 index 000000000..ed79223b6 --- /dev/null +++ b/internal/maintenancesentinel/verification_test.go @@ -0,0 +1,194 @@ +package maintenancesentinel + +import ( + "strings" + "testing" + "time" + + unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +func TestEvaluateVerification_HealthyWhenNoSignals(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + windowStart := now.Add(-30 * time.Minute) + windowEnd := now.Add(-10 * time.Minute) + inputs := VerificationInputs{ + ResourceID: "vm:101", + OperatorState: &unified.ResourceOperatorState{CanonicalID: "vm:101", MaintenanceStartAt: &windowStart, MaintenanceEndAt: &windowEnd}, + WindowStartedAt: windowStart, + WindowEndedAt: windowEnd, + Now: now, + MetricSourceAvailable: true, + PostWindowMetricSamples: []MetricSample{ + {Metric: "cpu", Value: 12, Timestamp: windowEnd.Add(time.Minute)}, + {Metric: "cpu", Value: 13, Timestamp: windowEnd.Add(2 * time.Minute)}, + {Metric: "memory", Value: 40, Timestamp: windowEnd.Add(time.Minute)}, + {Metric: "memory", Value: 41, Timestamp: windowEnd.Add(2 * time.Minute)}, + }, + } + report := EvaluateVerification(inputs) + if report.Status != unified.LoopReportStatusHealthy { + t.Fatalf("status = %q want %q (recommendation=%q)", report.Status, unified.LoopReportStatusHealthy, report.Recommendation) + } + if report.Type != unified.LoopReportTypeMaintenanceVerification { + t.Fatalf("type = %q want %q", report.Type, unified.LoopReportTypeMaintenanceVerification) + } + if report.Scope != "vm:101" { + t.Fatalf("scope = %q want vm:101", report.Scope) + } + if report.WindowEndedAt == nil || !report.WindowEndedAt.Equal(windowEnd) { + t.Fatalf("windowEndedAt = %v want %v", report.WindowEndedAt, windowEnd) + } + if !strings.HasPrefix(report.ID, "mv-vm_101-") { + t.Fatalf("report id = %q want prefix mv-vm_101-", report.ID) + } +} + +func TestEvaluateVerification_FailedOnCriticalAlert(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + windowStart := now.Add(-time.Hour) + windowEnd := now.Add(-15 * time.Minute) + inputs := VerificationInputs{ + ResourceID: "vm:101", + WindowStartedAt: windowStart, + WindowEndedAt: windowEnd, + Now: now, + ActiveAlerts: []AlertSummary{ + {ID: "alert-1", Severity: SeverityCritical, Type: "cpu"}, + {ID: "alert-2", Severity: SeverityWarning, Type: "memory"}, + }, + MetricSourceAvailable: true, + } + report := EvaluateVerification(inputs) + if report.Status != unified.LoopReportStatusFailedVerification { + t.Fatalf("status = %q want failed_verification", report.Status) + } + if report.Evidence.ActiveCriticalAlerts != 1 || report.Evidence.ActiveWarningAlerts != 1 { + t.Fatalf("alert counts = %d/%d want 1/1", report.Evidence.ActiveCriticalAlerts, report.Evidence.ActiveWarningAlerts) + } + if len(report.LinkedAlertIDs) != 2 { + t.Fatalf("linked alert ids = %v want 2 entries", report.LinkedAlertIDs) + } +} + +func TestEvaluateVerification_NeedsReviewOnNoMetrics(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + windowStart := now.Add(-time.Hour) + windowEnd := now.Add(-15 * time.Minute) + inputs := VerificationInputs{ + ResourceID: "ct:200", + WindowStartedAt: windowStart, + WindowEndedAt: windowEnd, + Now: now, + MetricSourceAvailable: true, + } + report := EvaluateVerification(inputs) + if report.Status != unified.LoopReportStatusNeedsReview { + t.Fatalf("status = %q want needs_review", report.Status) + } + if report.Evidence.MetricRecovery == nil || report.Evidence.MetricRecovery.SamplesAfterEnd != 0 { + t.Fatalf("metric recovery = %+v", report.Evidence.MetricRecovery) + } +} + +func TestEvaluateVerification_FailedOnFailedAction(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + windowStart := now.Add(-time.Hour) + windowEnd := now.Add(-15 * time.Minute) + inputs := VerificationInputs{ + ResourceID: "vm:101", + WindowStartedAt: windowStart, + WindowEndedAt: windowEnd, + Now: now, + RecentActions: []ActionSummary{ + {ID: "action-1", State: "failed", UpdatedAt: windowEnd.Add(-time.Minute)}, + {ID: "action-2", State: "succeeded", UpdatedAt: windowEnd.Add(time.Minute)}, + }, + MetricSourceAvailable: true, + } + report := EvaluateVerification(inputs) + if report.Status != unified.LoopReportStatusFailedVerification { + t.Fatalf("status = %q want failed_verification", report.Status) + } + if report.Evidence.FailedActionsSinceWindowStart != 1 { + t.Fatalf("failed actions = %d want 1", report.Evidence.FailedActionsSinceWindowStart) + } + if len(report.LinkedActionIDs) != 1 || report.LinkedActionIDs[0] != "action-1" { + t.Fatalf("linked action ids = %v want [action-1]", report.LinkedActionIDs) + } +} + +func TestEvaluateVerification_NeedsReviewWhenIntentionallyOffline(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + windowStart := now.Add(-time.Hour) + windowEnd := now.Add(-15 * time.Minute) + inputs := VerificationInputs{ + ResourceID: "vm:101", + OperatorState: &unified.ResourceOperatorState{ + CanonicalID: "vm:101", + IntentionallyOffline: true, + MaintenanceStartAt: &windowStart, + MaintenanceEndAt: &windowEnd, + }, + WindowStartedAt: windowStart, + WindowEndedAt: windowEnd, + Now: now, + MetricSourceAvailable: true, + PostWindowMetricSamples: []MetricSample{ + {Metric: "cpu", Value: 5, Timestamp: windowEnd.Add(time.Minute)}, + {Metric: "cpu", Value: 6, Timestamp: windowEnd.Add(2 * time.Minute)}, + }, + } + report := EvaluateVerification(inputs) + if report.Status != unified.LoopReportStatusNeedsReview { + t.Fatalf("status = %q want needs_review", report.Status) + } + if !strings.Contains(report.Evidence.OperatorStateSummary, "intentionally offline") { + t.Fatalf("operator state summary = %q want to mention intentionally offline", report.Evidence.OperatorStateSummary) + } +} + +func TestEvaluateVerification_FilesPatrolTODOWhenWarningsOnly(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + windowStart := now.Add(-time.Hour) + windowEnd := now.Add(-15 * time.Minute) + inputs := VerificationInputs{ + ResourceID: "vm:101", + WindowStartedAt: windowStart, + WindowEndedAt: windowEnd, + Now: now, + ActiveFindings: []FindingSummary{ + {ID: "finding-1", Severity: SeverityWarning, Category: "performance"}, + }, + MetricSourceAvailable: true, + } + report := EvaluateVerification(inputs) + if report.Status != unified.LoopReportStatusNeedsReview { + t.Fatalf("status = %q want needs_review", report.Status) + } + if report.Evidence.PatrolRunTODO == "" { + t.Fatalf("expected patrol run TODO breadcrumb on ambiguous evidence") + } + if len(report.LinkedFindingIDs) != 1 { + t.Fatalf("linked finding ids = %v want 1", report.LinkedFindingIDs) + } +} + +func TestEvaluateVerification_NeedsReviewWhenNoMetricSource(t *testing.T) { + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + windowStart := now.Add(-time.Hour) + windowEnd := now.Add(-15 * time.Minute) + inputs := VerificationInputs{ + ResourceID: "storage:tank", + WindowStartedAt: windowStart, + WindowEndedAt: windowEnd, + Now: now, + } + report := EvaluateVerification(inputs) + if report.Status != unified.LoopReportStatusNeedsReview { + t.Fatalf("status = %q want needs_review when no metric source", report.Status) + } + if report.Evidence.MetricRecovery != nil { + t.Fatalf("metric recovery = %+v want nil when source unavailable", report.Evidence.MetricRecovery) + } +} diff --git a/internal/unifiedresources/loop_reports.go b/internal/unifiedresources/loop_reports.go new file mode 100644 index 000000000..5b2c3d03b --- /dev/null +++ b/internal/unifiedresources/loop_reports.go @@ -0,0 +1,289 @@ +package unifiedresources + +import ( + "errors" + "fmt" + "strings" + "time" +) + +// LoopReportType is the discriminator that names the loop a report came +// from. Only one loop is implemented for the MVP — maintenance window +// verification — but the field stays in the contract so future loops can +// reuse the same persistence and review surfaces without a schema split. +type LoopReportType string + +const ( + // LoopReportTypeMaintenanceVerification is the durable summary the + // sentinel writes after a maintenance window ends for a resource. The + // product-facing name is "Maintenance Verification Report"; this is + // the internal type token used for storage and API filtering. + LoopReportTypeMaintenanceVerification LoopReportType = "maintenance_verification" +) + +// LoopReportStatus reports the verification outcome. +// +// - healthy — all deterministic checks passed. +// - needs_review — evidence is ambiguous; an operator should +// look. Used when there is no metric history to compare, when the +// resource is operator-marked intentionally offline, or when a +// warning-level alert/finding is active. +// - failed_verification — at least one strong negative signal +// (critical alert, critical finding, recent failed action) is +// present. The report does not auto-remediate; the operator +// decides next steps. +// +// The set is intentionally small. Adding more states is a contract +// change because the UI and review action depend on the enum. +type LoopReportStatus string + +const ( + LoopReportStatusPending LoopReportStatus = "pending" + LoopReportStatusHealthy LoopReportStatus = "healthy" + LoopReportStatusNeedsReview LoopReportStatus = "needs_review" + LoopReportStatusFailedVerification LoopReportStatus = "failed_verification" +) + +// LoopReportUserOutcome records the operator's review verdict after +// reading a report. "" means the operator has not yet reviewed. +type LoopReportUserOutcome string + +const ( + LoopReportUserOutcomeReviewed LoopReportUserOutcome = "reviewed" +) + +// LoopReportEvidence captures the deterministic checks the sentinel +// ran. Each field is read-only operational evidence — counts and +// summaries — not executable commands or stale plans. The shape is +// stable so future review tooling can render every report uniformly. +type LoopReportEvidence struct { + // OperatorStateSummary is a short human-readable summary of the + // operator state at evaluation time (e.g. + // "maintenance_window_ended", "intentionally_offline", "no + // operator state"). + OperatorStateSummary string `json:"operatorStateSummary,omitempty"` + + // ActiveCriticalAlerts and ActiveWarningAlerts count alerts that + // were active for this resource at evaluation time. + ActiveCriticalAlerts int `json:"activeCriticalAlerts"` + ActiveWarningAlerts int `json:"activeWarningAlerts"` + + // ActiveCriticalFindings and ActiveWarningFindings count Patrol + // findings active for the resource at evaluation time. + ActiveCriticalFindings int `json:"activeCriticalFindings"` + ActiveWarningFindings int `json:"activeWarningFindings"` + + // FailedActionsSinceWindowStart is the number of action audits + // targeting the resource whose state ended in "failed" with an + // updated_at after the maintenance window started. + FailedActionsSinceWindowStart int `json:"failedActionsSinceWindowStart"` + + // MetricRecovery describes the basic recent-metric check. Absent + // when no metric source was available for the resource at + // evaluation time. + MetricRecovery *MetricRecoveryEvidence `json:"metricRecovery,omitempty"` + + // PatrolRunTODO is set when the deterministic evidence was + // ambiguous and a scoped Patrol run would have helped, but + // triggering one was not safe in this build. The value is the + // reason string so the UI can surface what was missing. + // + // MVP: deterministic checks only; the Patrol run trigger is + // deferred until the Patrol API surfaces a scoped per-resource + // run entrypoint that does not race the global scheduler. + PatrolRunTODO string `json:"patrolRunTodo,omitempty"` +} + +// MetricRecoveryEvidence is a small summary of the metric values +// observed inside the maintenance window vs. immediately after it. +// "Recent metric recovery" is intentionally shallow for the MVP — a +// per-metric trend label rather than a model. +type MetricRecoveryEvidence struct { + // MetricsObserved lists the metric short names the sentinel + // inspected (e.g. "cpu", "memory"). + MetricsObserved []string `json:"metricsObserved,omitempty"` + + // SamplesAfterEnd is the number of metric samples observed after + // the window ended. Zero means the resource has not reported any + // metrics since the window closed — operator should look. + SamplesAfterEnd int `json:"samplesAfterEnd"` + + // Trend describes the post-window metric trend in plain English. + // Allowed values: "improving", "stable", "degrading", "unknown". + Trend string `json:"trend,omitempty"` + + // Note is an optional explanation surfaced to the operator + // alongside the trend. + Note string `json:"note,omitempty"` +} + +// LoopReport is the durable record the sentinel writes when a loop run +// produces an outcome an operator may want to review. Fields are +// minimal on purpose — this is shared substrate, not a generic agent +// framework. +// +// For the maintenance-verification loop: +// - Scope is the canonical resource ID. +// - Trigger is always "maintenance_window_end". +// - WindowStartedAt and WindowEndedAt mirror the operator-set window +// so future runs can de-duplicate against the (resource, window) +// pair. +type LoopReport struct { + ID string `json:"id"` + Type LoopReportType `json:"type"` + Scope string `json:"scope"` + Trigger string `json:"trigger"` + Goal string `json:"goal,omitempty"` + Status LoopReportStatus `json:"status"` + StartedAt time.Time `json:"startedAt"` + CompletedAt time.Time `json:"completedAt"` + WindowStartedAt *time.Time `json:"windowStartedAt,omitempty"` + WindowEndedAt *time.Time `json:"windowEndedAt,omitempty"` + Evidence LoopReportEvidence `json:"evidence"` + LinkedFindingIDs []string `json:"linkedFindingIds,omitempty"` + LinkedAlertIDs []string `json:"linkedAlertIds,omitempty"` + LinkedActionIDs []string `json:"linkedActionIds,omitempty"` + LinkedPatrolRunID string `json:"linkedPatrolRunId,omitempty"` + Recommendation string `json:"recommendation,omitempty"` + UserOutcome LoopReportUserOutcome `json:"userOutcome,omitempty"` + ReviewedAt *time.Time `json:"reviewedAt,omitempty"` + ReviewedBy string `json:"reviewedBy,omitempty"` + ReviewNote string `json:"reviewNote,omitempty"` +} + +// ErrLoopReportInvalid is returned when a record fails contract checks +// at the store boundary. +var ErrLoopReportInvalid = errors.New("loop_report_invalid") + +// IsValidLoopReportType reports whether the value names a known loop +// report type. Unknown types are rejected at the store boundary so +// freeform discriminators cannot accumulate. +func IsValidLoopReportType(t LoopReportType) bool { + switch t { + case LoopReportTypeMaintenanceVerification: + return true + } + return false +} + +// IsValidLoopReportStatus reports whether the value names a known +// terminal or in-flight status. +func IsValidLoopReportStatus(s LoopReportStatus) bool { + switch s { + case LoopReportStatusPending, + LoopReportStatusHealthy, + LoopReportStatusNeedsReview, + LoopReportStatusFailedVerification: + return true + } + return false +} + +// IsValidLoopReportUserOutcome reports whether the value names a known +// review verdict. Empty is valid — the operator has not reviewed yet. +func IsValidLoopReportUserOutcome(o LoopReportUserOutcome) bool { + switch o { + case "", LoopReportUserOutcomeReviewed: + return true + } + return false +} + +// NormalizeLoopReport trims string fields, canonicalizes the scope, +// and applies UTC to all timestamps. It does not validate — callers +// follow normalize with ValidateLoopReport. +func NormalizeLoopReport(r LoopReport) LoopReport { + r.ID = strings.TrimSpace(r.ID) + r.Scope = CanonicalResourceID(r.Scope) + r.Trigger = strings.TrimSpace(r.Trigger) + r.Goal = strings.TrimSpace(r.Goal) + r.Recommendation = strings.TrimSpace(r.Recommendation) + r.ReviewedBy = strings.TrimSpace(r.ReviewedBy) + r.ReviewNote = strings.TrimSpace(r.ReviewNote) + r.LinkedPatrolRunID = strings.TrimSpace(r.LinkedPatrolRunID) + r.Evidence.OperatorStateSummary = strings.TrimSpace(r.Evidence.OperatorStateSummary) + r.Evidence.PatrolRunTODO = strings.TrimSpace(r.Evidence.PatrolRunTODO) + if r.Evidence.MetricRecovery != nil { + r.Evidence.MetricRecovery.Trend = strings.TrimSpace(r.Evidence.MetricRecovery.Trend) + r.Evidence.MetricRecovery.Note = strings.TrimSpace(r.Evidence.MetricRecovery.Note) + } + if !r.StartedAt.IsZero() { + r.StartedAt = r.StartedAt.UTC() + } + if !r.CompletedAt.IsZero() { + r.CompletedAt = r.CompletedAt.UTC() + } + if r.WindowStartedAt != nil { + t := r.WindowStartedAt.UTC() + r.WindowStartedAt = &t + } + if r.WindowEndedAt != nil { + t := r.WindowEndedAt.UTC() + r.WindowEndedAt = &t + } + if r.ReviewedAt != nil { + t := r.ReviewedAt.UTC() + r.ReviewedAt = &t + } + r.LinkedFindingIDs = trimAndDedupeStrings(r.LinkedFindingIDs) + r.LinkedAlertIDs = trimAndDedupeStrings(r.LinkedAlertIDs) + r.LinkedActionIDs = trimAndDedupeStrings(r.LinkedActionIDs) + return r +} + +// ValidateLoopReport applies the contract checks the persistence layer +// must enforce before writing a record. Returns an +// ErrLoopReportInvalid-wrapped error on violation. +func ValidateLoopReport(r LoopReport) error { + if r.ID == "" { + return fmt.Errorf("%w: id is required", ErrLoopReportInvalid) + } + if !IsValidLoopReportType(r.Type) { + return fmt.Errorf("%w: unknown report type %q", ErrLoopReportInvalid, r.Type) + } + if r.Scope == "" { + return fmt.Errorf("%w: scope (canonical resource id) is required", ErrLoopReportInvalid) + } + if r.Trigger == "" { + return fmt.Errorf("%w: trigger is required", ErrLoopReportInvalid) + } + if !IsValidLoopReportStatus(r.Status) { + return fmt.Errorf("%w: unknown status %q", ErrLoopReportInvalid, r.Status) + } + if r.StartedAt.IsZero() { + return fmt.Errorf("%w: startedAt is required", ErrLoopReportInvalid) + } + if r.CompletedAt.IsZero() { + return fmt.Errorf("%w: completedAt is required", ErrLoopReportInvalid) + } + if r.CompletedAt.Before(r.StartedAt) { + return fmt.Errorf("%w: completedAt must not be before startedAt", ErrLoopReportInvalid) + } + if !IsValidLoopReportUserOutcome(r.UserOutcome) { + return fmt.Errorf("%w: unknown user outcome %q", ErrLoopReportInvalid, r.UserOutcome) + } + return nil +} + +func trimAndDedupeStrings(in []string) []string { + if len(in) == 0 { + return nil + } + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, v := range in { + v = strings.TrimSpace(v) + if v == "" { + continue + } + if _, ok := seen[v]; ok { + continue + } + seen[v] = struct{}{} + out = append(out, v) + } + if len(out) == 0 { + return nil + } + return out +} diff --git a/internal/unifiedresources/loop_reports_store.go b/internal/unifiedresources/loop_reports_store.go new file mode 100644 index 000000000..84a2c6089 --- /dev/null +++ b/internal/unifiedresources/loop_reports_store.go @@ -0,0 +1,510 @@ +package unifiedresources + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" +) + +// ListResourceOperatorStates returns every persisted operator-set +// state row. Order is not guaranteed. +func (s *SQLiteResourceStore) ListResourceOperatorStates() ([]ResourceOperatorState, error) { + rows, err := s.db.Query(` + SELECT canonical_id, intentionally_offline, never_auto_remediate, + maintenance_start_at, maintenance_end_at, maintenance_reason, + criticality, note, set_at, set_by + FROM resource_operator_state`) + if err != nil { + return nil, fmt.Errorf("query resource operator states: %w", err) + } + defer rows.Close() + + var out []ResourceOperatorState + for rows.Next() { + var ( + state ResourceOperatorState + intentional int + neverRemediate int + startAt, endAt sql.NullTime + reason sql.NullString + criticality sql.NullString + note sql.NullString + setBy sql.NullString + ) + if err := rows.Scan( + &state.CanonicalID, + &intentional, + &neverRemediate, + &startAt, + &endAt, + &reason, + &criticality, + ¬e, + &state.SetAt, + &setBy, + ); err != nil { + return nil, fmt.Errorf("scan resource operator state row: %w", err) + } + state.IntentionallyOffline = intentional != 0 + state.NeverAutoRemediate = neverRemediate != 0 + if startAt.Valid { + t := startAt.Time + state.MaintenanceStartAt = &t + } + if endAt.Valid { + t := endAt.Time + state.MaintenanceEndAt = &t + } + if reason.Valid { + state.MaintenanceReason = reason.String + } + if criticality.Valid { + state.Criticality = ResourceCriticality(criticality.String) + } + if note.Valid { + state.Note = note.String + } + if setBy.Valid { + state.SetBy = setBy.String + } + out = append(out, state) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate resource operator state rows: %w", err) + } + return out, nil +} + +// ListResourceOperatorStates returns every persisted operator-set state row. +func (m *MemoryStore) ListResourceOperatorStates() ([]ResourceOperatorState, error) { + m.mu.RLock() + defer m.mu.RUnlock() + out := make([]ResourceOperatorState, 0, len(m.resourceOperatorState)) + for _, state := range m.resourceOperatorState { + out = append(out, state) + } + return out, nil +} + +// ErrLoopReportNotFound is returned by store methods that target a +// specific report id which does not exist. +var ErrLoopReportNotFound = errors.New("loop_report_not_found") + +// RecordLoopReport persists a new loop report. Reports are immutable +// except for the user_outcome / reviewed_* / review_note fields, which +// are updated via UpdateLoopReportUserOutcome. Tick-vs-tick dedup for +// the (type, scope, window_ended_at) triple is enforced at the +// sentinel layer (mutex + FindLoopReportByWindow) so explicit reruns, +// which intentionally share the triple under a different id suffix, +// can land alongside the original. +func (s *SQLiteResourceStore) RecordLoopReport(report LoopReport) error { + report = NormalizeLoopReport(report) + if err := ValidateLoopReport(report); err != nil { + return err + } + evidenceJSON, err := json.Marshal(report.Evidence) + if err != nil { + return fmt.Errorf("marshal loop report evidence: %w", err) + } + findingIDsJSON, err := json.Marshal(report.LinkedFindingIDs) + if err != nil { + return fmt.Errorf("marshal loop report finding ids: %w", err) + } + alertIDsJSON, err := json.Marshal(report.LinkedAlertIDs) + if err != nil { + return fmt.Errorf("marshal loop report alert ids: %w", err) + } + actionIDsJSON, err := json.Marshal(report.LinkedActionIDs) + if err != nil { + return fmt.Errorf("marshal loop report action ids: %w", err) + } + var windowStart, windowEnd, reviewedAt sql.NullTime + if report.WindowStartedAt != nil { + windowStart.Time = *report.WindowStartedAt + windowStart.Valid = true + } + if report.WindowEndedAt != nil { + windowEnd.Time = *report.WindowEndedAt + windowEnd.Valid = true + } + if report.ReviewedAt != nil { + reviewedAt.Time = *report.ReviewedAt + reviewedAt.Valid = true + } + + s.mu.Lock() + defer s.mu.Unlock() + + _, err = s.db.Exec(` + INSERT INTO loop_reports ( + id, report_type, scope, trigger, goal, status, started_at, completed_at, + window_started_at, window_ended_at, evidence_json, + linked_finding_ids_json, linked_alert_ids_json, linked_action_ids_json, + linked_patrol_run_id, recommendation, + user_outcome, reviewed_at, reviewed_by, review_note + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + report.ID, + string(report.Type), + report.Scope, + report.Trigger, + report.Goal, + string(report.Status), + report.StartedAt, + report.CompletedAt, + windowStart, + windowEnd, + string(evidenceJSON), + string(findingIDsJSON), + string(alertIDsJSON), + string(actionIDsJSON), + report.LinkedPatrolRunID, + report.Recommendation, + string(report.UserOutcome), + reviewedAt, + report.ReviewedBy, + report.ReviewNote, + ) + if err != nil { + return fmt.Errorf("insert loop report: %w", err) + } + return nil +} + +// GetLoopReport returns a single report by id. +func (s *SQLiteResourceStore) GetLoopReport(reportID string) (LoopReport, bool, error) { + reportID = strings.TrimSpace(reportID) + if reportID == "" { + return LoopReport{}, false, nil + } + row := s.db.QueryRow(` + SELECT id, report_type, scope, trigger, goal, status, started_at, completed_at, + window_started_at, window_ended_at, evidence_json, + linked_finding_ids_json, linked_alert_ids_json, linked_action_ids_json, + linked_patrol_run_id, recommendation, + user_outcome, reviewed_at, reviewed_by, review_note + FROM loop_reports + WHERE id = ?`, reportID) + report, err := scanLoopReportRow(row) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return LoopReport{}, false, nil + } + return LoopReport{}, false, err + } + return report, true, nil +} + +// FindLoopReportByWindow looks up an existing report for a +// (type, scope, window-end) triple. Used by the sentinel to dedupe +// before writing a new report. +func (s *SQLiteResourceStore) FindLoopReportByWindow(reportType LoopReportType, canonicalID string, windowEndedAt time.Time) (LoopReport, bool, error) { + canonicalID = CanonicalResourceID(canonicalID) + if canonicalID == "" || !IsValidLoopReportType(reportType) || windowEndedAt.IsZero() { + return LoopReport{}, false, nil + } + row := s.db.QueryRow(` + SELECT id, report_type, scope, trigger, goal, status, started_at, completed_at, + window_started_at, window_ended_at, evidence_json, + linked_finding_ids_json, linked_alert_ids_json, linked_action_ids_json, + linked_patrol_run_id, recommendation, + user_outcome, reviewed_at, reviewed_by, review_note + FROM loop_reports + WHERE report_type = ? AND scope = ? AND window_ended_at = ? + ORDER BY started_at DESC + LIMIT 1`, string(reportType), canonicalID, windowEndedAt.UTC()) + report, err := scanLoopReportRow(row) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return LoopReport{}, false, nil + } + return LoopReport{}, false, err + } + return report, true, nil +} + +// ListLoopReportsForResource returns the most recent reports for a +// resource of the given type, newest first. limit=0 returns all rows. +func (s *SQLiteResourceStore) ListLoopReportsForResource(reportType LoopReportType, canonicalID string, limit int) ([]LoopReport, error) { + canonicalID = CanonicalResourceID(canonicalID) + if !IsValidLoopReportType(reportType) { + return nil, nil + } + query := ` + SELECT id, report_type, scope, trigger, goal, status, started_at, completed_at, + window_started_at, window_ended_at, evidence_json, + linked_finding_ids_json, linked_alert_ids_json, linked_action_ids_json, + linked_patrol_run_id, recommendation, + user_outcome, reviewed_at, reviewed_by, review_note + FROM loop_reports + WHERE report_type = ? AND scope = ? + ORDER BY started_at DESC` + args := []any{string(reportType), canonicalID} + if limit > 0 { + query += ` LIMIT ?` + args = append(args, limit) + } + rows, err := s.db.Query(query, args...) + if err != nil { + return nil, fmt.Errorf("query loop reports: %w", err) + } + defer rows.Close() + + var out []LoopReport + for rows.Next() { + report, err := scanLoopReportRow(rows) + if err != nil { + return nil, err + } + out = append(out, report) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate loop report rows: %w", err) + } + return out, nil +} + +// UpdateLoopReportUserOutcome records the operator's review verdict on +// a report. It does not change the underlying status or evidence — +// those are computed by the sentinel and remain immutable. +func (s *SQLiteResourceStore) UpdateLoopReportUserOutcome(reportID string, outcome LoopReportUserOutcome, reviewedBy, note string, reviewedAt time.Time) error { + reportID = strings.TrimSpace(reportID) + if reportID == "" { + return fmt.Errorf("%w: id is required", ErrLoopReportInvalid) + } + if !IsValidLoopReportUserOutcome(outcome) { + return fmt.Errorf("%w: unknown user outcome %q", ErrLoopReportInvalid, outcome) + } + if reviewedAt.IsZero() { + reviewedAt = time.Now().UTC() + } else { + reviewedAt = reviewedAt.UTC() + } + s.mu.Lock() + defer s.mu.Unlock() + res, err := s.db.Exec(` + UPDATE loop_reports + SET user_outcome = ?, reviewed_at = ?, reviewed_by = ?, review_note = ? + WHERE id = ?`, + string(outcome), + reviewedAt, + strings.TrimSpace(reviewedBy), + strings.TrimSpace(note), + reportID, + ) + if err != nil { + return fmt.Errorf("update loop report user outcome: %w", err) + } + affected, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("rows affected for update loop report: %w", err) + } + if affected == 0 { + return ErrLoopReportNotFound + } + return nil +} + +// loopReportScanner is the surface RowQuery / Rows share for +// loop-report scanning. +type loopReportScanner interface { + Scan(dest ...any) error +} + +func scanLoopReportRow(scanner loopReportScanner) (LoopReport, error) { + var ( + r LoopReport + typ, status, outcome string + windowStart, windowEnd sql.NullTime + reviewedAt sql.NullTime + evidenceJSON string + findingIDsJSON string + alertIDsJSON string + actionIDsJSON string + linkedPatrolRunID string + recommendation, goal string + reviewedBy, reviewNote string + trigger string + ) + if err := scanner.Scan( + &r.ID, + &typ, + &r.Scope, + &trigger, + &goal, + &status, + &r.StartedAt, + &r.CompletedAt, + &windowStart, + &windowEnd, + &evidenceJSON, + &findingIDsJSON, + &alertIDsJSON, + &actionIDsJSON, + &linkedPatrolRunID, + &recommendation, + &outcome, + &reviewedAt, + &reviewedBy, + &reviewNote, + ); err != nil { + return LoopReport{}, err + } + r.Type = LoopReportType(typ) + r.Trigger = trigger + r.Goal = goal + r.Status = LoopReportStatus(status) + r.UserOutcome = LoopReportUserOutcome(outcome) + r.LinkedPatrolRunID = linkedPatrolRunID + r.Recommendation = recommendation + r.ReviewedBy = reviewedBy + r.ReviewNote = reviewNote + if windowStart.Valid { + t := windowStart.Time.UTC() + r.WindowStartedAt = &t + } + if windowEnd.Valid { + t := windowEnd.Time.UTC() + r.WindowEndedAt = &t + } + if reviewedAt.Valid { + t := reviewedAt.Time.UTC() + r.ReviewedAt = &t + } + if evidenceJSON != "" { + if err := json.Unmarshal([]byte(evidenceJSON), &r.Evidence); err != nil { + return LoopReport{}, fmt.Errorf("unmarshal loop report evidence: %w", err) + } + } + if findingIDsJSON != "" && findingIDsJSON != "null" { + if err := json.Unmarshal([]byte(findingIDsJSON), &r.LinkedFindingIDs); err != nil { + return LoopReport{}, fmt.Errorf("unmarshal loop report finding ids: %w", err) + } + } + if alertIDsJSON != "" && alertIDsJSON != "null" { + if err := json.Unmarshal([]byte(alertIDsJSON), &r.LinkedAlertIDs); err != nil { + return LoopReport{}, fmt.Errorf("unmarshal loop report alert ids: %w", err) + } + } + if actionIDsJSON != "" && actionIDsJSON != "null" { + if err := json.Unmarshal([]byte(actionIDsJSON), &r.LinkedActionIDs); err != nil { + return LoopReport{}, fmt.Errorf("unmarshal loop report action ids: %w", err) + } + } + r.StartedAt = r.StartedAt.UTC() + r.CompletedAt = r.CompletedAt.UTC() + return r, nil +} + +// --- MemoryStore implementations --- + +// RecordLoopReport stores a loop report in memory. Duplicate (type, +// scope, window-end) triples are rejected with ErrLoopReportInvalid so +// the sentinel's dedupe contract matches the SQLite store. +func (m *MemoryStore) RecordLoopReport(report LoopReport) error { + report = NormalizeLoopReport(report) + if err := ValidateLoopReport(report); err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + if m.loopReports == nil { + m.loopReports = make(map[string]LoopReport) + } + if _, exists := m.loopReports[report.ID]; exists { + return fmt.Errorf("%w: id %q already exists", ErrLoopReportInvalid, report.ID) + } + m.loopReports[report.ID] = report + return nil +} + +// GetLoopReport returns the report by id, if present. +func (m *MemoryStore) GetLoopReport(reportID string) (LoopReport, bool, error) { + reportID = strings.TrimSpace(reportID) + if reportID == "" { + return LoopReport{}, false, nil + } + m.mu.RLock() + defer m.mu.RUnlock() + report, ok := m.loopReports[reportID] + return report, ok, nil +} + +// FindLoopReportByWindow scans the in-memory map for a matching report. +func (m *MemoryStore) FindLoopReportByWindow(reportType LoopReportType, canonicalID string, windowEndedAt time.Time) (LoopReport, bool, error) { + canonicalID = CanonicalResourceID(canonicalID) + if canonicalID == "" || !IsValidLoopReportType(reportType) || windowEndedAt.IsZero() { + return LoopReport{}, false, nil + } + target := windowEndedAt.UTC() + m.mu.RLock() + defer m.mu.RUnlock() + for _, report := range m.loopReports { + if report.Type != reportType || report.Scope != canonicalID { + continue + } + if report.WindowEndedAt == nil { + continue + } + if report.WindowEndedAt.Equal(target) { + return report, true, nil + } + } + return LoopReport{}, false, nil +} + +// ListLoopReportsForResource returns matching reports sorted by +// started_at DESC. +func (m *MemoryStore) ListLoopReportsForResource(reportType LoopReportType, canonicalID string, limit int) ([]LoopReport, error) { + canonicalID = CanonicalResourceID(canonicalID) + if !IsValidLoopReportType(reportType) { + return nil, nil + } + m.mu.RLock() + defer m.mu.RUnlock() + out := []LoopReport{} + for _, report := range m.loopReports { + if report.Type != reportType || report.Scope != canonicalID { + continue + } + out = append(out, report) + } + sort.Slice(out, func(i, j int) bool { + return out[i].StartedAt.After(out[j].StartedAt) + }) + if limit > 0 && len(out) > limit { + out = out[:limit] + } + return out, nil +} + +// UpdateLoopReportUserOutcome records the operator's review verdict. +func (m *MemoryStore) UpdateLoopReportUserOutcome(reportID string, outcome LoopReportUserOutcome, reviewedBy, note string, reviewedAt time.Time) error { + reportID = strings.TrimSpace(reportID) + if reportID == "" { + return fmt.Errorf("%w: id is required", ErrLoopReportInvalid) + } + if !IsValidLoopReportUserOutcome(outcome) { + return fmt.Errorf("%w: unknown user outcome %q", ErrLoopReportInvalid, outcome) + } + if reviewedAt.IsZero() { + reviewedAt = time.Now().UTC() + } else { + reviewedAt = reviewedAt.UTC() + } + m.mu.Lock() + defer m.mu.Unlock() + report, ok := m.loopReports[reportID] + if !ok { + return ErrLoopReportNotFound + } + report.UserOutcome = outcome + report.ReviewedAt = &reviewedAt + report.ReviewedBy = strings.TrimSpace(reviewedBy) + report.ReviewNote = strings.TrimSpace(note) + m.loopReports[reportID] = report + return nil +} diff --git a/internal/unifiedresources/loop_reports_store_test.go b/internal/unifiedresources/loop_reports_store_test.go new file mode 100644 index 000000000..af4bf12b6 --- /dev/null +++ b/internal/unifiedresources/loop_reports_store_test.go @@ -0,0 +1,185 @@ +package unifiedresources + +import ( + "testing" + "time" +) + +func newLoopReport(id, scope string, windowEnd time.Time, status LoopReportStatus) LoopReport { + return LoopReport{ + ID: id, + Type: LoopReportTypeMaintenanceVerification, + Scope: scope, + Trigger: "maintenance_window_end", + Goal: "verify recovery", + Status: status, + StartedAt: windowEnd.Add(time.Minute), + CompletedAt: windowEnd.Add(time.Minute), + WindowEndedAt: &windowEnd, + Evidence: LoopReportEvidence{OperatorStateSummary: "maintenance window ended"}, + } +} + +func TestSQLiteRecordLoopReport_RoundTrip(t *testing.T) { + dataDir := t.TempDir() + store, err := NewSQLiteResourceStore(dataDir, "default") + if err != nil { + t.Fatalf("new store: %v", err) + } + defer store.Close() + + windowEnd := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + report := newLoopReport("mv-vm_101-20260512T120000Z", "vm:101", windowEnd, LoopReportStatusHealthy) + if err := store.RecordLoopReport(report); err != nil { + t.Fatalf("record: %v", err) + } + got, found, err := store.GetLoopReport(report.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if !found { + t.Fatal("expected report to be found after record") + } + if got.Scope != "vm:101" || got.Status != LoopReportStatusHealthy { + t.Fatalf("round trip mismatch: scope=%q status=%q", got.Scope, got.Status) + } +} + +func TestSQLiteRecordLoopReport_AllowsRerunForSameWindow(t *testing.T) { + dataDir := t.TempDir() + store, err := NewSQLiteResourceStore(dataDir, "default") + if err != nil { + t.Fatalf("new store: %v", err) + } + defer store.Close() + + windowEnd := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + original := newLoopReport("mv-vm_101-20260512T120000Z", "vm:101", windowEnd, LoopReportStatusHealthy) + if err := store.RecordLoopReport(original); err != nil { + t.Fatalf("record original: %v", err) + } + rerun := newLoopReport(original.ID+"-rerun-1", "vm:101", windowEnd, LoopReportStatusNeedsReview) + if err := store.RecordLoopReport(rerun); err != nil { + t.Fatalf("record rerun: %v", err) + } + + reports, err := store.ListLoopReportsForResource(LoopReportTypeMaintenanceVerification, "vm:101", 0) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(reports) != 2 { + t.Fatalf("expected 2 reports (original + rerun), got %d", len(reports)) + } + + found, ok, err := store.FindLoopReportByWindow(LoopReportTypeMaintenanceVerification, "vm:101", windowEnd) + if err != nil || !ok { + t.Fatalf("find by window: ok=%v err=%v", ok, err) + } + if found.ID == "" { + t.Fatal("expected matching report id from FindLoopReportByWindow") + } +} + +func TestSQLiteRecordLoopReport_RejectsDuplicateID(t *testing.T) { + dataDir := t.TempDir() + store, err := NewSQLiteResourceStore(dataDir, "default") + if err != nil { + t.Fatalf("new store: %v", err) + } + defer store.Close() + + windowEnd := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + report := newLoopReport("mv-dup-id", "vm:101", windowEnd, LoopReportStatusHealthy) + if err := store.RecordLoopReport(report); err != nil { + t.Fatalf("first record: %v", err) + } + if err := store.RecordLoopReport(report); err == nil { + t.Fatal("expected error on duplicate id") + } +} + +func TestSQLiteUpdateLoopReportUserOutcome_RoundTrip(t *testing.T) { + dataDir := t.TempDir() + store, err := NewSQLiteResourceStore(dataDir, "default") + if err != nil { + t.Fatalf("new store: %v", err) + } + defer store.Close() + + windowEnd := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + report := newLoopReport("mv-review", "vm:101", windowEnd, LoopReportStatusNeedsReview) + if err := store.RecordLoopReport(report); err != nil { + t.Fatalf("record: %v", err) + } + reviewedAt := time.Date(2026, 5, 12, 13, 0, 0, 0, time.UTC) + if err := store.UpdateLoopReportUserOutcome(report.ID, LoopReportUserOutcomeReviewed, "rcourtman", "ack", reviewedAt); err != nil { + t.Fatalf("update outcome: %v", err) + } + got, _, err := store.GetLoopReport(report.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.UserOutcome != LoopReportUserOutcomeReviewed { + t.Fatalf("outcome = %q want reviewed", got.UserOutcome) + } + if got.ReviewedBy != "rcourtman" || got.ReviewNote != "ack" { + t.Fatalf("review fields = %q/%q want rcourtman/ack", got.ReviewedBy, got.ReviewNote) + } + if got.ReviewedAt == nil || !got.ReviewedAt.Equal(reviewedAt) { + t.Fatalf("reviewedAt = %v want %v", got.ReviewedAt, reviewedAt) + } + if got.Status != LoopReportStatusNeedsReview { + t.Fatalf("status mutated to %q; should remain needs_review", got.Status) + } +} + +func TestMemoryStoreRecordLoopReport_AllowsRerun(t *testing.T) { + store := NewMemoryStore() + windowEnd := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + original := newLoopReport("mv-mem-1", "vm:101", windowEnd, LoopReportStatusHealthy) + if err := store.RecordLoopReport(original); err != nil { + t.Fatalf("record original: %v", err) + } + rerun := newLoopReport("mv-mem-1-rerun-1", "vm:101", windowEnd, LoopReportStatusNeedsReview) + if err := store.RecordLoopReport(rerun); err != nil { + t.Fatalf("record rerun: %v", err) + } + reports, err := store.ListLoopReportsForResource(LoopReportTypeMaintenanceVerification, "vm:101", 0) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(reports) != 2 { + t.Fatalf("expected 2 reports, got %d", len(reports)) + } +} + +func TestSQLiteListResourceOperatorStates_ReturnsAllRows(t *testing.T) { + dataDir := t.TempDir() + store, err := NewSQLiteResourceStore(dataDir, "default") + if err != nil { + t.Fatalf("new store: %v", err) + } + defer store.Close() + + now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) + startA := now.Add(-time.Hour) + endA := now.Add(-time.Minute) + startB := now.Add(-30 * time.Minute) + endB := now.Add(30 * time.Minute) + + for _, s := range []ResourceOperatorState{ + {CanonicalID: "vm:101", MaintenanceStartAt: &startA, MaintenanceEndAt: &endA, SetAt: now, SetBy: "op"}, + {CanonicalID: "vm:102", MaintenanceStartAt: &startB, MaintenanceEndAt: &endB, SetAt: now, SetBy: "op"}, + } { + if err := store.SetResourceOperatorState(s); err != nil { + t.Fatalf("seed %s: %v", s.CanonicalID, err) + } + } + got, err := store.ListResourceOperatorStates() + if err != nil { + t.Fatalf("list: %v", err) + } + if len(got) != 2 { + t.Fatalf("expected 2 operator states, got %d", len(got)) + } +} diff --git a/internal/unifiedresources/store.go b/internal/unifiedresources/store.go index 00a7e2d0e..901103619 100644 --- a/internal/unifiedresources/store.go +++ b/internal/unifiedresources/store.go @@ -49,6 +49,22 @@ type ResourceStore interface { GetResourceOperatorState(canonicalID string) (ResourceOperatorState, bool, error) SetResourceOperatorState(state ResourceOperatorState) error ClearResourceOperatorState(canonicalID string) error + // ListResourceOperatorStates returns every persisted operator-set + // state row. Used by background loops (e.g. the maintenance + // verification sentinel) that need to sweep the full set on each + // tick. Order is implementation-defined; the caller sorts if it + // cares. + ListResourceOperatorStates() ([]ResourceOperatorState, error) + // Loop reports — durable summaries written by background loops + // (currently only the maintenance-verification sentinel). + RecordLoopReport(report LoopReport) error + GetLoopReport(reportID string) (LoopReport, bool, error) + ListLoopReportsForResource(reportType LoopReportType, canonicalID string, limit int) ([]LoopReport, error) + UpdateLoopReportUserOutcome(reportID string, outcome LoopReportUserOutcome, reviewedBy, note string, reviewedAt time.Time) error + // FindLoopReportByWindow looks up an existing report by the + // (type, scope, window-end) triple so the sentinel can dedupe + // on each tick without scanning the whole table. + FindLoopReportByWindow(reportType LoopReportType, canonicalID string, windowEndedAt time.Time) (LoopReport, bool, error) Close() error } @@ -369,6 +385,38 @@ func (s *SQLiteResourceStore) initSchema() error { set_by TEXT ); CREATE INDEX IF NOT EXISTS idx_resource_operator_state_maintenance ON resource_operator_state(maintenance_end_at); + + CREATE TABLE IF NOT EXISTS loop_reports ( + id TEXT PRIMARY KEY, + report_type TEXT NOT NULL, + scope TEXT NOT NULL, + trigger TEXT NOT NULL, + goal TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL, + started_at DATETIME NOT NULL, + completed_at DATETIME NOT NULL, + window_started_at DATETIME, + window_ended_at DATETIME, + evidence_json TEXT NOT NULL DEFAULT '{}', + linked_finding_ids_json TEXT NOT NULL DEFAULT '[]', + linked_alert_ids_json TEXT NOT NULL DEFAULT '[]', + linked_action_ids_json TEXT NOT NULL DEFAULT '[]', + linked_patrol_run_id TEXT NOT NULL DEFAULT '', + recommendation TEXT NOT NULL DEFAULT '', + user_outcome TEXT NOT NULL DEFAULT '', + reviewed_at DATETIME, + reviewed_by TEXT NOT NULL DEFAULT '', + review_note TEXT NOT NULL DEFAULT '' + ); + CREATE INDEX IF NOT EXISTS idx_loop_reports_scope_type_started + ON loop_reports(report_type, scope, started_at DESC); + -- Look-up index for FindLoopReportByWindow. Intentionally non-unique: + -- rerun records share (type, scope, window_ended_at) with the + -- original under a distinct id suffix. Tick-vs-tick dedup runs at + -- the sentinel layer (mutex + FindLoopReportByWindow check). + CREATE INDEX IF NOT EXISTS idx_loop_reports_window_lookup + ON loop_reports(report_type, scope, window_ended_at) + WHERE window_ended_at IS NOT NULL; ` _, err := s.db.Exec(schema) @@ -1565,11 +1613,13 @@ type MemoryStore struct { actionLifecycleEvents []ActionLifecycleEvent exportAudits []ExportAuditRecord resourceOperatorState map[string]ResourceOperatorState + loopReports map[string]LoopReport } func NewMemoryStore() *MemoryStore { return &MemoryStore{ resourceOperatorState: make(map[string]ResourceOperatorState), + loopReports: make(map[string]LoopReport), } }