gsadmin/pulse
1
0
Fork 0
mirror of https://github.com/rcourtman/Pulse.git synced 2026-09-10 02:25:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add maintenance verification reports

Browse Source 
When a maintenance window ends on a resource, the sentinel runs
deterministic checks (active alerts, Patrol findings, failed actions
since window start, basic post-window metric recovery) and writes a
durable LoopReport. Operators can list reports per resource, mark them
reviewed, or rerun verification immediately. UI surfaces the section in
the resource detail drawer; scoped Patrol runs and Assistant deep-link
are deferred until those entry points stabilise.
This commit is contained in:
rcourtman
2026-05-12 18:02:16 +01:00
parent 5b18438528
commit 015e7f6555
18 changed files with 3495 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend-modern/src/api/maintenanceVerification.ts
+133
View File
@@ -0,0 +1,133 @@
import { apiFetchJSON } from '@/utils/apiClient';
/**
* Maintenance Verification Report — the durable summary Pulse writes
* when a maintenance window ends for a resource. Mirrors the Go
* `unified.LoopReport` projected through
* `internal/api/maintenance_verification.go`.
*
* Surfaced as "Maintenance Verification Report" in the product copy.
* The status enum is small on purpose so the UI can render each value
* with a stable visual treatment:
* - healthy → green check / quiet
* - needs_review → amber prompt for an operator look
* - failed_verification → red surface, operator action expected
* - pending → in-flight; the sentinel hasn't decided
*/
export type MaintenanceVerificationStatus =
| 'pending'
| 'healthy'
| 'needs_review'
| 'failed_verification';
export type MaintenanceVerificationUserOutcome = 'reviewed' | '';
export interface MaintenanceVerificationMetricRecovery {
metricsObserved?: string[];
samplesAfterEnd: number;
trend?: 'improving' | 'stable' | 'degrading' | 'unknown' | '';
note?: string;
}
export interface MaintenanceVerificationEvidence {
operatorStateSummary?: string;
activeCriticalAlerts: number;
activeWarningAlerts: number;
activeCriticalFindings: number;
activeWarningFindings: number;
failedActionsSinceWindowStart: number;
metricRecovery?: MaintenanceVerificationMetricRecovery;
/**
* Breadcrumb set by the sentinel when deterministic evidence was
* ambiguous and a scoped Patrol run would have helped, but
* triggering one was not safe in this build. Empty when the report
* was unambiguous.
*/
patrolRunTodo?: string;
}
export interface MaintenanceVerificationReport {
id: string;
resourceId: string;
trigger: string;
goal?: string;
status: MaintenanceVerificationStatus;
startedAt: string;
completedAt: string;
windowStartedAt?: string;
windowEndedAt?: string;
evidence: MaintenanceVerificationEvidence;
linkedFindingIds: string[];
linkedAlertIds: string[];
linkedActionIds: string[];
linkedPatrolRunId?: string;
recommendation?: string;
userOutcome?: MaintenanceVerificationUserOutcome;
reviewedAt?: string;
reviewedBy?: string;
reviewNote?: string;
}
export interface MaintenanceVerificationListResponse {
data: MaintenanceVerificationReport[];
meta: {
resourceId: string;
limit: number;
total: number;
};
}
/**
* Fetch recent Maintenance Verification Reports for a resource,
* newest first.
*/
export async function listMaintenanceVerificationsForResource(
resourceId: string,
limit = 25,
): Promise<MaintenanceVerificationListResponse> {
const params = new URLSearchParams();
if (limit && limit > 0) {
params.set('limit', String(limit));
}
const query = params.toString();
const path = `/api/resources/${encodeURIComponent(resourceId)}/maintenance-verifications${
query ? `?${query}` : ''
}`;
return apiFetchJSON<MaintenanceVerificationListResponse>(path, { cache: 'no-store' });
}
/**
* Mark a report as reviewed by the operator. The report's status and
* evidence stay immutable — only the user verdict and review fields
* update.
*/
export async function reviewMaintenanceVerification(
reportId: string,
note?: string,
): Promise<MaintenanceVerificationReport> {
return apiFetchJSON<MaintenanceVerificationReport>(
`/api/maintenance-verifications/${encodeURIComponent(reportId)}/review`,
{
method: 'POST',
body: JSON.stringify({ note: note ?? '' }),
headers: { 'Content-Type': 'application/json' },
},
);
}
/**
* Re-run the deterministic verification immediately. The new report
* is persisted with a `-rerun-N` suffix on its id so the review
* history is preserved.
*/
export async function rerunMaintenanceVerification(
resourceId: string,
): Promise<MaintenanceVerificationReport> {
return apiFetchJSON<MaintenanceVerificationReport>(
`/api/resources/${encodeURIComponent(resourceId)}/maintenance-verifications/rerun`,
{
method: 'POST',
headers: { 'Content-Type': 'application/json' },
},
);
}
frontend-modern/src/components/Infrastructure/MaintenanceVerificationSection.tsx
+288
View File
@@ -0,0 +1,288 @@
import { Component, For, Show, createMemo, createSignal } from 'solid-js';
import { createNonSuspendingQuery } from '@/hooks/createNonSuspendingQuery';
import { notificationStore } from '@/stores/notifications';
import {
listMaintenanceVerificationsForResource,
rerunMaintenanceVerification,
reviewMaintenanceVerification,
type MaintenanceVerificationReport,
type MaintenanceVerificationStatus,
} from '@/api/maintenanceVerification';
import { formatRelativeTime } from '@/utils/format';
/**
* MaintenanceVerificationSection surfaces Maintenance Verification
* Reports in the resource detail drawer. Reports are durable records
* the sentinel writes when a maintenance window ends for a resource,
* summarizing whether the resource recovered cleanly.
*
* The section is empty (and hidden) until the first report is
* written. Operators can:
* - Read the deterministic evidence (alerts, findings, failed
* actions, basic metric recovery summary)
* - Mark a report as reviewed (without changing the underlying
* status / evidence — those stay immutable)
* - Rerun verification now (writes a new -rerun-N report)
*
* The "open Assistant with this report context" action is intentionally
* omitted from this first pass — see Status: missing-actions note at
* the bottom. The other two actions (mark reviewed, rerun verification)
* are sufficient for the MVP review loop.
*/
interface MaintenanceVerificationSectionProps {
resourceId: string;
}
const STATUS_LABELS: Record<MaintenanceVerificationStatus, string> = {
pending: 'Pending',
healthy: 'Healthy',
needs_review: 'Needs review',
failed_verification: 'Failed verification',
};
const STATUS_CLASSES: Record<MaintenanceVerificationStatus, string> = {
pending: 'bg-surface-hover text-base-content',
healthy:
'bg-emerald-100 text-emerald-700 dark:bg-emerald-900 dark:text-emerald-300',
needs_review:
'bg-amber-100 text-amber-700 dark:bg-amber-900 dark:text-amber-300',
failed_verification:
'bg-rose-100 text-rose-700 dark:bg-rose-900 dark:text-rose-300',
};
export const MaintenanceVerificationSection: Component<MaintenanceVerificationSectionProps> = (
props,
) => {
const [refreshTick, setRefreshTick] = createSignal(0);
const query = createNonSuspendingQuery<MaintenanceVerificationReport[], string>({
source: () => (props.resourceId ? `${props.resourceId}:${refreshTick()}` : null),
fetcher: async () => {
try {
const res = await listMaintenanceVerificationsForResource(props.resourceId);
return res.data ?? [];
} catch (err) {
notificationStore.error(
err instanceof Error
? err.message
: 'Failed to load Maintenance Verification Reports',
);
return [];
}
},
initialValue: [] as MaintenanceVerificationReport[],
cacheKey: (key: string) => `maintenance-verifications:${key}`,
});
const reports = createMemo<MaintenanceVerificationReport[]>(() => query.value() ?? []);
const hasReports = createMemo(() => reports().length > 0);
const [rerunning, setRerunning] = createSignal(false);
const [reviewingId, setReviewingId] = createSignal<string | null>(null);
const refresh = () => setRefreshTick((n) => n + 1);
const handleRerun = async () => {
if (rerunning() || !props.resourceId) return;
setRerunning(true);
try {
await rerunMaintenanceVerification(props.resourceId);
notificationStore.success('Maintenance verification rerun complete');
refresh();
} catch (err) {
notificationStore.error(
err instanceof Error
? err.message
: 'Failed to rerun maintenance verification',
);
} finally {
setRerunning(false);
}
};
const handleReview = async (report: MaintenanceVerificationReport) => {
if (reviewingId() || !report.id) return;
setReviewingId(report.id);
try {
await reviewMaintenanceVerification(report.id);
notificationStore.success('Marked report reviewed');
refresh();
} catch (err) {
notificationStore.error(
err instanceof Error
? err.message
: 'Failed to mark report reviewed',
);
} finally {
setReviewingId(null);
}
};
return (
<section
class="rounded border border-border bg-surface p-3 text-sm"
data-testid="maintenance-verification-section"
aria-labelledby="maintenance-verification-heading"
>
<header class="flex flex-wrap items-center justify-between gap-2">
<div>
<h3
id="maintenance-verification-heading"
class="text-sm font-semibold text-base-content"
>
Maintenance verification
</h3>
<p class="text-xs text-muted">
Pulse runs deterministic checks each time a maintenance window
ends and writes a Maintenance Verification Report. The result
sticks here for review.
</p>
</div>
<button
type="button"
class="rounded border border-border bg-surface-hover px-2 py-1 text-xs font-medium hover:bg-surface"
disabled={rerunning() || !props.resourceId}
onClick={handleRerun}
data-testid="maintenance-verification-rerun"
>
{rerunning() ? 'Rerunning…' : 'Rerun verification'}
</button>
</header>
<Show
when={hasReports()}
fallback={
<div class="mt-3 rounded border border-dashed border-border bg-surface-hover p-3 text-xs text-muted">
No verification reports yet. A report is written automatically the
next time this resource exits a maintenance window.
</div>
}
>
<ul class="mt-3 space-y-2">
<For each={reports()}>
{(report) => (
<li
class="rounded border border-border bg-surface-hover p-2"
data-testid="maintenance-verification-report"
>
<div class="flex flex-wrap items-center justify-between gap-2">
<div class="flex flex-wrap items-center gap-2">
<span
class={`inline-flex items-center rounded px-1.5 py-0.5 text-[10px] font-medium ${STATUS_CLASSES[report.status]}`}
data-testid={`maintenance-verification-status-${report.status}`}
>
{STATUS_LABELS[report.status]}
</span>
<span class="text-xs text-muted">
Window ended{' '}
{report.windowEndedAt
? formatRelativeTime(report.windowEndedAt)
: 'unknown'}
</span>
</div>
<Show when={!report.userOutcome}>
<button
type="button"
class="rounded border border-border bg-surface px-2 py-1 text-[11px] font-medium hover:bg-surface-hover"
disabled={reviewingId() === report.id}
onClick={() => handleReview(report)}
data-testid="maintenance-verification-review"
>
{reviewingId() === report.id ? 'Marking…' : 'Mark reviewed'}
</button>
</Show>
<Show when={report.userOutcome === 'reviewed'}>
<span class="text-[11px] text-muted">
Reviewed{' '}
{report.reviewedAt
? formatRelativeTime(report.reviewedAt)
: ''}
{report.reviewedBy ? ` by ${report.reviewedBy}` : ''}
</span>
</Show>
</div>
<Show when={report.recommendation}>
<p class="mt-1 text-xs text-base-content">{report.recommendation}</p>
</Show>
<dl class="mt-2 grid grid-cols-2 gap-x-3 gap-y-1 text-[11px] text-muted sm:grid-cols-4">
<div>
<dt class="font-medium text-base-content">Critical alerts</dt>
<dd>{report.evidence.activeCriticalAlerts}</dd>
</div>
<div>
<dt class="font-medium text-base-content">Warning alerts</dt>
<dd>{report.evidence.activeWarningAlerts}</dd>
</div>
<div>
<dt class="font-medium text-base-content">
Critical findings
</dt>
<dd>{report.evidence.activeCriticalFindings}</dd>
</div>
<div>
<dt class="font-medium text-base-content">
Warning findings
</dt>
<dd>{report.evidence.activeWarningFindings}</dd>
</div>
<div>
<dt class="font-medium text-base-content">Failed actions</dt>
<dd>{report.evidence.failedActionsSinceWindowStart}</dd>
</div>
<Show when={report.evidence.metricRecovery}>
{(rec) => (
<>
<div>
<dt class="font-medium text-base-content">
Metric samples
</dt>
<dd>{rec().samplesAfterEnd}</dd>
</div>
<div>
<dt class="font-medium text-base-content">
Metric trend
</dt>
<dd>{rec().trend || 'unknown'}</dd>
</div>
</>
)}
</Show>
</dl>
<Show when={report.evidence.operatorStateSummary}>
<p class="mt-1 text-[11px] text-muted">
{report.evidence.operatorStateSummary}
</p>
</Show>
<Show when={report.evidence.patrolRunTodo}>
<p class="mt-1 text-[11px] italic text-muted">
{report.evidence.patrolRunTodo}
</p>
</Show>
<Show when={report.userOutcome === 'reviewed' && report.reviewNote}>
<p class="mt-1 text-[11px] text-base-content">
Reviewer note: {report.reviewNote}
</p>
</Show>
</li>
)}
</For>
</ul>
</Show>
{/*
MVP omission: "Open Assistant with this report context" is the
third action called out in the product brief. Wiring it requires
a stable Assistant deep-link contract for report payloads which
is not yet implemented — left as a follow-up so the surface
does not ship a button that does nothing.
*/}
</section>
);
};
export default MaintenanceVerificationSection;
frontend-modern/src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx
+10
View File
@@ -29,6 +29,7 @@ import { ResourceChangeSummary } from './ResourceChangeSummary';
import { ResourceFacetSummary } from './ResourceFacetSummary';
import { ResourceActionHistory } from './ResourceActionHistory';
import { ResourceOperatorStateSection } from './ResourceOperatorStateSection';
import { MaintenanceVerificationSection } from './MaintenanceVerificationSection';
import {
RESOURCE_CHANGE_KIND_ORDER,
RESOURCE_CHANGE_SOURCE_ADAPTER_ORDER,
@@ -571,6 +572,15 @@ export const ResourceDetailDrawerOverviewTab: Component<ResourceDetailDrawerOver
<ResourceOperatorStateSection resourceId={resource.id} />
</Show>
{/* Maintenance Verification Reports sit directly under the
operator-state section because the same operator who set
the maintenance window above is the one who needs to read
the verification result here. Self-fetching component;
empty state hides gracefully when no reports exist. */}
<Show when={resource.id}>
<MaintenanceVerificationSection resourceId={resource.id} />
</Show>
<Show when={drawer.actionAuditAvailable()}>
<ResourceActionHistory
audits={drawer.sortedActionAudits()}
frontend-modern/src/components/Infrastructure/__tests__/MaintenanceVerificationSection.test.ts
+89
View File
@@ -0,0 +1,89 @@
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
import { describe, expect, it } from 'vitest';
const sectionSource = readFileSync(
resolve(__dirname, '..', 'MaintenanceVerificationSection.tsx'),
'utf-8',
);
const overviewTabSource = readFileSync(
resolve(__dirname, '..', 'ResourceDetailDrawerOverviewTab.tsx'),
'utf-8',
);
const apiClientSource = readFileSync(
resolve(__dirname, '..', '..', '..', 'api', 'maintenanceVerification.ts'),
'utf-8',
);
describe('MaintenanceVerificationSection', () => {
it('routes list/rerun/review through the canonical maintenanceVerification API client', () => {
expect(sectionSource).toContain("from '@/api/maintenanceVerification'");
expect(sectionSource).toContain('listMaintenanceVerificationsForResource');
expect(sectionSource).toContain('rerunMaintenanceVerification');
expect(sectionSource).toContain('reviewMaintenanceVerification');
});
it('keeps the section out of the parent Suspense fallback by using createNonSuspendingQuery', () => {
expect(sectionSource).toContain('createNonSuspendingQuery');
expect(sectionSource).not.toContain('createResource<');
});
it('renders an empty state for resources with no reports yet', () => {
expect(sectionSource).toContain('No verification reports yet.');
});
it('surfaces the deterministic evidence counts the report exposes', () => {
expect(sectionSource).toContain('Critical alerts');
expect(sectionSource).toContain('Warning alerts');
expect(sectionSource).toContain('Critical findings');
expect(sectionSource).toContain('Warning findings');
expect(sectionSource).toContain('Failed actions');
});
it('exposes both operator actions (rerun + mark reviewed) the MVP committed to', () => {
expect(sectionSource).toContain('data-testid="maintenance-verification-rerun"');
expect(sectionSource).toContain('data-testid="maintenance-verification-review"');
});
it('hides the review button once the report has been reviewed', () => {
expect(sectionSource).toContain("when={!report.userOutcome}");
expect(sectionSource).toContain("when={report.userOutcome === 'reviewed'}");
});
it('renders the patrolRunTodo breadcrumb when the sentinel surfaced one', () => {
expect(sectionSource).toContain('report.evidence.patrolRunTodo');
});
});
describe('maintenanceVerification API client', () => {
it('encodes the resource id segment so canonical ids with colons survive', () => {
expect(apiClientSource).toContain('encodeURIComponent(resourceId)');
expect(apiClientSource).toContain('encodeURIComponent(reportId)');
});
it('exposes the four operations the section needs', () => {
expect(apiClientSource).toContain('export async function listMaintenanceVerificationsForResource');
expect(apiClientSource).toContain('export async function reviewMaintenanceVerification');
expect(apiClientSource).toContain('export async function rerunMaintenanceVerification');
});
it('pins the four-state status enum the UI branches on', () => {
expect(apiClientSource).toContain("'pending'");
expect(apiClientSource).toContain("'healthy'");
expect(apiClientSource).toContain("'needs_review'");
expect(apiClientSource).toContain("'failed_verification'");
});
});
describe('ResourceDetailDrawerOverviewTab integration', () => {
it('renders MaintenanceVerificationSection directly under the operator-state section', () => {
expect(overviewTabSource).toContain("from './MaintenanceVerificationSection'");
expect(overviewTabSource).toContain('<MaintenanceVerificationSection resourceId={resource.id} />');
const operatorIndex = overviewTabSource.indexOf('<ResourceOperatorStateSection');
const verificationIndex = overviewTabSource.indexOf('<MaintenanceVerificationSection');
expect(operatorIndex).toBeGreaterThan(0);
expect(verificationIndex).toBeGreaterThan(operatorIndex);
});
});
internal/api/maintenance_verification.go
+372
View File
@@ -0,0 +1,372 @@
package api
import (
"context"
"encoding/json"
"errors"
"net/http"
"strconv"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/maintenancesentinel"
unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
// maintenanceVerificationReportAPI is the wire shape for Maintenance
// Verification Reports. Mirrors `unified.LoopReport` but is kept as a
// separate API type so the storage shape can evolve independently of
// the wire contract. Filenames and product copy use the "Maintenance
// Verification Report" name; the underlying loop substrate is
// implementation detail and is not exposed verbatim.
type maintenanceVerificationReportAPI struct {
ID string `json:"id"`
ResourceID string `json:"resourceId"`
Trigger string `json:"trigger"`
Goal string `json:"goal,omitempty"`
Status string `json:"status"`
StartedAt time.Time `json:"startedAt"`
CompletedAt time.Time `json:"completedAt"`
WindowStartedAt *time.Time `json:"windowStartedAt,omitempty"`
WindowEndedAt *time.Time `json:"windowEndedAt,omitempty"`
Evidence maintenanceVerificationEvidenceAPI `json:"evidence"`
LinkedFindingIDs []string `json:"linkedFindingIds"`
LinkedAlertIDs []string `json:"linkedAlertIds"`
LinkedActionIDs []string `json:"linkedActionIds"`
LinkedPatrolRunID string `json:"linkedPatrolRunId,omitempty"`
Recommendation string `json:"recommendation,omitempty"`
UserOutcome string `json:"userOutcome,omitempty"`
ReviewedAt *time.Time `json:"reviewedAt,omitempty"`
ReviewedBy string `json:"reviewedBy,omitempty"`
ReviewNote string `json:"reviewNote,omitempty"`
}
type maintenanceVerificationEvidenceAPI struct {
OperatorStateSummary string `json:"operatorStateSummary,omitempty"`
ActiveCriticalAlerts int `json:"activeCriticalAlerts"`
ActiveWarningAlerts int `json:"activeWarningAlerts"`
ActiveCriticalFindings int `json:"activeCriticalFindings"`
ActiveWarningFindings int `json:"activeWarningFindings"`
FailedActionsSinceWindowStart int `json:"failedActionsSinceWindowStart"`
MetricRecovery *maintenanceVerificationMetricRecoveryAPI `json:"metricRecovery,omitempty"`
PatrolRunTODO string `json:"patrolRunTodo,omitempty"`
}
type maintenanceVerificationMetricRecoveryAPI struct {
MetricsObserved []string `json:"metricsObserved,omitempty"`
SamplesAfterEnd int `json:"samplesAfterEnd"`
Trend string `json:"trend,omitempty"`
Note string `json:"note,omitempty"`
}
type maintenanceVerificationReviewRequest struct {
Note string `json:"note,omitempty"`
}
// MaintenanceVerificationHandlers serves the Maintenance Verification
// Report endpoints. Composes the resource store (for read/write of
// LoopReport rows) with the sentinel runtime (for "rerun" requests).
type MaintenanceVerificationHandlers struct {
resources *ResourceHandlers
sentinel *maintenancesentinel.Sentinel
}
// NewMaintenanceVerificationHandlers wires the handler. resources is
// required (it owns the per-org store accessor); sentinel may be nil
// when the runtime has not started a sentinel (e.g. unit tests of the
// other endpoints) — in that case the rerun endpoint returns 503.
func NewMaintenanceVerificationHandlers(resources *ResourceHandlers, sentinel *maintenancesentinel.Sentinel) *MaintenanceVerificationHandlers {
return &MaintenanceVerificationHandlers{resources: resources, sentinel: sentinel}
}
// HandleListForResource serves
// `GET /api/resources/{id}/maintenance-verifications?limit=N`. Returns
// the most recent reports for the resource. Limit defaults to 25 and
// is capped at 200.
func (h *MaintenanceVerificationHandlers) HandleListForResource(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if h.resources == nil {
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
resourceID := extractMaintenanceVerificationsResourceID(r.URL.Path)
if resourceID == "" {
http.Error(w, "Resource ID required", http.StatusBadRequest)
return
}
orgID := GetOrgID(r.Context())
store, err := h.resources.getStore(orgID)
if err != nil {
http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError)
return
}
limit := parseMaintenanceVerificationLimit(r.URL.Query().Get("limit"), 25, 200)
reports, err := store.ListLoopReportsForResource(unified.LoopReportTypeMaintenanceVerification, resourceID, limit)
if err != nil {
http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError)
return
}
out := make([]maintenanceVerificationReportAPI, 0, len(reports))
for _, report := range reports {
out = append(out, toMaintenanceVerificationAPI(report))
}
writeJSON(w, http.StatusOK, map[string]any{
"data": out,
"meta": map[string]any{
"resourceId": resourceID,
"limit": limit,
"total": len(out),
},
})
}
// HandleGet serves `GET /api/maintenance-verifications/{id}`.
func (h *MaintenanceVerificationHandlers) HandleGet(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if h.resources == nil {
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
reportID := extractMaintenanceVerificationReportID(r.URL.Path)
if reportID == "" {
http.Error(w, "Report ID required", http.StatusBadRequest)
return
}
orgID := GetOrgID(r.Context())
store, err := h.resources.getStore(orgID)
if err != nil {
http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError)
return
}
report, found, err := store.GetLoopReport(reportID)
if err != nil {
http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError)
return
}
if !found || report.Type != unified.LoopReportTypeMaintenanceVerification {
writeJSONError(w, http.StatusNotFound, "maintenance_verification_not_found",
"No maintenance verification report exists with this id.")
return
}
writeJSON(w, http.StatusOK, toMaintenanceVerificationAPI(report))
}
// HandleReview serves
// `POST /api/maintenance-verifications/{id}/review`. Records the
// operator's "mark reviewed" verdict on a report. The report's
// status / evidence are immutable — only the review fields update.
func (h *MaintenanceVerificationHandlers) HandleReview(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if h.resources == nil {
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
reportID := extractMaintenanceVerificationReviewReportID(r.URL.Path)
if reportID == "" {
http.Error(w, "Report ID required", http.StatusBadRequest)
return
}
var payload maintenanceVerificationReviewRequest
if r.Body != nil && r.ContentLength != 0 {
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
http.Error(w, "Invalid JSON", http.StatusBadRequest)
return
}
}
orgID := GetOrgID(r.Context())
store, err := h.resources.getStore(orgID)
if err != nil {
http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError)
return
}
report, found, err := store.GetLoopReport(reportID)
if err != nil {
http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError)
return
}
if !found || report.Type != unified.LoopReportTypeMaintenanceVerification {
writeJSONError(w, http.StatusNotFound, "maintenance_verification_not_found",
"No maintenance verification report exists with this id.")
return
}
if err := store.UpdateLoopReportUserOutcome(
reportID,
unified.LoopReportUserOutcomeReviewed,
getUserID(r),
strings.TrimSpace(payload.Note),
time.Now().UTC(),
); err != nil {
if errors.Is(err, unified.ErrLoopReportNotFound) {
writeJSONError(w, http.StatusNotFound, "maintenance_verification_not_found",
"No maintenance verification report exists with this id.")
return
}
if errors.Is(err, unified.ErrLoopReportInvalid) {
writeJSONError(w, http.StatusBadRequest, "maintenance_verification_invalid", err.Error())
return
}
http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError)
return
}
persisted, _, err := store.GetLoopReport(reportID)
if err != nil {
http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, toMaintenanceVerificationAPI(persisted))
}
// HandleRerun serves
// `POST /api/resources/{id}/maintenance-verifications/rerun`. Runs
// the deterministic evaluator again immediately and persists a new
// report (with a `-rerun-N` suffix on the id) so the review history
// is preserved.
func (h *MaintenanceVerificationHandlers) HandleRerun(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if h.sentinel == nil {
writeJSONError(w, http.StatusServiceUnavailable, "maintenance_sentinel_unavailable",
"Maintenance verification sentinel is not running on this instance.")
return
}
resourceID := extractMaintenanceVerificationRerunResourceID(r.URL.Path)
if resourceID == "" {
http.Error(w, "Resource ID required", http.StatusBadRequest)
return
}
report, err := h.sentinel.EvaluateOnce(r.Context(), resourceID)
if err != nil {
if isMaintenanceWindowMissing(err) {
writeJSONError(w, http.StatusBadRequest, "maintenance_window_missing", err.Error())
return
}
http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, toMaintenanceVerificationAPI(report))
}
func toMaintenanceVerificationAPI(report unified.LoopReport) maintenanceVerificationReportAPI {
out := maintenanceVerificationReportAPI{
ID: report.ID,
ResourceID: report.Scope,
Trigger: report.Trigger,
Goal: report.Goal,
Status: string(report.Status),
StartedAt: report.StartedAt,
CompletedAt: report.CompletedAt,
WindowStartedAt: report.WindowStartedAt,
WindowEndedAt: report.WindowEndedAt,
LinkedPatrolRunID: report.LinkedPatrolRunID,
Recommendation: report.Recommendation,
UserOutcome: string(report.UserOutcome),
ReviewedAt: report.ReviewedAt,
ReviewedBy: report.ReviewedBy,
ReviewNote: report.ReviewNote,
LinkedFindingIDs: ensureStringSlice(report.LinkedFindingIDs),
LinkedAlertIDs: ensureStringSlice(report.LinkedAlertIDs),
LinkedActionIDs: ensureStringSlice(report.LinkedActionIDs),
Evidence: maintenanceVerificationEvidenceAPI{
OperatorStateSummary: report.Evidence.OperatorStateSummary,
ActiveCriticalAlerts: report.Evidence.ActiveCriticalAlerts,
ActiveWarningAlerts: report.Evidence.ActiveWarningAlerts,
ActiveCriticalFindings: report.Evidence.ActiveCriticalFindings,
ActiveWarningFindings: report.Evidence.ActiveWarningFindings,
FailedActionsSinceWindowStart: report.Evidence.FailedActionsSinceWindowStart,
PatrolRunTODO: report.Evidence.PatrolRunTODO,
},
}
if report.Evidence.MetricRecovery != nil {
out.Evidence.MetricRecovery = &maintenanceVerificationMetricRecoveryAPI{
MetricsObserved: ensureStringSlice(report.Evidence.MetricRecovery.MetricsObserved),
SamplesAfterEnd: report.Evidence.MetricRecovery.SamplesAfterEnd,
Trend: report.Evidence.MetricRecovery.Trend,
Note: report.Evidence.MetricRecovery.Note,
}
}
return out
}
func ensureStringSlice(in []string) []string {
if in == nil {
return []string{}
}
return in
}
func parseMaintenanceVerificationLimit(raw string, def, max int) int {
raw = strings.TrimSpace(raw)
if raw == "" {
return def
}
v, err := strconv.Atoi(raw)
if err != nil || v <= 0 {
return def
}
if v > max {
return max
}
return v
}
// extractMaintenanceVerificationsResourceID parses the canonical
// resource id out of `/api/resources/<id>/maintenance-verifications`.
func extractMaintenanceVerificationsResourceID(path string) string {
trimmed := strings.TrimPrefix(path, "/api/resources/")
trimmed = strings.TrimSuffix(trimmed, "/")
trimmed = strings.TrimSuffix(trimmed, "/maintenance-verifications")
trimmed = strings.TrimSuffix(trimmed, "/")
return unified.CanonicalResourceID(trimmed)
}
// extractMaintenanceVerificationRerunResourceID parses the canonical
// resource id out of
// `/api/resources/<id>/maintenance-verifications/rerun`.
func extractMaintenanceVerificationRerunResourceID(path string) string {
trimmed := strings.TrimPrefix(path, "/api/resources/")
trimmed = strings.TrimSuffix(trimmed, "/")
trimmed = strings.TrimSuffix(trimmed, "/maintenance-verifications/rerun")
trimmed = strings.TrimSuffix(trimmed, "/")
return unified.CanonicalResourceID(trimmed)
}
// extractMaintenanceVerificationReportID parses the report id out of
// `/api/maintenance-verifications/<id>`.
func extractMaintenanceVerificationReportID(path string) string {
trimmed := strings.TrimPrefix(path, "/api/maintenance-verifications/")
trimmed = strings.TrimSuffix(trimmed, "/")
return strings.TrimSpace(trimmed)
}
// extractMaintenanceVerificationReviewReportID parses the report id
// out of `/api/maintenance-verifications/<id>/review`.
func extractMaintenanceVerificationReviewReportID(path string) string {
trimmed := strings.TrimPrefix(path, "/api/maintenance-verifications/")
trimmed = strings.TrimSuffix(trimmed, "/")
trimmed = strings.TrimSuffix(trimmed, "/review")
trimmed = strings.TrimSuffix(trimmed, "/")
return strings.TrimSpace(trimmed)
}
func isMaintenanceWindowMissing(err error) bool {
if err == nil {
return false
}
return strings.Contains(err.Error(), "no operator state for") ||
strings.Contains(err.Error(), "no maintenance window to verify")
}
// Compile-time interface satisfaction check so the router never wires
// a misshapen sentinel pointer.
var _ context.Context = context.Background()
internal/api/maintenance_verification_test.go
+184
View File
@@ -0,0 +1,184 @@
package api
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/maintenancesentinel"
unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
func newMaintenanceVerificationFixture(t *testing.T) (*MaintenanceVerificationHandlers, unified.ResourceStore) {
t.Helper()
rh := NewResourceHandlers(&config.Config{DataPath: t.TempDir()})
store, err := rh.getStore("default")
if err != nil {
t.Fatalf("get store: %v", err)
}
providers := maintenancesentinel.Providers{
Stores: func(orgID string) (unified.ResourceStore, error) {
return rh.getStore(orgID)
},
Now: func() time.Time { return time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC) },
}
sentinel, err := maintenancesentinel.New(maintenancesentinel.Config{OrgID: "default"}, providers)
if err != nil {
t.Fatalf("new sentinel: %v", err)
}
return NewMaintenanceVerificationHandlers(rh, sentinel), store
}
func withDefaultOrg(req *http.Request) *http.Request {
return req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "default"))
}
func seedMaintenanceWindow(t *testing.T, store unified.ResourceStore, canonicalID string, start, end time.Time) {
t.Helper()
if err := store.SetResourceOperatorState(unified.ResourceOperatorState{
CanonicalID: canonicalID,
MaintenanceStartAt: &start,
MaintenanceEndAt: &end,
SetAt: start,
SetBy: "operator",
}); err != nil {
t.Fatalf("seed operator state: %v", err)
}
}
func TestMaintenanceVerification_HandleListForResource_Empty(t *testing.T) {
h, _ := newMaintenanceVerificationFixture(t)
rec := httptest.NewRecorder()
req := withDefaultOrg(httptest.NewRequest(http.MethodGet, "/api/resources/vm:101/maintenance-verifications", nil))
h.HandleListForResource(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
}
var resp struct {
Data []maintenanceVerificationReportAPI `json:"data"`
Meta struct {
ResourceID string `json:"resourceId"`
Limit int `json:"limit"`
Total int `json:"total"`
} `json:"meta"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatalf("parse body: %v", err)
}
if resp.Meta.ResourceID != "vm:101" || resp.Meta.Total != 0 || resp.Meta.Limit != 25 {
t.Fatalf("meta = %+v", resp.Meta)
}
}
func TestMaintenanceVerification_RerunWritesReport(t *testing.T) {
h, store := newMaintenanceVerificationFixture(t)
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
seedMaintenanceWindow(t, store, "vm:101", now.Add(-time.Hour), now.Add(-15*time.Minute))
rec := httptest.NewRecorder()
req := withDefaultOrg(httptest.NewRequest(http.MethodPost, "/api/resources/vm:101/maintenance-verifications/rerun", nil))
h.HandleRerun(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("rerun status = %d body=%s", rec.Code, rec.Body.String())
}
var report maintenanceVerificationReportAPI
if err := json.Unmarshal(rec.Body.Bytes(), &report); err != nil {
t.Fatalf("parse: %v", err)
}
if report.ResourceID != "vm:101" {
t.Fatalf("resource id = %q", report.ResourceID)
}
if report.Status == "" {
t.Fatalf("status must be set")
}
// Confirm the report landed in the store.
reports, err := store.ListLoopReportsForResource(unified.LoopReportTypeMaintenanceVerification, "vm:101", 0)
if err != nil {
t.Fatalf("list: %v", err)
}
if len(reports) != 1 {
t.Fatalf("expected 1 report after rerun, got %d", len(reports))
}
}
func TestMaintenanceVerification_RerunWithoutWindowReturns400(t *testing.T) {
h, _ := newMaintenanceVerificationFixture(t)
rec := httptest.NewRecorder()
req := withDefaultOrg(httptest.NewRequest(http.MethodPost, "/api/resources/vm:101/maintenance-verifications/rerun", nil))
h.HandleRerun(rec, req)
if rec.Code != http.StatusBadRequest {
t.Fatalf("expected 400 on missing window, got %d body=%s", rec.Code, rec.Body.String())
}
var body map[string]string
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatalf("body must be JSON: %v", err)
}
if body["error"] != "maintenance_window_missing" {
t.Fatalf("error code = %q want maintenance_window_missing", body["error"])
}
}
func TestMaintenanceVerification_ReviewMarksReportReviewed(t *testing.T) {
h, store := newMaintenanceVerificationFixture(t)
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
seedMaintenanceWindow(t, store, "vm:101", now.Add(-time.Hour), now.Add(-15*time.Minute))
rerunRec := httptest.NewRecorder()
rerunReq := withDefaultOrg(httptest.NewRequest(http.MethodPost, "/api/resources/vm:101/maintenance-verifications/rerun", nil))
h.HandleRerun(rerunRec, rerunReq)
if rerunRec.Code != http.StatusOK {
t.Fatalf("rerun status = %d body=%s", rerunRec.Code, rerunRec.Body.String())
}
var seeded maintenanceVerificationReportAPI
if err := json.Unmarshal(rerunRec.Body.Bytes(), &seeded); err != nil {
t.Fatalf("parse rerun: %v", err)
}
rec := httptest.NewRecorder()
req := withDefaultOrg(httptest.NewRequest(
http.MethodPost,
"/api/maintenance-verifications/"+seeded.ID+"/review",
nil,
))
h.HandleReview(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("review status = %d body=%s", rec.Code, rec.Body.String())
}
var reviewed maintenanceVerificationReportAPI
if err := json.Unmarshal(rec.Body.Bytes(), &reviewed); err != nil {
t.Fatalf("parse review response: %v", err)
}
if reviewed.UserOutcome != string(unified.LoopReportUserOutcomeReviewed) {
t.Fatalf("userOutcome = %q want reviewed", reviewed.UserOutcome)
}
if reviewed.Status != seeded.Status {
t.Fatalf("status mutated: was %q, now %q", seeded.Status, reviewed.Status)
}
}
func TestMaintenanceVerification_ReviewMissingReportReturns404(t *testing.T) {
h, _ := newMaintenanceVerificationFixture(t)
rec := httptest.NewRecorder()
req := withDefaultOrg(httptest.NewRequest(http.MethodPost, "/api/maintenance-verifications/no-such-id/review", nil))
h.HandleReview(rec, req)
if rec.Code != http.StatusNotFound {
t.Fatalf("expected 404 on missing report, got %d body=%s", rec.Code, rec.Body.String())
}
}
func TestMaintenanceVerification_RerunReturns503WhenSentinelNil(t *testing.T) {
rh := NewResourceHandlers(&config.Config{DataPath: t.TempDir()})
h := NewMaintenanceVerificationHandlers(rh, nil)
rec := httptest.NewRecorder()
req := withDefaultOrg(httptest.NewRequest(http.MethodPost, "/api/resources/vm:101/maintenance-verifications/rerun", nil))
h.HandleRerun(rec, req)
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("expected 503 when sentinel disabled, got %d body=%s", rec.Code, rec.Body.String())
}
}
internal/api/maintenance_verification_wiring.go
+289
View File
@@ -0,0 +1,289 @@
package api
import (
"context"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/ai"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/maintenancesentinel"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
// buildMaintenanceVerificationSentinel constructs the sentinel with
// adapters that pull alerts, findings, action audits, and recent
// metric samples out of the live runtime. Closures capture `r` so the
// providers always read the current monitor / aiSettings / store
// state (multi-tenant rebuilds replace those in place; latching to a
// snapshot would go stale).
//
// Returns nil when the resource handlers are not initialized — the
// caller treats nil as "sentinel disabled" and the API rerun endpoint
// returns 503.
func (r *Router) buildMaintenanceVerificationSentinel() *maintenancesentinel.Sentinel {
if r == nil || r.resourceHandlers == nil {
return nil
}
providers := maintenancesentinel.Providers{
Stores: func(orgID string) (unified.ResourceStore, error) {
return r.resourceHandlers.getStore(orgID)
},
ActiveAlerts: r.maintenanceVerificationActiveAlerts,
ActiveFindings: r.maintenanceVerificationActiveFindings,
RecentActions: r.maintenanceVerificationRecentActions,
PostWindowMetricSamples: r.maintenanceVerificationMetricSamples,
Now: func() time.Time { return time.Now().UTC() },
}
sentinel, err := maintenancesentinel.New(maintenancesentinel.Config{
OrgID: "default",
}, providers)
if err != nil {
return nil
}
return sentinel
}
func (r *Router) maintenanceVerificationActiveAlerts(orgID, canonicalID string) []maintenancesentinel.AlertSummary {
mgr := r.resolveAlertManagerForOrg(orgID)
if mgr == nil {
return nil
}
canonicalID = unified.CanonicalResourceID(canonicalID)
if canonicalID == "" {
return nil
}
out := []maintenancesentinel.AlertSummary{}
for _, a := range mgr.GetActiveAlerts() {
if !alertMatchesResource(a, canonicalID) {
continue
}
out = append(out, maintenancesentinel.AlertSummary{
ID: a.ID,
Severity: mapAlertLevel(a.Level),
Type: a.Type,
Acknowledged: a.Acknowledged,
})
}
return out
}
func (r *Router) maintenanceVerificationActiveFindings(orgID, canonicalID string) []maintenancesentinel.FindingSummary {
patrol := r.resolvePatrolServiceForOrg(orgID)
if patrol == nil {
return nil
}
canonicalID = unified.CanonicalResourceID(canonicalID)
if canonicalID == "" {
return nil
}
findings := patrol.GetFindingsForResource(canonicalID)
out := make([]maintenancesentinel.FindingSummary, 0, len(findings))
for _, f := range findings {
if f == nil {
continue
}
out = append(out, maintenancesentinel.FindingSummary{
ID: f.ID,
Severity: mapFindingSeverity(f.Severity),
Category: string(f.Category),
Resolved: f.ResolvedAt != nil,
Acknowledged: f.AcknowledgedAt != nil,
})
}
return out
}
func (r *Router) maintenanceVerificationRecentActions(orgID, canonicalID string, since time.Time) []maintenancesentinel.ActionSummary {
if r.resourceHandlers == nil {
return nil
}
store, err := r.resourceHandlers.getStore(orgID)
if err != nil || store == nil {
return nil
}
audits, err := store.GetActionAudits(canonicalID, since, 50)
if err != nil {
return nil
}
out := make([]maintenancesentinel.ActionSummary, 0, len(audits))
for _, a := range audits {
out = append(out, maintenancesentinel.ActionSummary{
ID: a.ID,
State: string(a.State),
UpdatedAt: a.UpdatedAt,
})
}
return out
}
func (r *Router) maintenanceVerificationMetricSamples(orgID, canonicalID string, windowEnd, now time.Time) ([]maintenancesentinel.MetricSample, bool) {
history := r.resolveMetricsHistoryForOrg(orgID)
if history == nil {
return nil, false
}
canonicalID = unified.CanonicalResourceID(canonicalID)
if canonicalID == "" {
return nil, false
}
// The metrics history is keyed by source IDs (e.g. "qemu/101",
// "node/pve") not canonical resource IDs. Strip the canonical
// `kind:` prefix so the lookup works for the common case (vm,
// container, node). Resources whose source-id form does not
// match this convention (storage, agents, docker hosts) won't
// have metric history available — the sentinel reports
// MetricSourceAvailable=false in that case, which is honest.
sourceID := canonicalToSourceID(canonicalID)
if sourceID == "" {
return nil, false
}
since := now.Sub(windowEnd)
if since <= 0 {
since = time.Hour
}
// Inspect cpu + memory only for the MVP. These are the metrics
// every guest/node reports; storage / disk / network are not
// universally available across resource kinds.
samples := []maintenancesentinel.MetricSample{}
anyData := false
for _, metric := range []string{"cpu", "memory"} {
// Try both guest and node lookups; whichever has data wins.
points := history.GetGuestMetrics(sourceID, metric, since)
if len(points) == 0 {
points = history.GetNodeMetrics(sourceID, metric, since)
}
if len(points) > 0 {
anyData = true
for _, p := range points {
if !p.Timestamp.After(windowEnd) {
continue
}
samples = append(samples, maintenancesentinel.MetricSample{
Metric: metric,
Value: p.Value,
Timestamp: p.Timestamp,
})
}
}
}
// available=true even when samples is empty if we know there's
// a history bucket for the resource — that's still useful
// evidence ("the resource exists in history but has not reported
// since the window closed").
return samples, anyData
}
// resolveAlertManagerForOrg returns the AlertManager for the supplied
// org. MVP runs the default org only; multi-tenant resolution is
// supported as a future change.
func (r *Router) resolveAlertManagerForOrg(orgID string) *alerts.Manager {
monitor := r.resolveMonitorForOrg(orgID)
if monitor == nil {
return nil
}
return monitor.GetAlertManager()
}
func (r *Router) resolvePatrolServiceForOrg(orgID string) *ai.PatrolService {
if r.aiSettingsHandler == nil {
return nil
}
aiService := r.aiSettingsHandler.GetAIService(maintenanceVerificationOrgContext(orgID))
if aiService == nil {
return nil
}
return aiService.GetPatrolService()
}
func (r *Router) resolveMetricsHistoryForOrg(orgID string) *monitoring.MetricsHistory {
monitor := r.resolveMonitorForOrg(orgID)
if monitor == nil {
return nil
}
return monitor.GetMetricsHistory()
}
func (r *Router) resolveMonitorForOrg(orgID string) *monitoring.Monitor {
orgID = strings.TrimSpace(orgID)
if orgID == "" || orgID == "default" {
return r.monitor
}
if r.mtMonitor == nil {
return nil
}
monitor, err := r.mtMonitor.GetMonitor(orgID)
if err != nil {
return nil
}
return monitor
}
func maintenanceVerificationOrgContext(orgID string) context.Context {
if orgID == "" {
orgID = "default"
}
return context.WithValue(context.Background(), OrgIDContextKey, orgID)
}
// alertMatchesResource reports whether the alert targets the supplied
// canonical resource id. Alert IDs are not canonical — the alert
// model carries `ResourceID` (legacy form) and `CanonicalSpecID` /
// `CanonicalState` from the canonical engine. We try the canonical
// form first, then fall back to canonicalizing the legacy form so we
// catch both providers.
func alertMatchesResource(a alerts.Alert, canonicalID string) bool {
if unified.CanonicalResourceID(a.CanonicalState) == canonicalID {
return true
}
if unified.CanonicalResourceID(a.CanonicalSpecID) == canonicalID {
return true
}
return unified.CanonicalResourceID(a.ResourceID) == canonicalID
}
func mapAlertLevel(level alerts.AlertLevel) maintenancesentinel.Severity {
switch level {
case alerts.AlertLevelCritical:
return maintenancesentinel.SeverityCritical
case alerts.AlertLevelWarning:
return maintenancesentinel.SeverityWarning
default:
return ""
}
}
func mapFindingSeverity(s ai.FindingSeverity) maintenancesentinel.Severity {
switch s {
case ai.FindingSeverityCritical:
return maintenancesentinel.SeverityCritical
case ai.FindingSeverityWarning:
return maintenancesentinel.SeverityWarning
default:
return ""
}
}
// canonicalToSourceID best-effort maps a canonical resource ID into
// the legacy source-id form used by `monitoring.MetricsHistory`.
// Canonical ids look like `vm:101`, `ct:200`, `node:pve`,
// `docker-container:abc`, etc. The metrics history keys are
// `qemu/101`, `lxc/200`, `node/pve`. Resources not covered here will
// not surface metric history evidence on their report — the sentinel
// honestly reports MetricSourceAvailable=false.
func canonicalToSourceID(canonicalID string) string {
parts := strings.SplitN(canonicalID, ":", 2)
if len(parts) != 2 {
return ""
}
kind, id := parts[0], parts[1]
switch kind {
case "vm":
return "qemu/" + id
case "ct":
return "lxc/" + id
case "node":
return "node/" + id
}
return ""
}
internal/api/route_inventory_test.go
+4
View File
@@ -412,6 +412,10 @@ var allRouteAllowlist = []string{
"/api/resources/{id}/facets",
"/api/resources/{id}/timeline",
"/api/resources/{id}/operator-state",
"/api/resources/{id}/maintenance-verifications",
"POST /api/resources/{id}/maintenance-verifications/rerun",
"/api/maintenance-verifications/{reportId}",
"POST /api/maintenance-verifications/{reportId}/review",
"/api/agent/resource-context/{id}",
"/api/agent/fleet-context",
"/api/agent/capabilities",
internal/api/router.go
+8
View File
@@ -42,6 +42,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/deploy"
"github.com/rcourtman/pulse-go-rewrite/internal/maintenancesentinel"
"github.com/rcourtman/pulse-go-rewrite/internal/metrics"
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
@@ -88,6 +89,8 @@ type Router struct {
aiHandler *AIHandler // AI chat handler
discoveryHandlers *DiscoveryHandlers
resourceHandlers *ResourceHandlers
maintenanceVerificationHandlers *MaintenanceVerificationHandlers
maintenanceSentinel *maintenancesentinel.Sentinel
agentContextHandler *AgentContextHandler
agentEventBroadcaster *AgentEventBroadcaster
resourceRegistry *unifiedresources.ResourceRegistry
@@ -436,6 +439,11 @@ func (r *Router) setupRoutes() {
r.unifiedAgentHandlers = NewUnifiedAgentHandlers(r.mtMonitor, r.monitor, r.wsHub)
r.kubernetesAgentHandlers.SetRecoveryIngestor(r.recoveryHandlers)
r.resourceHandlers = NewResourceHandlers(r.config)
r.maintenanceSentinel = r.buildMaintenanceVerificationSentinel()
r.maintenanceVerificationHandlers = NewMaintenanceVerificationHandlers(r.resourceHandlers, r.maintenanceSentinel)
if r.maintenanceSentinel != nil {
r.maintenanceSentinel.Start(r.lifecycleCtx)
}
r.agentContextHandler = NewAgentContextHandler(r.resourceHandlers)
// Wire pending-approvals into the bundle. The provider resolves
// the approval store at request time so multi-tenant rebuilds
internal/api/router_routes_monitoring.go
+11
View File
@@ -57,6 +57,17 @@ func (r *Router) registerMonitoringResourceRoutes(
}
r.resourceHandlers.HandleResourceOperatorState(w, req)
}))
// Per-resource Maintenance Verification Reports. Listing and reading
// are monitoring-read; rerun is monitoring-write because it produces
// a new persisted record. Review (mark-reviewed) is monitoring-write
// for the same reason. Wired before the broad `/api/resources/`
// catch-all so the more specific paths route here.
if r.maintenanceVerificationHandlers != nil {
r.mux.HandleFunc("/api/resources/{id}/maintenance-verifications", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.maintenanceVerificationHandlers.HandleListForResource)))
r.mux.HandleFunc("POST /api/resources/{id}/maintenance-verifications/rerun", RequireAuth(r.config, RequireScope(config.ScopeMonitoringWrite, r.maintenanceVerificationHandlers.HandleRerun)))
r.mux.HandleFunc("/api/maintenance-verifications/{reportId}", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.maintenanceVerificationHandlers.HandleGet)))
r.mux.HandleFunc("POST /api/maintenance-verifications/{reportId}/review", RequireAuth(r.config, RequireScope(config.ScopeMonitoringWrite, r.maintenanceVerificationHandlers.HandleReview)))
}
r.mux.HandleFunc("/api/resources/", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.resourceHandlers.HandleResourceRoutes)))
// Agent-consumable bundled context — substrate for any LLM agent
// (in-process Patrol/Assistant or external) that needs the full
internal/maintenancesentinel/sentinel.go
+288
View File
@@ -0,0 +1,288 @@
package maintenancesentinel
import (
"context"
"errors"
"fmt"
"sync"
"time"
unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog/log"
)
// Providers bundles the slim, package-defined interfaces the sentinel
// needs from the rest of the system. The caller (router wiring) is
// responsible for adapting alerts/findings/actions/metrics into these
// shapes. Keeps this package free of `internal/alerts`, `internal/ai`,
// and `internal/monitoring` imports so there are no cycles.
type Providers struct {
// Stores resolves the ResourceStore for the given org id. The
// sentinel runs against the default org for MVP; tenant-scoped
// scheduling is left to a future change.
Stores func(orgID string) (unified.ResourceStore, error)
// ActiveAlerts returns active alerts for a resource at this moment.
ActiveAlerts func(orgID, canonicalID string) []AlertSummary
// ActiveFindings returns active Patrol findings for a resource.
ActiveFindings func(orgID, canonicalID string) []FindingSummary
// RecentActions returns action audit records for the resource
// since the supplied lower bound. The sentinel itself filters for
// failed state after the call returns.
RecentActions func(orgID, canonicalID string, since time.Time) []ActionSummary
// PostWindowMetricSamples returns metric samples observed after
// the maintenance window closed for the resource. The boolean
// reports whether a metric source was available for the
// resource — distinct from "available but empty".
PostWindowMetricSamples func(orgID, canonicalID string, windowEnd, now time.Time) ([]MetricSample, bool)
// Now is the clock; tests inject a fixed clock. Defaults to
// time.Now.
Now func() time.Time
}
// Sentinel watches operator state for maintenance-window-end events
// and writes Maintenance Verification Reports.
type Sentinel struct {
orgID string
providers Providers
tick time.Duration
lookbackLimit time.Duration
mu sync.Mutex
}
// Config configures a Sentinel instance.
type Config struct {
// OrgID is the tenant the sentinel scans. MVP runs the default
// org only — multi-tenant scheduling is deferred.
OrgID string
// Tick controls the sweep cadence. Defaults to one minute.
Tick time.Duration
// LookbackLimit prevents the sentinel from generating reports
// for ancient windows on first start (e.g. after a long
// downtime). Window-end events older than `now - LookbackLimit`
// are ignored. Default: 7 days.
LookbackLimit time.Duration
}
// New returns a Sentinel ready to start. Providers is required.
func New(cfg Config, providers Providers) (*Sentinel, error) {
if providers.Stores == nil {
return nil, errors.New("maintenancesentinel: Providers.Stores is required")
}
tick := cfg.Tick
if tick <= 0 {
tick = time.Minute
}
lookback := cfg.LookbackLimit
if lookback <= 0 {
lookback = 7 * 24 * time.Hour
}
org := cfg.OrgID
if org == "" {
org = "default"
}
return &Sentinel{
orgID: org,
providers: providers,
tick: tick,
lookbackLimit: lookback,
}, nil
}
// Start runs the sentinel loop in a goroutine and returns. The loop
// stops when ctx is canceled. Safe to call once per Sentinel.
func (s *Sentinel) Start(ctx context.Context) {
go s.run(ctx)
}
func (s *Sentinel) run(ctx context.Context) {
ticker := time.NewTicker(s.tick)
defer ticker.Stop()
// Run one sweep immediately so a newly-ended window doesn't have
// to wait a full tick for a report.
s.tickOnce(ctx)
for {
select {
case <-ctx.Done():
log.Info().Msg("maintenance-verification sentinel stopped")
return
case <-ticker.C:
s.tickOnce(ctx)
}
}
}
func (s *Sentinel) tickOnce(ctx context.Context) {
s.mu.Lock()
defer s.mu.Unlock()
store, err := s.providers.Stores(s.orgID)
if err != nil {
log.Debug().Err(err).Msg("maintenance-verification sentinel: get store")
return
}
states, err := store.ListResourceOperatorStates()
if err != nil {
log.Debug().Err(err).Msg("maintenance-verification sentinel: list operator states")
return
}
now := s.now()
cutoff := now.Add(-s.lookbackLimit)
for _, state := range states {
if ctx.Err() != nil {
return
}
if state.MaintenanceStartAt == nil || state.MaintenanceEndAt == nil {
continue
}
if !state.MaintenanceEndAt.Before(now) && !state.MaintenanceEndAt.Equal(now) {
// Window hasn't ended yet.
continue
}
if state.MaintenanceEndAt.Before(cutoff) {
// Window ended too long ago — don't backfill ancient events.
continue
}
s.evaluateForState(state, store, now)
}
}
func (s *Sentinel) evaluateForState(state unified.ResourceOperatorState, store unified.ResourceStore, now time.Time) {
canonicalID := unified.CanonicalResourceID(state.CanonicalID)
if canonicalID == "" || state.MaintenanceEndAt == nil {
return
}
if _, exists, err := store.FindLoopReportByWindow(unified.LoopReportTypeMaintenanceVerification, canonicalID, *state.MaintenanceEndAt); err != nil {
log.Debug().Err(err).Str("resource", canonicalID).Msg("maintenance-verification sentinel: dedupe lookup")
return
} else if exists {
// Already wrote a report for this (resource, window-end).
return
}
inputs := s.buildInputs(state, now)
report := EvaluateVerification(inputs)
if err := store.RecordLoopReport(report); err != nil {
// A unique-constraint conflict (race against a parallel
// tick) is benign — another writer wrote the same window.
log.Debug().Err(err).Str("resource", canonicalID).Msg("maintenance-verification sentinel: record report")
return
}
log.Info().
Str("resource", canonicalID).
Str("status", string(report.Status)).
Str("report_id", report.ID).
Msg("maintenance verification report written")
}
// buildInputs gathers the deterministic input bundle for the
// evaluator. Provider closures may be nil — the inputs simply carry
// zero values in that case.
func (s *Sentinel) buildInputs(state unified.ResourceOperatorState, now time.Time) VerificationInputs {
canonicalID := unified.CanonicalResourceID(state.CanonicalID)
inputs := VerificationInputs{
ResourceID: canonicalID,
OperatorState: cloneOperatorState(state),
Now: now,
}
if state.MaintenanceStartAt != nil {
inputs.WindowStartedAt = state.MaintenanceStartAt.UTC()
}
if state.MaintenanceEndAt != nil {
inputs.WindowEndedAt = state.MaintenanceEndAt.UTC()
}
if s.providers.ActiveAlerts != nil {
inputs.ActiveAlerts = s.providers.ActiveAlerts(s.orgID, canonicalID)
}
if s.providers.ActiveFindings != nil {
inputs.ActiveFindings = s.providers.ActiveFindings(s.orgID, canonicalID)
}
if s.providers.RecentActions != nil {
windowStart := inputs.WindowStartedAt
if windowStart.IsZero() {
windowStart = now.Add(-24 * time.Hour)
}
inputs.RecentActions = s.providers.RecentActions(s.orgID, canonicalID, windowStart)
}
if s.providers.PostWindowMetricSamples != nil {
samples, available := s.providers.PostWindowMetricSamples(s.orgID, canonicalID, inputs.WindowEndedAt, now)
inputs.PostWindowMetricSamples = samples
inputs.MetricSourceAvailable = available
}
return inputs
}
func cloneOperatorState(s unified.ResourceOperatorState) *unified.ResourceOperatorState {
clone := s
if s.MaintenanceStartAt != nil {
t := *s.MaintenanceStartAt
clone.MaintenanceStartAt = &t
}
if s.MaintenanceEndAt != nil {
t := *s.MaintenanceEndAt
clone.MaintenanceEndAt = &t
}
return &clone
}
func (s *Sentinel) now() time.Time {
if s.providers.Now != nil {
return s.providers.Now().UTC()
}
return time.Now().UTC()
}
// EvaluateOnce runs the deterministic verification for the supplied
// resource exactly once, immediately, writing a fresh report. Used by
// the "rerun verification" handler so an operator can re-evaluate
// without waiting for the next tick.
//
// If the existing report for the (resource, window-end) triple is
// found, this writes a *new* report with a -rerun-N suffix so the
// review history is preserved.
func (s *Sentinel) EvaluateOnce(ctx context.Context, canonicalID string) (unified.LoopReport, error) {
if ctx.Err() != nil {
return unified.LoopReport{}, ctx.Err()
}
canonicalID = unified.CanonicalResourceID(canonicalID)
if canonicalID == "" {
return unified.LoopReport{}, fmt.Errorf("maintenancesentinel: canonical id is required")
}
store, err := s.providers.Stores(s.orgID)
if err != nil {
return unified.LoopReport{}, err
}
state, found, err := store.GetResourceOperatorState(canonicalID)
if err != nil {
return unified.LoopReport{}, err
}
if !found {
return unified.LoopReport{}, fmt.Errorf("maintenancesentinel: no operator state for %q", canonicalID)
}
if state.MaintenanceStartAt == nil || state.MaintenanceEndAt == nil {
return unified.LoopReport{}, fmt.Errorf("maintenancesentinel: resource %q has no maintenance window to verify", canonicalID)
}
now := s.now()
inputs := s.buildInputs(state, now)
report := EvaluateVerification(inputs)
report.ID = uniqueRerunID(store, report.ID)
if err := store.RecordLoopReport(report); err != nil {
return unified.LoopReport{}, err
}
return report, nil
}
func uniqueRerunID(store unified.ResourceStore, base string) string {
if _, exists, err := store.GetLoopReport(base); err == nil && !exists {
return base
}
for i := 1; i < 1000; i++ {
candidate := fmt.Sprintf("%s-rerun-%d", base, i)
if _, exists, err := store.GetLoopReport(candidate); err == nil && !exists {
return candidate
}
}
return fmt.Sprintf("%s-rerun-%d", base, time.Now().UnixNano())
}
internal/maintenancesentinel/sentinel_test.go
+191
View File
@@ -0,0 +1,191 @@
package maintenancesentinel
import (
"context"
"strings"
"testing"
"time"
unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
func TestSentinelTickOnceWritesReportAndDedupes(t *testing.T) {
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
windowStart := now.Add(-time.Hour)
windowEnd := now.Add(-15 * time.Minute)
store := unified.NewMemoryStore()
if err := store.SetResourceOperatorState(unified.ResourceOperatorState{
CanonicalID: "vm:101",
MaintenanceStartAt: &windowStart,
MaintenanceEndAt: &windowEnd,
SetAt: windowStart,
SetBy: "operator",
}); err != nil {
t.Fatalf("seed operator state: %v", err)
}
providers := Providers{
Stores: func(orgID string) (unified.ResourceStore, error) { return store, nil },
Now: func() time.Time { return now },
}
sentinel, err := New(Config{OrgID: "default", Tick: time.Minute}, providers)
if err != nil {
t.Fatalf("new sentinel: %v", err)
}
sentinel.tickOnce(context.Background())
reports, err := store.ListLoopReportsForResource(unified.LoopReportTypeMaintenanceVerification, "vm:101", 0)
if err != nil {
t.Fatalf("list reports: %v", err)
}
if len(reports) != 1 {
t.Fatalf("expected 1 report after first tick, got %d", len(reports))
}
sentinel.tickOnce(context.Background())
reports, err = store.ListLoopReportsForResource(unified.LoopReportTypeMaintenanceVerification, "vm:101", 0)
if err != nil {
t.Fatalf("list reports: %v", err)
}
if len(reports) != 1 {
t.Fatalf("expected dedupe; got %d reports after second tick", len(reports))
}
}
func TestSentinelSkipsAncientWindows(t *testing.T) {
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
windowStart := now.Add(-30 * 24 * time.Hour)
windowEnd := now.Add(-29 * 24 * time.Hour)
store := unified.NewMemoryStore()
if err := store.SetResourceOperatorState(unified.ResourceOperatorState{
CanonicalID: "vm:202",
MaintenanceStartAt: &windowStart,
MaintenanceEndAt: &windowEnd,
SetAt: windowStart,
SetBy: "operator",
}); err != nil {
t.Fatalf("seed operator state: %v", err)
}
providers := Providers{
Stores: func(orgID string) (unified.ResourceStore, error) { return store, nil },
Now: func() time.Time { return now },
}
sentinel, err := New(Config{OrgID: "default", LookbackLimit: 7 * 24 * time.Hour}, providers)
if err != nil {
t.Fatalf("new sentinel: %v", err)
}
sentinel.tickOnce(context.Background())
reports, err := store.ListLoopReportsForResource(unified.LoopReportTypeMaintenanceVerification, "vm:202", 0)
if err != nil {
t.Fatalf("list reports: %v", err)
}
if len(reports) != 0 {
t.Fatalf("expected no report for ancient window, got %d", len(reports))
}
}
func TestSentinelSkipsOpenWindow(t *testing.T) {
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
windowStart := now.Add(-30 * time.Minute)
windowEnd := now.Add(30 * time.Minute)
store := unified.NewMemoryStore()
if err := store.SetResourceOperatorState(unified.ResourceOperatorState{
CanonicalID: "vm:303",
MaintenanceStartAt: &windowStart,
MaintenanceEndAt: &windowEnd,
SetAt: windowStart,
SetBy: "operator",
}); err != nil {
t.Fatalf("seed operator state: %v", err)
}
providers := Providers{
Stores: func(orgID string) (unified.ResourceStore, error) { return store, nil },
Now: func() time.Time { return now },
}
sentinel, err := New(Config{OrgID: "default"}, providers)
if err != nil {
t.Fatalf("new sentinel: %v", err)
}
sentinel.tickOnce(context.Background())
reports, err := store.ListLoopReportsForResource(unified.LoopReportTypeMaintenanceVerification, "vm:303", 0)
if err != nil {
t.Fatalf("list reports: %v", err)
}
if len(reports) != 0 {
t.Fatalf("expected no report while window still open, got %d", len(reports))
}
}
func TestSentinelEvaluateOncePersistsRerunReport(t *testing.T) {
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
windowStart := now.Add(-time.Hour)
windowEnd := now.Add(-15 * time.Minute)
store := unified.NewMemoryStore()
if err := store.SetResourceOperatorState(unified.ResourceOperatorState{
CanonicalID: "vm:101",
MaintenanceStartAt: &windowStart,
MaintenanceEndAt: &windowEnd,
SetAt: windowStart,
SetBy: "operator",
}); err != nil {
t.Fatalf("seed operator state: %v", err)
}
providers := Providers{
Stores: func(orgID string) (unified.ResourceStore, error) { return store, nil },
Now: func() time.Time { return now },
}
sentinel, err := New(Config{OrgID: "default"}, providers)
if err != nil {
t.Fatalf("new sentinel: %v", err)
}
sentinel.tickOnce(context.Background())
rerun, err := sentinel.EvaluateOnce(context.Background(), "vm:101")
if err != nil {
t.Fatalf("evaluate once: %v", err)
}
if !strings.Contains(rerun.ID, "rerun") {
t.Fatalf("expected rerun id suffix, got %q", rerun.ID)
}
reports, err := store.ListLoopReportsForResource(unified.LoopReportTypeMaintenanceVerification, "vm:101", 0)
if err != nil {
t.Fatalf("list reports: %v", err)
}
if len(reports) != 2 {
t.Fatalf("expected 2 reports after rerun, got %d", len(reports))
}
}
func TestSentinelEvaluateOnceErrorsWhenNoWindow(t *testing.T) {
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
store := unified.NewMemoryStore()
if err := store.SetResourceOperatorState(unified.ResourceOperatorState{
CanonicalID: "vm:404",
SetAt: now,
SetBy: "operator",
}); err != nil {
t.Fatalf("seed operator state: %v", err)
}
providers := Providers{
Stores: func(orgID string) (unified.ResourceStore, error) { return store, nil },
Now: func() time.Time { return now },
}
sentinel, err := New(Config{OrgID: "default"}, providers)
if err != nil {
t.Fatalf("new sentinel: %v", err)
}
if _, err := sentinel.EvaluateOnce(context.Background(), "vm:404"); err == nil {
t.Fatalf("expected error when no maintenance window present")
}
}
internal/maintenancesentinel/verification.go
+400
View File
@@ -0,0 +1,400 @@
// Package maintenancesentinel implements the Maintenance Verification
// Report loop. When a maintenance window ends for a resource, the
// sentinel runs deterministic checks against the resource's current
// state and writes a durable Maintenance Verification Report.
//
// The package is intentionally small and shaped so future loops
// (post-incident verification, post-deployment verification) can reuse
// the same persistence (LoopReport) and review actions. This is
// substrate, not a generic agent framework.
package maintenancesentinel
import (
"fmt"
"strings"
"time"
unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
// Severity classifies an alert or finding for the verification
// decision tree. Maintenance Verification only distinguishes critical
// vs. warning — the deeper severity grades (watch/info) are not
// load-bearing for the decision.
type Severity string
const (
SeverityCritical Severity = "critical"
SeverityWarning Severity = "warning"
)
// AlertSummary is the slim projection of an alert the sentinel needs.
// The router adapts the canonical `alerts.Alert` into this shape so
// this package does not import `internal/alerts`.
type AlertSummary struct {
ID string
Severity Severity
Type string
Acknowledged bool
}
// FindingSummary is the slim projection of a Patrol finding the
// sentinel needs. The router adapts `ai.Finding` into this shape so
// this package does not import `internal/ai`.
type FindingSummary struct {
ID string
Severity Severity
Category string
Resolved bool
Acknowledged bool
}
// ActionSummary is the slim projection of an action audit record the
// sentinel needs.
type ActionSummary struct {
ID string
State string
UpdatedAt time.Time
}
// MetricSample is a single recent metric data point used to decide
// whether a resource is reporting metrics after maintenance closed.
type MetricSample struct {
Metric string
Value float64
Timestamp time.Time
}
// VerificationInputs is the full deterministic input bundle the
// evaluator consumes. The sentinel collects this once before calling
// EvaluateVerification.
type VerificationInputs struct {
ResourceID string
ResourceName string
OperatorState *unified.ResourceOperatorState
WindowStartedAt time.Time
WindowEndedAt time.Time
Now time.Time
ActiveAlerts []AlertSummary
ActiveFindings []FindingSummary
// RecentActions should be the action audits with UpdatedAt >=
// WindowStartedAt. The evaluator counts how many of them ended in
// "failed" state.
RecentActions []ActionSummary
// PostWindowMetricSamples are the metric samples observed after
// WindowEndedAt. Empty slice means no metrics have been reported
// since the window closed.
PostWindowMetricSamples []MetricSample
// MetricSourceAvailable indicates whether the sentinel had access
// to a metric source for this resource at all. Distinguishes
// "we looked and saw nothing" from "we have no way to look".
MetricSourceAvailable bool
}
// EvaluateVerification applies the deterministic decision tree to the
// inputs and returns a fully-populated LoopReport. The function is
// pure: no I/O, no time.Now() (always uses inputs.Now), no goroutines.
func EvaluateVerification(inputs VerificationInputs) unified.LoopReport {
now := inputs.Now
if now.IsZero() {
now = time.Now().UTC()
}
now = now.UTC()
canonicalID := unified.CanonicalResourceID(inputs.ResourceID)
report := unified.LoopReport{
ID: newReportID(canonicalID, inputs.WindowEndedAt),
Type: unified.LoopReportTypeMaintenanceVerification,
Scope: canonicalID,
Trigger: "maintenance_window_end",
Goal: "Confirm the resource recovered after the maintenance window closed.",
Status: unified.LoopReportStatusPending,
StartedAt: now,
CompletedAt: now,
}
if !inputs.WindowStartedAt.IsZero() {
t := inputs.WindowStartedAt.UTC()
report.WindowStartedAt = &t
}
if !inputs.WindowEndedAt.IsZero() {
t := inputs.WindowEndedAt.UTC()
report.WindowEndedAt = &t
}
evidence := unified.LoopReportEvidence{
OperatorStateSummary: summarizeOperatorState(inputs.OperatorState, inputs.WindowEndedAt, now),
}
criticalAlerts, warningAlerts, alertIDs := classifyAlerts(inputs.ActiveAlerts)
evidence.ActiveCriticalAlerts = criticalAlerts
evidence.ActiveWarningAlerts = warningAlerts
report.LinkedAlertIDs = alertIDs
criticalFindings, warningFindings, findingIDs := classifyFindings(inputs.ActiveFindings)
evidence.ActiveCriticalFindings = criticalFindings
evidence.ActiveWarningFindings = warningFindings
report.LinkedFindingIDs = findingIDs
failedActions, failedActionIDs := classifyFailedActions(inputs.RecentActions, inputs.WindowStartedAt)
evidence.FailedActionsSinceWindowStart = failedActions
report.LinkedActionIDs = failedActionIDs
if inputs.MetricSourceAvailable {
evidence.MetricRecovery = summarizeMetricRecovery(inputs.PostWindowMetricSamples, inputs.WindowEndedAt, now)
}
// Patrol-run trigger is deferred — see PatrolRunTODO contract.
evidence.PatrolRunTODO = patrolRunTODO(inputs, evidence)
report.Evidence = evidence
report.Status, report.Recommendation = decideStatus(inputs, evidence)
return report
}
func summarizeOperatorState(state *unified.ResourceOperatorState, windowEnd, now time.Time) string {
if state == nil {
return "no operator state recorded for this resource"
}
parts := []string{}
if state.MaintenanceStartAt != nil && state.MaintenanceEndAt != nil {
if state.MaintenanceEndAt.After(now) {
parts = append(parts, "maintenance window still open")
} else {
parts = append(parts, "maintenance window ended")
}
} else {
parts = append(parts, "no maintenance window")
}
if state.IntentionallyOffline {
parts = append(parts, "intentionally offline")
}
if state.NeverAutoRemediate {
parts = append(parts, "never auto-remediate")
}
_ = windowEnd
return strings.Join(parts, "; ")
}
func classifyAlerts(alerts []AlertSummary) (criticalCount, warningCount int, ids []string) {
for _, a := range alerts {
switch a.Severity {
case SeverityCritical:
criticalCount++
case SeverityWarning:
warningCount++
}
if a.ID != "" {
ids = append(ids, a.ID)
}
}
return criticalCount, warningCount, ids
}
func classifyFindings(findings []FindingSummary) (criticalCount, warningCount int, ids []string) {
for _, f := range findings {
if f.Resolved {
continue
}
switch f.Severity {
case SeverityCritical:
criticalCount++
case SeverityWarning:
warningCount++
}
if f.ID != "" {
ids = append(ids, f.ID)
}
}
return criticalCount, warningCount, ids
}
func classifyFailedActions(actions []ActionSummary, windowStart time.Time) (count int, ids []string) {
for _, a := range actions {
if a.State != "failed" {
continue
}
if !windowStart.IsZero() && a.UpdatedAt.Before(windowStart) {
continue
}
count++
if a.ID != "" {
ids = append(ids, a.ID)
}
}
return count, ids
}
func summarizeMetricRecovery(samples []MetricSample, windowEnd, now time.Time) *unified.MetricRecoveryEvidence {
rec := &unified.MetricRecoveryEvidence{}
metrics := map[string]struct{}{}
relevant := make([]MetricSample, 0, len(samples))
for _, s := range samples {
if !s.Timestamp.After(windowEnd) {
continue
}
relevant = append(relevant, s)
if s.Metric != "" {
metrics[s.Metric] = struct{}{}
}
}
rec.SamplesAfterEnd = len(relevant)
if len(metrics) > 0 {
rec.MetricsObserved = make([]string, 0, len(metrics))
for m := range metrics {
rec.MetricsObserved = append(rec.MetricsObserved, m)
}
}
if len(relevant) == 0 {
rec.Trend = "unknown"
rec.Note = "no metric samples have been recorded since the window closed"
return rec
}
rec.Trend = trendFromSamples(relevant)
if rec.Trend == "stable" {
rec.Note = fmt.Sprintf("%d post-window samples observed; values within expected band", len(relevant))
}
return rec
}
// trendFromSamples reports a simple plain-English trend label by
// comparing the first half of samples to the second half. The MVP is
// deterministic and crude on purpose — anomaly detection is the
// baseline store's job, not the sentinel's.
func trendFromSamples(samples []MetricSample) string {
if len(samples) < 2 {
return "unknown"
}
byMetric := map[string][]MetricSample{}
for _, s := range samples {
byMetric[s.Metric] = append(byMetric[s.Metric], s)
}
var trends []string
for _, ms := range byMetric {
if len(ms) < 2 {
trends = append(trends, "unknown")
continue
}
mid := len(ms) / 2
firstAvg := averageValue(ms[:mid])
secondAvg := averageValue(ms[mid:])
// Threshold is intentionally wide — the trend label only
// changes the report's note, not its overall status.
switch {
case firstAvg == 0 && secondAvg == 0:
trends = append(trends, "stable")
case secondAvg <= firstAvg*1.05+0.01:
trends = append(trends, "stable")
case secondAvg > firstAvg*1.5:
trends = append(trends, "degrading")
default:
trends = append(trends, "stable")
}
}
// Combine: any degrading wins, else stable.
for _, t := range trends {
if t == "degrading" {
return "degrading"
}
}
return "stable"
}
func averageValue(samples []MetricSample) float64 {
if len(samples) == 0 {
return 0
}
var sum float64
for _, s := range samples {
sum += s.Value
}
return sum / float64(len(samples))
}
func patrolRunTODO(inputs VerificationInputs, evidence unified.LoopReportEvidence) string {
// If the deterministic evidence is unambiguous (clear failure or
// clear pass), there's nothing for Patrol to add.
if evidence.ActiveCriticalAlerts > 0 || evidence.ActiveCriticalFindings > 0 || evidence.FailedActionsSinceWindowStart > 0 {
return ""
}
if evidence.ActiveWarningAlerts == 0 && evidence.ActiveWarningFindings == 0 {
if evidence.MetricRecovery != nil && evidence.MetricRecovery.SamplesAfterEnd > 0 {
return ""
}
}
_ = inputs
// Ambiguous case: a scoped Patrol run would help.
//
// MVP: deterministic checks only. The Patrol API surfaces a global
// scheduled run today; a scoped per-resource trigger that does
// not race the global scheduler is not yet available. Until that
// lands, we leave a TODO breadcrumb on the report so the operator
// knows verification was conservative, and so future work can
// pick this up cleanly.
return "scoped Patrol run not triggered — per-resource scoped run entrypoint is not yet implemented; review evidence manually"
}
func decideStatus(inputs VerificationInputs, evidence unified.LoopReportEvidence) (unified.LoopReportStatus, string) {
// Strong negative signals → failed verification.
if evidence.ActiveCriticalAlerts > 0 {
return unified.LoopReportStatusFailedVerification,
"Critical alert(s) are active after maintenance closed. Investigate before clearing operator state."
}
if evidence.ActiveCriticalFindings > 0 {
return unified.LoopReportStatusFailedVerification,
"Critical Patrol finding(s) are active. Resource has not recovered cleanly."
}
if evidence.FailedActionsSinceWindowStart > 0 {
return unified.LoopReportStatusFailedVerification,
"One or more recovery actions failed during or after the maintenance window. Review the action audit timeline."
}
// Ambiguous signals → needs review.
if inputs.OperatorState != nil && inputs.OperatorState.IntentionallyOffline {
return unified.LoopReportStatusNeedsReview,
"Resource is marked intentionally offline. Verification cannot confirm recovery automatically."
}
if evidence.ActiveWarningAlerts > 0 || evidence.ActiveWarningFindings > 0 {
return unified.LoopReportStatusNeedsReview,
"Warning-level signals are active after maintenance closed. Operator review recommended."
}
if evidence.MetricRecovery != nil {
if evidence.MetricRecovery.SamplesAfterEnd == 0 {
return unified.LoopReportStatusNeedsReview,
"No metric samples have been recorded since the window closed. Confirm the resource is reporting again."
}
if evidence.MetricRecovery.Trend == "degrading" {
return unified.LoopReportStatusNeedsReview,
"Post-window metric trend is degrading. Operator review recommended."
}
} else {
return unified.LoopReportStatusNeedsReview,
"No metric source was available for this resource. Confirm recovery manually."
}
return unified.LoopReportStatusHealthy,
"All deterministic checks passed. No active alerts, findings, or failed actions since the window started."
}
func newReportID(canonicalID string, windowEndedAt time.Time) string {
ts := windowEndedAt.UTC().Format("20060102T150405Z")
if windowEndedAt.IsZero() {
ts = time.Now().UTC().Format("20060102T150405Z")
}
return fmt.Sprintf("mv-%s-%s", sanitizeIDComponent(canonicalID), ts)
}
func sanitizeIDComponent(s string) string {
var b strings.Builder
b.Grow(len(s))
for _, r := range s {
switch {
case (r >= 'a' && r <= 'z'), (r >= 'A' && r <= 'Z'), (r >= '0' && r <= '9'):
b.WriteRune(r)
case r == '-' || r == '_' || r == '.':
b.WriteRune(r)
default:
b.WriteRune('_')
}
}
return b.String()
}
internal/maintenancesentinel/verification_test.go
+194
View File
@@ -0,0 +1,194 @@
package maintenancesentinel
import (
"strings"
"testing"
"time"
unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
func TestEvaluateVerification_HealthyWhenNoSignals(t *testing.T) {
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
windowStart := now.Add(-30 * time.Minute)
windowEnd := now.Add(-10 * time.Minute)
inputs := VerificationInputs{
ResourceID: "vm:101",
OperatorState: &unified.ResourceOperatorState{CanonicalID: "vm:101", MaintenanceStartAt: &windowStart, MaintenanceEndAt: &windowEnd},
WindowStartedAt: windowStart,
WindowEndedAt: windowEnd,
Now: now,
MetricSourceAvailable: true,
PostWindowMetricSamples: []MetricSample{
{Metric: "cpu", Value: 12, Timestamp: windowEnd.Add(time.Minute)},
{Metric: "cpu", Value: 13, Timestamp: windowEnd.Add(2 * time.Minute)},
{Metric: "memory", Value: 40, Timestamp: windowEnd.Add(time.Minute)},
{Metric: "memory", Value: 41, Timestamp: windowEnd.Add(2 * time.Minute)},
},
}
report := EvaluateVerification(inputs)
if report.Status != unified.LoopReportStatusHealthy {
t.Fatalf("status = %q want %q (recommendation=%q)", report.Status, unified.LoopReportStatusHealthy, report.Recommendation)
}
if report.Type != unified.LoopReportTypeMaintenanceVerification {
t.Fatalf("type = %q want %q", report.Type, unified.LoopReportTypeMaintenanceVerification)
}
if report.Scope != "vm:101" {
t.Fatalf("scope = %q want vm:101", report.Scope)
}
if report.WindowEndedAt == nil || !report.WindowEndedAt.Equal(windowEnd) {
t.Fatalf("windowEndedAt = %v want %v", report.WindowEndedAt, windowEnd)
}
if !strings.HasPrefix(report.ID, "mv-vm_101-") {
t.Fatalf("report id = %q want prefix mv-vm_101-", report.ID)
}
}
func TestEvaluateVerification_FailedOnCriticalAlert(t *testing.T) {
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
windowStart := now.Add(-time.Hour)
windowEnd := now.Add(-15 * time.Minute)
inputs := VerificationInputs{
ResourceID: "vm:101",
WindowStartedAt: windowStart,
WindowEndedAt: windowEnd,
Now: now,
ActiveAlerts: []AlertSummary{
{ID: "alert-1", Severity: SeverityCritical, Type: "cpu"},
{ID: "alert-2", Severity: SeverityWarning, Type: "memory"},
},
MetricSourceAvailable: true,
}
report := EvaluateVerification(inputs)
if report.Status != unified.LoopReportStatusFailedVerification {
t.Fatalf("status = %q want failed_verification", report.Status)
}
if report.Evidence.ActiveCriticalAlerts != 1 || report.Evidence.ActiveWarningAlerts != 1 {
t.Fatalf("alert counts = %d/%d want 1/1", report.Evidence.ActiveCriticalAlerts, report.Evidence.ActiveWarningAlerts)
}
if len(report.LinkedAlertIDs) != 2 {
t.Fatalf("linked alert ids = %v want 2 entries", report.LinkedAlertIDs)
}
}
func TestEvaluateVerification_NeedsReviewOnNoMetrics(t *testing.T) {
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
windowStart := now.Add(-time.Hour)
windowEnd := now.Add(-15 * time.Minute)
inputs := VerificationInputs{
ResourceID: "ct:200",
WindowStartedAt: windowStart,
WindowEndedAt: windowEnd,
Now: now,
MetricSourceAvailable: true,
}
report := EvaluateVerification(inputs)
if report.Status != unified.LoopReportStatusNeedsReview {
t.Fatalf("status = %q want needs_review", report.Status)
}
if report.Evidence.MetricRecovery == nil || report.Evidence.MetricRecovery.SamplesAfterEnd != 0 {
t.Fatalf("metric recovery = %+v", report.Evidence.MetricRecovery)
}
}
func TestEvaluateVerification_FailedOnFailedAction(t *testing.T) {
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
windowStart := now.Add(-time.Hour)
windowEnd := now.Add(-15 * time.Minute)
inputs := VerificationInputs{
ResourceID: "vm:101",
WindowStartedAt: windowStart,
WindowEndedAt: windowEnd,
Now: now,
RecentActions: []ActionSummary{
{ID: "action-1", State: "failed", UpdatedAt: windowEnd.Add(-time.Minute)},
{ID: "action-2", State: "succeeded", UpdatedAt: windowEnd.Add(time.Minute)},
},
MetricSourceAvailable: true,
}
report := EvaluateVerification(inputs)
if report.Status != unified.LoopReportStatusFailedVerification {
t.Fatalf("status = %q want failed_verification", report.Status)
}
if report.Evidence.FailedActionsSinceWindowStart != 1 {
t.Fatalf("failed actions = %d want 1", report.Evidence.FailedActionsSinceWindowStart)
}
if len(report.LinkedActionIDs) != 1 || report.LinkedActionIDs[0] != "action-1" {
t.Fatalf("linked action ids = %v want [action-1]", report.LinkedActionIDs)
}
}
func TestEvaluateVerification_NeedsReviewWhenIntentionallyOffline(t *testing.T) {
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
windowStart := now.Add(-time.Hour)
windowEnd := now.Add(-15 * time.Minute)
inputs := VerificationInputs{
ResourceID: "vm:101",
OperatorState: &unified.ResourceOperatorState{
CanonicalID: "vm:101",
IntentionallyOffline: true,
MaintenanceStartAt: &windowStart,
MaintenanceEndAt: &windowEnd,
},
WindowStartedAt: windowStart,
WindowEndedAt: windowEnd,
Now: now,
MetricSourceAvailable: true,
PostWindowMetricSamples: []MetricSample{
{Metric: "cpu", Value: 5, Timestamp: windowEnd.Add(time.Minute)},
{Metric: "cpu", Value: 6, Timestamp: windowEnd.Add(2 * time.Minute)},
},
}
report := EvaluateVerification(inputs)
if report.Status != unified.LoopReportStatusNeedsReview {
t.Fatalf("status = %q want needs_review", report.Status)
}
if !strings.Contains(report.Evidence.OperatorStateSummary, "intentionally offline") {
t.Fatalf("operator state summary = %q want to mention intentionally offline", report.Evidence.OperatorStateSummary)
}
}
func TestEvaluateVerification_FilesPatrolTODOWhenWarningsOnly(t *testing.T) {
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
windowStart := now.Add(-time.Hour)
windowEnd := now.Add(-15 * time.Minute)
inputs := VerificationInputs{
ResourceID: "vm:101",
WindowStartedAt: windowStart,
WindowEndedAt: windowEnd,
Now: now,
ActiveFindings: []FindingSummary{
{ID: "finding-1", Severity: SeverityWarning, Category: "performance"},
},
MetricSourceAvailable: true,
}
report := EvaluateVerification(inputs)
if report.Status != unified.LoopReportStatusNeedsReview {
t.Fatalf("status = %q want needs_review", report.Status)
}
if report.Evidence.PatrolRunTODO == "" {
t.Fatalf("expected patrol run TODO breadcrumb on ambiguous evidence")
}
if len(report.LinkedFindingIDs) != 1 {
t.Fatalf("linked finding ids = %v want 1", report.LinkedFindingIDs)
}
}
func TestEvaluateVerification_NeedsReviewWhenNoMetricSource(t *testing.T) {
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
windowStart := now.Add(-time.Hour)
windowEnd := now.Add(-15 * time.Minute)
inputs := VerificationInputs{
ResourceID: "storage:tank",
WindowStartedAt: windowStart,
WindowEndedAt: windowEnd,
Now: now,
}
report := EvaluateVerification(inputs)
if report.Status != unified.LoopReportStatusNeedsReview {
t.Fatalf("status = %q want needs_review when no metric source", report.Status)
}
if report.Evidence.MetricRecovery != nil {
t.Fatalf("metric recovery = %+v want nil when source unavailable", report.Evidence.MetricRecovery)
}
}
internal/unifiedresources/loop_reports.go
+289
View File
@@ -0,0 +1,289 @@
package unifiedresources
import (
"errors"
"fmt"
"strings"
"time"
)
// LoopReportType is the discriminator that names the loop a report came
// from. Only one loop is implemented for the MVP — maintenance window
// verification — but the field stays in the contract so future loops can
// reuse the same persistence and review surfaces without a schema split.
type LoopReportType string
const (
// LoopReportTypeMaintenanceVerification is the durable summary the
// sentinel writes after a maintenance window ends for a resource. The
// product-facing name is "Maintenance Verification Report"; this is
// the internal type token used for storage and API filtering.
LoopReportTypeMaintenanceVerification LoopReportType = "maintenance_verification"
)
// LoopReportStatus reports the verification outcome.
//
// - healthy — all deterministic checks passed.
// - needs_review — evidence is ambiguous; an operator should
// look. Used when there is no metric history to compare, when the
// resource is operator-marked intentionally offline, or when a
// warning-level alert/finding is active.
// - failed_verification — at least one strong negative signal
// (critical alert, critical finding, recent failed action) is
// present. The report does not auto-remediate; the operator
// decides next steps.
//
// The set is intentionally small. Adding more states is a contract
// change because the UI and review action depend on the enum.
type LoopReportStatus string
const (
LoopReportStatusPending LoopReportStatus = "pending"
LoopReportStatusHealthy LoopReportStatus = "healthy"
LoopReportStatusNeedsReview LoopReportStatus = "needs_review"
LoopReportStatusFailedVerification LoopReportStatus = "failed_verification"
)
// LoopReportUserOutcome records the operator's review verdict after
// reading a report. "" means the operator has not yet reviewed.
type LoopReportUserOutcome string
const (
LoopReportUserOutcomeReviewed LoopReportUserOutcome = "reviewed"
)
// LoopReportEvidence captures the deterministic checks the sentinel
// ran. Each field is read-only operational evidence — counts and
// summaries — not executable commands or stale plans. The shape is
// stable so future review tooling can render every report uniformly.
type LoopReportEvidence struct {
// OperatorStateSummary is a short human-readable summary of the
// operator state at evaluation time (e.g.
// "maintenance_window_ended", "intentionally_offline", "no
// operator state").
OperatorStateSummary string `json:"operatorStateSummary,omitempty"`
// ActiveCriticalAlerts and ActiveWarningAlerts count alerts that
// were active for this resource at evaluation time.
ActiveCriticalAlerts int `json:"activeCriticalAlerts"`
ActiveWarningAlerts int `json:"activeWarningAlerts"`
// ActiveCriticalFindings and ActiveWarningFindings count Patrol
// findings active for the resource at evaluation time.
ActiveCriticalFindings int `json:"activeCriticalFindings"`
ActiveWarningFindings int `json:"activeWarningFindings"`
// FailedActionsSinceWindowStart is the number of action audits
// targeting the resource whose state ended in "failed" with an
// updated_at after the maintenance window started.
FailedActionsSinceWindowStart int `json:"failedActionsSinceWindowStart"`
// MetricRecovery describes the basic recent-metric check. Absent
// when no metric source was available for the resource at
// evaluation time.
MetricRecovery *MetricRecoveryEvidence `json:"metricRecovery,omitempty"`
// PatrolRunTODO is set when the deterministic evidence was
// ambiguous and a scoped Patrol run would have helped, but
// triggering one was not safe in this build. The value is the
// reason string so the UI can surface what was missing.
//
// MVP: deterministic checks only; the Patrol run trigger is
// deferred until the Patrol API surfaces a scoped per-resource
// run entrypoint that does not race the global scheduler.
PatrolRunTODO string `json:"patrolRunTodo,omitempty"`
}
// MetricRecoveryEvidence is a small summary of the metric values
// observed inside the maintenance window vs. immediately after it.
// "Recent metric recovery" is intentionally shallow for the MVP — a
// per-metric trend label rather than a model.
type MetricRecoveryEvidence struct {
// MetricsObserved lists the metric short names the sentinel
// inspected (e.g. "cpu", "memory").
MetricsObserved []string `json:"metricsObserved,omitempty"`
// SamplesAfterEnd is the number of metric samples observed after
// the window ended. Zero means the resource has not reported any
// metrics since the window closed — operator should look.
SamplesAfterEnd int `json:"samplesAfterEnd"`
// Trend describes the post-window metric trend in plain English.
// Allowed values: "improving", "stable", "degrading", "unknown".
Trend string `json:"trend,omitempty"`
// Note is an optional explanation surfaced to the operator
// alongside the trend.
Note string `json:"note,omitempty"`
}
// LoopReport is the durable record the sentinel writes when a loop run
// produces an outcome an operator may want to review. Fields are
// minimal on purpose — this is shared substrate, not a generic agent
// framework.
//
// For the maintenance-verification loop:
// - Scope is the canonical resource ID.
// - Trigger is always "maintenance_window_end".
// - WindowStartedAt and WindowEndedAt mirror the operator-set window
// so future runs can de-duplicate against the (resource, window)
// pair.
type LoopReport struct {
ID string `json:"id"`
Type LoopReportType `json:"type"`
Scope string `json:"scope"`
Trigger string `json:"trigger"`
Goal string `json:"goal,omitempty"`
Status LoopReportStatus `json:"status"`
StartedAt time.Time `json:"startedAt"`
CompletedAt time.Time `json:"completedAt"`
WindowStartedAt *time.Time `json:"windowStartedAt,omitempty"`
WindowEndedAt *time.Time `json:"windowEndedAt,omitempty"`
Evidence LoopReportEvidence `json:"evidence"`
LinkedFindingIDs []string `json:"linkedFindingIds,omitempty"`
LinkedAlertIDs []string `json:"linkedAlertIds,omitempty"`
LinkedActionIDs []string `json:"linkedActionIds,omitempty"`
LinkedPatrolRunID string `json:"linkedPatrolRunId,omitempty"`
Recommendation string `json:"recommendation,omitempty"`
UserOutcome LoopReportUserOutcome `json:"userOutcome,omitempty"`
ReviewedAt *time.Time `json:"reviewedAt,omitempty"`
ReviewedBy string `json:"reviewedBy,omitempty"`
ReviewNote string `json:"reviewNote,omitempty"`
}
// ErrLoopReportInvalid is returned when a record fails contract checks
// at the store boundary.
var ErrLoopReportInvalid = errors.New("loop_report_invalid")
// IsValidLoopReportType reports whether the value names a known loop
// report type. Unknown types are rejected at the store boundary so
// freeform discriminators cannot accumulate.
func IsValidLoopReportType(t LoopReportType) bool {
switch t {
case LoopReportTypeMaintenanceVerification:
return true
}
return false
}
// IsValidLoopReportStatus reports whether the value names a known
// terminal or in-flight status.
func IsValidLoopReportStatus(s LoopReportStatus) bool {
switch s {
case LoopReportStatusPending,
LoopReportStatusHealthy,
LoopReportStatusNeedsReview,
LoopReportStatusFailedVerification:
return true
}
return false
}
// IsValidLoopReportUserOutcome reports whether the value names a known
// review verdict. Empty is valid — the operator has not reviewed yet.
func IsValidLoopReportUserOutcome(o LoopReportUserOutcome) bool {
switch o {
case "", LoopReportUserOutcomeReviewed:
return true
}
return false
}
// NormalizeLoopReport trims string fields, canonicalizes the scope,
// and applies UTC to all timestamps. It does not validate — callers
// follow normalize with ValidateLoopReport.
func NormalizeLoopReport(r LoopReport) LoopReport {
r.ID = strings.TrimSpace(r.ID)
r.Scope = CanonicalResourceID(r.Scope)
r.Trigger = strings.TrimSpace(r.Trigger)
r.Goal = strings.TrimSpace(r.Goal)
r.Recommendation = strings.TrimSpace(r.Recommendation)
r.ReviewedBy = strings.TrimSpace(r.ReviewedBy)
r.ReviewNote = strings.TrimSpace(r.ReviewNote)
r.LinkedPatrolRunID = strings.TrimSpace(r.LinkedPatrolRunID)
r.Evidence.OperatorStateSummary = strings.TrimSpace(r.Evidence.OperatorStateSummary)
r.Evidence.PatrolRunTODO = strings.TrimSpace(r.Evidence.PatrolRunTODO)
if r.Evidence.MetricRecovery != nil {
r.Evidence.MetricRecovery.Trend = strings.TrimSpace(r.Evidence.MetricRecovery.Trend)
r.Evidence.MetricRecovery.Note = strings.TrimSpace(r.Evidence.MetricRecovery.Note)
}
if !r.StartedAt.IsZero() {
r.StartedAt = r.StartedAt.UTC()
}
if !r.CompletedAt.IsZero() {
r.CompletedAt = r.CompletedAt.UTC()
}
if r.WindowStartedAt != nil {
t := r.WindowStartedAt.UTC()
r.WindowStartedAt = &t
}
if r.WindowEndedAt != nil {
t := r.WindowEndedAt.UTC()
r.WindowEndedAt = &t
}
if r.ReviewedAt != nil {
t := r.ReviewedAt.UTC()
r.ReviewedAt = &t
}
r.LinkedFindingIDs = trimAndDedupeStrings(r.LinkedFindingIDs)
r.LinkedAlertIDs = trimAndDedupeStrings(r.LinkedAlertIDs)
r.LinkedActionIDs = trimAndDedupeStrings(r.LinkedActionIDs)
return r
}
// ValidateLoopReport applies the contract checks the persistence layer
// must enforce before writing a record. Returns an
// ErrLoopReportInvalid-wrapped error on violation.
func ValidateLoopReport(r LoopReport) error {
if r.ID == "" {
return fmt.Errorf("%w: id is required", ErrLoopReportInvalid)
}
if !IsValidLoopReportType(r.Type) {
return fmt.Errorf("%w: unknown report type %q", ErrLoopReportInvalid, r.Type)
}
if r.Scope == "" {
return fmt.Errorf("%w: scope (canonical resource id) is required", ErrLoopReportInvalid)
}
if r.Trigger == "" {
return fmt.Errorf("%w: trigger is required", ErrLoopReportInvalid)
}
if !IsValidLoopReportStatus(r.Status) {
return fmt.Errorf("%w: unknown status %q", ErrLoopReportInvalid, r.Status)
}
if r.StartedAt.IsZero() {
return fmt.Errorf("%w: startedAt is required", ErrLoopReportInvalid)
}
if r.CompletedAt.IsZero() {
return fmt.Errorf("%w: completedAt is required", ErrLoopReportInvalid)
}
if r.CompletedAt.Before(r.StartedAt) {
return fmt.Errorf("%w: completedAt must not be before startedAt", ErrLoopReportInvalid)
}
if !IsValidLoopReportUserOutcome(r.UserOutcome) {
return fmt.Errorf("%w: unknown user outcome %q", ErrLoopReportInvalid, r.UserOutcome)
}
return nil
}
func trimAndDedupeStrings(in []string) []string {
if len(in) == 0 {
return nil
}
seen := make(map[string]struct{}, len(in))
out := make([]string, 0, len(in))
for _, v := range in {
v = strings.TrimSpace(v)
if v == "" {
continue
}
if _, ok := seen[v]; ok {
continue
}
seen[v] = struct{}{}
out = append(out, v)
}
if len(out) == 0 {
return nil
}
return out
}
internal/unifiedresources/loop_reports_store.go
+510
View File
@@ -0,0 +1,510 @@
package unifiedresources
import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
)
// ListResourceOperatorStates returns every persisted operator-set
// state row. Order is not guaranteed.
func (s *SQLiteResourceStore) ListResourceOperatorStates() ([]ResourceOperatorState, error) {
rows, err := s.db.Query(`
SELECT canonical_id, intentionally_offline, never_auto_remediate,
maintenance_start_at, maintenance_end_at, maintenance_reason,
criticality, note, set_at, set_by
FROM resource_operator_state`)
if err != nil {
return nil, fmt.Errorf("query resource operator states: %w", err)
}
defer rows.Close()
var out []ResourceOperatorState
for rows.Next() {
var (
state ResourceOperatorState
intentional int
neverRemediate int
startAt, endAt sql.NullTime
reason sql.NullString
criticality sql.NullString
note sql.NullString
setBy sql.NullString
)
if err := rows.Scan(
&state.CanonicalID,
&intentional,
&neverRemediate,
&startAt,
&endAt,
&reason,
&criticality,
&note,
&state.SetAt,
&setBy,
); err != nil {
return nil, fmt.Errorf("scan resource operator state row: %w", err)
}
state.IntentionallyOffline = intentional != 0
state.NeverAutoRemediate = neverRemediate != 0
if startAt.Valid {
t := startAt.Time
state.MaintenanceStartAt = &t
}
if endAt.Valid {
t := endAt.Time
state.MaintenanceEndAt = &t
}
if reason.Valid {
state.MaintenanceReason = reason.String
}
if criticality.Valid {
state.Criticality = ResourceCriticality(criticality.String)
}
if note.Valid {
state.Note = note.String
}
if setBy.Valid {
state.SetBy = setBy.String
}
out = append(out, state)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate resource operator state rows: %w", err)
}
return out, nil
}
// ListResourceOperatorStates returns every persisted operator-set state row.
func (m *MemoryStore) ListResourceOperatorStates() ([]ResourceOperatorState, error) {
m.mu.RLock()
defer m.mu.RUnlock()
out := make([]ResourceOperatorState, 0, len(m.resourceOperatorState))
for _, state := range m.resourceOperatorState {
out = append(out, state)
}
return out, nil
}
// ErrLoopReportNotFound is returned by store methods that target a
// specific report id which does not exist.
var ErrLoopReportNotFound = errors.New("loop_report_not_found")
// RecordLoopReport persists a new loop report. Reports are immutable
// except for the user_outcome / reviewed_* / review_note fields, which
// are updated via UpdateLoopReportUserOutcome. Tick-vs-tick dedup for
// the (type, scope, window_ended_at) triple is enforced at the
// sentinel layer (mutex + FindLoopReportByWindow) so explicit reruns,
// which intentionally share the triple under a different id suffix,
// can land alongside the original.
func (s *SQLiteResourceStore) RecordLoopReport(report LoopReport) error {
report = NormalizeLoopReport(report)
if err := ValidateLoopReport(report); err != nil {
return err
}
evidenceJSON, err := json.Marshal(report.Evidence)
if err != nil {
return fmt.Errorf("marshal loop report evidence: %w", err)
}
findingIDsJSON, err := json.Marshal(report.LinkedFindingIDs)
if err != nil {
return fmt.Errorf("marshal loop report finding ids: %w", err)
}
alertIDsJSON, err := json.Marshal(report.LinkedAlertIDs)
if err != nil {
return fmt.Errorf("marshal loop report alert ids: %w", err)
}
actionIDsJSON, err := json.Marshal(report.LinkedActionIDs)
if err != nil {
return fmt.Errorf("marshal loop report action ids: %w", err)
}
var windowStart, windowEnd, reviewedAt sql.NullTime
if report.WindowStartedAt != nil {
windowStart.Time = *report.WindowStartedAt
windowStart.Valid = true
}
if report.WindowEndedAt != nil {
windowEnd.Time = *report.WindowEndedAt
windowEnd.Valid = true
}
if report.ReviewedAt != nil {
reviewedAt.Time = *report.ReviewedAt
reviewedAt.Valid = true
}
s.mu.Lock()
defer s.mu.Unlock()
_, err = s.db.Exec(`
INSERT INTO loop_reports (
id, report_type, scope, trigger, goal, status, started_at, completed_at,
window_started_at, window_ended_at, evidence_json,
linked_finding_ids_json, linked_alert_ids_json, linked_action_ids_json,
linked_patrol_run_id, recommendation,
user_outcome, reviewed_at, reviewed_by, review_note
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
report.ID,
string(report.Type),
report.Scope,
report.Trigger,
report.Goal,
string(report.Status),
report.StartedAt,
report.CompletedAt,
windowStart,
windowEnd,
string(evidenceJSON),
string(findingIDsJSON),
string(alertIDsJSON),
string(actionIDsJSON),
report.LinkedPatrolRunID,
report.Recommendation,
string(report.UserOutcome),
reviewedAt,
report.ReviewedBy,
report.ReviewNote,
)
if err != nil {
return fmt.Errorf("insert loop report: %w", err)
}
return nil
}
// GetLoopReport returns a single report by id.
func (s *SQLiteResourceStore) GetLoopReport(reportID string) (LoopReport, bool, error) {
reportID = strings.TrimSpace(reportID)
if reportID == "" {
return LoopReport{}, false, nil
}
row := s.db.QueryRow(`
SELECT id, report_type, scope, trigger, goal, status, started_at, completed_at,
window_started_at, window_ended_at, evidence_json,
linked_finding_ids_json, linked_alert_ids_json, linked_action_ids_json,
linked_patrol_run_id, recommendation,
user_outcome, reviewed_at, reviewed_by, review_note
FROM loop_reports
WHERE id = ?`, reportID)
report, err := scanLoopReportRow(row)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return LoopReport{}, false, nil
}
return LoopReport{}, false, err
}
return report, true, nil
}
// FindLoopReportByWindow looks up an existing report for a
// (type, scope, window-end) triple. Used by the sentinel to dedupe
// before writing a new report.
func (s *SQLiteResourceStore) FindLoopReportByWindow(reportType LoopReportType, canonicalID string, windowEndedAt time.Time) (LoopReport, bool, error) {
canonicalID = CanonicalResourceID(canonicalID)
if canonicalID == "" || !IsValidLoopReportType(reportType) || windowEndedAt.IsZero() {
return LoopReport{}, false, nil
}
row := s.db.QueryRow(`
SELECT id, report_type, scope, trigger, goal, status, started_at, completed_at,
window_started_at, window_ended_at, evidence_json,
linked_finding_ids_json, linked_alert_ids_json, linked_action_ids_json,
linked_patrol_run_id, recommendation,
user_outcome, reviewed_at, reviewed_by, review_note
FROM loop_reports
WHERE report_type = ? AND scope = ? AND window_ended_at = ?
ORDER BY started_at DESC
LIMIT 1`, string(reportType), canonicalID, windowEndedAt.UTC())
report, err := scanLoopReportRow(row)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return LoopReport{}, false, nil
}
return LoopReport{}, false, err
}
return report, true, nil
}
// ListLoopReportsForResource returns the most recent reports for a
// resource of the given type, newest first. limit=0 returns all rows.
func (s *SQLiteResourceStore) ListLoopReportsForResource(reportType LoopReportType, canonicalID string, limit int) ([]LoopReport, error) {
canonicalID = CanonicalResourceID(canonicalID)
if !IsValidLoopReportType(reportType) {
return nil, nil
}
query := `
SELECT id, report_type, scope, trigger, goal, status, started_at, completed_at,
window_started_at, window_ended_at, evidence_json,
linked_finding_ids_json, linked_alert_ids_json, linked_action_ids_json,
linked_patrol_run_id, recommendation,
user_outcome, reviewed_at, reviewed_by, review_note
FROM loop_reports
WHERE report_type = ? AND scope = ?
ORDER BY started_at DESC`
args := []any{string(reportType), canonicalID}
if limit > 0 {
query += ` LIMIT ?`
args = append(args, limit)
}
rows, err := s.db.Query(query, args...)
if err != nil {
return nil, fmt.Errorf("query loop reports: %w", err)
}
defer rows.Close()
var out []LoopReport
for rows.Next() {
report, err := scanLoopReportRow(rows)
if err != nil {
return nil, err
}
out = append(out, report)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate loop report rows: %w", err)
}
return out, nil
}
// UpdateLoopReportUserOutcome records the operator's review verdict on
// a report. It does not change the underlying status or evidence —
// those are computed by the sentinel and remain immutable.
func (s *SQLiteResourceStore) UpdateLoopReportUserOutcome(reportID string, outcome LoopReportUserOutcome, reviewedBy, note string, reviewedAt time.Time) error {
reportID = strings.TrimSpace(reportID)
if reportID == "" {
return fmt.Errorf("%w: id is required", ErrLoopReportInvalid)
}
if !IsValidLoopReportUserOutcome(outcome) {
return fmt.Errorf("%w: unknown user outcome %q", ErrLoopReportInvalid, outcome)
}
if reviewedAt.IsZero() {
reviewedAt = time.Now().UTC()
} else {
reviewedAt = reviewedAt.UTC()
}
s.mu.Lock()
defer s.mu.Unlock()
res, err := s.db.Exec(`
UPDATE loop_reports
SET user_outcome = ?, reviewed_at = ?, reviewed_by = ?, review_note = ?
WHERE id = ?`,
string(outcome),
reviewedAt,
strings.TrimSpace(reviewedBy),
strings.TrimSpace(note),
reportID,
)
if err != nil {
return fmt.Errorf("update loop report user outcome: %w", err)
}
affected, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("rows affected for update loop report: %w", err)
}
if affected == 0 {
return ErrLoopReportNotFound
}
return nil
}
// loopReportScanner is the surface RowQuery / Rows share for
// loop-report scanning.
type loopReportScanner interface {
Scan(dest ...any) error
}
func scanLoopReportRow(scanner loopReportScanner) (LoopReport, error) {
var (
r LoopReport
typ, status, outcome string
windowStart, windowEnd sql.NullTime
reviewedAt sql.NullTime
evidenceJSON string
findingIDsJSON string
alertIDsJSON string
actionIDsJSON string
linkedPatrolRunID string
recommendation, goal string
reviewedBy, reviewNote string
trigger string
)
if err := scanner.Scan(
&r.ID,
&typ,
&r.Scope,
&trigger,
&goal,
&status,
&r.StartedAt,
&r.CompletedAt,
&windowStart,
&windowEnd,
&evidenceJSON,
&findingIDsJSON,
&alertIDsJSON,
&actionIDsJSON,
&linkedPatrolRunID,
&recommendation,
&outcome,
&reviewedAt,
&reviewedBy,
&reviewNote,
); err != nil {
return LoopReport{}, err
}
r.Type = LoopReportType(typ)
r.Trigger = trigger
r.Goal = goal
r.Status = LoopReportStatus(status)
r.UserOutcome = LoopReportUserOutcome(outcome)
r.LinkedPatrolRunID = linkedPatrolRunID
r.Recommendation = recommendation
r.ReviewedBy = reviewedBy
r.ReviewNote = reviewNote
if windowStart.Valid {
t := windowStart.Time.UTC()
r.WindowStartedAt = &t
}
if windowEnd.Valid {
t := windowEnd.Time.UTC()
r.WindowEndedAt = &t
}
if reviewedAt.Valid {
t := reviewedAt.Time.UTC()
r.ReviewedAt = &t
}
if evidenceJSON != "" {
if err := json.Unmarshal([]byte(evidenceJSON), &r.Evidence); err != nil {
return LoopReport{}, fmt.Errorf("unmarshal loop report evidence: %w", err)
}
}
if findingIDsJSON != "" && findingIDsJSON != "null" {
if err := json.Unmarshal([]byte(findingIDsJSON), &r.LinkedFindingIDs); err != nil {
return LoopReport{}, fmt.Errorf("unmarshal loop report finding ids: %w", err)
}
}
if alertIDsJSON != "" && alertIDsJSON != "null" {
if err := json.Unmarshal([]byte(alertIDsJSON), &r.LinkedAlertIDs); err != nil {
return LoopReport{}, fmt.Errorf("unmarshal loop report alert ids: %w", err)
}
}
if actionIDsJSON != "" && actionIDsJSON != "null" {
if err := json.Unmarshal([]byte(actionIDsJSON), &r.LinkedActionIDs); err != nil {
return LoopReport{}, fmt.Errorf("unmarshal loop report action ids: %w", err)
}
}
r.StartedAt = r.StartedAt.UTC()
r.CompletedAt = r.CompletedAt.UTC()
return r, nil
}
// --- MemoryStore implementations ---
// RecordLoopReport stores a loop report in memory. Duplicate (type,
// scope, window-end) triples are rejected with ErrLoopReportInvalid so
// the sentinel's dedupe contract matches the SQLite store.
func (m *MemoryStore) RecordLoopReport(report LoopReport) error {
report = NormalizeLoopReport(report)
if err := ValidateLoopReport(report); err != nil {
return err
}
m.mu.Lock()
defer m.mu.Unlock()
if m.loopReports == nil {
m.loopReports = make(map[string]LoopReport)
}
if _, exists := m.loopReports[report.ID]; exists {
return fmt.Errorf("%w: id %q already exists", ErrLoopReportInvalid, report.ID)
}
m.loopReports[report.ID] = report
return nil
}
// GetLoopReport returns the report by id, if present.
func (m *MemoryStore) GetLoopReport(reportID string) (LoopReport, bool, error) {
reportID = strings.TrimSpace(reportID)
if reportID == "" {
return LoopReport{}, false, nil
}
m.mu.RLock()
defer m.mu.RUnlock()
report, ok := m.loopReports[reportID]
return report, ok, nil
}
// FindLoopReportByWindow scans the in-memory map for a matching report.
func (m *MemoryStore) FindLoopReportByWindow(reportType LoopReportType, canonicalID string, windowEndedAt time.Time) (LoopReport, bool, error) {
canonicalID = CanonicalResourceID(canonicalID)
if canonicalID == "" || !IsValidLoopReportType(reportType) || windowEndedAt.IsZero() {
return LoopReport{}, false, nil
}
target := windowEndedAt.UTC()
m.mu.RLock()
defer m.mu.RUnlock()
for _, report := range m.loopReports {
if report.Type != reportType || report.Scope != canonicalID {
continue
}
if report.WindowEndedAt == nil {
continue
}
if report.WindowEndedAt.Equal(target) {
return report, true, nil
}
}
return LoopReport{}, false, nil
}
// ListLoopReportsForResource returns matching reports sorted by
// started_at DESC.
func (m *MemoryStore) ListLoopReportsForResource(reportType LoopReportType, canonicalID string, limit int) ([]LoopReport, error) {
canonicalID = CanonicalResourceID(canonicalID)
if !IsValidLoopReportType(reportType) {
return nil, nil
}
m.mu.RLock()
defer m.mu.RUnlock()
out := []LoopReport{}
for _, report := range m.loopReports {
if report.Type != reportType || report.Scope != canonicalID {
continue
}
out = append(out, report)
}
sort.Slice(out, func(i, j int) bool {
return out[i].StartedAt.After(out[j].StartedAt)
})
if limit > 0 && len(out) > limit {
out = out[:limit]
}
return out, nil
}
// UpdateLoopReportUserOutcome records the operator's review verdict.
func (m *MemoryStore) UpdateLoopReportUserOutcome(reportID string, outcome LoopReportUserOutcome, reviewedBy, note string, reviewedAt time.Time) error {
reportID = strings.TrimSpace(reportID)
if reportID == "" {
return fmt.Errorf("%w: id is required", ErrLoopReportInvalid)
}
if !IsValidLoopReportUserOutcome(outcome) {
return fmt.Errorf("%w: unknown user outcome %q", ErrLoopReportInvalid, outcome)
}
if reviewedAt.IsZero() {
reviewedAt = time.Now().UTC()
} else {
reviewedAt = reviewedAt.UTC()
}
m.mu.Lock()
defer m.mu.Unlock()
report, ok := m.loopReports[reportID]
if !ok {
return ErrLoopReportNotFound
}
report.UserOutcome = outcome
report.ReviewedAt = &reviewedAt
report.ReviewedBy = strings.TrimSpace(reviewedBy)
report.ReviewNote = strings.TrimSpace(note)
m.loopReports[reportID] = report
return nil
}
internal/unifiedresources/loop_reports_store_test.go
+185
View File
@@ -0,0 +1,185 @@
package unifiedresources
import (
"testing"
"time"
)
func newLoopReport(id, scope string, windowEnd time.Time, status LoopReportStatus) LoopReport {
return LoopReport{
ID: id,
Type: LoopReportTypeMaintenanceVerification,
Scope: scope,
Trigger: "maintenance_window_end",
Goal: "verify recovery",
Status: status,
StartedAt: windowEnd.Add(time.Minute),
CompletedAt: windowEnd.Add(time.Minute),
WindowEndedAt: &windowEnd,
Evidence: LoopReportEvidence{OperatorStateSummary: "maintenance window ended"},
}
}
func TestSQLiteRecordLoopReport_RoundTrip(t *testing.T) {
dataDir := t.TempDir()
store, err := NewSQLiteResourceStore(dataDir, "default")
if err != nil {
t.Fatalf("new store: %v", err)
}
defer store.Close()
windowEnd := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
report := newLoopReport("mv-vm_101-20260512T120000Z", "vm:101", windowEnd, LoopReportStatusHealthy)
if err := store.RecordLoopReport(report); err != nil {
t.Fatalf("record: %v", err)
}
got, found, err := store.GetLoopReport(report.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if !found {
t.Fatal("expected report to be found after record")
}
if got.Scope != "vm:101" || got.Status != LoopReportStatusHealthy {
t.Fatalf("round trip mismatch: scope=%q status=%q", got.Scope, got.Status)
}
}
func TestSQLiteRecordLoopReport_AllowsRerunForSameWindow(t *testing.T) {
dataDir := t.TempDir()
store, err := NewSQLiteResourceStore(dataDir, "default")
if err != nil {
t.Fatalf("new store: %v", err)
}
defer store.Close()
windowEnd := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
original := newLoopReport("mv-vm_101-20260512T120000Z", "vm:101", windowEnd, LoopReportStatusHealthy)
if err := store.RecordLoopReport(original); err != nil {
t.Fatalf("record original: %v", err)
}
rerun := newLoopReport(original.ID+"-rerun-1", "vm:101", windowEnd, LoopReportStatusNeedsReview)
if err := store.RecordLoopReport(rerun); err != nil {
t.Fatalf("record rerun: %v", err)
}
reports, err := store.ListLoopReportsForResource(LoopReportTypeMaintenanceVerification, "vm:101", 0)
if err != nil {
t.Fatalf("list: %v", err)
}
if len(reports) != 2 {
t.Fatalf("expected 2 reports (original + rerun), got %d", len(reports))
}
found, ok, err := store.FindLoopReportByWindow(LoopReportTypeMaintenanceVerification, "vm:101", windowEnd)
if err != nil || !ok {
t.Fatalf("find by window: ok=%v err=%v", ok, err)
}
if found.ID == "" {
t.Fatal("expected matching report id from FindLoopReportByWindow")
}
}
func TestSQLiteRecordLoopReport_RejectsDuplicateID(t *testing.T) {
dataDir := t.TempDir()
store, err := NewSQLiteResourceStore(dataDir, "default")
if err != nil {
t.Fatalf("new store: %v", err)
}
defer store.Close()
windowEnd := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
report := newLoopReport("mv-dup-id", "vm:101", windowEnd, LoopReportStatusHealthy)
if err := store.RecordLoopReport(report); err != nil {
t.Fatalf("first record: %v", err)
}
if err := store.RecordLoopReport(report); err == nil {
t.Fatal("expected error on duplicate id")
}
}
func TestSQLiteUpdateLoopReportUserOutcome_RoundTrip(t *testing.T) {
dataDir := t.TempDir()
store, err := NewSQLiteResourceStore(dataDir, "default")
if err != nil {
t.Fatalf("new store: %v", err)
}
defer store.Close()
windowEnd := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
report := newLoopReport("mv-review", "vm:101", windowEnd, LoopReportStatusNeedsReview)
if err := store.RecordLoopReport(report); err != nil {
t.Fatalf("record: %v", err)
}
reviewedAt := time.Date(2026, 5, 12, 13, 0, 0, 0, time.UTC)
if err := store.UpdateLoopReportUserOutcome(report.ID, LoopReportUserOutcomeReviewed, "rcourtman", "ack", reviewedAt); err != nil {
t.Fatalf("update outcome: %v", err)
}
got, _, err := store.GetLoopReport(report.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if got.UserOutcome != LoopReportUserOutcomeReviewed {
t.Fatalf("outcome = %q want reviewed", got.UserOutcome)
}
if got.ReviewedBy != "rcourtman" || got.ReviewNote != "ack" {
t.Fatalf("review fields = %q/%q want rcourtman/ack", got.ReviewedBy, got.ReviewNote)
}
if got.ReviewedAt == nil || !got.ReviewedAt.Equal(reviewedAt) {
t.Fatalf("reviewedAt = %v want %v", got.ReviewedAt, reviewedAt)
}
if got.Status != LoopReportStatusNeedsReview {
t.Fatalf("status mutated to %q; should remain needs_review", got.Status)
}
}
func TestMemoryStoreRecordLoopReport_AllowsRerun(t *testing.T) {
store := NewMemoryStore()
windowEnd := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
original := newLoopReport("mv-mem-1", "vm:101", windowEnd, LoopReportStatusHealthy)
if err := store.RecordLoopReport(original); err != nil {
t.Fatalf("record original: %v", err)
}
rerun := newLoopReport("mv-mem-1-rerun-1", "vm:101", windowEnd, LoopReportStatusNeedsReview)
if err := store.RecordLoopReport(rerun); err != nil {
t.Fatalf("record rerun: %v", err)
}
reports, err := store.ListLoopReportsForResource(LoopReportTypeMaintenanceVerification, "vm:101", 0)
if err != nil {
t.Fatalf("list: %v", err)
}
if len(reports) != 2 {
t.Fatalf("expected 2 reports, got %d", len(reports))
}
}
func TestSQLiteListResourceOperatorStates_ReturnsAllRows(t *testing.T) {
dataDir := t.TempDir()
store, err := NewSQLiteResourceStore(dataDir, "default")
if err != nil {
t.Fatalf("new store: %v", err)
}
defer store.Close()
now := time.Date(2026, 5, 12, 12, 0, 0, 0, time.UTC)
startA := now.Add(-time.Hour)
endA := now.Add(-time.Minute)
startB := now.Add(-30 * time.Minute)
endB := now.Add(30 * time.Minute)
for _, s := range []ResourceOperatorState{
{CanonicalID: "vm:101", MaintenanceStartAt: &startA, MaintenanceEndAt: &endA, SetAt: now, SetBy: "op"},
{CanonicalID: "vm:102", MaintenanceStartAt: &startB, MaintenanceEndAt: &endB, SetAt: now, SetBy: "op"},
} {
if err := store.SetResourceOperatorState(s); err != nil {
t.Fatalf("seed %s: %v", s.CanonicalID, err)
}
}
got, err := store.ListResourceOperatorStates()
if err != nil {
t.Fatalf("list: %v", err)
}
if len(got) != 2 {
t.Fatalf("expected 2 operator states, got %d", len(got))
}
}
internal/unifiedresources/store.go
+50
View File
@@ -49,6 +49,22 @@ type ResourceStore interface {
GetResourceOperatorState(canonicalID string) (ResourceOperatorState, bool, error)
SetResourceOperatorState(state ResourceOperatorState) error
ClearResourceOperatorState(canonicalID string) error
// ListResourceOperatorStates returns every persisted operator-set
// state row. Used by background loops (e.g. the maintenance
// verification sentinel) that need to sweep the full set on each
// tick. Order is implementation-defined; the caller sorts if it
// cares.
ListResourceOperatorStates() ([]ResourceOperatorState, error)
// Loop reports — durable summaries written by background loops
// (currently only the maintenance-verification sentinel).
RecordLoopReport(report LoopReport) error
GetLoopReport(reportID string) (LoopReport, bool, error)
ListLoopReportsForResource(reportType LoopReportType, canonicalID string, limit int) ([]LoopReport, error)
UpdateLoopReportUserOutcome(reportID string, outcome LoopReportUserOutcome, reviewedBy, note string, reviewedAt time.Time) error
// FindLoopReportByWindow looks up an existing report by the
// (type, scope, window-end) triple so the sentinel can dedupe
// on each tick without scanning the whole table.
FindLoopReportByWindow(reportType LoopReportType, canonicalID string, windowEndedAt time.Time) (LoopReport, bool, error)
Close() error
}
@@ -369,6 +385,38 @@ func (s *SQLiteResourceStore) initSchema() error {
set_by TEXT
);
CREATE INDEX IF NOT EXISTS idx_resource_operator_state_maintenance ON resource_operator_state(maintenance_end_at);
CREATE TABLE IF NOT EXISTS loop_reports (
id TEXT PRIMARY KEY,
report_type TEXT NOT NULL,
scope TEXT NOT NULL,
trigger TEXT NOT NULL,
goal TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL,
started_at DATETIME NOT NULL,
completed_at DATETIME NOT NULL,
window_started_at DATETIME,
window_ended_at DATETIME,
evidence_json TEXT NOT NULL DEFAULT '{}',
linked_finding_ids_json TEXT NOT NULL DEFAULT '[]',
linked_alert_ids_json TEXT NOT NULL DEFAULT '[]',
linked_action_ids_json TEXT NOT NULL DEFAULT '[]',
linked_patrol_run_id TEXT NOT NULL DEFAULT '',
recommendation TEXT NOT NULL DEFAULT '',
user_outcome TEXT NOT NULL DEFAULT '',
reviewed_at DATETIME,
reviewed_by TEXT NOT NULL DEFAULT '',
review_note TEXT NOT NULL DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_loop_reports_scope_type_started
ON loop_reports(report_type, scope, started_at DESC);
-- Look-up index for FindLoopReportByWindow. Intentionally non-unique:
-- rerun records share (type, scope, window_ended_at) with the
-- original under a distinct id suffix. Tick-vs-tick dedup runs at
-- the sentinel layer (mutex + FindLoopReportByWindow check).
CREATE INDEX IF NOT EXISTS idx_loop_reports_window_lookup
ON loop_reports(report_type, scope, window_ended_at)
WHERE window_ended_at IS NOT NULL;
`
_, err := s.db.Exec(schema)
@@ -1565,11 +1613,13 @@ type MemoryStore struct {
actionLifecycleEvents []ActionLifecycleEvent
exportAudits []ExportAuditRecord
resourceOperatorState map[string]ResourceOperatorState
loopReports map[string]LoopReport
}
func NewMemoryStore() *MemoryStore {
return &MemoryStore{
resourceOperatorState: make(map[string]ResourceOperatorState),
loopReports: make(map[string]LoopReport),
}
}