Keep snapshot warnings when creation dates are unknown (#3145)

**Keep snapshot warnings when creation dates are unknown**

Distinguish a listed snapshot with an unusable date from removal. Retain
the last trusted canonical occurrence and evidence after current policy
resolution, including Proxmox's missing-snaptime Unix epoch
representation. Add lifecycle, exclusion, identity and real-consumer
browser proof fixtures.

Contract-Neutral: Repair existing snapshot warning preservation on
incomplete evidence; no API, configuration, schema or canonical identity
change. Dedicated regression and browser-producer tests accompany the
internal contract clarification.

**Keep the global alert pause contract in snapshot regression**

The focused source proof shows that global alerting disablement pauses
evaluation without resolving existing incidents. Assert exact trusted
occurrence preservation for that case; retain clearing assertions for
snapshot/guest disablement, exclusions, threshold changes and removal.
No production source changes.

---
Fleet candidate `20261023T184023Z-core-runtime` from the `core-runtime`
lane. Reviewed tip `9b607668b95bcd843c94aa8271bcd5d356abb985` on `main`.

[How Pulse handles
triage](https://github.com/rcourtman/Pulse/blob/main/docs/AI_TRANSPARENCY.md)
This commit is contained in:
pulse-triage[bot]
2026-10-23 19:45:42 +00:00
committed by GitHub
7 changed files with 518 additions and 10 deletions
@@ -33,6 +33,25 @@ Its opt-in browser frames come from `CheckBackups` and `GetActiveAlerts`.
This is age-evidence handling only: no change to backup execution, source
inventory readiness, snapshot alerts, native freeze/thaw safety or notification
delivery, and no proof of a reported installation's recovery.
### A listed snapshot with an unknown creation date is not recovered
Snapshot warning cleanup distinguishes an absent snapshot from a still-listed
snapshot whose creation time is missing, non-positive Unix time or in the future.
After current guest policy and overrides are applied, an unknown date retains
only that snapshot's existing canonical occurrence, including its trusted
age/size evidence, acknowledgement and observation time. It neither creates a
new occurrence nor refreshes old evidence. A trustworthy dated observation,
authoritative removal, exclusion or disablement still follows normal recovery.
The global alerting switch remains an evaluation pause, not evidence of recovery.
This is deliberately conservative for both age and size warnings: an incomplete
snapshot observation does not independently update either metric. It introduces
no guest, snapshot, backup, filesystem or freeze/thaw operation.
`snapshot_date_observation_test.go` checks repeated gaps, epoch conversion,
exact occurrence/evidence preservation, lifecycle events, independent instances,
policy changes, trustworthy recovery and removal. The real manager's exported
frames feed the production Alerts API adapter and Overview/card browser fixture
at desktop and narrow viewports; this is not installed guest recovery.
### Legacy alert-history compaction releases removed-row storage
@@ -0,0 +1,70 @@
// Compile the production consumer outside the bounded browser process tree.
import { build } from 'vite';
import solid from 'vite-plugin-solid';
import tailwindcss from '@tailwindcss/vite';
import { stop } from 'esbuild';
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { fileURLToPath } from 'node:url';
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const outDir = path.resolve(root, '../tmp/snapshot-date-static');
const hash = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex');
const sources = {};
const artifacts = {};
try {
await build({
root,
base: '/',
configFile: false,
publicDir: false,
resolve: { alias: { '@': path.join(root, 'src') } },
plugins: [
{
name: 'bind-snapshot-date-consumer',
generateBundle() {
for (const id of this.getModuleIds()) {
const file = id.split('?')[0];
if (
file.startsWith(root + '/') &&
!file.includes('/node_modules/') &&
fs.existsSync(file) &&
fs.statSync(file).isFile()
) {
sources[path.relative(root, file)] = hash(fs.readFileSync(file));
}
}
},
},
tailwindcss(),
solid(),
],
build: {
target: 'esnext',
outDir,
emptyOutDir: true,
rollupOptions: { input: path.join(root, 'browser-tests/snapshot-date-observation.html') },
},
});
const walk = (dir) => {
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
const file = path.join(dir, entry.name);
if (entry.isDirectory()) walk(file);
else artifacts[path.relative(outDir, file)] = hash(fs.readFileSync(file));
}
};
walk(outDir);
fs.writeFileSync(
path.join(outDir, 'binding.json'),
JSON.stringify({
graph: hash(fs.readFileSync(path.join(root, 'package-lock.json'))),
builder: hash(fs.readFileSync(fileURLToPath(import.meta.url))),
sources,
artifacts,
}, null, 2) + '\n',
);
} finally {
stop();
}
@@ -0,0 +1,12 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Snapshot observation evidence</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/browser-tests/snapshot-date-observation.tsx"></script>
</body>
</html>
@@ -0,0 +1,60 @@
// Real active-alert adapter and Overview, fed only by the Go manager's frames.
import { createSignal, onMount } from 'solid-js';
import { render } from 'solid-js/web';
import { Route, Router } from '@solidjs/router';
import { AlertsAPI } from '../src/api/alerts';
import { OverviewTab } from '../src/features/alerts/OverviewTab';
import { DarkModeContext } from '../src/contexts/appRuntime';
import { aiAvailabilityStore } from '../src/stores/aiAvailability';
import type { Alert } from '../src/types/api';
import '../src/index.css';
aiAvailabilityStore.setEnabled(false);
function Fixture() {
const [active, setActive] = createSignal<Record<string, Alert>>({});
const [showAcknowledged, setShowAcknowledged] = createSignal(true);
const [generation, setGeneration] = createSignal(0);
const reload = async () => {
const alerts = await AlertsAPI.getActive();
setActive(Object.fromEntries(alerts.map((alert) => [alert.id, alert])));
setGeneration((value) => value + 1);
};
onMount(() => void reload());
Object.assign(window, { __snapshotDate: { reload, active, generation } });
return (
<main class="mx-auto min-h-screen max-w-5xl space-y-4 bg-surface p-3 text-base-content">
<h1 class="text-base font-semibold">Snapshot observations</h1>
<p class="text-xs text-muted">Synthetic monitoring evidence. No backup or guest operation.</p>
<button
class="min-h-11 rounded-sm border border-border px-3 text-sm"
onClick={() => void reload()}
>
Refresh monitoring evidence
</button>
<OverviewTab
overrides={[]}
activeAlerts={active()}
updateAlert={(id, updates) =>
setActive((previous) => ({ ...previous, [id]: { ...previous[id], ...updates } }))
}
showQuickTip={() => false}
dismissQuickTip={() => {}}
showAcknowledged={showAcknowledged}
setShowAcknowledged={setShowAcknowledged}
alertsDisabled={() => false}
/>
</main>
);
}
render(
() => (
<DarkModeContext.Provider value={() => document.documentElement.classList.contains('dark')}>
<Router>
<Route path="*" component={Fixture} />
</Router>
</DarkModeContext.Provider>
),
document.getElementById('root')!,
);
+13 -10
View File
@@ -369,16 +369,6 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
if instanceName != "" && snapshot.Instance != "" && snapshot.Instance != instanceName {
continue
}
if snapshot.Time.IsZero() {
continue
}
ageHours := now.Sub(snapshot.Time).Hours()
if ageHours < 0 {
continue
}
ageDays := ageHours / 24
const gib = 1024.0 * 1024 * 1024
sizeGiB := 0.0
if snapshot.SizeBytes > 0 {
@@ -429,6 +419,19 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod
continue
}
// The listing still contains this snapshot. An unusable creation time
// is not evidence of deletion or recovery: keep the last trusted
// occurrence unchanged, without refreshing its evidence or age. Do
// this after policy resolution so exclusions and disablement still win.
// Proxmox's absent snaptime arrives as Unix epoch, not Go's zero time.
if snapshot.Time.IsZero() || snapshot.Time.Unix() <= 0 || snapshot.Time.After(now) {
validAlerts[buildCanonicalStateID(resourceID, resourceID+"/snapshot:"+snapshot.ID)] = struct{}{}
continue
}
ageHours := now.Sub(snapshot.Time).Hours()
ageDays := ageHours / 24
var ageLevel AlertLevel
var ageThreshold int
var sizeLevel AlertLevel
@@ -0,0 +1,203 @@
package alerts
import (
"bytes"
"encoding/json"
"fmt"
"os"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
func snapshotDateManager(t *testing.T) *Manager {
t.Helper()
m := newEventLogManager(t)
cfg := m.GetConfig()
cfg.SnapshotDefaults = SnapshotAlertConfig{Enabled: true, WarningDays: 7, CriticalDays: 14, WarningSizeGiB: 50, CriticalSizeGiB: 100}
cfg.TimeThresholds = map[string]int{}
m.UpdateConfig(cfg)
return m
}
func snapshotDateFixture(kind string) (models.GuestSnapshot, map[string]GuestLookup) {
s := models.GuestSnapshot{ID: "east-pve-101-weekly", Name: "weekly", Instance: "east", Node: "pve", VMID: 101, Type: kind, Time: time.Now().Add(-16 * 24 * time.Hour), SizeBytes: 120 << 30}
g := GuestLookup{ResourceID: "east:pve:101", Name: "database", Instance: "east", Node: "pve", VMID: 101, Type: kind}
return s, map[string]GuestLookup{g.ResourceID: g}
}
func snapshotDateJSON(t *testing.T, v any) []byte {
t.Helper()
b, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return b
}
func TestSnapshotDateObservations(t *testing.T) {
invalid := map[string]time.Time{
"zero": {}, "epoch": time.Unix(0, 0), "negative": time.Unix(-1, 0), "future": time.Now().Add(24 * time.Hour),
}
for _, kind := range []string{"qemu", "lxc"} {
for _, metric := range []string{"age", "size", "both"} {
for label, date := range invalid {
t.Run(kind+"/"+metric+"/"+label, func(t *testing.T) {
m := snapshotDateManager(t)
s, guests := snapshotDateFixture(kind)
if metric == "size" {
s.Time = time.Now().Add(-time.Hour)
}
if metric == "age" {
s.SizeBytes = 1 << 30
}
m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests)
active := m.GetActiveAlerts()
if len(active) != 1 || active[0].Level != AlertLevelCritical {
t.Fatalf("control must establish one critical incident: %+v", active)
}
if err := m.AcknowledgeAlert(active[0].ID, "operator"); err != nil {
t.Fatal(err)
}
before := snapshotDateJSON(t, m.GetActiveAlerts())
s.Time, s.SizeBytes = date, 0
for range 4 {
m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests)
}
if after := snapshotDateJSON(t, m.GetActiveAlerts()); !bytes.Equal(before, after) {
t.Fatalf("unknown date changed occurrence/evidence/acknowledgement: before=%s after=%s", before, after)
}
if got := len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeFired, eventlog.TypeRefired, eventlog.TypeResolved}})); got != 1 {
t.Fatalf("unknown date invented a lifecycle transition: events=%d", got)
}
s.Time, s.SizeBytes = time.Now().Add(-time.Hour), 1<<30
m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests)
m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests)
if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeResolved}})) != 1 {
t.Fatal("trustworthy normal observation must resolve exactly once")
}
})
}
}
}
t.Run("never-observed", func(t *testing.T) {
m := snapshotDateManager(t)
s, guests := snapshotDateFixture("qemu")
for _, date := range invalid {
s.Time = date
m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests)
}
if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeFired, eventlog.TypeResolved}})) != 0 {
t.Fatal("an unknown date cannot invent snapshot age or a lifecycle event")
}
})
for _, policy := range []string{"global-disabled", "guest-disabled", "snapshot-disabled", "override-disabled", "threshold-raised", "no-alerts-tag", "removed"} {
t.Run("policy/"+policy, func(t *testing.T) {
m := snapshotDateManager(t)
s, guests := snapshotDateFixture("qemu")
m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests)
if len(m.GetActiveAlerts()) != 1 {
t.Fatal("control must establish an incident")
}
before := snapshotDateJSON(t, m.GetActiveAlerts())
s.Time = time.Time{}
snapshots := []models.GuestSnapshot{s}
cfg := m.GetConfig()
switch policy {
case "global-disabled":
cfg.Enabled = false
case "guest-disabled":
cfg.Overrides = map[string]ThresholdConfig{"east:pve:101": {Disabled: true}}
case "snapshot-disabled":
cfg.SnapshotDefaults.Enabled = false
case "override-disabled":
cfg.Overrides = map[string]ThresholdConfig{"east:pve:101": {Snapshot: &SnapshotAlertConfig{Enabled: false}}}
case "threshold-raised":
cfg.SnapshotDefaults = SnapshotAlertConfig{Enabled: true, WarningDays: 30, CriticalDays: 60, WarningSizeGiB: 200, CriticalSizeGiB: 400}
case "no-alerts-tag":
g := guests["east:pve:101"]
g.Tags = []string{"pulse-no-alerts"}
guests[g.ResourceID] = g
case "removed":
snapshots = nil
}
m.UpdateConfig(cfg)
m.CheckSnapshotsForInstance("east", snapshots, guests)
if policy == "global-disabled" {
// The global switch pauses evaluation; it is not snapshot
// recovery or per-resource disablement. Preserve that contract.
if !bytes.Equal(before, snapshotDateJSON(t, m.GetActiveAlerts())) {
t.Fatal("global evaluation pause changed the trusted occurrence")
}
return
}
if len(m.GetActiveAlerts()) != 0 {
t.Fatal("unknown date overrode explicit policy or authoritative removal")
}
})
}
t.Run("independent-instances-and-snapshots", func(t *testing.T) {
m := snapshotDateManager(t)
s, guests := snapshotDateFixture("qemu")
other := s
other.ID, other.Name = "east-pve-101-daily", "daily"
west := s
west.ID, west.Instance = "west-pve-101-weekly", "west"
g := guests["east:pve:101"]
g.ResourceID, g.Instance, g.Name = "west:pve:101", "west", "west-database"
guests[g.ResourceID] = g
m.CheckSnapshotsForInstance("", []models.GuestSnapshot{s, other, west}, guests)
if len(m.GetActiveAlerts()) != 3 {
t.Fatal("control must establish three independent incidents")
}
s.Time = time.Time{}
// One same-guest snapshot is removed and the other instance recovers.
m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests)
if len(m.GetActiveAlerts()) != 2 {
t.Fatal("unknown date protected a removed sibling or cleared another instance")
}
west.Time, west.SizeBytes = time.Now().Add(-time.Hour), 1<<30
m.CheckSnapshotsForInstance("west", []models.GuestSnapshot{west}, guests)
active := m.GetActiveAlerts()
if len(active) != 1 || active[0].Instance != "east" || active[0].Metadata["snapshotName"] != "weekly" {
t.Fatalf("unknown date leaked across independent snapshot identity: %+v", active)
}
})
}
// Actual manager output is the only alert payload used by the browser fixture.
func TestSnapshotDateBrowserProducer(t *testing.T) {
m := snapshotDateManager(t)
s, guests := snapshotDateFixture("qemu")
frames := map[string]json.RawMessage{}
for _, phase := range []string{"stale", "missing", "future", "recovered"} {
switch phase {
case "missing":
s.Time = time.Unix(0, 0) // real absent-snaptime conversion
case "future":
s.Time = time.Now().Add(24 * time.Hour)
case "recovered":
s.Time, s.SizeBytes = time.Now().Add(-time.Hour), 1<<30
}
m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests)
frames[phase] = snapshotDateJSON(t, m.GetActiveAlerts())
want := 1
if phase == "recovered" {
want = 0
}
if len(m.GetActiveAlerts()) != want {
t.Fatalf("phase %s: want %d active alerts", phase, want)
}
}
if !bytes.Equal(frames["stale"], frames["missing"]) || !bytes.Equal(frames["stale"], frames["future"]) {
t.Fatal("browser frames changed the retained occurrence across a date gap")
}
if os.Getenv("PULSE_SNAPSHOT_DATE_FRAMES") == "1" {
fmt.Printf("SNAPSHOT_DATE_FRAMES=%s\n", snapshotDateJSON(t, frames))
}
}
+141
View File
@@ -0,0 +1,141 @@
// Run from the assigned root with pulse-worker-browser. No external services.
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const http = require('node:http');
const crypto = require('node:crypto');
const { chromium, webkit } = require('playwright');
const root = '/workspace';
const compiled = path.join(root, 'tmp/snapshot-date-static');
const output = path.join(root, 'tmp/snapshot-date-browser');
const origin = 'http://127.0.0.1:5371';
const hash = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex');
const binding = JSON.parse(fs.readFileSync(path.join(compiled, 'binding.json')));
const producer = JSON.parse(fs.readFileSync(path.join(root, 'tmp/snapshot-date-producer.json')));
const frames = producer.frames;
const result = {
result: 'incomplete',
base: '7374abf19b434c9a275df730d28234bea3943774',
producer,
consumer: binding,
harness: hash(fs.readFileSync(__filename)),
playwright: require('playwright/package.json').version,
cases: [],
requests: [],
captures: [],
cleanup: { browser: false, server: false },
limits: 'Real CheckSnapshotsForInstance/GetActiveAlerts -> AlertsAPI -> Overview/card; synthetic snapshot listings only. No installed recovery, guest action, backup execution, notification delivery or reporter acceptance.',
};
let server;
let browser;
(async () => {
fs.mkdirSync(output, { recursive: true });
assert.equal(result.playwright, JSON.parse(fs.readFileSync(path.join(root, 'tests/integration/package-lock.json'))).packages['node_modules/@playwright/test'].version);
assert.equal(binding.graph, hash(fs.readFileSync(path.join(root, 'frontend-modern/package-lock.json'))));
assert.equal(binding.builder, hash(fs.readFileSync(path.join(root, 'frontend-modern/browser-tests/build-snapshot-date-observation.mjs'))));
assert.equal(producer.runtime_sha256, hash(fs.readFileSync(path.join(root, 'internal/alerts/backup_snapshot.go'))));
assert.deepEqual(frames.stale, frames.missing);
assert.deepEqual(frames.stale, frames.future);
assert.equal(frames.stale.length, 1);
assert.deepEqual(frames.recovered, []);
for (const [file, digest] of Object.entries(binding.sources)) {
assert.equal(hash(fs.readFileSync(path.join(root, 'frontend-modern', file))), digest, file);
}
for (const file of ['src/api/alerts.ts', 'src/features/alerts/OverviewTab.tsx', 'src/features/alerts/AlertOverviewAlertCard.tsx']) {
assert(binding.sources[file], `real consumer not compiled: ${file}`);
}
const assets = new Map();
for (const [file, digest] of Object.entries(binding.artifacts)) {
const bytes = fs.readFileSync(path.join(compiled, file));
assert.equal(hash(bytes), digest, file);
assets.set('/' + file, bytes);
}
server = http.createServer((request, response) => {
const pathname = new URL(request.url, origin).pathname;
const bytes = assets.get(pathname);
if (!bytes || request.method !== 'GET') {
response.writeHead(404).end();
return;
}
response.writeHead(200, { 'Content-Type': { '.html': 'text/html', '.js': 'text/javascript', '.css': 'text/css' }[path.extname(pathname)] || 'application/octet-stream' });
response.end(bytes);
});
await new Promise((resolve, reject) => {
server.once('error', reject);
server.listen(5371, '127.0.0.1', resolve);
});
for (const [name, engine, width, height] of [
['desktop', chromium, 1280, 900],
['narrow', webkit, 320, 812],
]) {
browser = await engine.launch(engine === chromium ? { headless: true, channel: 'chromium', args: ['--no-sandbox'] } : { headless: true });
const page = await browser.newPage({ viewport: { width, height }, isMobile: width < 768, hasTouch: width < 768, locale: 'en-GB', timezoneId: 'UTC' });
page.setDefaultTimeout(10000);
const record = { name, width, height, browser: browser.version(), errors: [], phases: [] };
page.on('pageerror', (error) => record.errors.push(error.message));
let phase = 'stale';
await page.route('**/*', async (route) => {
const request = route.request();
const url = new URL(request.url());
assert.equal(url.origin, origin, 'external request');
assert.equal(request.method(), 'GET', 'unexpected mutation');
if (!url.pathname.startsWith('/api/')) return route.continue();
result.requests.push({ viewport: name, phase, path: url.pathname });
let body;
if (url.pathname === '/api/alerts/active') body = frames[phase];
else if (url.pathname === '/api/alerts/delivery-diagnosis') body = [];
else if (url.pathname === '/api/notifications/health') body = { queue: { status: 'unavailable' } };
else if (url.pathname === '/api/alerts/incidents') body = null;
else throw new Error(`unexpected API read: ${url.pathname}`);
return route.fulfill({ json: body });
});
await page.goto(origin + '/browser-tests/snapshot-date-observation.html');
await page.waitForFunction(() => window.__snapshotDate?.generation() === 1);
for (phase of ['stale', 'missing', 'future', 'recovered']) {
if (phase !== 'stale') {
const previous = await page.evaluate(() => window.__snapshotDate.generation());
await page.getByRole('button', { name: 'Refresh monitoring evidence', exact: true }).click();
await page.waitForFunction((generation) => window.__snapshotDate.generation() > generation, previous);
}
const active = await page.evaluate(() => Object.values(window.__snapshotDate.active()));
assert.deepEqual(active, frames[phase], `producer/consumer mismatch in ${name}/${phase}`);
if (phase !== 'recovered') {
await page.getByText(frames.stale[0].message, { exact: true }).waitFor();
const more = page.getByRole('button', { name: 'More', exact: true });
if (await more.count()) {
await more.click();
await page.getByRole('button', { name: 'Less', exact: true }).click();
}
} else {
assert.equal(await page.getByText(frames.stale[0].message, { exact: true }).count(), 0);
}
await page.evaluate(async () => {
await document.fonts.ready;
await new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve)));
});
const overflow = await page.evaluate(() => document.documentElement.scrollWidth > window.innerWidth);
assert.equal(overflow, false, `${name}/${phase} horizontal overflow`);
const capture = `${name}-${phase}.png`;
await page.screenshot({ path: path.join(output, capture), fullPage: true });
result.captures.push({ file: capture, sha256: hash(fs.readFileSync(path.join(output, capture))) });
record.phases.push({ phase, active: active.length, overflow });
}
assert.deepEqual(record.errors, []);
result.cases.push(record);
await browser.close();
browser = undefined;
}
result.result = 'passed';
})().catch((error) => {
result.result = 'failed';
result.error = String(error.stack || error);
process.exitCode = 1;
}).finally(async () => {
if (browser) await browser.close();
result.cleanup.browser = true;
if (server) await new Promise((resolve) => server.close(resolve));
result.cleanup.server = true;
fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(result, null, 2) + '\n');
});