diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index d43854cb27..b20715e0d2 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -33,6 +33,25 @@ Its opt-in browser frames come from `CheckBackups` and `GetActiveAlerts`. This is age-evidence handling only: no change to backup execution, source inventory readiness, snapshot alerts, native freeze/thaw safety or notification delivery, and no proof of a reported installation's recovery. +### A listed snapshot with an unknown creation date is not recovered + +Snapshot warning cleanup distinguishes an absent snapshot from a still-listed +snapshot whose creation time is missing, non-positive Unix time or in the future. +After current guest policy and overrides are applied, an unknown date retains +only that snapshot's existing canonical occurrence, including its trusted +age/size evidence, acknowledgement and observation time. It neither creates a +new occurrence nor refreshes old evidence. A trustworthy dated observation, +authoritative removal, exclusion or disablement still follows normal recovery. +The global alerting switch remains an evaluation pause, not evidence of recovery. + +This is deliberately conservative for both age and size warnings: an incomplete +snapshot observation does not independently update either metric. It introduces +no guest, snapshot, backup, filesystem or freeze/thaw operation. +`snapshot_date_observation_test.go` checks repeated gaps, epoch conversion, +exact occurrence/evidence preservation, lifecycle events, independent instances, +policy changes, trustworthy recovery and removal. The real manager's exported +frames feed the production Alerts API adapter and Overview/card browser fixture +at desktop and narrow viewports; this is not installed guest recovery. ### Legacy alert-history compaction releases removed-row storage diff --git a/frontend-modern/browser-tests/build-snapshot-date-observation.mjs b/frontend-modern/browser-tests/build-snapshot-date-observation.mjs new file mode 100644 index 0000000000..c4ce129e24 --- /dev/null +++ b/frontend-modern/browser-tests/build-snapshot-date-observation.mjs @@ -0,0 +1,70 @@ +// Compile the production consumer outside the bounded browser process tree. +import { build } from 'vite'; +import solid from 'vite-plugin-solid'; +import tailwindcss from '@tailwindcss/vite'; +import { stop } from 'esbuild'; +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { fileURLToPath } from 'node:url'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const outDir = path.resolve(root, '../tmp/snapshot-date-static'); +const hash = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex'); +const sources = {}; +const artifacts = {}; + +try { + await build({ + root, + base: '/', + configFile: false, + publicDir: false, + resolve: { alias: { '@': path.join(root, 'src') } }, + plugins: [ + { + name: 'bind-snapshot-date-consumer', + generateBundle() { + for (const id of this.getModuleIds()) { + const file = id.split('?')[0]; + if ( + file.startsWith(root + '/') && + !file.includes('/node_modules/') && + fs.existsSync(file) && + fs.statSync(file).isFile() + ) { + sources[path.relative(root, file)] = hash(fs.readFileSync(file)); + } + } + }, + }, + tailwindcss(), + solid(), + ], + build: { + target: 'esnext', + outDir, + emptyOutDir: true, + rollupOptions: { input: path.join(root, 'browser-tests/snapshot-date-observation.html') }, + }, + }); + const walk = (dir) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const file = path.join(dir, entry.name); + if (entry.isDirectory()) walk(file); + else artifacts[path.relative(outDir, file)] = hash(fs.readFileSync(file)); + } + }; + walk(outDir); + fs.writeFileSync( + path.join(outDir, 'binding.json'), + JSON.stringify({ + graph: hash(fs.readFileSync(path.join(root, 'package-lock.json'))), + builder: hash(fs.readFileSync(fileURLToPath(import.meta.url))), + sources, + artifacts, + }, null, 2) + '\n', + ); +} finally { + stop(); +} diff --git a/frontend-modern/browser-tests/snapshot-date-observation.html b/frontend-modern/browser-tests/snapshot-date-observation.html new file mode 100644 index 0000000000..c8e8175b5b --- /dev/null +++ b/frontend-modern/browser-tests/snapshot-date-observation.html @@ -0,0 +1,12 @@ + + + + + + Snapshot observation evidence + + +
+ + + diff --git a/frontend-modern/browser-tests/snapshot-date-observation.tsx b/frontend-modern/browser-tests/snapshot-date-observation.tsx new file mode 100644 index 0000000000..a83701f0a7 --- /dev/null +++ b/frontend-modern/browser-tests/snapshot-date-observation.tsx @@ -0,0 +1,60 @@ +// Real active-alert adapter and Overview, fed only by the Go manager's frames. +import { createSignal, onMount } from 'solid-js'; +import { render } from 'solid-js/web'; +import { Route, Router } from '@solidjs/router'; +import { AlertsAPI } from '../src/api/alerts'; +import { OverviewTab } from '../src/features/alerts/OverviewTab'; +import { DarkModeContext } from '../src/contexts/appRuntime'; +import { aiAvailabilityStore } from '../src/stores/aiAvailability'; +import type { Alert } from '../src/types/api'; +import '../src/index.css'; + +aiAvailabilityStore.setEnabled(false); + +function Fixture() { + const [active, setActive] = createSignal>({}); + const [showAcknowledged, setShowAcknowledged] = createSignal(true); + const [generation, setGeneration] = createSignal(0); + const reload = async () => { + const alerts = await AlertsAPI.getActive(); + setActive(Object.fromEntries(alerts.map((alert) => [alert.id, alert]))); + setGeneration((value) => value + 1); + }; + onMount(() => void reload()); + Object.assign(window, { __snapshotDate: { reload, active, generation } }); + return ( +
+

Snapshot observations

+

Synthetic monitoring evidence. No backup or guest operation.

+ + + setActive((previous) => ({ ...previous, [id]: { ...previous[id], ...updates } })) + } + showQuickTip={() => false} + dismissQuickTip={() => {}} + showAcknowledged={showAcknowledged} + setShowAcknowledged={setShowAcknowledged} + alertsDisabled={() => false} + /> +
+ ); +} + +render( + () => ( + document.documentElement.classList.contains('dark')}> + + + + + ), + document.getElementById('root')!, +); diff --git a/internal/alerts/backup_snapshot.go b/internal/alerts/backup_snapshot.go index ddd92e21a7..ca3df0caea 100644 --- a/internal/alerts/backup_snapshot.go +++ b/internal/alerts/backup_snapshot.go @@ -369,16 +369,6 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod if instanceName != "" && snapshot.Instance != "" && snapshot.Instance != instanceName { continue } - if snapshot.Time.IsZero() { - continue - } - - ageHours := now.Sub(snapshot.Time).Hours() - if ageHours < 0 { - continue - } - ageDays := ageHours / 24 - const gib = 1024.0 * 1024 * 1024 sizeGiB := 0.0 if snapshot.SizeBytes > 0 { @@ -429,6 +419,19 @@ func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []mod continue } + // The listing still contains this snapshot. An unusable creation time + // is not evidence of deletion or recovery: keep the last trusted + // occurrence unchanged, without refreshing its evidence or age. Do + // this after policy resolution so exclusions and disablement still win. + // Proxmox's absent snaptime arrives as Unix epoch, not Go's zero time. + if snapshot.Time.IsZero() || snapshot.Time.Unix() <= 0 || snapshot.Time.After(now) { + validAlerts[buildCanonicalStateID(resourceID, resourceID+"/snapshot:"+snapshot.ID)] = struct{}{} + continue + } + + ageHours := now.Sub(snapshot.Time).Hours() + ageDays := ageHours / 24 + var ageLevel AlertLevel var ageThreshold int var sizeLevel AlertLevel diff --git a/internal/alerts/snapshot_date_observation_test.go b/internal/alerts/snapshot_date_observation_test.go new file mode 100644 index 0000000000..79b2a1b3bc --- /dev/null +++ b/internal/alerts/snapshot_date_observation_test.go @@ -0,0 +1,203 @@ +package alerts + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog" + "github.com/rcourtman/pulse-go-rewrite/internal/models" +) + +func snapshotDateManager(t *testing.T) *Manager { + t.Helper() + m := newEventLogManager(t) + cfg := m.GetConfig() + cfg.SnapshotDefaults = SnapshotAlertConfig{Enabled: true, WarningDays: 7, CriticalDays: 14, WarningSizeGiB: 50, CriticalSizeGiB: 100} + cfg.TimeThresholds = map[string]int{} + m.UpdateConfig(cfg) + return m +} + +func snapshotDateFixture(kind string) (models.GuestSnapshot, map[string]GuestLookup) { + s := models.GuestSnapshot{ID: "east-pve-101-weekly", Name: "weekly", Instance: "east", Node: "pve", VMID: 101, Type: kind, Time: time.Now().Add(-16 * 24 * time.Hour), SizeBytes: 120 << 30} + g := GuestLookup{ResourceID: "east:pve:101", Name: "database", Instance: "east", Node: "pve", VMID: 101, Type: kind} + return s, map[string]GuestLookup{g.ResourceID: g} +} + +func snapshotDateJSON(t *testing.T, v any) []byte { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return b +} + +func TestSnapshotDateObservations(t *testing.T) { + invalid := map[string]time.Time{ + "zero": {}, "epoch": time.Unix(0, 0), "negative": time.Unix(-1, 0), "future": time.Now().Add(24 * time.Hour), + } + for _, kind := range []string{"qemu", "lxc"} { + for _, metric := range []string{"age", "size", "both"} { + for label, date := range invalid { + t.Run(kind+"/"+metric+"/"+label, func(t *testing.T) { + m := snapshotDateManager(t) + s, guests := snapshotDateFixture(kind) + if metric == "size" { + s.Time = time.Now().Add(-time.Hour) + } + if metric == "age" { + s.SizeBytes = 1 << 30 + } + m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests) + active := m.GetActiveAlerts() + if len(active) != 1 || active[0].Level != AlertLevelCritical { + t.Fatalf("control must establish one critical incident: %+v", active) + } + if err := m.AcknowledgeAlert(active[0].ID, "operator"); err != nil { + t.Fatal(err) + } + before := snapshotDateJSON(t, m.GetActiveAlerts()) + s.Time, s.SizeBytes = date, 0 + for range 4 { + m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests) + } + if after := snapshotDateJSON(t, m.GetActiveAlerts()); !bytes.Equal(before, after) { + t.Fatalf("unknown date changed occurrence/evidence/acknowledgement: before=%s after=%s", before, after) + } + if got := len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeFired, eventlog.TypeRefired, eventlog.TypeResolved}})); got != 1 { + t.Fatalf("unknown date invented a lifecycle transition: events=%d", got) + } + s.Time, s.SizeBytes = time.Now().Add(-time.Hour), 1<<30 + m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests) + m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests) + if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeResolved}})) != 1 { + t.Fatal("trustworthy normal observation must resolve exactly once") + } + }) + } + } + } + + t.Run("never-observed", func(t *testing.T) { + m := snapshotDateManager(t) + s, guests := snapshotDateFixture("qemu") + for _, date := range invalid { + s.Time = date + m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests) + } + if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeFired, eventlog.TypeResolved}})) != 0 { + t.Fatal("an unknown date cannot invent snapshot age or a lifecycle event") + } + }) + + for _, policy := range []string{"global-disabled", "guest-disabled", "snapshot-disabled", "override-disabled", "threshold-raised", "no-alerts-tag", "removed"} { + t.Run("policy/"+policy, func(t *testing.T) { + m := snapshotDateManager(t) + s, guests := snapshotDateFixture("qemu") + m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests) + if len(m.GetActiveAlerts()) != 1 { + t.Fatal("control must establish an incident") + } + before := snapshotDateJSON(t, m.GetActiveAlerts()) + s.Time = time.Time{} + snapshots := []models.GuestSnapshot{s} + cfg := m.GetConfig() + switch policy { + case "global-disabled": + cfg.Enabled = false + case "guest-disabled": + cfg.Overrides = map[string]ThresholdConfig{"east:pve:101": {Disabled: true}} + case "snapshot-disabled": + cfg.SnapshotDefaults.Enabled = false + case "override-disabled": + cfg.Overrides = map[string]ThresholdConfig{"east:pve:101": {Snapshot: &SnapshotAlertConfig{Enabled: false}}} + case "threshold-raised": + cfg.SnapshotDefaults = SnapshotAlertConfig{Enabled: true, WarningDays: 30, CriticalDays: 60, WarningSizeGiB: 200, CriticalSizeGiB: 400} + case "no-alerts-tag": + g := guests["east:pve:101"] + g.Tags = []string{"pulse-no-alerts"} + guests[g.ResourceID] = g + case "removed": + snapshots = nil + } + m.UpdateConfig(cfg) + m.CheckSnapshotsForInstance("east", snapshots, guests) + if policy == "global-disabled" { + // The global switch pauses evaluation; it is not snapshot + // recovery or per-resource disablement. Preserve that contract. + if !bytes.Equal(before, snapshotDateJSON(t, m.GetActiveAlerts())) { + t.Fatal("global evaluation pause changed the trusted occurrence") + } + return + } + if len(m.GetActiveAlerts()) != 0 { + t.Fatal("unknown date overrode explicit policy or authoritative removal") + } + }) + } + + t.Run("independent-instances-and-snapshots", func(t *testing.T) { + m := snapshotDateManager(t) + s, guests := snapshotDateFixture("qemu") + other := s + other.ID, other.Name = "east-pve-101-daily", "daily" + west := s + west.ID, west.Instance = "west-pve-101-weekly", "west" + g := guests["east:pve:101"] + g.ResourceID, g.Instance, g.Name = "west:pve:101", "west", "west-database" + guests[g.ResourceID] = g + m.CheckSnapshotsForInstance("", []models.GuestSnapshot{s, other, west}, guests) + if len(m.GetActiveAlerts()) != 3 { + t.Fatal("control must establish three independent incidents") + } + s.Time = time.Time{} + // One same-guest snapshot is removed and the other instance recovers. + m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests) + if len(m.GetActiveAlerts()) != 2 { + t.Fatal("unknown date protected a removed sibling or cleared another instance") + } + west.Time, west.SizeBytes = time.Now().Add(-time.Hour), 1<<30 + m.CheckSnapshotsForInstance("west", []models.GuestSnapshot{west}, guests) + active := m.GetActiveAlerts() + if len(active) != 1 || active[0].Instance != "east" || active[0].Metadata["snapshotName"] != "weekly" { + t.Fatalf("unknown date leaked across independent snapshot identity: %+v", active) + } + }) +} + +// Actual manager output is the only alert payload used by the browser fixture. +func TestSnapshotDateBrowserProducer(t *testing.T) { + m := snapshotDateManager(t) + s, guests := snapshotDateFixture("qemu") + frames := map[string]json.RawMessage{} + for _, phase := range []string{"stale", "missing", "future", "recovered"} { + switch phase { + case "missing": + s.Time = time.Unix(0, 0) // real absent-snaptime conversion + case "future": + s.Time = time.Now().Add(24 * time.Hour) + case "recovered": + s.Time, s.SizeBytes = time.Now().Add(-time.Hour), 1<<30 + } + m.CheckSnapshotsForInstance("east", []models.GuestSnapshot{s}, guests) + frames[phase] = snapshotDateJSON(t, m.GetActiveAlerts()) + want := 1 + if phase == "recovered" { + want = 0 + } + if len(m.GetActiveAlerts()) != want { + t.Fatalf("phase %s: want %d active alerts", phase, want) + } + } + if !bytes.Equal(frames["stale"], frames["missing"]) || !bytes.Equal(frames["stale"], frames["future"]) { + t.Fatal("browser frames changed the retained occurrence across a date gap") + } + if os.Getenv("PULSE_SNAPSHOT_DATE_FRAMES") == "1" { + fmt.Printf("SNAPSHOT_DATE_FRAMES=%s\n", snapshotDateJSON(t, frames)) + } +} diff --git a/scripts/check-snapshot-date-observation.cjs b/scripts/check-snapshot-date-observation.cjs new file mode 100644 index 0000000000..1b225fcf51 --- /dev/null +++ b/scripts/check-snapshot-date-observation.cjs @@ -0,0 +1,141 @@ +// Run from the assigned root with pulse-worker-browser. No external services. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const http = require('node:http'); +const crypto = require('node:crypto'); +const { chromium, webkit } = require('playwright'); + +const root = '/workspace'; +const compiled = path.join(root, 'tmp/snapshot-date-static'); +const output = path.join(root, 'tmp/snapshot-date-browser'); +const origin = 'http://127.0.0.1:5371'; +const hash = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex'); +const binding = JSON.parse(fs.readFileSync(path.join(compiled, 'binding.json'))); +const producer = JSON.parse(fs.readFileSync(path.join(root, 'tmp/snapshot-date-producer.json'))); +const frames = producer.frames; +const result = { + result: 'incomplete', + base: '7374abf19b434c9a275df730d28234bea3943774', + producer, + consumer: binding, + harness: hash(fs.readFileSync(__filename)), + playwright: require('playwright/package.json').version, + cases: [], + requests: [], + captures: [], + cleanup: { browser: false, server: false }, + limits: 'Real CheckSnapshotsForInstance/GetActiveAlerts -> AlertsAPI -> Overview/card; synthetic snapshot listings only. No installed recovery, guest action, backup execution, notification delivery or reporter acceptance.', +}; +let server; +let browser; + +(async () => { + fs.mkdirSync(output, { recursive: true }); + assert.equal(result.playwright, JSON.parse(fs.readFileSync(path.join(root, 'tests/integration/package-lock.json'))).packages['node_modules/@playwright/test'].version); + assert.equal(binding.graph, hash(fs.readFileSync(path.join(root, 'frontend-modern/package-lock.json')))); + assert.equal(binding.builder, hash(fs.readFileSync(path.join(root, 'frontend-modern/browser-tests/build-snapshot-date-observation.mjs')))); + assert.equal(producer.runtime_sha256, hash(fs.readFileSync(path.join(root, 'internal/alerts/backup_snapshot.go')))); + assert.deepEqual(frames.stale, frames.missing); + assert.deepEqual(frames.stale, frames.future); + assert.equal(frames.stale.length, 1); + assert.deepEqual(frames.recovered, []); + for (const [file, digest] of Object.entries(binding.sources)) { + assert.equal(hash(fs.readFileSync(path.join(root, 'frontend-modern', file))), digest, file); + } + for (const file of ['src/api/alerts.ts', 'src/features/alerts/OverviewTab.tsx', 'src/features/alerts/AlertOverviewAlertCard.tsx']) { + assert(binding.sources[file], `real consumer not compiled: ${file}`); + } + const assets = new Map(); + for (const [file, digest] of Object.entries(binding.artifacts)) { + const bytes = fs.readFileSync(path.join(compiled, file)); + assert.equal(hash(bytes), digest, file); + assets.set('/' + file, bytes); + } + server = http.createServer((request, response) => { + const pathname = new URL(request.url, origin).pathname; + const bytes = assets.get(pathname); + if (!bytes || request.method !== 'GET') { + response.writeHead(404).end(); + return; + } + response.writeHead(200, { 'Content-Type': { '.html': 'text/html', '.js': 'text/javascript', '.css': 'text/css' }[path.extname(pathname)] || 'application/octet-stream' }); + response.end(bytes); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(5371, '127.0.0.1', resolve); + }); + for (const [name, engine, width, height] of [ + ['desktop', chromium, 1280, 900], + ['narrow', webkit, 320, 812], + ]) { + browser = await engine.launch(engine === chromium ? { headless: true, channel: 'chromium', args: ['--no-sandbox'] } : { headless: true }); + const page = await browser.newPage({ viewport: { width, height }, isMobile: width < 768, hasTouch: width < 768, locale: 'en-GB', timezoneId: 'UTC' }); + page.setDefaultTimeout(10000); + const record = { name, width, height, browser: browser.version(), errors: [], phases: [] }; + page.on('pageerror', (error) => record.errors.push(error.message)); + let phase = 'stale'; + await page.route('**/*', async (route) => { + const request = route.request(); + const url = new URL(request.url()); + assert.equal(url.origin, origin, 'external request'); + assert.equal(request.method(), 'GET', 'unexpected mutation'); + if (!url.pathname.startsWith('/api/')) return route.continue(); + result.requests.push({ viewport: name, phase, path: url.pathname }); + let body; + if (url.pathname === '/api/alerts/active') body = frames[phase]; + else if (url.pathname === '/api/alerts/delivery-diagnosis') body = []; + else if (url.pathname === '/api/notifications/health') body = { queue: { status: 'unavailable' } }; + else if (url.pathname === '/api/alerts/incidents') body = null; + else throw new Error(`unexpected API read: ${url.pathname}`); + return route.fulfill({ json: body }); + }); + await page.goto(origin + '/browser-tests/snapshot-date-observation.html'); + await page.waitForFunction(() => window.__snapshotDate?.generation() === 1); + for (phase of ['stale', 'missing', 'future', 'recovered']) { + if (phase !== 'stale') { + const previous = await page.evaluate(() => window.__snapshotDate.generation()); + await page.getByRole('button', { name: 'Refresh monitoring evidence', exact: true }).click(); + await page.waitForFunction((generation) => window.__snapshotDate.generation() > generation, previous); + } + const active = await page.evaluate(() => Object.values(window.__snapshotDate.active())); + assert.deepEqual(active, frames[phase], `producer/consumer mismatch in ${name}/${phase}`); + if (phase !== 'recovered') { + await page.getByText(frames.stale[0].message, { exact: true }).waitFor(); + const more = page.getByRole('button', { name: 'More', exact: true }); + if (await more.count()) { + await more.click(); + await page.getByRole('button', { name: 'Less', exact: true }).click(); + } + } else { + assert.equal(await page.getByText(frames.stale[0].message, { exact: true }).count(), 0); + } + await page.evaluate(async () => { + await document.fonts.ready; + await new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve))); + }); + const overflow = await page.evaluate(() => document.documentElement.scrollWidth > window.innerWidth); + assert.equal(overflow, false, `${name}/${phase} horizontal overflow`); + const capture = `${name}-${phase}.png`; + await page.screenshot({ path: path.join(output, capture), fullPage: true }); + result.captures.push({ file: capture, sha256: hash(fs.readFileSync(path.join(output, capture))) }); + record.phases.push({ phase, active: active.length, overflow }); + } + assert.deepEqual(record.errors, []); + result.cases.push(record); + await browser.close(); + browser = undefined; + } + result.result = 'passed'; +})().catch((error) => { + result.result = 'failed'; + result.error = String(error.stack || error); + process.exitCode = 1; +}).finally(async () => { + if (browser) await browser.close(); + result.cleanup.browser = true; + if (server) await new Promise((resolve) => server.close(resolve)); + result.cleanup.server = true; + fs.writeFileSync(path.join(output, 'result.json'), JSON.stringify(result, null, 2) + '\n'); +});