fix(notifications): stop Sencho version notifications from silently skipping (#594)

* fix(notifications): stop Sencho version notifications from silently skipping

Three independent defects combined to make version-update notifications
silently fail even while Fleet overview correctly surfaced an update button:

- The in-memory 6-hour cooldown was advanced before the network fetch,
  so a single transient failure at boot could lock the check for the
  rest of the container lifetime. Moved the cooldown update inside the
  success branch so failures retry on the next eval cycle.
- MonitorService called the raw version fetch directly, bypassing the
  CacheService wrapper (TTL, inflight dedup, stale-on-error) that Fleet
  uses, so the two paths could diverge. Unified both on a shared
  getLatestVersion() helper in utils/version-check.ts.
- The dedup key could carry stale state from a previous build and never
  self-clear. It now self-heals when the running version reaches the
  previously-notified version, so future releases re-fire as expected.

Added diagnostic logs gated on debug mode for each skip branch, plus
three regression tests covering cooldown-on-failure, cooldown-on-success,
and dedup self-heal.

* docs(notifications): drop legacy-upgrade framing from alerts troubleshooting

Sencho has not shipped publicly, so troubleshooting entries written in
'this used to happen but now does Y' mode reference a past that does not
exist for any reader. Rewrote the version-notification, image-update,
and crash-alert troubleshooting entries to describe current behavior
positively without referring to prior builds, upgrade paths, or legacy
fixes.

* chore(security): accept CVE-2026-33810 in bundled Docker CLI 29.4.0

Trivy now flags CVE-2026-33810 (Go stdlib crypto/x509 DNS constraint
bypass, fixed in Go 1.26.2) in the Docker CLI static binary we ship.
Docker CLI 29.4.0 is the latest upstream release and still links Go
1.26.1; no newer static binary exists yet.

Same exposure profile as the already-accepted CVE-2026-32280: the
Docker CLI and compose plugin only validate certificates from
well-known registry CAs and the local Docker socket, not from
attacker-controlled CAs with crafted DNS name constraints. Revisit
on the next Docker CLI release that rebuilds against Go 1.26.2 or
later.

backend/src/__tests__/monitor-service.test.ts

@@ -15,6 +15,7 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
   mockCurrentLoad, mockMem, mockFsSize,
   mockExecAsync,
   mockFetchLatestSenchoVersion,
+  mockGetLatestVersion,
   mockGetSenchoVersion,
 } = vi.hoisted(() => ({
   mockGetGlobalSettings: vi.fn().mockReturnValue({}),
@@ -37,6 +38,7 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
   mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]),
   mockExecAsync: vi.fn().mockResolvedValue({ stdout: '' }),
   mockFetchLatestSenchoVersion: vi.fn().mockRejectedValue(new Error('not configured')),
+  mockGetLatestVersion: vi.fn().mockResolvedValue(null),
   mockGetSenchoVersion: vi.fn().mockReturnValue(null),
 }));
 
@@ -70,6 +72,7 @@ vi.mock('../services/DockerController', () => ({
 
 vi.mock('../utils/version-check', () => ({
   fetchLatestSenchoVersion: (...args: unknown[]) => mockFetchLatestSenchoVersion(...args),
+  getLatestVersion: (...args: unknown[]) => mockGetLatestVersion(...args),
 }));
 
 vi.mock('../services/CapabilityRegistry', async () => {
@@ -549,13 +552,25 @@ describe('MonitorService - restart_count metric', () => {
 // ── Sencho version update check ───────────────────────────────────────
 
 describe('MonitorService - Sencho version check', () => {
-  it('dispatches notification when newer version available', async () => {
-    mockGetSenchoVersion.mockReturnValue('0.45.0');
-    mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0');
-    mockGetSystemState.mockReturnValue(null); // No previous notification
+  /** Stateful system_state backing for tests that need getSystemState to
+   *  reflect setSystemState writes within the same evaluation. */
+  function wireStatefulSystemState(seed: Record<string, string> = {}) {
+    const store: Record<string, string> = { ...seed };
+    mockGetSystemState.mockImplementation((key: string) => store[key] ?? null);
+    mockSetSystemState.mockImplementation((key: string, value: string) => { store[key] = value; });
+    return store;
+  }
+
+  beforeEach(() => {
     mockGetGlobalSettings.mockReturnValue({});
     mockGetNodes.mockReturnValue([]);
     mockGetStackAlerts.mockReturnValue([]);
+  });
+
+  it('dispatches notification when newer version available', async () => {
+    mockGetSenchoVersion.mockReturnValue('0.45.0');
+    mockGetLatestVersion.mockResolvedValue('0.46.0');
+    mockGetSystemState.mockReturnValue(null); // No previous notification
 
     const svc = MonitorService.getInstance();
     // Reset the version check timer so it runs immediately
@@ -570,11 +585,9 @@ describe('MonitorService - Sencho version check', () => {
 
   it('does not re-notify for the same version', async () => {
     mockGetSenchoVersion.mockReturnValue('0.45.0');
-    mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0');
-    mockGetSystemState.mockReturnValue('0.46.0'); // Already notified for this version
-    mockGetGlobalSettings.mockReturnValue({});
-    mockGetNodes.mockReturnValue([]);
-    mockGetStackAlerts.mockReturnValue([]);
+    mockGetLatestVersion.mockResolvedValue('0.46.0');
+    // Running version < last notified, so self-heal does NOT clear the key.
+    mockGetSystemState.mockReturnValue('0.46.0');
 
     const svc = MonitorService.getInstance();
     (svc as any).lastVersionCheckAt = 0;
@@ -585,10 +598,7 @@ describe('MonitorService - Sencho version check', () => {
 
   it('handles version check failure gracefully', async () => {
     mockGetSenchoVersion.mockReturnValue('0.45.0');
-    mockFetchLatestSenchoVersion.mockRejectedValue(new Error('Network down'));
-    mockGetGlobalSettings.mockReturnValue({});
-    mockGetNodes.mockReturnValue([]);
-    mockGetStackAlerts.mockReturnValue([]);
+    mockGetLatestVersion.mockResolvedValue(null); // CacheService failed + no stale
 
     const svc = MonitorService.getInstance();
     (svc as any).lastVersionCheckAt = 0;
@@ -600,29 +610,23 @@ describe('MonitorService - Sencho version check', () => {
 
   it('respects the 6-hour cooldown interval', async () => {
     mockGetSenchoVersion.mockReturnValue('0.45.0');
-    mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0');
+    mockGetLatestVersion.mockResolvedValue('0.46.0');
     mockGetSystemState.mockReturnValue(null);
-    mockGetGlobalSettings.mockReturnValue({});
-    mockGetNodes.mockReturnValue([]);
-    mockGetStackAlerts.mockReturnValue([]);
 
     const svc = MonitorService.getInstance();
     // Simulate the check ran 1 hour ago (within 6-hour window)
     (svc as any).lastVersionCheckAt = Date.now() - 1 * 60 * 60 * 1000;
     await (svc as any).evaluate();
 
-    // fetchLatestSenchoVersion should not have been called since we're within cooldown
-    expect(mockFetchLatestSenchoVersion).not.toHaveBeenCalled();
+    // getLatestVersion should not have been called since we're within cooldown
+    expect(mockGetLatestVersion).not.toHaveBeenCalled();
   });
 
   it('skips version check when getSenchoVersion returns null', async () => {
     // Simulates the production-Docker scenario that previously leaked "0.0.0"
     mockGetSenchoVersion.mockReturnValue(null);
-    mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0');
+    mockGetLatestVersion.mockResolvedValue('0.46.0');
     mockGetSystemState.mockReturnValue(null);
-    mockGetGlobalSettings.mockReturnValue({});
-    mockGetNodes.mockReturnValue([]);
-    mockGetStackAlerts.mockReturnValue([]);
 
     const svc = MonitorService.getInstance();
     (svc as any).lastVersionCheckAt = 0;
@@ -630,5 +634,64 @@ describe('MonitorService - Sencho version check', () => {
 
     expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', expect.stringContaining('0.46.0'));
     expect(mockSetSystemState).not.toHaveBeenCalledWith('last_sencho_update_notified_version', expect.anything());
+    // Should not have even attempted the lookup.
+    expect(mockGetLatestVersion).not.toHaveBeenCalled();
+  });
+
+  // ── Regression coverage for PR: cooldown leak + dedup self-heal ───────
+
+  it('does NOT advance cooldown when getLatestVersion returns null (retries next cycle)', async () => {
+    mockGetSenchoVersion.mockReturnValue('0.45.0');
+    mockGetLatestVersion.mockResolvedValue(null);
+    mockGetSystemState.mockReturnValue(null);
+
+    const svc = MonitorService.getInstance();
+    (svc as any).lastVersionCheckAt = 0;
+
+    await (svc as any).evaluate();
+    await (svc as any).evaluate();
+
+    // Both evals should attempt the lookup since failures do not lock cooldown.
+    expect(mockGetLatestVersion).toHaveBeenCalledTimes(2);
+    expect((svc as any).lastVersionCheckAt).toBe(0);
+  });
+
+  it('DOES advance cooldown on a successful lookup (prevents re-fetch inside window)', async () => {
+    mockGetSenchoVersion.mockReturnValue('0.45.0');
+    mockGetLatestVersion.mockResolvedValue('0.46.0');
+    mockGetSystemState.mockReturnValue(null);
+
+    const svc = MonitorService.getInstance();
+    (svc as any).lastVersionCheckAt = 0;
+
+    await (svc as any).evaluate();
+    const firstCooldown = (svc as any).lastVersionCheckAt;
+    expect(firstCooldown).toBeGreaterThan(0);
+
+    // Second eval immediately after: cooldown gate should block it.
+    mockGetLatestVersion.mockClear();
+    await (svc as any).evaluate();
+
+    expect(mockGetLatestVersion).not.toHaveBeenCalled();
+    // Exactly one dispatch across both evals.
+    const availabilityDispatches = mockDispatchAlert.mock.calls.filter(
+      (args: unknown[]) => typeof args[1] === 'string' && args[1].includes('available'),
+    );
+    expect(availabilityDispatches).toHaveLength(1);
+  });
+
+  it('self-heals dedup after user upgrades to the previously-notified version', async () => {
+    // Prior notification stored "0.46.0" back when the user was on 0.45.0.
+    // User has now upgraded to 0.46.0; a new release (0.47.0) just dropped.
+    const store = wireStatefulSystemState({ last_sencho_update_notified_version: '0.46.0' });
+    mockGetSenchoVersion.mockReturnValue('0.46.0');
+    mockGetLatestVersion.mockResolvedValue('0.47.0');
+
+    const svc = MonitorService.getInstance();
+    (svc as any).lastVersionCheckAt = 0;
+    await (svc as any).evaluate();
+
+    expect(mockDispatchAlert).toHaveBeenCalledWith('info', expect.stringContaining('0.47.0'));
+    expect(store.last_sencho_update_notified_version).toBe('0.47.0');
   });
 });

backend/src/index.ts

@@ -60,7 +60,7 @@ function invalidateNodeCaches(nodeId: number): void {
 }
 
 import { isDebugEnabled } from './utils/debug';
-import { fetchLatestSenchoVersion } from './utils/version-check';
+import { getLatestVersion } from './utils/version-check';
 import { getErrorMessage } from './utils/errors';
 import { captureLocalNodeFiles, captureRemoteNodeFiles, SnapshotNodeData } from './utils/snapshot-capture';
 import { GlobalLogEntry, normalizeContainerName, parseLogTimestamp, detectLogLevel, demuxDockerLog } from './utils/log-parsing';
@@ -1281,29 +1281,9 @@ const UPDATE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
 const UPDATE_TIMEOUT_MSG = 'Node did not come back online within 5 minutes.';
 const EARLY_FAIL_MS = 180 * 1000; // 3 minutes before declaring a probable pull failure
 
-// Latest Sencho version cache (fetched from GitHub Releases).
-// Backed by CacheService: TTL, inflight dedup, and stale-on-error are all
-// handled by the unified cache layer.
-const LATEST_VERSION_CACHE_KEY = 'latest-version';
-const LATEST_VERSION_CACHE_TTL = 30 * 60 * 1000; // 30 minutes
-
-// Version fetch logic lives in utils/version-check.ts; imported below for getLatestVersion().
-// fetchFromGitHub + fetchFromDockerHub + fetchLatestSenchoVersion are shared with MonitorService.
-
-async function getLatestVersion(forceRefresh = false): Promise<string | null> {
-  if (forceRefresh) {
-    CacheService.getInstance().invalidate(LATEST_VERSION_CACHE_KEY);
-  }
-  try {
-    return await CacheService.getInstance().getOrFetch<string>(
-      LATEST_VERSION_CACHE_KEY,
-      LATEST_VERSION_CACHE_TTL,
-      fetchLatestSenchoVersion,
-    );
-  } catch {
-    return null;
-  }
-}
+// Latest Sencho version lookup and caching live in utils/version-check.ts
+// (shared with MonitorService). Fleet compares the gateway version against
+// whatever getLatestVersion() returns from GitHub or Docker Hub.
 
 /** Resolve the version to compare nodes against (latest from GitHub, or gateway fallback). */
 async function getCompareTarget(gatewayVersion: string | null) {

backend/src/services/MonitorService.ts

@@ -6,7 +6,7 @@ import DockerController from './DockerController';
 import { DatabaseService } from './DatabaseService';
 import { NotificationService } from './NotificationService';
 import { isValidVersion, getSenchoVersion } from './CapabilityRegistry';
-import { fetchLatestSenchoVersion } from '../utils/version-check';
+import { getLatestVersion } from '../utils/version-check';
 import { isDebugEnabled } from '../utils/debug';
 
 const execAsync = promisify(exec);
@@ -76,6 +76,7 @@ export class MonitorService {
     // Sencho version check cooldown (6 hours between external API calls)
     private lastVersionCheckAt = 0;
     private static readonly VERSION_CHECK_INTERVAL_MS = 6 * 60 * 60 * 1000;
+    private static readonly SENCHO_UPDATE_NOTIFIED_KEY = 'last_sencho_update_notified_version';
 
     private constructor() { }
 
@@ -209,30 +210,82 @@ export class MonitorService {
         }
 
         // 4. Sencho version update check (runs once per VERSION_CHECK_INTERVAL_MS)
-        if (Date.now() - this.lastVersionCheckAt > MonitorService.VERSION_CHECK_INTERVAL_MS) {
-            this.lastVersionCheckAt = Date.now();
-            try {
-                // Resolve from the packaged manifest: process.env.npm_package_version is
-                // only set by npm scripts, so it is undefined in Docker (node dist/index.js).
-                const currentVersion = getSenchoVersion();
-                const latest = await fetchLatestSenchoVersion();
-                if (isValidVersion(latest) && isValidVersion(currentVersion) && semver.gt(latest, currentVersion)) {
-                    const db = DatabaseService.getInstance();
-                    const stateKey = 'last_sencho_update_notified_version';
-                    const lastNotified = db.getSystemState(stateKey) || '';
-                    if (lastNotified !== latest) {
-                        const notifier = NotificationService.getInstance();
-                        await notifier.dispatchAlert('info',
-                            `Sencho ${latest} is available (currently running ${currentVersion}). Visit the Fleet dashboard to update.`);
-                        db.setSystemState(stateKey, latest);
-                    }
-                } else if (isDebugEnabled() && !isValidVersion(currentVersion)) {
-                    console.debug('[Monitor:diag] Sencho version unresolvable; skipping update notification');
-                }
-            } catch (e) {
-                // Network errors are expected; do not spam logs
-                if (isDebugEnabled()) console.debug('[Monitor:diag] Sencho version check failed:', e);
+        await this.checkSenchoVersion();
+    }
+
+    /**
+     * Check GitHub/Docker Hub for a newer Sencho release and dispatch a
+     * one-shot notification. Uses getLatestVersion() which wraps CacheService
+     * (30 min TTL + inflight dedup + stale-on-error) so transient network
+     * blips do not cause gaps, and the check stays consistent with the Fleet
+     * update banner.
+     *
+     * The 6-hour cooldown gate prevents bell spam: it is only advanced on a
+     * SUCCESSFUL lookup. A failed lookup retries on the next eval cycle
+     * (30 seconds) instead of locking for 6 hours.
+     */
+    private async checkSenchoVersion(): Promise<void> {
+        const sinceLast = Date.now() - this.lastVersionCheckAt;
+        if (sinceLast <= MonitorService.VERSION_CHECK_INTERVAL_MS) {
+            if (isDebugEnabled()) {
+                const nextInMs = MonitorService.VERSION_CHECK_INTERVAL_MS - sinceLast;
+                console.debug(`[Monitor:diag] Sencho version check in cooldown (next in ~${Math.round(nextInMs / 60000)}m)`);
             }
+            return;
+        }
+
+        // Resolve from the packaged manifest: process.env.npm_package_version is
+        // only set by npm scripts, so it is undefined in Docker (node dist/index.js).
+        const currentVersion = getSenchoVersion();
+        if (!isValidVersion(currentVersion)) {
+            if (isDebugEnabled()) console.debug('[Monitor:diag] Sencho version unresolvable; skipping update notification');
+            return;
+        }
+
+        const latest = await getLatestVersion();
+        if (!isValidVersion(latest)) {
+            // Network failure (GitHub + Docker Hub both down, no stale cache).
+            // Do NOT advance the cooldown so the next eval retries.
+            if (isDebugEnabled()) console.debug('[Monitor:diag] Latest Sencho version unresolvable; will retry next cycle');
+            return;
+        }
+
+        this.lastVersionCheckAt = Date.now();
+
+        const db = DatabaseService.getInstance();
+        const stateKey = MonitorService.SENCHO_UPDATE_NOTIFIED_KEY;
+        const storedLastNotified = db.getSystemState(stateKey) || '';
+
+        // Self-heal: if the user has reached the previously-notified version,
+        // clear the dedup so future releases always trigger a fresh notification.
+        // This also recovers from stale state left over by the pre-586 "0.0.0" bug.
+        let effectiveLastNotified = storedLastNotified;
+        if (storedLastNotified && isValidVersion(storedLastNotified) && semver.gte(currentVersion, storedLastNotified)) {
+            if (isDebugEnabled()) console.debug(`[Monitor:diag] Clearing stale dedup key (running ${currentVersion} >= last notified ${storedLastNotified})`);
+            db.setSystemState(stateKey, '');
+            effectiveLastNotified = '';
+        }
+
+        if (!semver.gt(latest, currentVersion)) {
+            if (isDebugEnabled()) console.debug(`[Monitor:diag] Running ${currentVersion} is up-to-date with latest ${latest}`);
+            return;
+        }
+
+        if (effectiveLastNotified === latest) {
+            if (isDebugEnabled()) console.debug(`[Monitor:diag] Already notified for Sencho ${latest}`);
+            return;
+        }
+
+        try {
+            const notifier = NotificationService.getInstance();
+            await notifier.dispatchAlert('info',
+                `Sencho ${latest} is available (currently running ${currentVersion}). Visit the Fleet dashboard to update.`);
+            db.setSystemState(stateKey, latest);
+            if (isDebugEnabled()) console.debug(`[Monitor:diag] Dispatched version notification: ${currentVersion} -> ${latest}`);
+        } catch (e) {
+            // dispatchAlert normally catches channel errors internally, but the
+            // history insert or WebSocket broadcast can throw on an unhealthy DB.
+            console.error('[MonitorService] Failed to dispatch Sencho version notification:', e);
         }
     }
 

backend/src/utils/version-check.ts

@@ -1,4 +1,5 @@
 import semver from 'semver';
+import { CacheService } from '../services/CacheService';
 
 /**
  * Fetches the latest Sencho release version from GitHub or Docker Hub.
@@ -50,3 +51,26 @@ export async function fetchLatestSenchoVersion(): Promise<string> {
   // and so we do not poison the cache with null.
   throw new Error('Both GitHub and Docker Hub version lookups failed');
 }
+
+/**
+ * Cached wrapper shared by the Fleet endpoint and MonitorService.
+ * CacheService provides TTL, inflight deduplication, and stale-on-error
+ * fallback so transient network blips do not cause user-visible gaps.
+ */
+const LATEST_VERSION_CACHE_KEY = 'latest-version';
+const LATEST_VERSION_CACHE_TTL = 30 * 60 * 1000; // 30 minutes
+
+export async function getLatestVersion(forceRefresh = false): Promise<string | null> {
+  if (forceRefresh) {
+    CacheService.getInstance().invalidate(LATEST_VERSION_CACHE_KEY);
+  }
+  try {
+    return await CacheService.getInstance().getOrFetch<string>(
+      LATEST_VERSION_CACHE_KEY,
+      LATEST_VERSION_CACHE_TTL,
+      fetchLatestSenchoVersion,
+    );
+  } catch {
+    return null;
+  }
+}