diff --git a/.trivyignore b/.trivyignore index faf6e788..f2a99aef 100644 --- a/.trivyignore +++ b/.trivyignore @@ -43,6 +43,16 @@ CVE-2026-32282 # Go rebuild; revisit on next CLI/Compose release. CVE-2026-32280 +# Justification: Go stdlib crypto/x509 certificate validation bypass due to +# incorrect DNS name constraint handling. Fixed in Go 1.26.2, but Docker CLI +# 29.4.0 ships Go 1.26.1 and no newer upstream static binary is available. +# Same exposure profile as CVE-2026-32280: the Docker CLI and compose plugin +# only validate certificates from well-known registry CAs and the local +# Docker socket in our usage, not from attacker-controlled CAs with crafted +# DNS name constraints. Blocked on upstream Go rebuild; revisit on next +# CLI/Compose release. +CVE-2026-33810 + # --------------------------------------------------------------------------- # Bundled inside /usr/local/lib/docker/cli-plugins/docker-compose (v5.1.2) # --------------------------------------------------------------------------- diff --git a/backend/src/__tests__/monitor-service.test.ts b/backend/src/__tests__/monitor-service.test.ts index 141c61d5..e8bd0f0d 100644 --- a/backend/src/__tests__/monitor-service.test.ts +++ b/backend/src/__tests__/monitor-service.test.ts @@ -15,6 +15,7 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine mockCurrentLoad, mockMem, mockFsSize, mockExecAsync, mockFetchLatestSenchoVersion, + mockGetLatestVersion, mockGetSenchoVersion, } = vi.hoisted(() => ({ mockGetGlobalSettings: vi.fn().mockReturnValue({}), @@ -37,6 +38,7 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]), mockExecAsync: vi.fn().mockResolvedValue({ stdout: '' }), mockFetchLatestSenchoVersion: vi.fn().mockRejectedValue(new Error('not configured')), + mockGetLatestVersion: vi.fn().mockResolvedValue(null), mockGetSenchoVersion: vi.fn().mockReturnValue(null), })); @@ -70,6 +72,7 @@ vi.mock('../services/DockerController', () => ({ vi.mock('../utils/version-check', () => ({ fetchLatestSenchoVersion: (...args: unknown[]) => mockFetchLatestSenchoVersion(...args), + getLatestVersion: (...args: unknown[]) => mockGetLatestVersion(...args), })); vi.mock('../services/CapabilityRegistry', async () => { @@ -549,13 +552,25 @@ describe('MonitorService - restart_count metric', () => { // ── Sencho version update check ─────────────────────────────────────── describe('MonitorService - Sencho version check', () => { - it('dispatches notification when newer version available', async () => { - mockGetSenchoVersion.mockReturnValue('0.45.0'); - mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0'); - mockGetSystemState.mockReturnValue(null); // No previous notification + /** Stateful system_state backing for tests that need getSystemState to + * reflect setSystemState writes within the same evaluation. */ + function wireStatefulSystemState(seed: Record = {}) { + const store: Record = { ...seed }; + mockGetSystemState.mockImplementation((key: string) => store[key] ?? null); + mockSetSystemState.mockImplementation((key: string, value: string) => { store[key] = value; }); + return store; + } + + beforeEach(() => { mockGetGlobalSettings.mockReturnValue({}); mockGetNodes.mockReturnValue([]); mockGetStackAlerts.mockReturnValue([]); + }); + + it('dispatches notification when newer version available', async () => { + mockGetSenchoVersion.mockReturnValue('0.45.0'); + mockGetLatestVersion.mockResolvedValue('0.46.0'); + mockGetSystemState.mockReturnValue(null); // No previous notification const svc = MonitorService.getInstance(); // Reset the version check timer so it runs immediately @@ -570,11 +585,9 @@ describe('MonitorService - Sencho version check', () => { it('does not re-notify for the same version', async () => { mockGetSenchoVersion.mockReturnValue('0.45.0'); - mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0'); - mockGetSystemState.mockReturnValue('0.46.0'); // Already notified for this version - mockGetGlobalSettings.mockReturnValue({}); - mockGetNodes.mockReturnValue([]); - mockGetStackAlerts.mockReturnValue([]); + mockGetLatestVersion.mockResolvedValue('0.46.0'); + // Running version < last notified, so self-heal does NOT clear the key. + mockGetSystemState.mockReturnValue('0.46.0'); const svc = MonitorService.getInstance(); (svc as any).lastVersionCheckAt = 0; @@ -585,10 +598,7 @@ describe('MonitorService - Sencho version check', () => { it('handles version check failure gracefully', async () => { mockGetSenchoVersion.mockReturnValue('0.45.0'); - mockFetchLatestSenchoVersion.mockRejectedValue(new Error('Network down')); - mockGetGlobalSettings.mockReturnValue({}); - mockGetNodes.mockReturnValue([]); - mockGetStackAlerts.mockReturnValue([]); + mockGetLatestVersion.mockResolvedValue(null); // CacheService failed + no stale const svc = MonitorService.getInstance(); (svc as any).lastVersionCheckAt = 0; @@ -600,29 +610,23 @@ describe('MonitorService - Sencho version check', () => { it('respects the 6-hour cooldown interval', async () => { mockGetSenchoVersion.mockReturnValue('0.45.0'); - mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0'); + mockGetLatestVersion.mockResolvedValue('0.46.0'); mockGetSystemState.mockReturnValue(null); - mockGetGlobalSettings.mockReturnValue({}); - mockGetNodes.mockReturnValue([]); - mockGetStackAlerts.mockReturnValue([]); const svc = MonitorService.getInstance(); // Simulate the check ran 1 hour ago (within 6-hour window) (svc as any).lastVersionCheckAt = Date.now() - 1 * 60 * 60 * 1000; await (svc as any).evaluate(); - // fetchLatestSenchoVersion should not have been called since we're within cooldown - expect(mockFetchLatestSenchoVersion).not.toHaveBeenCalled(); + // getLatestVersion should not have been called since we're within cooldown + expect(mockGetLatestVersion).not.toHaveBeenCalled(); }); it('skips version check when getSenchoVersion returns null', async () => { // Simulates the production-Docker scenario that previously leaked "0.0.0" mockGetSenchoVersion.mockReturnValue(null); - mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0'); + mockGetLatestVersion.mockResolvedValue('0.46.0'); mockGetSystemState.mockReturnValue(null); - mockGetGlobalSettings.mockReturnValue({}); - mockGetNodes.mockReturnValue([]); - mockGetStackAlerts.mockReturnValue([]); const svc = MonitorService.getInstance(); (svc as any).lastVersionCheckAt = 0; @@ -630,5 +634,64 @@ describe('MonitorService - Sencho version check', () => { expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', expect.stringContaining('0.46.0')); expect(mockSetSystemState).not.toHaveBeenCalledWith('last_sencho_update_notified_version', expect.anything()); + // Should not have even attempted the lookup. + expect(mockGetLatestVersion).not.toHaveBeenCalled(); + }); + + // ── Regression coverage for PR: cooldown leak + dedup self-heal ─────── + + it('does NOT advance cooldown when getLatestVersion returns null (retries next cycle)', async () => { + mockGetSenchoVersion.mockReturnValue('0.45.0'); + mockGetLatestVersion.mockResolvedValue(null); + mockGetSystemState.mockReturnValue(null); + + const svc = MonitorService.getInstance(); + (svc as any).lastVersionCheckAt = 0; + + await (svc as any).evaluate(); + await (svc as any).evaluate(); + + // Both evals should attempt the lookup since failures do not lock cooldown. + expect(mockGetLatestVersion).toHaveBeenCalledTimes(2); + expect((svc as any).lastVersionCheckAt).toBe(0); + }); + + it('DOES advance cooldown on a successful lookup (prevents re-fetch inside window)', async () => { + mockGetSenchoVersion.mockReturnValue('0.45.0'); + mockGetLatestVersion.mockResolvedValue('0.46.0'); + mockGetSystemState.mockReturnValue(null); + + const svc = MonitorService.getInstance(); + (svc as any).lastVersionCheckAt = 0; + + await (svc as any).evaluate(); + const firstCooldown = (svc as any).lastVersionCheckAt; + expect(firstCooldown).toBeGreaterThan(0); + + // Second eval immediately after: cooldown gate should block it. + mockGetLatestVersion.mockClear(); + await (svc as any).evaluate(); + + expect(mockGetLatestVersion).not.toHaveBeenCalled(); + // Exactly one dispatch across both evals. + const availabilityDispatches = mockDispatchAlert.mock.calls.filter( + (args: unknown[]) => typeof args[1] === 'string' && args[1].includes('available'), + ); + expect(availabilityDispatches).toHaveLength(1); + }); + + it('self-heals dedup after user upgrades to the previously-notified version', async () => { + // Prior notification stored "0.46.0" back when the user was on 0.45.0. + // User has now upgraded to 0.46.0; a new release (0.47.0) just dropped. + const store = wireStatefulSystemState({ last_sencho_update_notified_version: '0.46.0' }); + mockGetSenchoVersion.mockReturnValue('0.46.0'); + mockGetLatestVersion.mockResolvedValue('0.47.0'); + + const svc = MonitorService.getInstance(); + (svc as any).lastVersionCheckAt = 0; + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledWith('info', expect.stringContaining('0.47.0')); + expect(store.last_sencho_update_notified_version).toBe('0.47.0'); }); }); diff --git a/backend/src/index.ts b/backend/src/index.ts index 74364c32..8a516a6c 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -60,7 +60,7 @@ function invalidateNodeCaches(nodeId: number): void { } import { isDebugEnabled } from './utils/debug'; -import { fetchLatestSenchoVersion } from './utils/version-check'; +import { getLatestVersion } from './utils/version-check'; import { getErrorMessage } from './utils/errors'; import { captureLocalNodeFiles, captureRemoteNodeFiles, SnapshotNodeData } from './utils/snapshot-capture'; import { GlobalLogEntry, normalizeContainerName, parseLogTimestamp, detectLogLevel, demuxDockerLog } from './utils/log-parsing'; @@ -1281,29 +1281,9 @@ const UPDATE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes const UPDATE_TIMEOUT_MSG = 'Node did not come back online within 5 minutes.'; const EARLY_FAIL_MS = 180 * 1000; // 3 minutes before declaring a probable pull failure -// Latest Sencho version cache (fetched from GitHub Releases). -// Backed by CacheService: TTL, inflight dedup, and stale-on-error are all -// handled by the unified cache layer. -const LATEST_VERSION_CACHE_KEY = 'latest-version'; -const LATEST_VERSION_CACHE_TTL = 30 * 60 * 1000; // 30 minutes - -// Version fetch logic lives in utils/version-check.ts; imported below for getLatestVersion(). -// fetchFromGitHub + fetchFromDockerHub + fetchLatestSenchoVersion are shared with MonitorService. - -async function getLatestVersion(forceRefresh = false): Promise { - if (forceRefresh) { - CacheService.getInstance().invalidate(LATEST_VERSION_CACHE_KEY); - } - try { - return await CacheService.getInstance().getOrFetch( - LATEST_VERSION_CACHE_KEY, - LATEST_VERSION_CACHE_TTL, - fetchLatestSenchoVersion, - ); - } catch { - return null; - } -} +// Latest Sencho version lookup and caching live in utils/version-check.ts +// (shared with MonitorService). Fleet compares the gateway version against +// whatever getLatestVersion() returns from GitHub or Docker Hub. /** Resolve the version to compare nodes against (latest from GitHub, or gateway fallback). */ async function getCompareTarget(gatewayVersion: string | null) { diff --git a/backend/src/services/MonitorService.ts b/backend/src/services/MonitorService.ts index 18e01609..550a0844 100644 --- a/backend/src/services/MonitorService.ts +++ b/backend/src/services/MonitorService.ts @@ -6,7 +6,7 @@ import DockerController from './DockerController'; import { DatabaseService } from './DatabaseService'; import { NotificationService } from './NotificationService'; import { isValidVersion, getSenchoVersion } from './CapabilityRegistry'; -import { fetchLatestSenchoVersion } from '../utils/version-check'; +import { getLatestVersion } from '../utils/version-check'; import { isDebugEnabled } from '../utils/debug'; const execAsync = promisify(exec); @@ -76,6 +76,7 @@ export class MonitorService { // Sencho version check cooldown (6 hours between external API calls) private lastVersionCheckAt = 0; private static readonly VERSION_CHECK_INTERVAL_MS = 6 * 60 * 60 * 1000; + private static readonly SENCHO_UPDATE_NOTIFIED_KEY = 'last_sencho_update_notified_version'; private constructor() { } @@ -209,30 +210,82 @@ export class MonitorService { } // 4. Sencho version update check (runs once per VERSION_CHECK_INTERVAL_MS) - if (Date.now() - this.lastVersionCheckAt > MonitorService.VERSION_CHECK_INTERVAL_MS) { - this.lastVersionCheckAt = Date.now(); - try { - // Resolve from the packaged manifest: process.env.npm_package_version is - // only set by npm scripts, so it is undefined in Docker (node dist/index.js). - const currentVersion = getSenchoVersion(); - const latest = await fetchLatestSenchoVersion(); - if (isValidVersion(latest) && isValidVersion(currentVersion) && semver.gt(latest, currentVersion)) { - const db = DatabaseService.getInstance(); - const stateKey = 'last_sencho_update_notified_version'; - const lastNotified = db.getSystemState(stateKey) || ''; - if (lastNotified !== latest) { - const notifier = NotificationService.getInstance(); - await notifier.dispatchAlert('info', - `Sencho ${latest} is available (currently running ${currentVersion}). Visit the Fleet dashboard to update.`); - db.setSystemState(stateKey, latest); - } - } else if (isDebugEnabled() && !isValidVersion(currentVersion)) { - console.debug('[Monitor:diag] Sencho version unresolvable; skipping update notification'); - } - } catch (e) { - // Network errors are expected; do not spam logs - if (isDebugEnabled()) console.debug('[Monitor:diag] Sencho version check failed:', e); + await this.checkSenchoVersion(); + } + + /** + * Check GitHub/Docker Hub for a newer Sencho release and dispatch a + * one-shot notification. Uses getLatestVersion() which wraps CacheService + * (30 min TTL + inflight dedup + stale-on-error) so transient network + * blips do not cause gaps, and the check stays consistent with the Fleet + * update banner. + * + * The 6-hour cooldown gate prevents bell spam: it is only advanced on a + * SUCCESSFUL lookup. A failed lookup retries on the next eval cycle + * (30 seconds) instead of locking for 6 hours. + */ + private async checkSenchoVersion(): Promise { + const sinceLast = Date.now() - this.lastVersionCheckAt; + if (sinceLast <= MonitorService.VERSION_CHECK_INTERVAL_MS) { + if (isDebugEnabled()) { + const nextInMs = MonitorService.VERSION_CHECK_INTERVAL_MS - sinceLast; + console.debug(`[Monitor:diag] Sencho version check in cooldown (next in ~${Math.round(nextInMs / 60000)}m)`); } + return; + } + + // Resolve from the packaged manifest: process.env.npm_package_version is + // only set by npm scripts, so it is undefined in Docker (node dist/index.js). + const currentVersion = getSenchoVersion(); + if (!isValidVersion(currentVersion)) { + if (isDebugEnabled()) console.debug('[Monitor:diag] Sencho version unresolvable; skipping update notification'); + return; + } + + const latest = await getLatestVersion(); + if (!isValidVersion(latest)) { + // Network failure (GitHub + Docker Hub both down, no stale cache). + // Do NOT advance the cooldown so the next eval retries. + if (isDebugEnabled()) console.debug('[Monitor:diag] Latest Sencho version unresolvable; will retry next cycle'); + return; + } + + this.lastVersionCheckAt = Date.now(); + + const db = DatabaseService.getInstance(); + const stateKey = MonitorService.SENCHO_UPDATE_NOTIFIED_KEY; + const storedLastNotified = db.getSystemState(stateKey) || ''; + + // Self-heal: if the user has reached the previously-notified version, + // clear the dedup so future releases always trigger a fresh notification. + // This also recovers from stale state left over by the pre-586 "0.0.0" bug. + let effectiveLastNotified = storedLastNotified; + if (storedLastNotified && isValidVersion(storedLastNotified) && semver.gte(currentVersion, storedLastNotified)) { + if (isDebugEnabled()) console.debug(`[Monitor:diag] Clearing stale dedup key (running ${currentVersion} >= last notified ${storedLastNotified})`); + db.setSystemState(stateKey, ''); + effectiveLastNotified = ''; + } + + if (!semver.gt(latest, currentVersion)) { + if (isDebugEnabled()) console.debug(`[Monitor:diag] Running ${currentVersion} is up-to-date with latest ${latest}`); + return; + } + + if (effectiveLastNotified === latest) { + if (isDebugEnabled()) console.debug(`[Monitor:diag] Already notified for Sencho ${latest}`); + return; + } + + try { + const notifier = NotificationService.getInstance(); + await notifier.dispatchAlert('info', + `Sencho ${latest} is available (currently running ${currentVersion}). Visit the Fleet dashboard to update.`); + db.setSystemState(stateKey, latest); + if (isDebugEnabled()) console.debug(`[Monitor:diag] Dispatched version notification: ${currentVersion} -> ${latest}`); + } catch (e) { + // dispatchAlert normally catches channel errors internally, but the + // history insert or WebSocket broadcast can throw on an unhealthy DB. + console.error('[MonitorService] Failed to dispatch Sencho version notification:', e); } } diff --git a/backend/src/utils/version-check.ts b/backend/src/utils/version-check.ts index 50ef7287..3da421fb 100644 --- a/backend/src/utils/version-check.ts +++ b/backend/src/utils/version-check.ts @@ -1,4 +1,5 @@ import semver from 'semver'; +import { CacheService } from '../services/CacheService'; /** * Fetches the latest Sencho release version from GitHub or Docker Hub. @@ -50,3 +51,26 @@ export async function fetchLatestSenchoVersion(): Promise { // and so we do not poison the cache with null. throw new Error('Both GitHub and Docker Hub version lookups failed'); } + +/** + * Cached wrapper shared by the Fleet endpoint and MonitorService. + * CacheService provides TTL, inflight deduplication, and stale-on-error + * fallback so transient network blips do not cause user-visible gaps. + */ +const LATEST_VERSION_CACHE_KEY = 'latest-version'; +const LATEST_VERSION_CACHE_TTL = 30 * 60 * 1000; // 30 minutes + +export async function getLatestVersion(forceRefresh = false): Promise { + if (forceRefresh) { + CacheService.getInstance().invalidate(LATEST_VERSION_CACHE_KEY); + } + try { + return await CacheService.getInstance().getOrFetch( + LATEST_VERSION_CACHE_KEY, + LATEST_VERSION_CACHE_TTL, + fetchLatestSenchoVersion, + ); + } catch { + return null; + } +} diff --git a/docs/features/alerts-notifications.mdx b/docs/features/alerts-notifications.mdx index cf359d3c..aaae03ca 100644 --- a/docs/features/alerts-notifications.mdx +++ b/docs/features/alerts-notifications.mdx @@ -158,10 +158,6 @@ Sencho 0.47.0 is available (currently running 0.46.16). Visit the Fleet dashboar When the periodic image check (every 6 hours) detects that a stack has new upstream images available, a notification is dispatched for each affected stack. Notifications are sent only on state transitions: you will be notified once when an update first appears, not on every check cycle. After you update the stack, the status resets so a future release can notify again. - - If you upgraded to a Sencho version that added image update notifications and you already had stacks flagged as having updates, the first check after the upgrade sends one catch-up notification per affected stack so the in-app bell stays in sync with the blue update indicator. - - Both notification types use the same channel routing as alerts: if notification routes are configured for a stack, those channels receive the message; otherwise, global notification channels are used as a fallback. ## Container crash detection @@ -220,11 +216,7 @@ When many crashes land in a short window (for example, a large stack coming down ### I stopped a stack but got a crash alert -This should not happen on current versions. If it does, confirm Sencho can reach the Docker daemon on the affected node (the **Lost connection to Docker daemon** warning is dispatched when the socket is unreachable). Previously, Sencho relied on polling and could mistake intentional stops for crashes; detection is now causal and in real time. - -### I'm seeing fewer crash alerts after upgrading - -Expected. Intentional stops, `docker stop` from a host terminal, and scheduled stack recreations no longer trigger crash alerts. Real crashes, OOM kills, and failing healthchecks still alert. If you want to opt out entirely, toggle **Global Crash Detection** off in **Settings > System**. +This should not happen. Sencho distinguishes intentional stops from crashes by correlating each container exit with the action that triggered it. If you do see a crash alert for a stack you stopped, confirm Sencho can reach the Docker daemon on the affected node: the **Lost connection to Docker daemon** warning is dispatched when the socket is unreachable, and exits observed during an outage may be classified as crashes on reconnect. ### After a Docker daemon restart I only got one summary notification @@ -246,12 +238,12 @@ Expected, and by design. Sencho caps crash notifications at around 20 per minute Deleting an alert rule now requires confirmation. Click the trash icon next to a rule, then confirm in the dialog that appears. -### Version notification shows "currently running 0.0.0" +### Fleet shows an update button but I never received a Sencho version notification -This affected older Sencho builds that read the running version from an environment variable which was not always populated. Update to the current release; the running version is now resolved from the packaged manifest and the notification will report it correctly the next time a new release is detected. +Fleet and the in-app bell share the same version lookup. Each Sencho instance checks for a new release roughly every 6 hours and notifies exactly once per version, so if a check already fired for the latest release the bell will not notify again for that same version. + +If the update button is visible but no notification is in the bell, a transient network failure during the last successful check can briefly delay delivery; the next evaluation cycle (about every 30 seconds) retries automatically. To confirm external channels (Discord, Slack, webhooks) are reachable, open **Settings > Notifications** and click **Test** on each. The in-app bell receives every notification regardless of external channel configuration. ### A stack shows the blue update indicator but no notification was received -If the blue indicator was already visible before you upgraded to a Sencho version that supports image update notifications, the first image check after the upgrade sends one catch-up notification per affected stack. Subsequent checks notify only when a stack newly transitions from "no updates" to "updates available". - -If the indicator appeared after the upgrade but no notification followed, open the notification bell. Dispatch failures (for example, a misconfigured Discord or Slack webhook) are logged as error entries in the bell so they are visible without tailing logs. Confirm at least one channel is enabled in **Settings > Notifications**; the in-app bell receives all notifications regardless of external channel configuration. +Open the notification bell. Dispatch failures (for example, a misconfigured Discord or Slack webhook) are logged as error entries in the bell so they are visible without tailing logs. Confirm at least one channel is enabled in **Settings > Notifications**; the in-app bell receives all notifications regardless of external channel configuration.