diff --git a/backend/src/__tests__/stack-activity-metrics.test.ts b/backend/src/__tests__/stack-activity-metrics.test.ts new file mode 100644 index 00000000..edd562a9 --- /dev/null +++ b/backend/src/__tests__/stack-activity-metrics.test.ts @@ -0,0 +1,69 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { StackActivityMetricsService } from '../services/StackActivityMetricsService'; + +beforeEach(() => { + StackActivityMetricsService.resetForTests(); +}); + +describe('StackActivityMetricsService', () => { + it('counts success and error per (nodeId, op) independently', () => { + const m = StackActivityMetricsService.getInstance(); + m.record(1, 'read', 5, true); + m.record(1, 'read', 10, true); + m.record(1, 'read', 8, false); + m.record(1, 'write', 2, true); + m.record(2, 'read', 3, true); + + const snap = m.snapshot(); + const oneRead = snap.entries.find(e => e.nodeId === 1 && e.op === 'read'); + expect(oneRead).toMatchObject({ count: 3, successCount: 2, errorCount: 1 }); + + const oneWrite = snap.entries.find(e => e.nodeId === 1 && e.op === 'write'); + expect(oneWrite).toMatchObject({ count: 1, successCount: 1, errorCount: 0 }); + + const twoRead = snap.entries.find(e => e.nodeId === 2 && e.op === 'read'); + expect(twoRead).toMatchObject({ count: 1, successCount: 1 }); + }); + + it('computes avg, p50, p95 over the ring buffer', () => { + const m = StackActivityMetricsService.getInstance(); + for (let i = 1; i <= 100; i++) m.record(0, 'read', i, true); + const snap = m.snapshot(); + const e = snap.entries[0]; + expect(e.count).toBe(100); + expect(e.avgMs).toBe(Math.round((1 + 100) / 2)); + expect(e.p50Ms).toBeGreaterThanOrEqual(48); + expect(e.p50Ms).toBeLessThanOrEqual(52); + expect(e.p95Ms).toBeGreaterThanOrEqual(94); + expect(e.p95Ms).toBeLessThanOrEqual(96); + }); + + it('drops oldest samples once the ring buffer is full', () => { + const m = StackActivityMetricsService.getInstance(); + for (let i = 0; i < 1500; i++) m.record(0, 'read', i, true); + const snap = m.snapshot(); + // count is unbounded; only the latency window is capped. + expect(snap.entries[0].count).toBe(1500); + // Samples 500..1499 are retained (oldest 500 evicted), so p50 is 999. + expect(snap.entries[0].p50Ms).toBeGreaterThanOrEqual(990); + expect(snap.entries[0].p50Ms).toBeLessThanOrEqual(1010); + }); + + it('ignores invalid durations', () => { + const m = StackActivityMetricsService.getInstance(); + m.record(0, 'read', Number.NaN, true); + m.record(0, 'read', -5, true); + m.record(0, 'read', Number.POSITIVE_INFINITY, true); + expect(m.snapshot().entries).toEqual([]); + }); + + it('orders snapshot entries by nodeId then op', () => { + const m = StackActivityMetricsService.getInstance(); + m.record(2, 'write', 1, true); + m.record(2, 'read', 1, true); + m.record(1, 'write', 1, true); + m.record(1, 'read', 1, true); + const order = m.snapshot().entries.map(e => `${e.nodeId}:${e.op}`); + expect(order).toEqual(['1:read', '1:write', '2:read', '2:write']); + }); +}); diff --git a/backend/src/index.ts b/backend/src/index.ts index b08642f1..0524b100 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -51,6 +51,7 @@ import { stacksRouter } from './routes/stacks'; import { stackActivityRouter } from './routes/stackActivity'; import { stackMetricsRouter } from './routes/stackMetrics'; import { fileExplorerMetricsRouter } from './routes/fileExplorerMetrics'; +import { stackActivityMetricsRouter } from './routes/stackActivityMetrics'; import { secretsRouter } from './routes/secrets'; // Suppress [DEP0060] DeprecationWarning emitted by http-proxy@1.18.1 which calls @@ -145,6 +146,7 @@ app.use('/api/stacks', stackActivityRouter); app.use('/api/stacks', stacksRouter); app.use('/api/stack-metrics', stackMetricsRouter); app.use('/api/file-explorer-metrics', fileExplorerMetricsRouter); +app.use('/api/stack-activity-metrics', stackActivityMetricsRouter); const { server, wss, pilotTunnelWss } = createServer(app); attachUpgrade(server, { wss, pilotTunnelWss }); diff --git a/backend/src/routes/stackActivity.ts b/backend/src/routes/stackActivity.ts index 44a98d0b..1420c8f9 100644 --- a/backend/src/routes/stackActivity.ts +++ b/backend/src/routes/stackActivity.ts @@ -2,9 +2,15 @@ import { Router, type Request, type Response } from 'express'; import { DatabaseService } from '../services/DatabaseService'; import { requirePermission } from '../middleware/permissions'; import { isValidStackName } from '../utils/validation'; +import { isDebugEnabled } from '../utils/debug'; +import { StackActivityMetricsService } from '../services/StackActivityMetricsService'; export const stackActivityRouter = Router(); +function dlog(message: string, details: Record): void { + if (isDebugEnabled()) console.log(`[StackActivity:diag] ${message}`, details); +} + function parseStrictPositiveInt(raw: unknown): number | null { if (raw === undefined || raw === null) return null; const s = String(raw).trim(); @@ -14,6 +20,7 @@ function parseStrictPositiveInt(raw: unknown): number | null { } stackActivityRouter.get('/:stackName/activity', (req: Request, res: Response): void => { + const t0 = Date.now(); const stackName = req.params.stackName as string; if (!isValidStackName(stackName)) { res.status(400).json({ error: 'Invalid stack name' }); @@ -57,6 +64,13 @@ stackActivityRouter.get('/:stackName/activity', (req: Request, res: Response): v beforeId = parsed; } - const events = DatabaseService.getInstance().getStackActivity(req.nodeId, stackName, { limit, before, beforeId }); - res.json({ events }); + let ok = false; + try { + const events = DatabaseService.getInstance().getStackActivity(req.nodeId, stackName, { limit, before, beforeId }); + ok = true; + res.json({ events }); + dlog('read', { stackName, nodeId: req.nodeId, limit, before, beforeId, returned: events.length, elapsedMs: Date.now() - t0 }); + } finally { + StackActivityMetricsService.getInstance().record(req.nodeId, 'read', Date.now() - t0, ok); + } }); diff --git a/backend/src/routes/stackActivityMetrics.ts b/backend/src/routes/stackActivityMetrics.ts new file mode 100644 index 00000000..ec78a5c6 --- /dev/null +++ b/backend/src/routes/stackActivityMetrics.ts @@ -0,0 +1,19 @@ +import { Router, type Request, type Response } from 'express'; +import { requireAdmin } from '../middleware/tierGates'; +import { StackActivityMetricsService } from '../services/StackActivityMetricsService'; + +export const stackActivityMetricsRouter = Router(); + +/** + * Admin-only snapshot of in-process stack activity metrics. No external + * export: this endpoint exists so an operator debugging "why is the + * activity tab slow?" or "are we logging unexpected error rates on the + * dispatch path?" can pull per-(nodeId, op) counts and latencies without + * scrolling logs. + * + * Mounted at /api/stack-activity-metrics after the global authGate. + */ +stackActivityMetricsRouter.get('/', (req: Request, res: Response): void => { + if (!requireAdmin(req, res)) return; + res.json(StackActivityMetricsService.getInstance().snapshot()); +}); diff --git a/backend/src/services/NotificationService.ts b/backend/src/services/NotificationService.ts index ce2455b6..cb6dede4 100644 --- a/backend/src/services/NotificationService.ts +++ b/backend/src/services/NotificationService.ts @@ -5,6 +5,7 @@ import { isDebugEnabled } from '../utils/debug'; import { getErrorMessage } from '../utils/errors'; import { sanitizeForLog } from '../utils/safeLog'; import { sanitizeNotificationMessage } from '../utils/notificationMessage'; +import { StackActivityMetricsService } from './StackActivityMetricsService'; export type NotificationCategory = | 'deploy_success' @@ -115,25 +116,42 @@ export class NotificationService { message: string, options?: { stackName?: string; containerName?: string; actor?: string }, ) { + const t0 = Date.now(); const { stackName, containerName, actor } = options ?? {}; // Internal writes use the middleware default so they share a row key // with user-initiated requests; otherwise the UI and monitors split // between different node_id buckets. const localNodeId = NodeRegistry.getInstance().getDefaultNodeId(); - // Use the full resolution chain (node.compose_dir → env → default) + // Use the full resolution chain (node.compose_dir, env, default) // so messages mentioning a per-node compose override get collapsed. const sanitized = sanitizeNotificationMessage(message, { composeDir: NodeRegistry.getInstance().getComposeDir(localNodeId), }); - const notification = this.dbService.addNotificationHistory(localNodeId, { - level, - category, - message: sanitized, - timestamp: Date.now(), - stack_name: stackName, - container_name: containerName, - actor_username: actor ?? null, - }); + + let notification: NotificationHistory; + try { + notification = this.dbService.addNotificationHistory(localNodeId, { + level, + category, + message: sanitized, + timestamp: Date.now(), + stack_name: stackName, + container_name: containerName, + actor_username: actor ?? null, + }); + } catch (err) { + StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, false); + throw err; + } + StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, true); + // Separate [StackActivity:diag] namespace from the [Notify:diag] lines + // below so a single grep can pull every per-stack timeline write across + // route reads and dispatch writes. + if (isDebugEnabled()) { + console.log('[StackActivity:diag] write', { + category, stackName, nodeId: localNodeId, actor: actor ?? null, messageLen: sanitized.length, + }); + } // 2. Push to connected browser clients via WebSocket this.broadcastToSubscribers(notification); diff --git a/backend/src/services/StackActivityMetricsService.ts b/backend/src/services/StackActivityMetricsService.ts new file mode 100644 index 00000000..e886363c --- /dev/null +++ b/backend/src/services/StackActivityMetricsService.ts @@ -0,0 +1,111 @@ +/** + * In-memory counters + latency samples for the per-stack activity timeline. + * Internal-only; never exported to any external system. Surfaced to admins + * via GET /api/stack-activity-metrics so operators can answer "why is the + * activity tab slow?" or "is the timeline dropping events?" without + * scrolling logs. + * + * State is process-local. A Sencho restart clears everything; persisting to + * SQLite would add write amplification to the hot dispatchAlert path for + * very little operator value. Each node tracks its own ops because the + * remote-node HTTP proxy (proxy/remoteNodeProxy.ts) short-circuits + * cross-node requests to the target's own router before this service sees + * them. + */ + +export type StackActivityOp = 'read' | 'write'; + +interface StackActivityOpStats { + count: number; + successCount: number; + errorCount: number; + totalMs: number; + recentSamples: number[]; +} + +export interface StackActivitySnapshotEntry { + nodeId: number; + op: StackActivityOp; + count: number; + successCount: number; + errorCount: number; + avgMs: number; + p50Ms: number; + p95Ms: number; +} + +export interface StackActivitySnapshot { + entries: StackActivitySnapshotEntry[]; +} + +const MAX_SAMPLES = 1000; + +function percentile(sorted: readonly number[], p: number): number { + if (sorted.length === 0) return 0; + const idx = Math.min(sorted.length - 1, Math.floor((sorted.length - 1) * p)); + return sorted[idx]; +} + +export class StackActivityMetricsService { + private static instance: StackActivityMetricsService; + private readonly stats = new Map(); + + public static getInstance(): StackActivityMetricsService { + if (!StackActivityMetricsService.instance) { + StackActivityMetricsService.instance = new StackActivityMetricsService(); + } + return StackActivityMetricsService.instance; + } + + public static resetForTests(): void { + this.instance = new StackActivityMetricsService(); + } + + private key(nodeId: number, op: StackActivityOp): string { + return `${nodeId}:${op}`; + } + + public record(nodeId: number, op: StackActivityOp, durationMs: number, ok: boolean): void { + if (!Number.isFinite(durationMs) || durationMs < 0) return; + const k = this.key(nodeId, op); + let s = this.stats.get(k); + if (!s) { + s = { count: 0, successCount: 0, errorCount: 0, totalMs: 0, recentSamples: [] }; + this.stats.set(k, s); + } + s.count += 1; + if (ok) s.successCount += 1; + else s.errorCount += 1; + s.totalMs += durationMs; + s.recentSamples.push(durationMs); + if (s.recentSamples.length > MAX_SAMPLES) { + s.recentSamples.shift(); + } + } + + public snapshot(): StackActivitySnapshot { + const entries: StackActivitySnapshotEntry[] = []; + for (const [key, s] of this.stats.entries()) { + const [nodeIdStr, op] = key.split(':'); + const nodeId = Number(nodeIdStr); + if (!Number.isFinite(nodeId)) continue; + const sorted = [...s.recentSamples].sort((a, b) => a - b); + entries.push({ + nodeId, + op: op as StackActivityOp, + count: s.count, + successCount: s.successCount, + errorCount: s.errorCount, + avgMs: s.count === 0 ? 0 : Math.round(s.totalMs / s.count), + p50Ms: percentile(sorted, 0.5), + p95Ms: percentile(sorted, 0.95), + }); + } + entries.sort((a, b) => a.nodeId - b.nodeId || a.op.localeCompare(b.op)); + return { entries }; + } + + public size(): number { + return this.stats.size; + } +} diff --git a/docs/features/stack-activity.mdx b/docs/features/stack-activity.mdx index ee851116..10473fc9 100644 --- a/docs/features/stack-activity.mdx +++ b/docs/features/stack-activity.mdx @@ -1,6 +1,6 @@ --- title: Stack Activity -description: Per-stack event log showing deploys, restarts, config changes, and alerts, each attributed to the user or system that triggered them. +description: Per-stack event log showing deploys, restarts, config changes, and alerts, each attributed to the user or system process that triggered them. --- The **Activity** tab in the right-hand **Anatomy** panel gives you a timestamped record of everything that happened to the selected stack: deploys, restarts, starts, stops, and image updates. Each entry shows who triggered the event so you always have operational context. The `files` and `edit` actions on the same strip belong to the panel as a whole and stay available regardless of which tab is active. @@ -16,13 +16,13 @@ Each entry in the activity list contains: | Field | Description | |-------|-------------| | **Icon** | Category of the event (deploy, restart, stop, start, image update applied) | -| **Message** | What happened (for example, `dozzle restarted`) | -| **Attributed to** | `by ` for user-initiated actions; omitted for events without an actor and for system-driven events such as auto-update or auto-heal | +| **Message** | What happened (for example, `dozzle restarted`). Sensitive values such as tokens, secrets, basic-auth credentials, and bearer tokens are redacted before storage | +| **Attributed to** | `by ` for user-initiated actions; `via ` for events generated by background processes (Auto-Heal, Scheduler, Image Update, Docker, Blueprint, Monitor, Policy) | | **Time** | Relative time since the event: `just now`, `5m ago`, `2h ago`, `3d ago` | ## Event categories -The five operational categories below have a dedicated icon. Other notification categories that scope to a stack (for example, deploy failures, available image updates, auto-heal triggers, monitor alerts, or scan findings) also flow into the timeline and render with a generic activity icon. +The five operational categories below have a dedicated icon. Other notification categories that scope to a stack (deploy failures, available image updates, auto-heal triggers, monitor alerts, scan findings) also flow into the timeline and render with a generic activity icon. | Category | Icon | Example message | |----------|------|-----------------| @@ -34,19 +34,24 @@ The five operational categories below have a dedicated icon. Other notification ## Day grouping -Events are grouped by day under three headers: **Today**, **Yesterday**, and **Earlier**. Within each group, events are ordered most-recent first. +Events are grouped by day under three headers: **Today**, **Yesterday**, and **Earlier**. Within each group, events are ordered most-recent first. The grouping recomputes every minute so a panel left open across midnight rolls the **Today** bucket into **Yesterday** automatically. ## Live updates -New events stream in over the same WebSocket that powers the notification bell, so the list updates without a page refresh while the tab is open. +New events stream in over the same WebSocket that powers the notification bell, so the list updates without a page refresh while the tab is open. Duplicate events are deduplicated by ID and the timeline is sorted by `(timestamp, id)` so events emitted within the same millisecond stay in a stable order. ## Pagination -The tab loads the most recent 50 events on open. If a stack has more recorded events, a **Load more** button appears at the bottom of the list. Each click fetches the next 50 older events. +The tab loads the most recent 50 events on open. If a stack has more recorded events, a **Load more** button appears at the bottom of the list. Each click fetches the next 50 older events. The button stops appearing once the timeline reaches the end of the stored history. -## Empty state +The underlying API endpoint, `GET /api/stacks/{stackName}/activity`, accepts an optional `before` (timestamp) and `beforeId` (row id) pair for cursor-based pagination. The two parameters must move together; sending `beforeId` without `before` is rejected. -When a stack has no recorded operational events yet, the tab displays a heartbeat icon with the message `No activity recorded yet`. +## Empty state vs unavailable + +Two empty-looking states exist and they mean different things: + +- **No activity recorded yet** appears when the stack genuinely has no recorded events. +- **Activity unavailable** appears when the request to fetch events fails (for example, the node became unreachable mid-load). A **Retry** button is shown next to it. Stack Activity tab in its empty state with a heartbeat icon and the message 'No activity recorded yet' @@ -54,7 +59,23 @@ When a stack has no recorded operational events yet, the tab displays a heartbea ## Who triggered an event -Sencho attributes every recorded event to the user who was authenticated when the action was taken. Events without an actor, and events triggered by automated processes such as auto-update polling or auto-heal, are recorded as system actions; the attribution label is omitted from their display. +Sencho attributes every recorded event to whichever actor performed it: + +- **User-initiated actions** (deploy, restart, stop, start, image update from a button click or the API): the actor is the authenticated username, shown as `by `. +- **Background subsystems** (Auto-Heal, Scheduler, Image Update, Docker event monitoring, Blueprint reconciler, Host Monitor, Policy enforcement): the actor is the subsystem name, shown as `via Auto-Heal`, `via Scheduler`, and so on. + +An autoheal restart and a manual restart are visually distinct in the timeline because the actor differs. + +## Retention + +Activity is stored in a per-instance SQLite table. Two limits apply, whichever comes first: + +- **Time-based**: events older than the configured retention window (30 days by default) are removed during the next periodic cleanup. +- **Per-stack cap**: the most recent 500 events per (node, stack) are retained. Older events for that stack are removed during the next periodic cleanup. + +Events without an associated stack (system-level notifications) are kept up to 1000 per node before the oldest are removed. + +Cleanup runs on the same schedule as container-metrics cleanup, so retention applies even on long-running instances. ## Accessing the Activity tab @@ -67,6 +88,15 @@ Sencho attributes every recorded event to the user who was authenticated when th The activity log is populated by operations performed through Sencho on this stack: deploys, restarts, starts, stops, and image updates. Trigger one of those actions and the entry appears within a few seconds. + + The request to fetch activity failed, usually because the node is currently unreachable or the backend returned a server error. Click **Retry** to attempt the request again. If the node has been removed or its credentials are stale, the activity for that node is no longer accessible. + + + Two reasons this can happen. First, the event may be older than the 30-day retention window. Second, if the stack has had more than 500 events recorded, the oldest are pruned automatically. To preserve a longer history, route the events to an external channel via **Notification Routes**. + + + These are two distinct events. The first is your manual click; the second is Auto-Heal restarting the same container after a separate unhealthy-duration breach. Each event is its own row with its own actor. + The notification dropdown surfaces system alerts and error-level events. User-initiated operational events (start, stop, restart, deploy, update) appear exclusively in the per-stack Activity tab, keeping the global tray focused on conditions that need your attention.