From 698b7d0713919b9f91a63e05d84b44057b7f1d12 Mon Sep 17 00:00:00 2001 From: Anso Date: Wed, 22 Jul 2026 20:00:45 -0400 Subject: [PATCH 01/13] fix: purge deleted-stack notifications from panel and ticker (#1674) * fix: purge deleted-stack notifications from panel and ticker Stack delete already cascaded scans and drift but left notification_history rows, so the bell and Activity ticker kept showing the deleted stack. Purge those rows in the shared deletion lifecycle, invalidate connected clients, and drop node-scoped in-memory rows immediately. * fix: target remote notification purge by hub node id Remote stack-deleted invalidations always reconcile with hub rn.id, and notification refetch preserves failed node slices instead of wiping them. --- .../stack-delete-purges-notifications.test.ts | 132 +++++++ backend/src/services/DatabaseService.ts | 6 + .../services/DeployedStackDeletionService.ts | 10 + docs/reference/settings.mdx | 2 +- frontend/src/components/EditorLayout.tsx | 2 + .../hooks/useNotifications.test.ts | 361 ++++++++++++++++++ .../EditorLayout/hooks/useNotifications.ts | 89 ++++- .../hooks/useStackActions.test.ts | 41 +- .../EditorLayout/hooks/useStackActions.ts | 15 +- 9 files changed, 644 insertions(+), 14 deletions(-) create mode 100644 backend/src/__tests__/stack-delete-purges-notifications.test.ts diff --git a/backend/src/__tests__/stack-delete-purges-notifications.test.ts b/backend/src/__tests__/stack-delete-purges-notifications.test.ts new file mode 100644 index 00000000..10aae69b --- /dev/null +++ b/backend/src/__tests__/stack-delete-purges-notifications.test.ts @@ -0,0 +1,132 @@ +/** + * DELETE /api/stacks/:stackName must purge notification_history for that + * (node_id, stack_name) so the bell panel, Activity ticker, Stack Activity, + * and Networking recent activity no longer show the deleted stack. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import request from 'supertest'; +import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; + +let tmpDir: string; +let app: import('express').Express; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let ComposeService: typeof import('../services/ComposeService').ComposeService; +let FileSystemService: typeof import('../services/FileSystemService').FileSystemService; +let MeshService: typeof import('../services/MeshService').MeshService; +let NotificationService: typeof import('../services/NotificationService').NotificationService; +let adminCookie: string; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ ComposeService } = await import('../services/ComposeService')); + ({ FileSystemService } = await import('../services/FileSystemService')); + ({ MeshService } = await import('../services/MeshService')); + ({ NotificationService } = await import('../services/NotificationService')); + ({ app } = await import('../index')); + adminCookie = await loginAsTestAdmin(app); +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +beforeEach(() => { + vi.restoreAllMocks(); + vi.spyOn(ComposeService.prototype, 'downStack').mockResolvedValue(undefined); + vi.spyOn(FileSystemService.prototype, 'deleteStack').mockResolvedValue(undefined); + vi.spyOn(MeshService.getInstance(), 'optOutStack').mockResolvedValue(undefined); + const raw = DatabaseService.getInstance().getDb(); + raw.prepare('DELETE FROM notification_history').run(); +}); + +function seedNotif( + nodeId: number, + opts: { stack_name?: string; message?: string; timestamp?: number }, +): void { + DatabaseService.getInstance().addNotificationHistory(nodeId, { + level: 'error', + message: opts.message ?? 'test', + timestamp: opts.timestamp ?? Date.now(), + stack_name: opts.stack_name ?? 'web', + category: 'deploy_failure', + }); +} + +function stackNamesForNode(nodeId: number): Array { + const rows = DatabaseService.getInstance() + .getDb() + .prepare('SELECT stack_name FROM notification_history WHERE node_id = ? ORDER BY id') + .all(nodeId) as Array<{ stack_name: string | null }>; + return rows.map((r) => r.stack_name); +} + +describe('DELETE /api/stacks/:stackName purges notifications', () => { + it('removes only the target node/stack rows and emits one invalidation', async () => { + const db = DatabaseService.getInstance(); + const nodeA = db.getNodes()[0].id; + const nodeB = db.addNode({ + name: 'other-node', + type: 'remote', + api_url: 'http://other:1852', + api_token: 't', + compose_dir: '/tmp', + is_default: false, + }); + + seedNotif(nodeA, { stack_name: 'web', message: 'a-web' }); + seedNotif(nodeA, { stack_name: 'db', message: 'a-db' }); + // Unattached (null stack_name) via SQL; addNotificationHistory omits null. + db.getDb() + .prepare( + `INSERT INTO notification_history (node_id, level, message, timestamp, is_read, stack_name, category) + VALUES (?, 'info', 'a-unattached', ?, 0, NULL, 'system')`, + ) + .run(nodeA, Date.now()); + seedNotif(nodeB, { stack_name: 'web', message: 'b-web' }); + + const broadcastSpy = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent'); + + const res = await request(app).delete('/api/stacks/web').set('Cookie', adminCookie); + + expect(res.status).toBe(200); + expect(stackNamesForNode(nodeA).sort((a, b) => String(a).localeCompare(String(b)))).toEqual([ + 'db', + null, + ]); + expect(stackNamesForNode(nodeB)).toEqual(['web']); + + const notifInvalidations = broadcastSpy.mock.calls + .map((c) => c[0]) + .filter( + (e) => + e.type === 'state-invalidate' && + e.scope === 'notifications' && + e.action === 'stack-deleted', + ); + expect(notifInvalidations).toHaveLength(1); + expect(notifInvalidations[0]).toMatchObject({ + type: 'state-invalidate', + scope: 'notifications', + action: 'stack-deleted', + nodeId: nodeA, + stackName: 'web', + }); + expect(typeof notifInvalidations[0].ts).toBe('number'); + }); + + it('emits no notification invalidation when delete fails before success', async () => { + vi.spyOn(FileSystemService.prototype, 'deleteStack').mockRejectedValue(new Error('fs boom')); + const broadcastSpy = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent'); + const db = DatabaseService.getInstance(); + const nodeId = db.getNodes()[0].id; + seedNotif(nodeId, { stack_name: 'web' }); + + const res = await request(app).delete('/api/stacks/web').set('Cookie', adminCookie); + + expect(res.status).toBe(500); + expect(stackNamesForNode(nodeId)).toEqual(['web']); + const notifInvalidations = broadcastSpy.mock.calls + .map((c) => c[0]) + .filter((e) => e.scope === 'notifications' && e.action === 'stack-deleted'); + expect(notifInvalidations).toHaveLength(0); + }); +}); diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 03663fc0..92090b04 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -4046,6 +4046,12 @@ export class DatabaseService { stmt.run(nodeId); } + /** Purge stack-scoped notification history when a stack is deleted. */ + public deleteNotificationsForStack(nodeId: number, stackName: string): number { + const stmt = this.db.prepare('DELETE FROM notification_history WHERE node_id = ? AND stack_name = ?'); + return stmt.run(nodeId, stackName).changes; + } + public updateNotificationDispatchError(id: number, error: string): void { this.db.prepare('UPDATE notification_history SET dispatch_error = ? WHERE id = ?').run(error, id); } diff --git a/backend/src/services/DeployedStackDeletionService.ts b/backend/src/services/DeployedStackDeletionService.ts index 3bddbd1a..dce2d133 100644 --- a/backend/src/services/DeployedStackDeletionService.ts +++ b/backend/src/services/DeployedStackDeletionService.ts @@ -18,6 +18,7 @@ import DockerController from './DockerController'; import { FileSystemService } from './FileSystemService'; import { NodeRegistry } from './NodeRegistry'; import { MeshService } from './MeshService'; +import { NotificationService } from './NotificationService'; import { StackOpLockService, stackOpSkipMessage } from './StackOpLockService'; import { getErrorMessage } from '../utils/errors'; import { sanitizeForLog } from '../utils/safeLog'; @@ -248,6 +249,7 @@ export class DeployedStackDeletionService { db.deleteStackExposure(nodeId, stackName); db.deleteStackProjectEnvFiles(nodeId, stackName); db.deleteStackScans(nodeId, stackName); + db.deleteNotificationsForStack(nodeId, stackName); } catch (dbErr) { console.error( '[DeployedStackDeletion] Secondary DB cleanup failed for %s; recovery rows already retired:', @@ -272,6 +274,14 @@ export class DeployedStackDeletionService { } await this.sweepReadyIntent(intentId); + NotificationService.getInstance().broadcastEvent({ + type: 'state-invalidate', + scope: 'notifications', + action: 'stack-deleted', + nodeId, + stackName, + ts: Date.now(), + }); return { ok: true }; } diff --git a/docs/reference/settings.mdx b/docs/reference/settings.mdx index f9427856..a60f61bf 100644 --- a/docs/reference/settings.mdx +++ b/docs/reference/settings.mdx @@ -554,7 +554,7 @@ How long Sencho keeps historical data on this node before pruning it. | Setting | Default | Max | Description | |---------|---------|-----|-------------| | **Container metrics** | 24 hrs | 8,760 (1 year) | How long to keep per-container CPU, RAM, and network history for dashboard charts. | -| **Notification log** | 30 days | 365 | How long to keep alert and notification history. | +| **Notification log** | 30 days | 365 | Maximum how long to keep alert and notification history. Stack-associated entries are also removed when that stack is deleted. | | **Scan history per digest** | 50 scans | 1,000 | How many vulnerability scans to keep per image digest (or per image reference when no digest is stored). Older scans beyond the cap are pruned. | | **Remove scans for deleted images and stacks** | On | - | When on, scan results are deleted once their image is gone from this node or their stack is deleted, so the Security Overview stays tied to what still exists. Turn it off to keep scan history for removed images and stacks. | | **Audit log** | 90 days | 365 | How long to keep audit trail entries. Requires Admiral. | diff --git a/frontend/src/components/EditorLayout.tsx b/frontend/src/components/EditorLayout.tsx index 6ceec945..be11b41e 100644 --- a/frontend/src/components/EditorLayout.tsx +++ b/frontend/src/components/EditorLayout.tsx @@ -256,6 +256,7 @@ export default function EditorLayout() { markAllRead, deleteNotification, clearAllNotifications, + removeNotificationsForStack, } = useNotifications({ nodes, onStateInvalidate: scheduleStateInvalidateRefresh, @@ -289,6 +290,7 @@ export default function EditorLayout() { }, canOfferVolumeRemoval, onDeletedOpenStack: () => onDeletedOpenStackRef.current(), + removeNotificationsForStack, }); // Wire the ref now that stackActions is available diff --git a/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts b/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts index 5550164e..fe5acd96 100644 --- a/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts +++ b/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts @@ -22,6 +22,9 @@ const makeNotif = (overrides: Partial = {}): NotificationItem id: 1, level: 'info', message: 'test', timestamp: 1000, is_read: 0, ...overrides, }); +const nodeMessageKeys = (items: NotificationItem[]): string[] => + items.map((n) => `${n.nodeId}:${n.message}`).sort(); + class MockWS { static instances: MockWS[] = []; static readonly CONNECTING = 0; @@ -251,4 +254,362 @@ describe('useNotifications', () => { expect(MockWS.instances).toHaveLength(2); await waitFor(() => expect(fetchForNode).toHaveBeenCalledWith('/notifications', 2)); }); + + it('removeNotificationsForStack drops only the matching node and stack', async () => { + (apiFetch as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); + const remote = makeRemoteNode('online', { id: 2, name: 'Remote-B' }); + const { result } = renderHook(() => + useNotifications({ nodes: [localNode, remote], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + ); + await waitFor(() => expect(apiFetch).toHaveBeenCalled()); + + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'notification', + payload: makeNotif({ id: 1, stack_name: 'web', message: 'a-web' }), + }), + }); + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'notification', + payload: makeNotif({ id: 2, stack_name: 'db', message: 'a-db' }), + }), + }); + MockWS.instances[1]?.onmessage?.({ + data: JSON.stringify({ + type: 'notification', + payload: makeNotif({ id: 3, stack_name: 'web', message: 'b-web' }), + }), + }); + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'notification', + payload: makeNotif({ id: 4, message: 'a-unattached' }), + }), + }); + }); + + expect(result.current.notifications).toHaveLength(4); + + act(() => { + result.current.removeNotificationsForStack(localNode.id, 'web'); + }); + + expect(result.current.notifications.map((n) => `${n.nodeId}:${n.stack_name ?? ''}:${n.message}`).sort()).toEqual([ + '1::a-unattached', + '1:db:a-db', + '2:web:b-web', + ]); + + act(() => { + result.current.removeNotificationsForStack(localNode.id, 'web'); + }); + expect(result.current.notifications).toHaveLength(3); + }); + + it('refetches notifications on scope=notifications invalidate after mount fetch', async () => { + (apiFetch as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); + const nodes = [localNode]; + const onStateInvalidate = vi.fn(); + const onImageUpdatesChange = vi.fn(); + renderHook(() => + useNotifications({ nodes, onStateInvalidate, onImageUpdatesChange }), + ); + await waitFor(() => expect(apiFetch).toHaveBeenCalled()); + const afterMount = (apiFetch as ReturnType).mock.calls.filter( + (c) => c[0] === '/notifications', + ).length; + + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', + scope: 'notifications', + // No stack-deleted action: this case only asserts refetch delta. + nodeId: 1, + ts: 1000, + }), + }); + }); + + await waitFor(() => { + const after = (apiFetch as ReturnType).mock.calls.filter( + (c) => c[0] === '/notifications', + ).length; + expect(after).toBe(afterMount + 1); + }); + }); + + it('remote notifications invalidate triggers one additional remote fetch', async () => { + const remote = makeRemoteNode('online', { id: 2, name: 'sencho-sat-qa' }); + const nodes = [localNode, remote]; + const onStateInvalidate = vi.fn(); + const onImageUpdatesChange = vi.fn(); + (apiFetch as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); + (fetchForNode as ReturnType).mockResolvedValue({ + ok: true, + json: async () => [{ id: 9, level: 'error', message: 'remote row', timestamp: 2000, is_read: 0, stack_name: 'db' }], + }); + + const { result } = renderHook(() => + useNotifications({ nodes, onStateInvalidate, onImageUpdatesChange }), + ); + await waitFor(() => expect(fetchForNode).toHaveBeenCalledWith('/notifications', 2)); + const afterMount = (fetchForNode as ReturnType).mock.calls.filter( + (c) => c[0] === '/notifications' && c[1] === 2, + ).length; + + act(() => { + MockWS.instances[1]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', + scope: 'notifications', + nodeId: 2, + ts: 1000, + }), + }); + }); + + await waitFor(() => { + const after = (fetchForNode as ReturnType).mock.calls.filter( + (c) => c[0] === '/notifications' && c[1] === 2, + ).length; + expect(after).toBe(afterMount + 1); + }); + await waitFor(() => expect(result.current.notifications.some((n) => n.nodeId === 2 && n.nodeName === 'sencho-sat-qa')).toBe(true)); + }); + + it('unrelated state-invalidate scopes do not refetch notifications', async () => { + (apiFetch as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); + const nodes = [localNode]; + renderHook(() => + useNotifications({ nodes, onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + ); + await waitFor(() => expect(apiFetch).toHaveBeenCalled()); + const afterMount = (apiFetch as ReturnType).mock.calls.filter( + (c) => c[0] === '/notifications', + ).length; + + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', + scope: 'stack', + nodeId: 1, + stackName: 'web', + action: 'start', + ts: 1000, + }), + }); + }); + + expect( + (apiFetch as ReturnType).mock.calls.filter((c) => c[0] === '/notifications').length, + ).toBe(afterMount); + }); + + it('remote stack-deleted purge uses hub rn.id when payload nodeId collides with local', async () => { + (apiFetch as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); + (fetchForNode as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); + const remote = makeRemoteNode('online', { id: 7, name: 'Remote-7' }); + const nodes = [localNode, remote]; + const { result } = renderHook(() => + useNotifications({ nodes, onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + ); + await waitFor(() => expect(fetchForNode).toHaveBeenCalledWith('/notifications', 7)); + + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'notification', + payload: makeNotif({ id: 1, stack_name: 'web', message: 'local-web' }), + }), + }); + MockWS.instances[1]?.onmessage?.({ + data: JSON.stringify({ + type: 'notification', + payload: makeNotif({ id: 2, stack_name: 'web', message: 'remote-web' }), + }), + }); + }); + expect(nodeMessageKeys(result.current.notifications)).toEqual(['1:local-web', '7:remote-web']); + + // Production remotes broadcast their own local DB id (often 1), which collides + // with the hub local node. The remote socket must purge hub rn.id=7 only. + act(() => { + MockWS.instances[1]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', + scope: 'notifications', + action: 'stack-deleted', + nodeId: 1, + stackName: 'web', + ts: Date.now(), + }), + }); + }); + + expect(nodeMessageKeys(result.current.notifications)).toEqual(['1:local-web']); + }); + + it('preserves cached notifications for a failed refetch leg after invalidate', async () => { + const remote = makeRemoteNode('online', { id: 7, name: 'Remote-7' }); + // Stable identity so the nodes effect does not re-fetch on every render. + const nodes = [localNode, remote]; + let remoteFail = false; + (apiFetch as ReturnType).mockResolvedValue({ + ok: true, + json: async () => [ + { id: 10, level: 'info', message: 'local-kept', timestamp: 5000, is_read: 0, stack_name: 'other' }, + ], + }); + (fetchForNode as ReturnType).mockImplementation(() => { + if (remoteFail) { + return Promise.resolve({ ok: false, status: 502, json: async () => [] }); + } + return Promise.resolve({ + ok: true, + json: async () => [ + { id: 20, level: 'error', message: 'remote-web', timestamp: 4000, is_read: 0, stack_name: 'web' }, + { id: 21, level: 'info', message: 'remote-db', timestamp: 3000, is_read: 0, stack_name: 'db' }, + ], + }); + }); + + const { result } = renderHook(() => + useNotifications({ nodes, onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + ); + await waitFor(() => expect(result.current.notifications.some((n) => n.message === 'remote-web')).toBe(true)); + expect(nodeMessageKeys(result.current.notifications)).toEqual([ + '1:local-kept', + '7:remote-db', + '7:remote-web', + ]); + + remoteFail = true; + (apiFetch as ReturnType).mockResolvedValue({ + ok: true, + json: async () => [ + { id: 11, level: 'info', message: 'local-kept-v2', timestamp: 6000, is_read: 0, stack_name: 'other' }, + ], + }); + + act(() => { + MockWS.instances[1]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', + scope: 'notifications', + action: 'stack-deleted', + nodeId: 1, + stackName: 'web', + ts: Date.now(), + }), + }); + }); + + // Wait for post-invalidate merge (local slice replaced), not only optimistic purge. + await waitFor(() => { + expect(result.current.notifications.some((n) => n.message === 'local-kept-v2')).toBe(true); + }); + expect(result.current.notifications.some((n) => n.message === 'remote-web')).toBe(false); + expect(nodeMessageKeys(result.current.notifications)).toEqual([ + '1:local-kept-v2', + '7:remote-db', + ]); + }); + + it('does not restore a deleted stack notification from a stale pre-removal fetch', async () => { + let resolveStale!: (value: { ok: boolean; status: number; json: () => Promise }) => void; + const stalePromise = new Promise<{ ok: boolean; status: number; json: () => Promise }>((resolve) => { + resolveStale = resolve; + }); + let call = 0; + let releaseStale = false; + (apiFetch as ReturnType).mockImplementation(() => { + call += 1; + if (!releaseStale && call >= 2) return stalePromise; + if (releaseStale) { + return Promise.resolve({ + ok: true, + status: 200, + json: async () => [ + { id: 2, level: 'info', message: 'fresh-db', timestamp: 4000, is_read: 0, stack_name: 'db' }, + ], + }); + } + return Promise.resolve({ ok: true, status: 200, json: async () => [] }); + }); + + // Stable identity so the nodes effect does not re-fetch on every render. + const nodes = [localNode]; + const onStateInvalidate = vi.fn(); + const onImageUpdatesChange = vi.fn(); + const { result } = renderHook(() => + useNotifications({ nodes, onStateInvalidate, onImageUpdatesChange }), + ); + await waitFor(() => expect(apiFetch).toHaveBeenCalled()); + const afterMount = call; + + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'notification', + payload: makeNotif({ id: 1, stack_name: 'web', message: 'keep-until-remove' }), + }), + }); + }); + expect(result.current.notifications.some((n) => n.stack_name === 'web')).toBe(true); + + // Start an in-flight fetch without act: React's act waits on the deferred promise. + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', + scope: 'notifications', + action: 'stack-deleted', + nodeId: 1, + stackName: 'other', + ts: Date.now(), + }), + }); + await waitFor(() => expect(call).toBeGreaterThan(afterMount)); + + act(() => { + result.current.removeNotificationsForStack(localNode.id, 'web'); + }); + expect(result.current.notifications.some((n) => n.stack_name === 'web')).toBe(false); + + resolveStale({ + ok: true, + status: 200, + json: async () => [ + { id: 1, level: 'error', message: 'stale-web', timestamp: 3000, is_read: 0, stack_name: 'web' }, + ], + }); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); + + expect(result.current.notifications.some((n) => n.stack_name === 'web')).toBe(false); + + releaseStale = true; + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', + scope: 'notifications', + action: 'stack-deleted', + nodeId: 1, + stackName: 'web', + ts: Date.now(), + }), + }); + }); + + await waitFor(() => { + expect(result.current.notifications.some((n) => n.stack_name === 'db' && n.message === 'fresh-db')).toBe(true); + }); + expect(result.current.notifications.some((n) => n.stack_name === 'web')).toBe(false); + }); }); diff --git a/frontend/src/components/EditorLayout/hooks/useNotifications.ts b/frontend/src/components/EditorLayout/hooks/useNotifications.ts index f0a8e2a0..573d1656 100644 --- a/frontend/src/components/EditorLayout/hooks/useNotifications.ts +++ b/frontend/src/components/EditorLayout/hooks/useNotifications.ts @@ -28,8 +28,12 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang // Its spans instrument only that first fetch so later polls do not pollute the // report. const notificationsReadyRef = useRef(false); + // Monotonic generation so a slow pre-removal fetch cannot overwrite state + // after removeNotificationsForStack (or a newer fetch) has already run. + const notificationFetchGeneration = useRef(0); const fetchNotifications = async () => { + const generation = ++notificationFetchGeneration.current; try { const currentNodes = nodesRef.current; const localNode = currentNodes.find(n => n.type === 'local'); @@ -41,7 +45,10 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang const localPromise = apiFetch('/notifications', { localOnly: true } as Parameters[1]); const remotePromises = remoteNodes.map(n => fetchForNode('/notifications', n.id)); - const all: NotificationItem[] = []; + // Only successful legs replace their node slice. A failed/non-OK/decode-failed + // leg is not treated as an empty list; that node's cached rows stay until a + // later successful response (including an intentional empty list). + const successfulSlices = new Map(); const instrument = !notificationsReadyRef.current; const headersSpan = instrument ? beginSpan('fetch_headers', { background: true }) : null; @@ -55,8 +62,15 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang if (bodySpan !== null) endSpan(bodySpan); bodySpan = null; const dispatchSpan = instrument ? beginSpan('state_dispatch', { background: true }) : null; - data.forEach(n => all.push({ ...n, nodeId: localNode?.id ?? -1, nodeName: localNode?.name ?? 'Local' })); + const localId = localNode?.id ?? -1; + const localName = localNode?.name ?? 'Local'; + successfulSlices.set( + localId, + data.map(n => ({ ...n, nodeId: localId, nodeName: localName })), + ); if (dispatchSpan !== null) endSpan(dispatchSpan); + } else { + console.error(`[Notifications] local fetch HTTP ${localRes.status}`); } } catch (e) { if (headersSpan !== null) endSpan(headersSpan, { outcome: 'error' }); @@ -71,17 +85,38 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang // Remotes settle in the background; they never gate the milestone above. const remoteNodeResults = await Promise.allSettled(remotePromises); - for (let i = 0; i < remoteNodes.length; i++) { + for (const [i, rn] of remoteNodes.entries()) { const result = remoteNodeResults[i]; - if (result?.status === 'fulfilled' && result.value.ok) { + if (result.status !== 'fulfilled') { + console.error(`[Notifications] remote fetch error (${rn.name}):`, result.reason); + continue; + } + if (!result.value.ok) { + console.error(`[Notifications] remote fetch HTTP ${result.value.status} (${rn.name})`); + continue; + } + try { const data = await result.value.json() as Omit[]; - const rn = remoteNodes[i]; - data.forEach(n => all.push({ ...n, nodeId: rn.id, nodeName: rn.name })); + successfulSlices.set( + rn.id, + data.map(n => ({ ...n, nodeId: rn.id, nodeName: rn.name })), + ); + } catch (e) { + console.error(`[Notifications] remote fetch decode error (${rn.name}):`, e); } } - all.sort((a, b) => b.timestamp - a.timestamp); - setNotifications(all); + if (generation !== notificationFetchGeneration.current) return; + // Keep failed/offline node slices; drop rows for nodes no longer in the roster. + const activeNodeIds = new Set(currentNodes.map(n => n.id)); + setNotifications(prev => { + const preserved = prev.filter(n => + n.nodeId == null + || (activeNodeIds.has(n.nodeId) && !successfulSlices.has(n.nodeId)), + ); + const replaced = [...successfulSlices.values()].flat(); + return [...preserved, ...replaced].sort((a, b) => b.timestamp - a.timestamp); + }); } catch (e) { console.error('[Notifications] fetch error:', e); } @@ -90,6 +125,33 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang const fetchNotificationsRef = useRef(fetchNotifications); fetchNotificationsRef.current = fetchNotifications; + const purgeStackNotifications = (nodeId: number, stackName: string) => { + notificationFetchGeneration.current += 1; + setNotifications(prev => + prev.filter(n => n.nodeId !== nodeId || n.stack_name !== stackName), + ); + }; + + const tryPurgeDeletedStackFromInvalidate = ( + msg: { action?: unknown; nodeId?: unknown; stackName?: unknown }, + fallbackNodeId?: number, + ) => { + if (msg.action !== 'stack-deleted' || typeof msg.stackName !== 'string') return; + const nodeId = typeof msg.nodeId === 'number' ? msg.nodeId : fallbackNodeId; + if (nodeId === undefined) return; + purgeStackNotifications(nodeId, msg.stackName); + }; + + const reconcileNotificationsInvalidate = ( + msg: { action?: unknown; nodeId?: unknown; stackName?: unknown }, + fallbackNodeId?: number, + ) => { + tryPurgeDeletedStackFromInvalidate(msg, fallbackNodeId); + void fetchNotificationsRef.current(); + }; + const reconcileNotificationsInvalidateRef = useRef(reconcileNotificationsInvalidate); + reconcileNotificationsInvalidateRef.current = reconcileNotificationsInvalidate; + // Local notification WebSocket with exponential-backoff reconnect. useEffect(() => { const wsProtocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; @@ -125,7 +187,9 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang } else if (msg.type === 'state-invalidate') { window.dispatchEvent(new CustomEvent('sencho:state-invalidate', { detail: msg })); onStateInvalidateRef.current(); - if (msg.scope === 'image-updates' && msg.action === 'stack-updated') { + if (msg.scope === 'notifications') { + reconcileNotificationsInvalidateRef.current(msg); + } else if (msg.scope === 'image-updates' && msg.action === 'stack-updated') { onImageUpdatesChangeRef.current(); } } @@ -203,7 +267,11 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang } else if (msg.type === 'state-invalidate') { window.dispatchEvent(new CustomEvent('sencho:state-invalidate', { detail: { ...msg, nodeId: rn.id } })); onStateInvalidateRef.current(); - if (msg.scope === 'image-updates' && msg.action === 'stack-updated') { + if (msg.scope === 'notifications') { + // Remote payloads use the remote's local DB node ID. Hub UI state + // is keyed by rn.id, so always reconcile with the hub node ID. + reconcileNotificationsInvalidateRef.current({ ...msg, nodeId: rn.id }); + } else if (msg.scope === 'image-updates' && msg.action === 'stack-updated') { onImageUpdatesChangeRef.current(); } } @@ -324,5 +392,6 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang markAllRead, deleteNotification, clearAllNotifications, + removeNotificationsForStack: purgeStackNotifications, } as const; } diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts index 56812468..50441939 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts @@ -130,6 +130,7 @@ function setup(over: { activeNode?: Parameters[0]['activeNode']; setActiveNode?: Parameters[0]['setActiveNode']; onDeletedOpenStack?: () => void; + removeNotificationsForStack?: (nodeId: number, stackName: string) => void; } = {}) { const editorState = makeEditorState(over.editorState); const stackListState = makeStackListState(over.stackList); @@ -141,6 +142,7 @@ function setup(over: { const overlayState = makeOverlay(over.overlay); const setActiveNode = over.setActiveNode ?? vi.fn(); const onDeletedOpenStack = over.onDeletedOpenStack ?? vi.fn(); + const removeNotificationsForStack = over.removeNotificationsForStack ?? vi.fn(); const { result } = renderHook(() => useStackActions({ @@ -157,9 +159,10 @@ function setup(over: { hasUpdateGuard: over.hasUpdateGuard ?? false, canEditStack: over.canEditStack ?? (() => true), onDeletedOpenStack, + removeNotificationsForStack, }), ); - return { result, editorState, stackListState, overlayState, navState, setActiveNode, onDeletedOpenStack }; + return { result, editorState, stackListState, overlayState, navState, setActiveNode, onDeletedOpenStack, removeNotificationsForStack }; } describe('useStackActions.saveFile', () => { @@ -1617,4 +1620,40 @@ describe('useStackActions.deleteStack', () => { expect(onDeletedOpenStack).not.toHaveBeenCalled(); expect(stackListState.refreshStacks).not.toHaveBeenCalled(); }); + + it('calls removeNotificationsForStack with node id and canonical name on success', async () => { + vi.mocked(apiFetch).mockResolvedValue(new Response(null, { status: 200 })); + const removeNotificationsForStack = vi.fn(); + const { result } = setup({ + overlay: { stackToDelete: 'web.yml' }, + stackList: { selectedFile: 'web.yml', files: ['web.yml'] }, + navState: { activeView: 'editor' }, + activeNode: { id: 7, type: 'local' } as Parameters[0]['activeNode'], + removeNotificationsForStack, + }); + + await act(async () => { + await result.current.deleteStack(false); + }); + + expect(removeNotificationsForStack).toHaveBeenCalledTimes(1); + expect(removeNotificationsForStack).toHaveBeenCalledWith(7, 'web'); + }); + + it('does not call removeNotificationsForStack on a non-OK delete', async () => { + vi.mocked(apiFetch).mockResolvedValue(new Response('boom', { status: 500 })); + const removeNotificationsForStack = vi.fn(); + const { result } = setup({ + overlay: { stackToDelete: 'web.yml' }, + stackList: { selectedFile: 'web.yml', files: ['web.yml'] }, + navState: { activeView: 'editor' }, + removeNotificationsForStack, + }); + + await act(async () => { + await result.current.deleteStack(false); + }); + + expect(removeNotificationsForStack).not.toHaveBeenCalled(); + }); }); diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.ts index 18f9beff..10023e33 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.ts @@ -176,6 +176,12 @@ interface UseStackActionsOptions { * owns that state, and an optional callback would silently skip it. */ onDeletedOpenStack: () => void; + /** + * Drop in-memory notifications for a deleted stack on the active node. + * Caller must pass the node id and canonical stack basename (no .yml/.yaml). + * Optional so unit tests that do not exercise delete can omit it. + */ + removeNotificationsForStack?: (nodeId: number, stackName: string) => void; } const isRecord = (value: unknown): value is Record => @@ -399,6 +405,7 @@ export function useStackActions(options: UseStackActionsOptions) { canEditStack, canOfferVolumeRemoval = false, onDeletedOpenStack, + removeNotificationsForStack, } = options; const pendingStackLoadRef = useRef(null); @@ -1895,6 +1902,7 @@ export function useStackActions(options: UseStackActionsOptions) { stackListState.files.find( f => f === stackToDelete || f.replace(/\.(yml|yaml)$/, '') === stackToDelete, ) ?? stackToDelete; + const canonicalName = deleteKey.replace(/\.(yml|yaml)$/, ''); if (stackListState.isStackBusy(deleteKey)) return; stackListState.setStackAction(deleteKey, 'delete'); try { @@ -1914,11 +1922,14 @@ export function useStackActions(options: UseStackActionsOptions) { toast.success('Stack deleted successfully!'); overlayState.closeDeleteDialog(); const selected = stackListState.selectedFile; - const stripExt = (name: string) => name.replace(/\.(yml|yaml)$/, ''); + const nodeId = activeNode?.id; + if (nodeId != null && removeNotificationsForStack) { + removeNotificationsForStack(nodeId, canonicalName); + } // Always clear a deleted selection, even when another top-level view is // visible (Resources, Networking, etc.). Leaving that view is gated on // the editor being the active surface below. - if (selected != null && stripExt(selected) === stripExt(deleteKey)) { + if (selected != null && selected.replace(/\.(yml|yaml)$/, '') === canonicalName) { resetEditorState(); if (navState.activeView === 'editor') { navState.setActiveView('dashboard'); From 6484c79015b5da7321379a76bc34d43478eb7dd0 Mon Sep 17 00:00:00 2001 From: Anso Date: Wed, 22 Jul 2026 20:29:08 -0400 Subject: [PATCH 02/13] fix(rbac): cover stack assignment cleanup on blueprint withdraw (#1664) Main already clears stack-scoped role_assignments through DeployedStackDeletionService (used by local blueprint withdraw and stack DELETE). Add call-path regressions for successful cleanup, ENOENT-as-absent, non-ENOENT filesystem failure, and db_failed when RBAC cleanup throws, plus remote hub isolation. --- .../blueprints-remote-deploy.test.ts | 38 +++++++ backend/src/__tests__/blueprints.test.ts | 104 ++++++++++++++++++ ...stack-delete-cascades-mesh-opt-out.test.ts | 29 +++++ backend/src/__tests__/users-rbac.test.ts | 23 ++++ 4 files changed, 194 insertions(+) diff --git a/backend/src/__tests__/blueprints-remote-deploy.test.ts b/backend/src/__tests__/blueprints-remote-deploy.test.ts index 2ce5ad3c..d9a145b7 100644 --- a/backend/src/__tests__/blueprints-remote-deploy.test.ts +++ b/backend/src/__tests__/blueprints-remote-deploy.test.ts @@ -214,4 +214,42 @@ describe('BlueprintService remote deploy', () => { expect(delSpy.mock.calls[0][0]).toMatch(/\/api\/stacks\//); expect(DatabaseService.getInstance().getDeployment(bp.id, node.id)).toBeUndefined(); }); + + it('does not clear hub role assignments when withdrawing a remote deployment', async () => { + const bcrypt = await import('bcrypt'); + const db = DatabaseService.getInstance(); + const node = seedRemoteNode(); + const bp = seedBlueprint([node.id]); + const nodeObj = db.getNode(node.id)!; + const bpObj = db.getBlueprint(bp.id)!; + db.upsertDeployment({ + blueprint_id: bp.id, + node_id: node.id, + status: 'active', + applied_revision: bpObj.revision, + }); + + const hash = await bcrypt.hash('password123', 1); + const userId = db.addUser({ + username: `remote-wd-rbac-${counter}`, password_hash: hash, role: 'viewer', + }); + db.addRoleAssignment({ + user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: bpObj.name, + }); + + vi.spyOn(axios, 'get').mockResolvedValue({ status: 404, data: {} }); + vi.spyOn(axios, 'post').mockResolvedValue({ status: 200, data: {} }); + const delSpy = vi.spyOn(axios, 'delete').mockResolvedValue({ status: 200, data: {} }); + const rbacSpy = vi.spyOn(db, 'deleteRoleAssignmentsByResource'); + + const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj); + + expect(result.status).toBe('withdrawn'); + expect(delSpy.mock.calls[0][0]).toMatch(/\/api\/stacks\//); + expect(rbacSpy).not.toHaveBeenCalled(); + expect(db.getAllRoleAssignments(userId) + .some((a) => a.resource_type === 'stack' && a.resource_id === bpObj.name)).toBe(true); + + db.deleteUser(userId); + }); }); diff --git a/backend/src/__tests__/blueprints.test.ts b/backend/src/__tests__/blueprints.test.ts index c9f91c28..935230af 100644 --- a/backend/src/__tests__/blueprints.test.ts +++ b/backend/src/__tests__/blueprints.test.ts @@ -13,6 +13,7 @@ * lifecycle in the plan. */ import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import bcrypt from 'bcrypt'; import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; import type { Blueprint, Node } from '../services/DatabaseService'; import type { ReconcileDecision } from '../services/BlueprintReconciler'; @@ -40,6 +41,7 @@ afterAll(() => cleanupTestDb(tmpDir)); beforeEach(() => { const db = DatabaseService.getInstance().getDb(); + db.prepare('DELETE FROM role_assignments').run(); db.prepare('DELETE FROM blueprint_deployments').run(); db.prepare('DELETE FROM blueprints').run(); db.prepare('DELETE FROM node_labels').run(); @@ -466,6 +468,108 @@ describe('BlueprintService per-stack lock', () => { }); }); +describe('BlueprintService local withdraw clears stack-scoped role assignments', () => { + function hasAssignment(userId: number, resourceType: string, resourceId: string): boolean { + return DatabaseService.getInstance().getAllRoleAssignments(userId) + .some((a) => a.resource_type === resourceType && a.resource_id === resourceId); + } + + async function arrangeLocalWithdraw() { + const db = DatabaseService.getInstance(); + const nodeId = seedNode(); + const bp = seedBlueprint({ name: `rbac-wd-${counter}`, classification: 'stateless', nodeIds: [nodeId] }); + const node = db.getNode(nodeId)!; + db.upsertDeployment({ + blueprint_id: bp.id, + node_id: nodeId, + status: 'active', + applied_revision: bp.revision, + }); + + const hash = await bcrypt.hash('password123', 1); + const userId = db.addUser({ username: `bp-rbac-${counter}`, password_hash: hash, role: 'viewer' }); + const otherNodeId = seedNode(); + db.addRoleAssignment({ + user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: bp.name, + }); + db.addRoleAssignment({ + user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'other-stack', + }); + db.addRoleAssignment({ + user_id: userId, role: 'deployer', resource_type: 'node', resource_id: String(otherNodeId), + }); + + vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue(null); + const { ComposeService } = await import('../services/ComposeService'); + const { FileSystemService } = await import('../services/FileSystemService'); + vi.spyOn(ComposeService.prototype, 'downStack').mockResolvedValue(undefined); + const deleteStackSpy = vi.spyOn(FileSystemService.prototype, 'deleteStack').mockResolvedValue(undefined); + + return { bp, node, nodeId, userId, deleteStackSpy, db }; + } + + it('clears the target assignment after successful filesystem deletion and preserves unrelated grants', async () => { + const { bp, node, userId, deleteStackSpy, db } = await arrangeLocalWithdraw(); + + const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node); + + expect(outcome.status).toBe('withdrawn'); + expect(deleteStackSpy).toHaveBeenCalledWith(bp.name); + expect(db.getDeployment(bp.id, node.id)).toBeUndefined(); + expect(hasAssignment(userId, 'stack', bp.name)).toBe(false); + expect(hasAssignment(userId, 'stack', 'other-stack')).toBe(true); + expect(db.getAllRoleAssignments(userId).some((a) => a.resource_type === 'node')).toBe(true); + db.deleteUser(userId); + }); + + it('clears the target assignment when deleteStack treats the directory as already absent (ENOENT)', async () => { + // Shared deletion always calls deleteStack; FileSystemService maps ENOENT to success. + const { bp, node, userId, deleteStackSpy, db } = await arrangeLocalWithdraw(); + deleteStackSpy.mockResolvedValue(undefined); + + const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node); + + expect(outcome.status).toBe('withdrawn'); + expect(deleteStackSpy).toHaveBeenCalledWith(bp.name); + expect(hasAssignment(userId, 'stack', bp.name)).toBe(false); + expect(db.getAllRoleAssignments(userId)).toHaveLength(2); + db.deleteUser(userId); + }); + + it('preserves the grant when filesystem deletion fails with a non-ENOENT error', async () => { + const { bp, node, nodeId, userId, deleteStackSpy, db } = await arrangeLocalWithdraw(); + const fsErr = Object.assign(new Error('permission denied'), { code: 'EACCES' }); + deleteStackSpy.mockRejectedValue(fsErr); + const rbacSpy = vi.spyOn(db, 'deleteRoleAssignmentsByResource'); + + const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node); + + expect(outcome.status).toBe('failed'); + expect(rbacSpy).not.toHaveBeenCalled(); + expect(db.getDeployment(bp.id, nodeId)?.status).toBe('failed'); + expect(hasAssignment(userId, 'stack', bp.name)).toBe(true); + db.deleteUser(userId); + }); + + it('fails withdraw and keeps the deployment when role-assignment cleanup throws', async () => { + const { bp, node, nodeId, userId, db } = await arrangeLocalWithdraw(); + vi.spyOn(db, 'deleteRoleAssignmentsByResource') + .mockImplementation(() => { throw new Error('simulated rbac cleanup failure'); }); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node); + + expect(outcome.status).toBe('failed'); + expect(db.getDeployment(bp.id, nodeId)).toBeDefined(); + expect(db.getDeployment(bp.id, nodeId)?.status).toBe('failed'); + expect(errorSpy.mock.calls.some((args) => + typeof args[0] === 'string' && args[0].includes('Secondary DB cleanup failed'), + )).toBe(true); + expect(hasAssignment(userId, 'stack', bp.name)).toBe(true); + db.deleteUser(userId); + }); +}); + describe('BlueprintService marker parsing + name-conflict guard', () => { it('parseMarker accepts a well-formed marker', () => { const marker = BlueprintService.parseMarker(JSON.stringify({ blueprintId: 7, revision: 3, lastApplied: 12345 })); diff --git a/backend/src/__tests__/stack-delete-cascades-mesh-opt-out.test.ts b/backend/src/__tests__/stack-delete-cascades-mesh-opt-out.test.ts index 037d6ed5..e1ad3b0a 100644 --- a/backend/src/__tests__/stack-delete-cascades-mesh-opt-out.test.ts +++ b/backend/src/__tests__/stack-delete-cascades-mesh-opt-out.test.ts @@ -131,3 +131,32 @@ describe('DELETE /api/stacks/:stackName mesh opt-out cascade (F-1 / F-14)', () = expect(sawCascadeWarning).toBe(true); }); }); + +describe('DELETE /api/stacks/:stackName clears stack-scoped role assignments', () => { + it('removes only the deleted stack assignment and preserves unrelated grants', async () => { + const db = DatabaseService.getInstance(); + const bcrypt = await import('bcrypt'); + const hash = await bcrypt.hash('password123', 1); + const userId = db.addUser({ username: 'stack-del-rbac', password_hash: hash, role: 'viewer' }); + const otherNodeId = db.addNode({ + name: 'stack-del-rbac-node', type: 'remote', api_url: 'http://test:1852', + api_token: '', compose_dir: '/tmp', is_default: false, + }); + db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'api' }); + db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'other-stack' }); + db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'node', resource_id: String(otherNodeId) }); + + const res = await request(app) + .delete('/api/stacks/api') + .set('Cookie', adminCookie); + + expect(res.status).toBe(200); + const remaining = db.getAllRoleAssignments(userId); + expect(remaining.some((a) => a.resource_type === 'stack' && a.resource_id === 'api')).toBe(false); + expect(remaining.some((a) => a.resource_type === 'stack' && a.resource_id === 'other-stack')).toBe(true); + expect(remaining.some((a) => a.resource_type === 'node' && a.resource_id === String(otherNodeId))).toBe(true); + + db.deleteUser(userId); + db.deleteNode(otherNodeId); + }); +}); diff --git a/backend/src/__tests__/users-rbac.test.ts b/backend/src/__tests__/users-rbac.test.ts index a6d01eca..c5917576 100644 --- a/backend/src/__tests__/users-rbac.test.ts +++ b/backend/src/__tests__/users-rbac.test.ts @@ -714,6 +714,29 @@ describe('Orphaned role assignment cleanup', () => { // Cleanup db.deleteUser(userId); }); + + it('deleteRoleAssignmentsByResource removes only the matching resource tuple', async () => { + const db = DatabaseService.getInstance(); + const hash = await bcrypt.hash('password123', 1); + const userId = db.addUser({ username: 'tupleorphan', password_hash: hash, role: 'viewer' }); + const nodeId = db.addNode({ + name: 'tuple-cleanup-node', type: 'remote', api_url: 'http://test:1852', + api_token: '', compose_dir: '/tmp', is_default: false, + }); + db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'target-stack' }); + db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'keep-stack' }); + db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'node', resource_id: String(nodeId) }); + + db.deleteRoleAssignmentsByResource('stack', 'target-stack'); + + const after = db.getAllRoleAssignments(userId); + expect(after.some((a) => a.resource_type === 'stack' && a.resource_id === 'target-stack')).toBe(false); + expect(after.some((a) => a.resource_type === 'stack' && a.resource_id === 'keep-stack')).toBe(true); + expect(after.some((a) => a.resource_type === 'node' && a.resource_id === String(nodeId))).toBe(true); + + db.deleteUser(userId); + db.deleteNode(nodeId); + }); }); // ---- Role-Based Permission Checks (via API) ---- From 8d60d491e808503231fd655fded8401c54bbbf0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:55:22 -0400 Subject: [PATCH 03/13] chore(deps-dev): bump fast-uri (#1682) Bumps the all-npm-root-security group with 1 update in the / directory: [fast-uri](https://github.com/fastify/fast-uri). Updates `fast-uri` from 3.1.3 to 3.1.4 - [Release notes](https://github.com/fastify/fast-uri/releases) - [Commits](https://github.com/fastify/fast-uri/compare/v3.1.3...v3.1.4) --- updated-dependencies: - dependency-name: fast-uri dependency-version: 3.1.4 dependency-type: indirect dependency-group: all-npm-root-security ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 26366d72..d40bb592 100644 --- a/package-lock.json +++ b/package-lock.json @@ -692,9 +692,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz", - "integrity": "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==", + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", + "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", "dev": true, "funding": [ { From bf7a8b57341d1e6653952e5f085402e286546a60 Mon Sep 17 00:00:00 2001 From: Anso Date: Wed, 22 Jul 2026 21:17:13 -0400 Subject: [PATCH 04/13] ci: refresh bug report subsystem checkboxes (#1683) Replace the outdated Admiral-heavy subsystem list with hybrid product-surface buckets that match current bug-prone areas. --- .github/ISSUE_TEMPLATE/bug_report.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index a019c923..99d72646 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -78,10 +78,10 @@ body: label: Subsystems involved description: Tick any that apply. options: - - label: Trivy / vulnerability scanning - - label: Pilot Agent (NAT tunnel) - - label: Sencho Mesh (cross-node networking) - - label: Fleet Actions / Secrets / Snapshots - - label: Cloud Backup - - label: SSO / LDAP + - label: Stacks, Compose & Deploy + - label: Multi-node, Pilot & Mesh + - label: Fleet (actions, sync, secrets, backups, blueprints) + - label: Alerts, Notifications & Observability + - label: Security, Scanning & Auth (Trivy, SSO, MFA, RBAC) + - label: Updates, Schedules & Automation - label: None of the above From ed5ca9c4f690d07dc1ccf25c3a8557e57780410d Mon Sep 17 00:00:00 2001 From: Anso Date: Thu, 23 Jul 2026 08:25:54 -0400 Subject: [PATCH 05/13] fix(fleet): refresh prune reclaimable estimate after successful run (#1675) * fix(fleet): refresh prune reclaimable estimate after successful run The live estimate only re-ran when targets or scope changed, so the toolbar and per-node reclaimable figures stayed stale after a real prune. Invalidate via estimateEpoch when any non-dry-run target succeeds. * fix(fleet): toast partial prune success honestly A reachable node with mixed per-target outcomes made okNodes zero and showed a total-failure toast even when some targets mutated Docker. Gate the failure toast on zero successful targets anywhere. --- docs/features/fleet-actions.mdx | 2 +- .../cards/FleetPruneCard.test.tsx | 165 +++++++++++++++++- .../FleetActions/cards/FleetPruneCard.tsx | 25 ++- 3 files changed, 185 insertions(+), 7 deletions(-) diff --git a/docs/features/fleet-actions.mdx b/docs/features/fleet-actions.mdx index 4cbe6556..43b70633 100644 --- a/docs/features/fleet-actions.mdx +++ b/docs/features/fleet-actions.mdx @@ -139,7 +139,7 @@ Scope is a segmented control with two options: ### Live estimate and behaviour -- Changing a target or the scope re-triggers a debounced call to `POST /api/fleet/prune/estimate`, which walks the same Docker enumeration the destructive path uses so the estimate matches what pruning would actually reclaim. **Prune fleet** stays disabled until the estimate resolves: you cannot confirm a destructive fleet-wide prune with no context on what it will reclaim. +- Changing a target or the scope re-triggers a debounced call to `POST /api/fleet/prune/estimate`, which walks the same Docker enumeration the destructive path uses so the estimate matches what pruning would actually reclaim. A completed real prune that succeeds on at least one target re-triggers the same estimate so the toolbar total and per-node list reflect post-prune Docker state. **Prune fleet** stays disabled until the estimate resolves: you cannot confirm a destructive fleet-wide prune with no context on what it will reclaim. - Each remote node receives one `POST /api/system/prune/system` call per selected target, with a 120-second timeout. If a transport error fires for one target, the remaining targets on that node are short-circuited with the same error rather than retried, so a dead node doesn't absorb the full multi-target timeout budget. - Local nodes serialize against a per-node lock (`bulk-prune:`). A second fleet prune launched against the same local node while the first is still in flight returns *A prune is already running on this node* for each target. - Reclaimed bytes are reported by the Docker daemon and are approximate. Per-node rows in the results panel sum the per-target reclaim; the per-target children show how much each individual prune actually freed. diff --git a/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.test.tsx b/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.test.tsx index df538584..6ad8850b 100644 --- a/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.test.tsx +++ b/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.test.tsx @@ -13,12 +13,13 @@ vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() })); const toastError = vi.fn(); const toastSuccess = vi.fn(); +const toastWarning = vi.fn(); vi.mock('@/components/ui/toast-store', () => ({ toast: { error: (...a: unknown[]) => toastError(...a), success: (...a: unknown[]) => toastSuccess(...a), info: vi.fn(), - warning: vi.fn(), + warning: (...a: unknown[]) => toastWarning(...a), loading: vi.fn(() => 'toast-id'), dismiss: vi.fn(), }, @@ -112,3 +113,165 @@ it('surfaces an error toast when the prune returns non-ok', async () => { await user.click(screen.getByRole('button', { name: 'Dry run' })); await waitFor(() => expect(toastError).toHaveBeenCalledWith('prune blew up')); }); + +function estimateCallCount(): number { + return mockedFetch.mock.calls.filter(c => c[0] === '/fleet/prune/estimate').length; +} + +it('re-fetches the estimate after a partial-success real prune', async () => { + const user = userEvent.setup(); + let estimatePhase: 'pre' | 'post' = 'pre'; + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/prune/estimate') { + if (estimatePhase === 'pre') { + return Promise.resolve(jsonResponse(200, { + totalBytes: 1024, + perNode: [{ nodeId: 1, nodeName: 'central', reclaimableBytes: 1024, reachable: true }], + })); + } + return Promise.resolve(jsonResponse(200, { + totalBytes: 0, + perNode: [{ nodeId: 1, nodeName: 'central', reclaimableBytes: 0, reachable: true }], + })); + } + if (url === '/fleet/labels/fleet-prune') { + // One target succeeded before a later failure left the node unreachable. + // reachable:false must not suppress the refresh (target.success is the signal). + return Promise.resolve(jsonResponse(200, { + results: [{ + nodeId: 1, + nodeName: 'central', + reachable: false, + error: 'transport failed after images', + targets: [ + { target: 'images', success: true, reclaimedBytes: 1024 }, + { target: 'volumes', success: false, reclaimedBytes: 0, error: 'unreachable' }, + ], + }], + })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + + render(); + await waitFor(() => expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeEnabled()); + expect(screen.getByText('~ 1 KB reclaimable')).toBeInTheDocument(); + expect(screen.getByText('1 KB')).toBeInTheDocument(); + const callsBeforePrune = estimateCallCount(); + + await user.click(screen.getByRole('button', { name: 'Prune fleet' })); + const dialog = await screen.findByRole('alertdialog'); + estimatePhase = 'post'; + await user.click(within(dialog).getByRole('button', { name: 'Prune managed' })); + + await waitFor(() => expect(estimateCallCount()).toBe(callsBeforePrune + 1)); + const postEstimateCall = [...mockedFetch.mock.calls].reverse().find(c => c[0] === '/fleet/prune/estimate'); + expect(postEstimateCall).toBeTruthy(); + expect(JSON.parse(postEstimateCall![1].body)).toEqual({ targets: ['images'], scope: 'managed' }); + await waitFor(() => expect(screen.getByText('0 reclaimable')).toBeInTheDocument()); + await waitFor(() => expect(screen.getByText('0 Bytes')).toBeInTheDocument()); +}); + +it('does not re-fetch the estimate after a dry run', async () => { + const user = userEvent.setup(); + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/prune/estimate') { + return Promise.resolve(jsonResponse(200, { + totalBytes: 1024, + perNode: [{ nodeId: 1, nodeName: 'central', reclaimableBytes: 1024, reachable: true }], + })); + } + if (url === '/fleet/labels/fleet-prune') { + return Promise.resolve(jsonResponse(200, { + results: [{ + nodeId: 1, + nodeName: 'central', + reachable: true, + targets: [{ target: 'images', success: true, reclaimedBytes: 4096, dryRun: true }], + }], + })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + + render(); + await waitFor(() => expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeEnabled()); + const callsBefore = estimateCallCount(); + await user.click(screen.getByRole('button', { name: 'Dry run' })); + await waitFor(() => expect(toastSuccess).toHaveBeenCalled()); + // Debounce window: a refresh would schedule another estimate within 350ms. + await new Promise(r => setTimeout(r, 500)); + expect(estimateCallCount()).toBe(callsBefore); +}); + +it('does not re-fetch the estimate when every target fails on a 2xx response', async () => { + const user = userEvent.setup(); + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/prune/estimate') { + return Promise.resolve(jsonResponse(200, { + totalBytes: 1024, + perNode: [{ nodeId: 1, nodeName: 'central', reclaimableBytes: 1024, reachable: true }], + })); + } + if (url === '/fleet/labels/fleet-prune') { + return Promise.resolve(jsonResponse(200, { + results: [{ + nodeId: 1, + nodeName: 'central', + reachable: true, + targets: [{ target: 'images', success: false, reclaimedBytes: 0, error: 'A prune is already running on this node' }], + }], + })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + + render(); + await waitFor(() => expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeEnabled()); + const callsBefore = estimateCallCount(); + await user.click(screen.getByRole('button', { name: 'Prune fleet' })); + const dialog = await screen.findByRole('alertdialog'); + await user.click(within(dialog).getByRole('button', { name: 'Prune managed' })); + await waitFor(() => expect(toastError).toHaveBeenCalled()); + await new Promise(r => setTimeout(r, 500)); + expect(estimateCallCount()).toBe(callsBefore); +}); + +it('warns instead of claiming total failure when a reachable node has mixed per-target outcomes', async () => { + const user = userEvent.setup(); + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/prune/estimate') { + return Promise.resolve(jsonResponse(200, { + totalBytes: 1024, + perNode: [{ nodeId: 1, nodeName: 'central', reclaimableBytes: 1024, reachable: true }], + })); + } + if (url === '/fleet/labels/fleet-prune') { + // Images succeed, volumes fail on the same reachable node. Pre-fix toast + // logic treated the node as fully failed (okNodes=0) and said every node + // failed even though Docker mutated for images. + return Promise.resolve(jsonResponse(200, { + results: [{ + nodeId: 1, + nodeName: 'central', + reachable: true, + targets: [ + { target: 'images', success: true, reclaimedBytes: 1024 }, + { target: 'volumes', success: false, reclaimedBytes: 0, error: 'volume prune failed' }, + ], + }], + })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + + render(); + await waitFor(() => expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeEnabled()); + await user.click(screen.getByRole('button', { name: 'Prune fleet' })); + const dialog = await screen.findByRole('alertdialog'); + await user.click(within(dialog).getByRole('button', { name: 'Prune managed' })); + + await waitFor(() => expect(toastWarning).toHaveBeenCalled()); + expect(toastWarning.mock.calls[0][0]).toMatch(/1\/1 nodes reclaimed space/); + expect(toastError).not.toHaveBeenCalledWith('Prune failed on every node. See results below.'); +}); diff --git a/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.tsx b/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.tsx index a2f8cebb..34729407 100644 --- a/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.tsx +++ b/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.tsx @@ -48,6 +48,9 @@ export function FleetPruneCard({ nodes }: Props) { const [running, setRunning] = useState(false); const [results, setResults] = useState([]); const [estimate, setEstimate] = useState({ kind: 'idle' }); + // Bumped after a real prune mutates at least one target so the estimate + // effect re-runs without changing targets/scope. + const [estimateEpoch, setEstimateEpoch] = useState(0); const toggleTarget = (target: PruneTarget) => { setTargets(prev => { @@ -92,7 +95,7 @@ export function FleetPruneCard({ nodes }: Props) { cancelled = true; if (estimateDebounceRef.current) clearTimeout(estimateDebounceRef.current); }; - }, [targets, scope]); + }, [targets, scope, estimateEpoch]); async function run(opts: { dryRun: boolean }) { if (targets.size === 0) return; @@ -113,6 +116,13 @@ export function FleetPruneCard({ nodes }: Props) { return; } const apiResults = (body.results as FleetPruneNodeResult[]) ?? []; + // HTTP 200 still arrives when every target fails; only a successful + // target means Docker state changed and the live estimate is stale. + // Scan targets independently of node.reachable (an early remote target + // can succeed before a later transport error marks the node unreachable). + if (!opts.dryRun && apiResults.some(node => node.targets.some(target => target.success))) { + setEstimateEpoch(e => e + 1); + } const rows: ResultRow[] = apiResults.map((node) => { const totalBytes = node.targets.reduce((sum, t) => sum + (t.reclaimedBytes ?? 0), 0); const allOk = node.reachable && node.targets.every(t => t.success); @@ -133,19 +143,24 @@ export function FleetPruneCard({ nodes }: Props) { }); setResults(rows); const totalNodes = apiResults.length; - const okNodes = apiResults.filter(n => n.reachable && n.targets.every(t => t.success)).length; + // Fully OK: every target on a reachable node succeeded. Partial: at + // least one target succeeded anywhere (mixed per-target on one node + // still counts). "Failed on every node" only when zero targets succeeded. + const fullyOkNodes = apiResults.filter(n => n.reachable && n.targets.every(t => t.success)).length; + const nodesWithAnySuccess = apiResults.filter(n => n.targets.some(t => t.success)).length; + const anyTargetSucceeded = nodesWithAnySuccess > 0; const totalReclaimed = apiResults.reduce( (sum, n) => sum + n.targets.reduce((s, t) => s + (t.reclaimedBytes ?? 0), 0), 0, ); if (opts.dryRun) { toast.success(`Dry run: ${formatBytes(totalReclaimed)} would be reclaimed across ${totalNodes} node${totalNodes === 1 ? '' : 's'}.`); - } else if (okNodes === totalNodes && totalNodes > 0) { + } else if (fullyOkNodes === totalNodes && totalNodes > 0) { toast.success(`Reclaimed ${formatBytes(totalReclaimed)} across ${totalNodes} node${totalNodes === 1 ? '' : 's'}.`); - } else if (okNodes === 0) { + } else if (!anyTargetSucceeded) { toast.error('Prune failed on every node. See results below.'); } else { - toast.warning(`${okNodes}/${totalNodes} nodes succeeded · ${formatBytes(totalReclaimed)} reclaimed. See results below.`); + toast.warning(`${nodesWithAnySuccess}/${totalNodes} nodes reclaimed space · ${formatBytes(totalReclaimed)} reclaimed. See results below.`); } } catch (err) { toast.dismiss(toastId); From dd54a2e483f4432c59f602ea1129a47ec87c1ec8 Mon Sep 17 00:00:00 2001 From: Anso Date: Thu, 23 Jul 2026 12:59:53 -0400 Subject: [PATCH 06/13] feat: graduate Host Console to Community admins (#1669) * feat: graduate Host Console to Community admins Make Host Console available to Community and Admiral admins (system:console), add host-console-community for mixed fleets, and keep opaque API tokens off the host shell. * docs: document Host Console deep links Cover root and stack-scoped Console URLs, correct the phone treatment note, and pin parse/build round-trips in senchoRoute tests. * fix: bind Host Console socket to the resolved node Treat unresolved activeNode as loading, target the WebSocket with an explicit nodeId, and wait for stack deep-link hydration so the shell cannot open on the wrong node or compose root. Add regression coverage for node/stack retargeting and fail-closed directory resolution. * fix: harden Host Console node binding, audit acting_as, and console_session tokens Reject unknown or malformed nodeIds before spawning a PTY. Record hub operators in audit_log.acting_as for remote console_session bridges. Path-scope and one-time-consume console_session JWTs so Host Console mints cannot open container exec or be replayed. * test: expect acting_as in audit CSV export header Align the CSV export assertion with the P0-2B acting_as column added to audit log exports. --- .../src/__tests__/api-token-ws-scope.test.ts | 4 + backend/src/__tests__/audit-log.test.ts | 2 +- backend/src/__tests__/auth.test.ts | 3 +- .../capability-registry-pilot.test.ts | 8 +- backend/src/__tests__/host-console-ws.test.ts | 136 ++++++++++++- backend/src/__tests__/host-console.test.ts | 35 +++- .../__tests__/remote-console-session.test.ts | 18 +- .../__tests__/remote-ws-forward-gate.test.ts | 7 + backend/src/__tests__/upgrade-order.test.ts | 96 +++++++++- backend/src/helpers/consoleSession.ts | 74 +++++++- backend/src/routes/auditLog.ts | 2 +- backend/src/routes/console.ts | 25 ++- backend/src/services/CapabilityRegistry.ts | 8 + backend/src/services/DatabaseService.ts | 45 ++++- backend/src/services/HostTerminalService.ts | 3 + backend/src/websocket/generic.ts | 7 +- backend/src/websocket/hostConsole.ts | 70 ++++--- backend/src/websocket/remoteForwarder.ts | 25 ++- backend/src/websocket/upgradeHandler.ts | 96 +++++++++- docs/features/api-tokens.mdx | 4 +- docs/features/deep-links.mdx | 5 +- docs/features/global-search.mdx | 2 +- docs/features/host-console.mdx | 18 +- docs/features/multi-node.mdx | 2 +- docs/features/node-compatibility.mdx | 5 +- docs/features/overview.mdx | 2 +- docs/getting-started/introduction.mdx | 2 +- docs/getting-started/quickstart.mdx | 2 +- frontend/src/components/AuditLogView.tsx | 13 ++ .../components/EditorLayout/ViewRouter.tsx | 64 +++++-- .../__tests__/ViewRouter.test.tsx | 179 ++++++++++++++++++ .../__tests__/useViewNavigationState.test.tsx | 46 ++--- frontend/src/components/HostConsole.tsx | 23 ++- .../components/__tests__/HostConsole.test.tsx | 141 ++++++++++++++ frontend/src/lib/capabilities.ts | 7 + .../navigation/buildNavigationModel.test.ts | 32 +++- .../lib/navigation/buildNavigationModel.ts | 6 - frontend/src/lib/router/readUrlRouteState.ts | 16 +- frontend/src/lib/router/senchoRoute.test.ts | 17 ++ .../lib/routing/hostConsoleCapability.test.ts | 94 +++++++++ .../src/lib/routing/hostConsoleCapability.ts | 49 +++++ frontend/src/lib/routing/reachability.test.ts | 28 +-- frontend/src/lib/routing/reachability.ts | 8 +- 43 files changed, 1230 insertions(+), 199 deletions(-) create mode 100644 frontend/src/components/EditorLayout/__tests__/ViewRouter.test.tsx create mode 100644 frontend/src/components/__tests__/HostConsole.test.tsx create mode 100644 frontend/src/lib/routing/hostConsoleCapability.test.ts create mode 100644 frontend/src/lib/routing/hostConsoleCapability.ts diff --git a/backend/src/__tests__/api-token-ws-scope.test.ts b/backend/src/__tests__/api-token-ws-scope.test.ts index 9d297b3e..eb61959f 100644 --- a/backend/src/__tests__/api-token-ws-scope.test.ts +++ b/backend/src/__tests__/api-token-ws-scope.test.ts @@ -76,4 +76,8 @@ describe('WebSocket API-token scope enforcement', () => { it('does not scope-block a full-admin token from a generic socket', async () => { expect(await upgradeStatus(createToken('full-admin'), '/ws')).not.toBe(403); }); + + it('blocks a full-admin token from the host console (403)', async () => { + expect(await upgradeStatus(createToken('full-admin'), '/api/system/host-console')).toBe(403); + }); }); diff --git a/backend/src/__tests__/audit-log.test.ts b/backend/src/__tests__/audit-log.test.ts index 2ecfa57b..c983c578 100644 --- a/backend/src/__tests__/audit-log.test.ts +++ b/backend/src/__tests__/audit-log.test.ts @@ -715,7 +715,7 @@ describe('GET /api/audit-log/export', () => { const csvText = res.text; const lines = csvText.split('\n'); - expect(lines[0]).toBe('id,timestamp,username,method,path,status_code,node_id,ip_address,summary'); + expect(lines[0]).toBe('id,timestamp,username,acting_as,method,path,status_code,node_id,ip_address,summary'); expect(lines.length).toBeGreaterThan(1); }); diff --git a/backend/src/__tests__/auth.test.ts b/backend/src/__tests__/auth.test.ts index 412ff3f6..0672ea22 100644 --- a/backend/src/__tests__/auth.test.ts +++ b/backend/src/__tests__/auth.test.ts @@ -115,7 +115,8 @@ describe('POST /api/system/console-token', () => { const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' }); const res = await request(app) .post('/api/system/console-token') - .set('Authorization', `Bearer ${token}`); + .set('Authorization', `Bearer ${token}`) + .send({ path: 'host-console' }); expect(res.status).toBe(200); expect(typeof res.body.token).toBe('string'); }); diff --git a/backend/src/__tests__/capability-registry-pilot.test.ts b/backend/src/__tests__/capability-registry-pilot.test.ts index b60ffb68..04231a62 100644 --- a/backend/src/__tests__/capability-registry-pilot.test.ts +++ b/backend/src/__tests__/capability-registry-pilot.test.ts @@ -70,15 +70,18 @@ describe('fetchRemoteMeta Authorization header', () => { describe('applyPilotModeCapabilityFilter', () => { afterEach(() => { enableCapability('host-console'); + enableCapability('host-console-community'); }); - it('removes host-console from active capabilities', () => { + it('removes host-console and host-console-community from active capabilities', () => { expect(CAPABILITIES).toContain('host-console'); + expect(CAPABILITIES).toContain('host-console-community'); applyPilotModeCapabilityFilter(); const active = getActiveCapabilities(); expect(active).not.toContain('host-console'); + expect(active).not.toContain('host-console-community'); expect(active).toContain('stacks'); }); @@ -97,6 +100,7 @@ describe('applyPilotModeCapabilityFilter', () => { const active = getActiveCapabilities(); expect(active).not.toContain('host-console'); - expect(active.length).toBe(CAPABILITIES.length - 1); + expect(active).not.toContain('host-console-community'); + expect(active.length).toBe(CAPABILITIES.length - 2); }); }); diff --git a/backend/src/__tests__/host-console-ws.test.ts b/backend/src/__tests__/host-console-ws.test.ts index 3b0656ce..973ffa91 100644 --- a/backend/src/__tests__/host-console-ws.test.ts +++ b/backend/src/__tests__/host-console-ws.test.ts @@ -21,8 +21,8 @@ describe('WebSocket upgrade - host console auth enforcement', () => { beforeAll(async () => { vi.restoreAllMocks(); tmpDir = await setupTestDb(); - // Host console requires the paid tier; mock the license so the tier gate - // passes for the admin/accepted cases. Individual tests override as needed. + // Host console is available on every tier for admins; mock paid so other + // suites that share LicenseService state stay stable. const { LicenseService } = await import('../services/LicenseService'); getTierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); const mod = await import('../index'); @@ -78,10 +78,28 @@ describe('WebSocket upgrade - host console auth enforcement', () => { expect(await expectRejected(ws)).toBe(403); }); - it('rejects an admin on the Community tier (403)', async () => { + it('accepts a Community-tier admin', async () => { getTierSpy.mockReturnValueOnce('community'); const ws = new WebSocket(wsUrl(), { headers: { Cookie: `sencho_token=${adminToken()}` } }); - expect(await expectRejected(ws)).toBe(403); + const opened = await new Promise((resolve) => { + ws.on('open', () => { ws.close(); resolve(true); }); + ws.on('error', () => resolve(false)); + ws.on('unexpected-response', () => resolve(false)); + }); + expect(opened).toBe(true); + }); + + it('accepts a console_session Bearer on Community (remote bridge mint path)', async () => { + getTierSpy.mockReturnValueOnce('community'); + const { mintConsoleSession } = await import('../helpers/consoleSession'); + const token = mintConsoleSession({ path: 'host-console', actingAs: TEST_USERNAME }); + const ws = new WebSocket(wsUrl(), { headers: { Authorization: `Bearer ${token}` } }); + const opened = await new Promise((resolve) => { + ws.on('open', () => { ws.close(); resolve(true); }); + ws.on('error', () => resolve(false)); + ws.on('unexpected-response', () => resolve(false)); + }); + expect(opened).toBe(true); }); it('accepts an admin on Admiral and records an open audit row', async () => { @@ -127,6 +145,116 @@ describe('WebSocket upgrade - host console auth enforcement', () => { expect(firstMessage).toContain('Invalid stack path'); ws.close(); }); + + + it('rejects an unknown nodeId without spawning a PTY (404)', async () => { + const { HostTerminalService } = await import('../services/HostTerminalService'); + const spawnSpy = vi.spyOn(HostTerminalService, 'spawnTerminal'); + try { + const ws = new WebSocket(wsUrl('?nodeId=99999999'), { + headers: { Cookie: `sencho_token=${adminToken()}` }, + }); + expect(await expectRejected(ws)).toBe(404); + expect(spawnSpy).not.toHaveBeenCalled(); + } finally { + spawnSpy.mockRestore(); + } + }); + + it('rejects a malformed nodeId that parseInt would coerce (404)', async () => { + const ws = new WebSocket(wsUrl('?nodeId=1abc'), { + headers: { Cookie: `sencho_token=${adminToken()}` }, + }); + expect(await expectRejected(ws)).toBe(404); + }); + + it('rejects zero and negative nodeId values (404)', async () => { + for (const id of ['0', '-1']) { + const ws = new WebSocket(wsUrl(`?nodeId=${id}`), { + headers: { Cookie: `sencho_token=${adminToken()}` }, + }); + expect(await expectRejected(ws)).toBe(404); + } + }); + + it('rejects a replayed console_session token on second Host Console upgrade', async () => { + const { mintConsoleSession } = await import('../helpers/consoleSession'); + const token = mintConsoleSession({ path: 'host-console', actingAs: 'qaadmin' }); + const first = new WebSocket(wsUrl(), { headers: { Authorization: `Bearer ${token}` } }); + const firstOpened = await new Promise((resolve) => { + first.on('open', () => { first.close(); resolve(true); }); + first.on('error', () => resolve(false)); + first.on('unexpected-response', () => resolve(false)); + }); + expect(firstOpened).toBe(true); + const second = new WebSocket(wsUrl(), { headers: { Authorization: `Bearer ${token}` } }); + expect(await expectRejected(second)).toBe(401); + }); + + it('rejects a host-console console_session on the container-exec /ws path', async () => { + const { mintConsoleSession } = await import('../helpers/consoleSession'); + const token = mintConsoleSession({ path: 'host-console' }); + const addr = server.address(); + if (!addr || typeof addr === 'string') throw new Error('Server not listening'); + const ws = new WebSocket(`ws://127.0.0.1:${addr.port}/ws`, { + headers: { Authorization: `Bearer ${token}` }, + }); + expect(await expectRejected(ws)).toBe(403); + }); + + it('records acting_as on console_session open audit rows', async () => { + const { mintConsoleSession } = await import('../helpers/consoleSession'); + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance(); + const insertSpy = vi.spyOn(db, 'insertAuditLog'); + const token = mintConsoleSession({ path: 'host-console', actingAs: 'qaadmin' }); + try { + const ws = new WebSocket(wsUrl(), { headers: { Authorization: `Bearer ${token}` } }); + const opened = await new Promise((resolve) => { + ws.on('open', () => { ws.close(); resolve(true); }); + ws.on('error', () => resolve(false)); + ws.on('unexpected-response', () => resolve(false)); + }); + expect(opened).toBe(true); + await waitFor(() => insertSpy.mock.calls.some((c) => { + const e = c[0] as { summary?: string }; + return typeof e.summary === 'string' && e.summary.includes('Opened host console'); + })); + const openEntry = insertSpy.mock.calls + .map((c) => c[0] as { username: string; acting_as?: string | null; summary: string }) + .find((e) => e.summary.includes('Opened host console')); + expect(openEntry?.username).toBe('console_session'); + expect(openEntry?.acting_as).toBe('qaadmin'); + } finally { + insertSpy.mockRestore(); + } + }); + + it('closes without spawning a PTY when directory resolution throws (no default-node fallback)', async () => { + const { FileSystemService } = await import('../services/FileSystemService'); + const { HostTerminalService } = await import('../services/HostTerminalService'); + const spawnSpy = vi.spyOn(HostTerminalService, 'spawnTerminal'); + const getInstanceSpy = vi.spyOn(FileSystemService, 'getInstance').mockImplementation(() => { + throw new Error('compose dir unavailable'); + }); + + try { + const ws = new WebSocket(wsUrl(), { headers: { Cookie: `sencho_token=${adminToken()}` } }); + const firstMessage = await new Promise((resolve) => { + ws.on('message', (data) => resolve(data.toString())); + ws.on('error', () => resolve('')); + ws.on('unexpected-response', () => resolve('')); + }); + expect(firstMessage).toMatch(/Failed to resolve console directory/i); + expect(spawnSpy).not.toHaveBeenCalled(); + // getInstance must not be retried against a fallback/default node id. + expect(getInstanceSpy).toHaveBeenCalledTimes(1); + ws.close(); + } finally { + spawnSpy.mockRestore(); + getInstanceSpy.mockRestore(); + } + }); }); /** Poll a predicate up to ~1s; resolve true as soon as it passes. */ diff --git a/backend/src/__tests__/host-console.test.ts b/backend/src/__tests__/host-console.test.ts index fcbc3510..1187113e 100644 --- a/backend/src/__tests__/host-console.test.ts +++ b/backend/src/__tests__/host-console.test.ts @@ -375,9 +375,42 @@ describe('POST /api/system/console-token', () => { const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' }); const res = await request(app) .post('/api/system/console-token') - .set('Authorization', `Bearer ${token}`); + .set('Authorization', `Bearer ${token}`) + .send({ path: 'host-console' }); expect(res.status).toBe(200); expect(typeof res.body.token).toBe('string'); + const decoded = jwt.verify(res.body.token, TEST_JWT_SECRET) as { scope?: string; path?: string; jti?: string }; + expect(decoded.scope).toBe('console_session'); + expect(decoded.path).toBe('host-console'); + expect(typeof decoded.jti).toBe('string'); + }); + + it('returns 400 when path is missing', async () => { + const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' }); + const res = await request(app) + .post('/api/system/console-token') + .set('Authorization', `Bearer ${token}`) + .send({}); + expect(res.status).toBe(400); + }); + + it('returns 200 for an admin on Community (no paid gate)', async () => { + const { LicenseService } = await import('../services/LicenseService'); + const spy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + try { + const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' }); + const res = await request(app) + .post('/api/system/console-token') + .set('Authorization', `Bearer ${token}`) + .send({ path: 'container-exec' }); + expect(res.status).toBe(200); + expect(typeof res.body.token).toBe('string'); + const decoded = jwt.verify(res.body.token, TEST_JWT_SECRET) as { scope?: string; path?: string }; + expect(decoded.scope).toBe('console_session'); + expect(decoded.path).toBe('container-exec'); + } finally { + spy.mockReturnValue('paid'); + } }); it('returns 403 for non-admin user (viewer role)', async () => { diff --git a/backend/src/__tests__/remote-console-session.test.ts b/backend/src/__tests__/remote-console-session.test.ts index db53cbaa..ffc4f18a 100644 --- a/backend/src/__tests__/remote-console-session.test.ts +++ b/backend/src/__tests__/remote-console-session.test.ts @@ -30,10 +30,9 @@ describe('console_session token parity (HTTP route vs mint helper)', () => { ({ app } = await import('../index')); adminCookie = await loginAsTestAdmin(app); - // POST /api/system/console-token is paid-gated. Seed an active license so the - // parity assertion can observe the token the route returns. The - // license_last_validated fallback is skipped when the state key is absent, so - // the active status alone drives the paid tier. + // POST /api/system/console-token is admin-gated (available on every tier). + // Seed an active license only so other suites that share LicenseService + // state stay stable if they expect paid. const { DatabaseService } = await import('../services/DatabaseService'); DatabaseService.getInstance().setSystemState('license_status', 'active'); }); @@ -43,12 +42,13 @@ describe('console_session token parity (HTTP route vs mint helper)', () => { }); it('POST /api/system/console-token produces a token with the same shape as mintConsoleSession()', async () => { - const directToken = mintConsoleSession(); + const directToken = mintConsoleSession({ path: 'host-console', actingAs: 'testadmin' }); const directDecoded = jwt.verify(directToken, TEST_JWT_SECRET) as Record; const res = await request(app) .post('/api/system/console-token') - .set('Cookie', adminCookie); + .set('Cookie', adminCookie) + .send({ path: 'host-console' }); expect(res.status).toBe(200); expect(typeof res.body.token).toBe('string'); @@ -57,6 +57,12 @@ describe('console_session token parity (HTTP route vs mint helper)', () => { // Identical scope so the remote's upgrade handler treats both the same. expect(routeDecoded.scope).toBe('console_session'); expect(directDecoded.scope).toBe('console_session'); + expect(routeDecoded.path).toBe('host-console'); + expect(directDecoded.path).toBe('host-console'); + expect(typeof routeDecoded.jti).toBe('string'); + expect(typeof directDecoded.jti).toBe('string'); + expect(routeDecoded.acting_as).toBe('testadmin'); + expect(directDecoded.acting_as).toBe('testadmin'); // Same claim keys: if somebody adds or drops a claim on one path but not // the other, remote upgrade behavior will diverge. diff --git a/backend/src/__tests__/remote-ws-forward-gate.test.ts b/backend/src/__tests__/remote-ws-forward-gate.test.ts index 1b752ce8..6bd2594d 100644 --- a/backend/src/__tests__/remote-ws-forward-gate.test.ts +++ b/backend/src/__tests__/remote-ws-forward-gate.test.ts @@ -58,6 +58,13 @@ describe('remoteWsForwardAllowed', () => { expect(remoteWsForwardAllowed(EXEC, { ...base, wsApiTokenScope: 'deploy-only', decoded: { scope: 'api_token' } })).toBe(false); }); + it('denies every API-token scope for Host Console while preserving full-admin for container exec', () => { + expect(remoteWsForwardAllowed(CONSOLE, { ...base, wsApiTokenScope: 'full-admin', decoded: { scope: 'api_token' } })).toBe(false); + expect(remoteWsForwardAllowed(CONSOLE, { ...base, wsApiTokenScope: 'deploy-only', decoded: { scope: 'api_token' } })).toBe(false); + expect(remoteWsForwardAllowed(CONSOLE, { ...base, wsApiTokenScope: 'read-only', decoded: { scope: 'api_token' } })).toBe(false); + expect(remoteWsForwardAllowed(EXEC, { ...base, wsApiTokenScope: 'full-admin', decoded: { scope: 'api_token' } })).toBe(true); + }); + it('allows an interactive path for a pre-gated console_session token', () => { expect(remoteWsForwardAllowed(CONSOLE, { ...base, decoded: { scope: 'console_session' } })).toBe(true); }); diff --git a/backend/src/__tests__/upgrade-order.test.ts b/backend/src/__tests__/upgrade-order.test.ts index dc5a6056..1f1cf15a 100644 --- a/backend/src/__tests__/upgrade-order.test.ts +++ b/backend/src/__tests__/upgrade-order.test.ts @@ -8,7 +8,7 @@ * product paths. This test pins each position by observing behavior that * could only originate from the expected handler. */ -import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest'; import WebSocket from 'ws'; import jwt from 'jsonwebtoken'; import bcrypt from 'bcrypt'; @@ -420,6 +420,100 @@ describe('WebSocket upgrade dispatch order', () => { expect(outcome.kind).toBe('unexpected'); if (outcome.kind === 'unexpected') expect(outcome.status).toBe(403); }); + + it('rejects a full-admin API token remote Host Console with 403 (container exec stays allowed)', async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const adminId = DatabaseService.getInstance().getUserByUsername(TEST_USERNAME)!.id; + const rawToken = createTestApiToken({ + db: DatabaseService, + scope: 'full-admin', + userId: adminId, + name: `host-console-api-deny-${Date.now()}`, + }); + const consoleWs = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { bearer: rawToken }); + const consoleOutcome = await waitForOutcome(consoleWs); + expect(consoleOutcome.kind).toBe('unexpected'); + if (consoleOutcome.kind === 'unexpected') expect(consoleOutcome.status).toBe(403); + + const execWs = connect(`/ws?nodeId=${remoteNodeId}`, { bearer: rawToken }); + const execOutcome = await waitForOutcome(execWs); + if (execOutcome.kind === 'unexpected') expect(execOutcome.status).not.toBe(403); + try { execWs.terminate(); } catch { /* ignore */ } + }); + }); + + describe('remote Host Console Community capability probe', () => { + // Community hubs must probe host-console-community before forwarding Host + // Console; Admiral hubs skip the probe so legacy remotes still work. + // Fail closed when the probe errors or the capability is missing. + + const meta = (capabilities: string[]): import('../services/CapabilityRegistry').RemoteMeta => ({ + version: '0.95.0', + capabilities, + startedAt: 1, + updateError: null, + online: true, + imagePinKind: null, + updateBlocked: false, + imageChannel: null, + }); + + async function setLicense(status: string): Promise { + const { DatabaseService } = await import('../services/DatabaseService'); + DatabaseService.getInstance().setSystemState('license_status', status); + } + + afterEach(async () => { + vi.restoreAllMocks(); + await setLicense('active'); + }); + + it('rejects Community hub remote Host Console when the remote lacks host-console-community', async () => { + await setLicense('community'); + const { NodeRegistry } = await import('../services/NodeRegistry'); + vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue(meta(['host-console'])); + + const ws = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { cookie: sessionCookie }); + const outcome = await waitForOutcome(ws); + expect(outcome.kind).toBe('unexpected'); + if (outcome.kind === 'unexpected') expect(outcome.status).toBe(403); + }); + + it('rejects Community hub remote Host Console when the capability probe fails', async () => { + await setLicense('community'); + const { NodeRegistry } = await import('../services/NodeRegistry'); + vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockRejectedValue(new Error('offline')); + + const ws = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { cookie: sessionCookie }); + const outcome = await waitForOutcome(ws); + expect(outcome.kind).toBe('unexpected'); + if (outcome.kind === 'unexpected') expect(outcome.status).toBe(403); + }); + + it('forwards Community hub remote Host Console when the remote advertises host-console-community', async () => { + await setLicense('community'); + const { NodeRegistry } = await import('../services/NodeRegistry'); + vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue( + meta(['host-console-community']), + ); + + const ws = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { cookie: sessionCookie }); + const outcome = await waitForOutcome(ws); + if (outcome.kind === 'unexpected') expect(outcome.status).not.toBe(403); + try { ws.terminate(); } catch { /* ignore */ } + }); + + it('skips the capability probe on an Admiral hub even when the remote lacks host-console-community', async () => { + await setLicense('active'); + const { NodeRegistry } = await import('../services/NodeRegistry'); + const spy = vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue(meta([])); + + const ws = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { cookie: sessionCookie }); + const outcome = await waitForOutcome(ws); + if (outcome.kind === 'unexpected') expect(outcome.status).not.toBe(403); + expect(spy).not.toHaveBeenCalled(); + try { ws.terminate(); } catch { /* ignore */ } + }); }); it('dispatches /api/pilot/tunnel to the pilot handler (rejects non-pilot bearer before path-based dispatch)', async () => { diff --git a/backend/src/helpers/consoleSession.ts b/backend/src/helpers/consoleSession.ts index 90565e31..6b32706b 100644 --- a/backend/src/helpers/consoleSession.ts +++ b/backend/src/helpers/consoleSession.ts @@ -1,17 +1,55 @@ +import { randomUUID } from 'crypto'; import jwt from 'jsonwebtoken'; import { DatabaseService } from '../services/DatabaseService'; /** - * Console session token lifetime. Short on purpose: these tokens are only - * used to bridge an already-authenticated HTTP request into a WebSocket - * upgrade, and each one is consumed by a single `wss:ws(target)` call. + * Console session token lifetime. Short on purpose: these tokens bridge an + * already-authenticated hub request into a remote WebSocket upgrade. Each + * token is path-scoped and consumed on first interactive WebSocket upgrade + * acceptance (before PTY spawn). */ const CONSOLE_SESSION_TTL_SECONDS = 60; const CONSOLE_SESSION_SCOPE = 'console_session'; +/** Interactive surfaces that may mint or accept a console_session JWT. */ +export type ConsoleSessionPath = 'host-console' | 'container-exec'; + +export interface MintConsoleSessionOptions { + path: ConsoleSessionPath; + /** Hub operator identity for remote audit (option B). Never used as the session principal. */ + actingAs?: string; +} + export interface ConsoleSessionClaims { scope: typeof CONSOLE_SESSION_SCOPE; - username?: string; + path: ConsoleSessionPath; + acting_as?: string; +} + +const ACTING_AS_MAX = 64; +const ACTING_AS_RE = /^[A-Za-z0-9._@+-]+$/; + +/** Sanitize an optional hub operator name for the acting_as claim. */ +export function sanitizeActingAs(raw: unknown): string | undefined { + if (typeof raw !== 'string') return undefined; + const trimmed = raw.trim(); + if (!trimmed || trimmed.length > ACTING_AS_MAX) return undefined; + if (!ACTING_AS_RE.test(trimmed)) return undefined; + return trimmed; +} + +export function isConsoleSessionPath(value: unknown): value is ConsoleSessionPath { + return value === 'host-console' || value === 'container-exec'; +} + +/** + * Map a WebSocket pathname to the console_session path claim it requires. + * Returns null for surfaces that must not accept console_session tokens. + */ +export function consoleSessionPathForPathname(pathname: string): ConsoleSessionPath | null { + if (pathname.startsWith('/api/system/host-console')) return 'host-console'; + if (pathname === '/ws') return 'container-exec'; + return null; } /** @@ -19,13 +57,19 @@ export interface ConsoleSessionClaims { * instance without leaking the long-lived node api_token onto a * machine-to-machine WebSocket. Throws if the JWT secret is not configured. */ -export function mintConsoleSession(username?: string): string { +export function mintConsoleSession(opts: MintConsoleSessionOptions): string { + if (!isConsoleSessionPath(opts.path)) { + throw new Error('Invalid console session path'); + } const jwtSecret = DatabaseService.getInstance().getGlobalSettings().auth_jwt_secret; if (!jwtSecret) throw new Error('No JWT secret configured'); - const payload: ConsoleSessionClaims = username - ? { scope: CONSOLE_SESSION_SCOPE, username } - : { scope: CONSOLE_SESSION_SCOPE }; - return jwt.sign(payload, jwtSecret, { expiresIn: CONSOLE_SESSION_TTL_SECONDS }); + const payload: ConsoleSessionClaims = { scope: CONSOLE_SESSION_SCOPE, path: opts.path }; + const actingAs = sanitizeActingAs(opts.actingAs); + if (actingAs) payload.acting_as = actingAs; + return jwt.sign(payload, jwtSecret, { + expiresIn: CONSOLE_SESSION_TTL_SECONDS, + jwtid: randomUUID(), + }); } /** True when `decoded.scope` is the console_session scope. */ @@ -33,4 +77,14 @@ export function isConsoleSessionScope(scope: unknown): boolean { return scope === CONSOLE_SESSION_SCOPE; } -export { CONSOLE_SESSION_SCOPE }; +/** + * Mark a console_session jti as used. Returns false when the jti is missing, + * already consumed, or cannot be recorded (replay / malformed token). + */ +export function consumeConsoleSessionJti(jti: unknown, expiresAtMs: number): boolean { + if (typeof jti !== 'string' || !jti) return false; + if (!Number.isFinite(expiresAtMs)) return false; + return DatabaseService.getInstance().consumeConsoleSessionJti(jti, expiresAtMs); +} + +export { CONSOLE_SESSION_SCOPE, CONSOLE_SESSION_TTL_SECONDS }; diff --git a/backend/src/routes/auditLog.ts b/backend/src/routes/auditLog.ts index d4780b10..5f0760fd 100644 --- a/backend/src/routes/auditLog.ts +++ b/backend/src/routes/auditLog.ts @@ -103,7 +103,7 @@ auditLogRouter.get('/export', async (req: Request, res: Response): Promise res.setHeader('Content-Type', 'text/csv'); res.setHeader('Content-Disposition', `attachment; filename="audit-log-${timestamp}.csv"`); - const headers = ['id', 'timestamp', 'username', 'method', 'path', 'status_code', 'node_id', 'ip_address', 'summary']; + const headers = ['id', 'timestamp', 'username', 'acting_as', 'method', 'path', 'status_code', 'node_id', 'ip_address', 'summary']; const rows = result.entries.map(e => headers.map(h => escapeCsvField(e[h as keyof typeof e])).join(','), ); diff --git a/backend/src/routes/console.ts b/backend/src/routes/console.ts index ae7fbe7f..7986c6bc 100644 --- a/backend/src/routes/console.ts +++ b/backend/src/routes/console.ts @@ -1,8 +1,12 @@ import { Router, type Request, type Response } from 'express'; import { authMiddleware } from '../middleware/auth'; -import { requireAdmin, requirePaid } from '../middleware/tierGates'; +import { requireAdmin } from '../middleware/tierGates'; import { rejectApiTokenScope } from '../middleware/apiTokenScope'; -import { mintConsoleSession } from '../helpers/consoleSession'; +import { + isConsoleSessionPath, + mintConsoleSession, + sanitizeActingAs, +} from '../helpers/consoleSession'; /** * Mint a short-lived `console_session` JWT. Used by the gateway when it @@ -11,15 +15,28 @@ import { mintConsoleSession } from '../helpers/consoleSession'; * remote's upgrade handler on interactive paths, so the gateway authenticates * with the long-lived token, asks for this short-lived one, then forwards * the WS upgrade using it. + * + * Body: + * - `path` (required): `host-console` | `container-exec` + * - `acting_as` (optional): hub operator for remote audit. On node_proxy mints + * the body value is used (sanitized). On browser admin mints the signed-in + * username is stamped so a Bearer console_session cannot erase attribution. */ export const consoleRouter = Router(); consoleRouter.post('/console-token', authMiddleware, (req: Request, res: Response): void => { if (rejectApiTokenScope(req, res, 'API tokens cannot generate console tokens.')) return; if (!requireAdmin(req, res)) return; - if (!requirePaid(req, res)) return; try { - res.json({ token: mintConsoleSession() }); + const path = req.body?.path; + if (!isConsoleSessionPath(path)) { + res.status(400).json({ error: 'Invalid or missing console path' }); + return; + } + const actingAs = req.machineAuthScope === 'node_proxy' + ? sanitizeActingAs(req.body?.acting_as) + : sanitizeActingAs(req.user?.username); + res.json({ token: mintConsoleSession({ path, actingAs }) }); } catch (error) { console.error('Failed to issue console token:', error); res.status(500).json({ error: 'Failed to issue console token' }); diff --git a/backend/src/services/CapabilityRegistry.ts b/backend/src/services/CapabilityRegistry.ts index 1ce65b55..578155a9 100644 --- a/backend/src/services/CapabilityRegistry.ts +++ b/backend/src/services/CapabilityRegistry.ts @@ -38,6 +38,7 @@ export const CAPABILITIES = [ 'notification-suppression', 'notification-suppression-schedule', 'host-console', + 'host-console-community', 'container-exec', 'audit-log', 'scheduled-ops', @@ -71,6 +72,12 @@ export const CROSS_NODE_RBAC_CAPABILITY = 'cross-node-rbac'; export type Capability = (typeof CAPABILITIES)[number]; +/** Legacy Host Console advertisement (Admiral hubs still accept this on remotes). */ +export const HOST_CONSOLE_CAPABILITY = 'host-console' as const satisfies Capability; + +/** Host Console works without a paid license on this node. */ +export const HOST_CONSOLE_COMMUNITY_CAPABILITY = 'host-console-community' as const satisfies Capability; + /** Remotes that evaluate weekly maintenance windows on mute/suppression replicas. */ export const NOTIFICATION_SUPPRESSION_SCHEDULE_CAPABILITY = 'notification-suppression-schedule' as const satisfies Capability; @@ -165,6 +172,7 @@ export function getActiveCapabilities(): readonly string[] { */ const PILOT_DISABLED_CAPABILITIES: readonly Capability[] = [ 'host-console', + 'host-console-community', ]; /** Disable capabilities that require a central->pilot path that is not yet wired. */ diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 92090b04..2d36dd49 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -613,6 +613,8 @@ export interface AuditLogEntry { node_id: number | null; ip_address: string; summary: string; + /** Hub operator for remote console_session bridges; null/absent for direct sessions. */ + acting_as?: string | null; } export interface SecretRow { @@ -1240,12 +1242,20 @@ export class DatabaseService { status_code INTEGER NOT NULL DEFAULT 0, node_id INTEGER, ip_address TEXT NOT NULL DEFAULT '', - summary TEXT NOT NULL DEFAULT '' + summary TEXT NOT NULL DEFAULT '', + acting_as TEXT ); CREATE INDEX IF NOT EXISTS idx_audit_log_timestamp ON audit_log(timestamp); CREATE INDEX IF NOT EXISTS idx_audit_log_username ON audit_log(username); + CREATE TABLE IF NOT EXISTS console_session_jtis ( + jti TEXT PRIMARY KEY, + used_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_console_session_jtis_expires ON console_session_jtis(expires_at); + CREATE TABLE IF NOT EXISTS api_tokens ( id INTEGER PRIMARY KEY AUTOINCREMENT, token_hash TEXT NOT NULL UNIQUE, @@ -1788,6 +1798,21 @@ export class DatabaseService { try { this.db.prepare(`ALTER TABLE ${table} ADD COLUMN ${col} ${def}`).run(); } catch (e) { /* ignore */ } }; + // Remote Host Console bridges record the hub operator separately from + // the console_session principal (username stays console_session). + maybeAddCol('audit_log', 'acting_as', 'TEXT'); + // Cached INSERT may predate the column; rebuild on next flush. + this.auditLogInsertStmt = null; + + this.db.exec(` + CREATE TABLE IF NOT EXISTS console_session_jtis ( + jti TEXT PRIMARY KEY, + used_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_console_session_jtis_expires ON console_session_jtis(expires_at); + `); + maybeAddCol('stack_update_recovery_generations', 'artifacts_retired', 'INTEGER NOT NULL DEFAULT 0'); // Distributed API model columns @@ -4462,6 +4487,21 @@ export class DatabaseService { this.db.prepare('DELETE FROM pilot_enrollments WHERE node_id = ?').run(nodeId); } + /** + * One-time consume for a console_session JWT id. Inserts the jti on first + * use; returns false when the jti was already recorded (replay). + */ + public consumeConsoleSessionJti(jti: string, expiresAtMs: number): boolean { + const now = Date.now(); + return this.db.transaction(() => { + this.db.prepare('DELETE FROM console_session_jtis WHERE expires_at < ?').run(now); + const result = this.db.prepare( + 'INSERT OR IGNORE INTO console_session_jtis (jti, used_at, expires_at) VALUES (?, ?, ?)', + ).run(jti, now, expiresAtMs); + return result.changes > 0; + })(); + } + // --- Stack Update Status --- /** @@ -5165,7 +5205,7 @@ export class DatabaseService { this.auditLogBuffer = []; if (!this.auditLogInsertStmt) { this.auditLogInsertStmt = this.db.prepare( - 'INSERT INTO audit_log (timestamp, username, method, path, status_code, node_id, ip_address, summary) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + 'INSERT INTO audit_log (timestamp, username, method, path, status_code, node_id, ip_address, summary, acting_as) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', ); } const stmt = this.auditLogInsertStmt; @@ -5180,6 +5220,7 @@ export class DatabaseService { entry.node_id, entry.ip_address, entry.summary, + entry.acting_as ?? null, ); } }); diff --git a/backend/src/services/HostTerminalService.ts b/backend/src/services/HostTerminalService.ts index d5c8d927..20627322 100644 --- a/backend/src/services/HostTerminalService.ts +++ b/backend/src/services/HostTerminalService.ts @@ -18,6 +18,8 @@ export interface ConsoleAuditContext { // path always supplies a concrete id. readonly nodeId: number | null; readonly ipAddress: string; + /** Hub operator for remote console_session bridges; null for direct sessions. */ + readonly actingAs?: string | null; } const CONSOLE_AUDIT_PATH = '/api/system/host-console'; @@ -97,6 +99,7 @@ export class HostTerminalService { node_id: audit.nodeId, ip_address: audit.ipAddress, summary, + acting_as: audit.actingAs ?? null, }); } catch (err) { console.error('[HostConsole] Failed to write session audit log:', err); diff --git a/backend/src/websocket/generic.ts b/backend/src/websocket/generic.ts index 9e2edfaf..bf93c84b 100644 --- a/backend/src/websocket/generic.ts +++ b/backend/src/websocket/generic.ts @@ -54,9 +54,10 @@ export function handleGenericWs( if (isProxyToken) return reject(socket, 403, 'Forbidden'); // Admin enforcement: container exec requires admin role. - // console_session tokens are already admin-gated at creation time. - // API tokens reaching this point have full-admin scope (read-only / - // deploy-only are blocked by the upgrade handler's scope gate). + // console_session tokens are already admin-gated at creation time and + // path/jti-gated in upgradeHandler. API tokens reaching this point have + // full-admin scope (read-only / deploy-only are blocked by the upgrade + // handler's scope gate). if (!decoded.scope) { const execUser = decoded.username ? DatabaseService.getInstance().getUserByUsername(decoded.username) : undefined; if (!execUser) { diff --git a/backend/src/websocket/hostConsole.ts b/backend/src/websocket/hostConsole.ts index eaf26ade..08c137d3 100644 --- a/backend/src/websocket/hostConsole.ts +++ b/backend/src/websocket/hostConsole.ts @@ -2,22 +2,16 @@ import type { IncomingMessage } from 'http'; import type { Duplex } from 'stream'; import WebSocket, { WebSocketServer } from 'ws'; import { FileSystemService } from '../services/FileSystemService'; -import { NodeRegistry } from '../services/NodeRegistry'; import { HostTerminalService } from '../services/HostTerminalService'; -import { PROXY_TIER_HEADER } from '../services/license-headers'; -import { - isLicenseTier, - normalizeTier, -} from '../services/license-normalize'; -import { LicenseService } from '../services/LicenseService'; -import { ROLE_PERMISSIONS, type PermissionAction } from '../middleware/permissions'; +import { ROLE_PERMISSIONS } from '../middleware/permissions'; import type { UserRole } from '../services/DatabaseService'; import { getErrorMessage } from '../utils/errors'; import { rejectUpgrade as reject } from './reject'; +import { isConsoleSessionScope } from '../helpers/consoleSession'; interface HostConsoleContext { nodeId: number; - decoded: { scope?: string; username?: string }; + decoded: { scope?: string; username?: string; acting_as?: string }; isProxyToken: boolean; wsResolvedUser: { username: string; role: UserRole; token_version: number } | undefined; stackParam: string | null; @@ -26,15 +20,16 @@ interface HostConsoleContext { /** * Handle `/api/system/host-console` WebSocket upgrades. * - * Enforces three gates before spawning the host PTY: + * Enforces two gates before spawning the host PTY: * 1. Machine-credential rejection: node_proxy tokens cannot reach an - * interactive host shell. + * interactive host shell directly (remote forwarding mints a + * console_session via POST /console-token instead). * 2. RBAC: user session tokens require the `system:console` permission. * console_session tokens are pre-gated at issuance (see * `routes/console.ts`) and skip this check. - * 3. License: host console requires the paid tier. For console_session - * tokens the tier is trusted from the gateway-supplied header; - * otherwise the local LicenseService is consulted. + * + * Path scoping and one-time jti consumption are enforced in upgradeHandler + * before this handler runs. */ export function handleHostConsoleWs( req: IncomingMessage, @@ -46,11 +41,10 @@ export function handleHostConsoleWs( if (isProxyToken) return reject(socket, 403, 'Forbidden'); - const isConsoleSession = decoded.scope === 'console_session'; + const isConsoleSession = isConsoleSessionScope(decoded.scope); if (!isConsoleSession) { const userRole = wsResolvedUser?.role; - const consolePermission: PermissionAction = 'system:console'; - if (!userRole || !ROLE_PERMISSIONS[userRole]?.includes(consolePermission)) { + if (!userRole || !ROLE_PERMISSIONS[userRole]?.includes('system:console')) { console.log('[HostConsole] Access denied: insufficient permissions', { username: wsResolvedUser?.username || decoded.username, role: userRole, @@ -59,18 +53,18 @@ export function handleHostConsoleWs( } } - const consoleTierHeader = req.headers[PROXY_TIER_HEADER] as string | undefined; - const ls = LicenseService.getInstance(); - const consoleTier = (isConsoleSession && isLicenseTier(consoleTierHeader)) - ? normalizeTier(consoleTierHeader) - : ls.getTier(); - if (consoleTier !== 'paid') { - return reject(socket, 403, 'Forbidden'); - } + // Option B: console_session principal stays "console_session"; hub operator + // is recorded separately as acting_as. Browser sessions use the real user. + const consoleUsername = isConsoleSession + ? 'console_session' + : (wsResolvedUser?.username || decoded.username || 'unknown'); + const actingAs = isConsoleSession && typeof decoded.acting_as === 'string' && decoded.acting_as + ? decoded.acting_as + : null; - const consoleUsername = wsResolvedUser?.username || decoded.username || 'console_session'; console.log('[HostConsole] WebSocket upgrade accepted', { username: consoleUsername, + actingAs, nodeId, stack: stackParam || '(root)', }); @@ -86,10 +80,6 @@ export function handleHostConsoleWs( hostConsoleWss.handleUpgrade(req, socket, head, (ws) => { hostConsoleWss.close(); let targetDirectory: string; - // The shell may end up rooted at a different node than requested if the - // requested node's directory cannot be resolved; the audit row must name - // the node the shell actually runs in, so track it alongside the directory. - let auditNodeId: number = nodeId; try { const baseDir = FileSystemService.getInstance(nodeId).getBaseDir(); const resolved = HostTerminalService.resolveConsoleDirectory(baseDir, stackParam); @@ -100,22 +90,28 @@ export function handleHostConsoleWs( } targetDirectory = resolved; } catch (error) { - const fallbackNodeId = NodeRegistry.getInstance().getDefaultNodeId(); - console.error('[HostConsole] Failed to resolve console directory; falling back to the default node base dir', { + console.error('[HostConsole] Failed to resolve console directory', { user: consoleUsername, + actingAs, nodeId, - fallbackNodeId, stack: stackParam || '(root)', error: getErrorMessage(error, 'unknown'), }); - targetDirectory = FileSystemService.getInstance(fallbackNodeId).getBaseDir(); - auditNodeId = fallbackNodeId; + if (ws.readyState === WebSocket.OPEN) { + ws.send('Error: Failed to resolve console directory.\r\n'); + ws.close(); + } + return; } - const auditCtx = { username: consoleUsername, nodeId: auditNodeId, ipAddress }; + const auditCtx = { username: consoleUsername, actingAs, nodeId, ipAddress }; try { HostTerminalService.spawnTerminal(ws, targetDirectory, auditCtx); } catch (error) { - console.error('[HostConsole] Unhandled spawn error:', { user: consoleUsername, error: getErrorMessage(error, 'unknown') }); + console.error('[HostConsole] Unhandled spawn error:', { + user: consoleUsername, + actingAs, + error: getErrorMessage(error, 'unknown'), + }); if (ws.readyState === WebSocket.OPEN) { ws.send('Error: Failed to start terminal session.\r\n'); ws.close(); diff --git a/backend/src/websocket/remoteForwarder.ts b/backend/src/websocket/remoteForwarder.ts index 60fcbd93..37059c56 100644 --- a/backend/src/websocket/remoteForwarder.ts +++ b/backend/src/websocket/remoteForwarder.ts @@ -5,6 +5,7 @@ import { LicenseService } from '../services/LicenseService'; import { wsProxyServer } from '../proxy/websocketProxy'; import { getErrorMessage } from '../utils/errors'; import { rejectUpgrade as reject } from './reject'; +import { consoleSessionPathForPathname } from '../helpers/consoleSession'; /** * Forward a WebSocket upgrade to a remote Sencho instance. Handles the @@ -23,9 +24,14 @@ export async function handleRemoteForwarder( req: IncomingMessage, socket: Duplex, head: Buffer, - opts: { pathname: string; target: { apiUrl: string; apiToken: string } }, + opts: { + pathname: string; + target: { apiUrl: string; apiToken: string }; + /** Hub browser operator; recorded as acting_as on the remote audit trail. */ + actingAs?: string; + }, ): Promise { - const { pathname, target } = opts; + const { pathname, target, actingAs } = opts; if (!target.apiUrl) return reject(socket, 503, 'Service Unavailable'); const wsTarget = target.apiUrl.replace(/\/$/, '').replace(/^https?/, (m) => m === 'https' ? 'wss' : 'ws'); @@ -38,24 +44,33 @@ export async function handleRemoteForwarder( // direct api_token access. Pilot loopback targets skip this: there is no // long-lived api_token to exchange, and host-console is disabled on pilot // mode at the capability registry anyway. - const isInteractiveConsolePath = pathname === '/api/system/host-console' || pathname === '/ws'; + const sessionPath = consoleSessionPathForPathname(pathname); let bearerTokenForProxy = target.apiToken; - if (isInteractiveConsolePath && !isPilotLoopback) { + if (sessionPath && !isPilotLoopback) { try { const consoleHeaders = LicenseService.getInstance().getProxyHeaders(); const tokenRes = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/system/console-token`, { method: 'POST', headers: { 'Authorization': `Bearer ${target.apiToken}`, + 'Content-Type': 'application/json', [PROXY_TIER_HEADER]: consoleHeaders.tier, }, + body: JSON.stringify({ + path: sessionPath, + ...(actingAs ? { acting_as: actingAs } : {}), + }), }); if (!tokenRes.ok) { console.error(`[WS Proxy] Remote console-token request failed: ${tokenRes.status}`); return reject(socket, 502, 'Bad Gateway'); } const data = await tokenRes.json() as { token?: string }; - if (typeof data.token === 'string') bearerTokenForProxy = data.token; + if (typeof data.token !== 'string' || !data.token) { + console.error('[WS Proxy] Remote console-token response missing token'); + return reject(socket, 502, 'Bad Gateway'); + } + bearerTokenForProxy = data.token; } catch (e) { console.error('[WS Proxy] Failed to fetch remote console token:', getErrorMessage(e, 'unknown')); return reject(socket, 502, 'Bad Gateway'); diff --git a/backend/src/websocket/upgradeHandler.ts b/backend/src/websocket/upgradeHandler.ts index e23a7b55..201ba34a 100644 --- a/backend/src/websocket/upgradeHandler.ts +++ b/backend/src/websocket/upgradeHandler.ts @@ -20,6 +20,14 @@ import { validateApiToken, touchApiTokenLastUsed } from '../utils/apiTokenAuth'; import { isDebugEnabled } from '../utils/debug'; import { PROXY_TIER_HEADER } from '../services/license-headers'; import { isLicenseTier, normalizeTier } from '../services/license-normalize'; +import { HOST_CONSOLE_COMMUNITY_CAPABILITY } from '../services/CapabilityRegistry'; +import { remoteAdvertisesCapability } from '../helpers/remoteCapabilities'; + +import { + consoleSessionPathForPathname, + consumeConsoleSessionJti, + isConsoleSessionScope, +} from '../helpers/consoleSession'; function parseCookies(req: IncomingMessage): Record { const header = req.headers.cookie || ''; @@ -31,6 +39,17 @@ function parseCookies(req: IncomingMessage): Record { ); } +/** + * Parse ?nodeId= as a strict positive integer. Returns null when the param is + * absent (caller should use the default node). Returns undefined when the + * param is present but malformed (caller should 404). + */ +function parseStrictNodeIdParam(raw: string | null): number | null | undefined { + if (raw == null || raw === '') return null; + if (!/^[1-9][0-9]*$/.test(raw)) return undefined; + return Number(raw); +} + // The two WebSocket paths open to any authenticated user (read-only/deploy-only // API tokens and non-admin sessions on a remote node). Defined once so the // scope gate and the remote-forward gate cannot drift apart. @@ -68,8 +87,15 @@ export function remoteWsForwardAllowed( if (ctx.isProxyToken) return false; // console_session is pre-gated as admin at issuance (routes/console.ts). if (ctx.decoded.scope === 'console_session') return true; + // Reject opaque API tokens on remote Host Console forward (would mint + // console_session and get a host shell). Local denial is enforced in + // attachUpgrade before handleHostConsoleWs. Container exec (/ws) keeps + // its existing full-admin API-token allow. + if (pathname.startsWith('/api/system/host-console') && ctx.wsApiTokenScope) { + return false; + } // A read-only/deploy-only api_token is already rejected for these paths by - // the scope gate; only full-admin survives to here. + // the scope gate; only full-admin survives to here (container exec /ws). if (ctx.wsApiTokenScope) return ctx.wsApiTokenScope === 'full-admin'; const role = ctx.wsResolvedUser?.role; if (!role) return false; @@ -136,7 +162,16 @@ export function attachUpgrade( try { // Opaque sen_sk_ API tokens: handled before jwt.verify. Prefix + // length + checksum reject malformed keys without touching SQLite. - let decoded: { username?: string; scope?: string; role?: string; tv?: number }; + let decoded: { + username?: string; + scope?: string; + role?: string; + tv?: number; + path?: string; + acting_as?: string; + jti?: string; + exp?: number; + }; let wsApiTokenScope: string | null = null; if (looksLikeApiToken(token)) { const validation = validateApiToken(token); @@ -151,7 +186,7 @@ export function attachUpgrade( const settings = DatabaseService.getInstance().getGlobalSettings(); const jwtSecret = settings.auth_jwt_secret; if (!jwtSecret) throw new Error('No JWT secret'); - decoded = jwt.verify(token, jwtSecret) as { username?: string; scope?: string; role?: string; tv?: number }; + decoded = jwt.verify(token, jwtSecret) as typeof decoded; } // Node proxy tokens are machine-to-machine credentials and must never be @@ -180,6 +215,21 @@ export function attachUpgrade( const parsedUrl = new URL(req.url || '', `http://${req.headers.host || 'localhost'}`); const pathname = parsedUrl.pathname; + // Path-scoped, one-time console_session tokens for interactive surfaces. + // Consume runs at upgrade acceptance (before PTY spawn); missing exp/jti fail closed. + if (isConsoleSessionScope(decoded.scope)) { + const requiredPath = consoleSessionPathForPathname(pathname); + if (!requiredPath || decoded.path !== requiredPath) { + return reject(socket, 403, 'Forbidden'); + } + if (typeof decoded.exp !== 'number' || typeof decoded.jti !== 'string' || !decoded.jti) { + return reject(socket, 401, 'Unauthorized'); + } + if (!consumeConsoleSessionJti(decoded.jti, decoded.exp * 1000)) { + return reject(socket, 401, 'Unauthorized'); + } + } + // Gate WebSocket paths by API token scope if (wsApiTokenScope) { if (wsApiTokenScope === 'read-only' || wsApiTokenScope === 'deploy-only') { @@ -219,8 +269,11 @@ export function attachUpgrade( return; } - const nodeIdParam = parsedUrl.searchParams.get('nodeId'); - const nodeId = nodeIdParam ? parseInt(nodeIdParam, 10) : NodeRegistry.getInstance().getDefaultNodeId(); + const parsedNodeId = parseStrictNodeIdParam(parsedUrl.searchParams.get('nodeId')); + if (parsedNodeId === undefined) { + return reject(socket, 404, 'Not Found'); + } + const nodeId = parsedNodeId ?? NodeRegistry.getInstance().getDefaultNodeId(); const node = NodeRegistry.getInstance().getNode(nodeId); // Notification push channel: local only when no remote nodeId is @@ -243,6 +296,23 @@ export function attachUpgrade( if (!remoteWsForwardAllowed(pathname, { wsResolvedUser, wsApiTokenScope, isProxyToken, decoded })) { return reject(socket, 403, 'Forbidden'); } + // Community hubs require host-console-community before forwarding Host + // Console (marks remotes that no longer paid-gate the console path). + // Admiral hubs skip the probe for legacy host-console peers. Fail + // closed on probe errors; never fall through to local handlers for a + // named remote. + if ( + pathname.startsWith('/api/system/host-console') + && LicenseService.getInstance().getTier() !== 'paid' + ) { + const communityCapable = await remoteAdvertisesCapability( + nodeId, + HOST_CONSOLE_COMMUNITY_CAPABILITY, + ); + if (!communityCapable) { + return reject(socket, 403, 'Forbidden'); + } + } // Resolve the proxy target through NodeRegistry so pilot-mode nodes // (empty api_url + api_token, loopback bridge instead) and proxy-mode // nodes share one dispatch path. Mirrors proxy/remoteNodeProxy.ts. @@ -253,7 +323,11 @@ export function attachUpgrade( // serve gateway-local data for a request that named a remote node. return reject(socket, 503, 'Service Unavailable'); } - await handleRemoteForwarder(req, socket, head, { pathname, target }); + await handleRemoteForwarder(req, socket, head, { + pathname, + target, + actingAs: wsResolvedUser?.username || decoded.username, + }); return; } @@ -264,6 +338,16 @@ export function attachUpgrade( } if (pathname.startsWith('/api/system/host-console')) { + // Opaque API tokens never open a local host shell (parity with the + // remote forward gate). Do not rely on missing wsResolvedUser alone. + if (wsApiTokenScope) { + return reject(socket, 403, 'Forbidden'); + } + // Unknown / deleted nodeIds must not fall through to the hub compose + // root via getComposeDir's env default. + if (!node) { + return reject(socket, 404, 'Not Found'); + } handleHostConsoleWs(req, socket, head, { nodeId, decoded, diff --git a/docs/features/api-tokens.mdx b/docs/features/api-tokens.mdx index d81a5056..60b412aa 100644 --- a/docs/features/api-tokens.mdx +++ b/docs/features/api-tokens.mdx @@ -171,7 +171,7 @@ Authorises every read and every write the API exposes, **except** the universal | `/api/stacks/:stack/logs` (stack log stream) | yes | yes | yes | | `/ws/notifications` (notification stream) | yes | yes | yes | | `/ws` (generic exec / stats) | no | no | yes | -| `/api/system/host-console` (host shell) | no | no | yes | +| `/api/system/host-console` (host shell) | no | no | no | A Read Only or Deploy Only token attempting an out-of-scope WebSocket upgrade is rejected with `403 Forbidden` before any frames flow. @@ -268,7 +268,7 @@ API tokens are issued and revoked on the hub that authenticates the call. When y - **One owner per token.** Tokens are not shareable across users at the data model; only the creating user can revoke a token through the UI. - **Owner deletion breaks the token.** If the user who created a token is deleted, subsequent calls with that token return `401` because the auth path requires the creator to still exist. Rotate before deleting accounts. - **No token introspection endpoint.** There is no `GET /api/me` for an API token; client code must know its own scope. -- **WebSocket scope restrictions.** Read Only and Deploy Only tokens cannot reach the generic `/ws` exec/stats endpoint or the host console. Full Admin can. +- **WebSocket scope restrictions.** Read Only and Deploy Only tokens cannot reach the generic `/ws` exec/stats endpoint or the host console. Full Admin can reach `/ws` (container exec / stats) but not Host Console; Host Console always requires a signed-in browser session. ## Common workflows diff --git a/docs/features/deep-links.mdx b/docs/features/deep-links.mdx index 875a2a49..5e1ec9e1 100644 --- a/docs/features/deep-links.mdx +++ b/docs/features/deep-links.mdx @@ -23,7 +23,8 @@ Sencho encodes the active node, the view you are on, and the deep state that vie | App Store | `/nodes/local/templates` | App Store on the active node | | Logs | `/nodes/local/logs` | Cross-fleet log aggregation | | Update | `/nodes/local/updates` | Fleet-wide update check | -| Console | `/nodes/local/host-console` | Host Console, a limited-availability operator surface documented on its own page when enabled on an instance | +| Console | `/nodes/local/host-console` | Host Console (admin role) | +| Console in a stack | `/nodes/local/host-console/radarr` | Host Console rooted in the Radarr stack directory | | Audit | `/nodes/local/audit` | Audit history | | Settings section | `/nodes/local/settings/nodes` | Settings on the Nodes section | | Fleet tab | `/nodes/local/fleet/snapshots` | Fleet on the Snapshots tab (desktop) | @@ -86,7 +87,7 @@ On desktop, opening the stack list at `/nodes/local/stacks` canonicalizes to `/n Settings follows a similar split, but only on a phone: `/nodes/local/settings` opens the section list, and `/nodes/local/settings/
` opens a section directly. On desktop, `/nodes/local/settings` always normalizes straight to `/nodes/local/settings/appearance`; there is no bare section-list state to land on. -Fleet, Resources, Security, App Store, Logs, Update, Audit, and Schedules keep the exact same URL between desktop and phone, each rendering a phone-optimized screen at that path. Console also keeps the same URL on a phone, reflowing the terminal rather than switching to a dedicated phone screen. +Fleet, Resources, Security, App Store, Logs, Update, Audit, and Schedules keep the exact same URL between desktop and phone, each rendering a phone-optimized screen at that path. Console also keeps the same URL on a phone (`/nodes//host-console`), but Host Console is a desktop-oriented terminal: open it on a wider screen for a full interactive session. On a phone, `/nodes/local/stacks//files` opens the compose editor instead. Sencho does not expose a separate file-browser URL on a phone. diff --git a/docs/features/global-search.mdx b/docs/features/global-search.mdx index 8d0d4f25..4f14d137 100644 --- a/docs/features/global-search.mdx +++ b/docs/features/global-search.mdx @@ -26,7 +26,7 @@ The palette groups results into three sections. | Group | What it contains | What happens when you pick one | |-------|------------------|--------------------------------| -| **Pages** | The reachable page destinations for your tier and role (the same set Classic / Smart / mobile navigation use). **Home**, **Resources**, **Networking**, **Security**, and **App Store** appear for signed-in operators; **Fleet** appears when your role holds the `node:read` permission; **Logs**, **Update**, and **Schedules** appear for admins; **Console** is a limited-availability operator surface shown when enabled on an instance; **Audit** appears on Admiral for any role with the `system:audit` permission. See [RBAC & User Management](/features/rbac) for the full permission matrix. | Navigates to that page | +| **Pages** | The reachable page destinations for your tier and role (the same set Classic / Smart / mobile navigation use). **Home**, **Resources**, **Networking**, **Security**, and **App Store** appear for signed-in operators; **Fleet** appears when your role holds the `node:read` permission; **Logs**, **Update**, **Schedules**, and **Console** appear for admins; **Audit** appears on Admiral for any role with the `system:audit` permission. See [RBAC & User Management](/features/rbac) for the full permission matrix. | Navigates to that page | | **Nodes** | Every node in your fleet, with a green dot for online and a grey dot for offline. The currently active node carries a small **ACTIVE** chip on the right. | Switches the active node without leaving the current page | | **Stacks** | Every compose stack on every online node, matched on the compose filename (extension included). | Switches to the stack's node and opens it in the editor | diff --git a/docs/features/host-console.mdx b/docs/features/host-console.mdx index 23ee6ea4..306fbd04 100644 --- a/docs/features/host-console.mdx +++ b/docs/features/host-console.mdx @@ -6,7 +6,7 @@ description: An interactive terminal on your host OS, directly in the browser - The **Console** tab opens a full interactive terminal session on the machine running Sencho. It behaves exactly like an SSH session, but without needing an SSH server, client, or key management. - The Host Console is a limited-availability operator surface. When it is present on an instance, it requires the **admin** role. [Learn more about licensing](/features/licensing). + Host Console requires the **admin** role. [Learn more about roles](/features/rbac). @@ -27,6 +27,14 @@ The Host Console gives you a real terminal session on the Sencho host, streamed Click **Console** in the top navigation bar. The session starts immediately in the `COMPOSE_DIR` root. If a stack is selected in the sidebar, the terminal opens directly inside that stack's directory instead, and a small back button appears in the masthead so you can return to the stack editor. +You can also open Console from a bookmark or shared link. Every Console view has a stable URL under [Deep links and URLs](/features/deep-links): + +- `/nodes/local/host-console` opens Host Console on the local node at the compose root +- `/nodes/local/host-console/radarr` opens the same terminal rooted in the Radarr stack directory +- Remote nodes use their node slug the same way (for example `/nodes/nas-box-42/host-console`) + +Console requires the **admin** role. The URL stays in the address bar on a phone, but Host Console is meant for a desktop browser. + ## Cockpit layout The Console page is a vertical stack of two surfaces. @@ -74,9 +82,9 @@ Environment variables whose names suggest secrets (passwords, tokens, keys, cred The Host Console is one of the most powerful features in Sencho and is treated as such: -- **Admin role required when present.** The Host Console is a limited-availability surface; only users with the **admin** role can open a console session. +- **Admin role required.** Only users with the **admin** role can open a console session. - **Browser sessions only.** Console sessions are only available from a signed-in browser session, not from API tokens. -- **Audited.** Every console session is recorded in the audit log. Opening and closing a session each write an entry capturing the user, node, client IP, and timestamp, so shell access is fully accountable. +- **Audited.** Every console session is recorded in the audit log. Opening and closing a session each write an entry with the session principal, node, client IP, and timestamp. When the session is opened through a remote hub bridge, the principal is `console_session` and the hub operator is recorded in `acting_as`, so remote shell access stays accountable to the signed-in admin. The Host Console provides unrestricted shell access to the machine running Sencho. Do not expose Sencho on a public network without HTTPS and strong authentication. @@ -97,7 +105,7 @@ The Host Console is one of the most powerful features in Sencho and is treated a - This typically means the shell could not be found on the host. In a Docker deployment, ensure `bash` or `sh` is available inside the container. You can check by running `docker exec which bash` from the host. Also verify that your user has the **admin** role and that Console is available on this instance. + This typically means the shell could not be found on the host. In a Docker deployment, ensure `bash` or `sh` is available inside the container. You can check by running `docker exec which bash` from the host. Also verify that your user has the **admin** role. @@ -105,6 +113,6 @@ The Host Console is one of the most powerful features in Sencho and is treated a - Console appears only when the surface is present on the instance, and only for users with the **admin** role. Verify those conditions are met, then refresh the page. + Console appears for users with the **admin** role. If you are an admin and still do not see it, refresh the page. On a remote node, the Console tab may show a lock card when that node does not support Host Console (for example a Pilot Agent node, or a Distributed API Proxy node that does not advertise Host Console). diff --git a/docs/features/multi-node.mdx b/docs/features/multi-node.mdx index 90d138a2..3b4b719e 100644 --- a/docs/features/multi-node.mdx +++ b/docs/features/multi-node.mdx @@ -236,7 +236,7 @@ Bearer tokens grant full control over the remote Sencho instance. Treat them lik - **Rotate immediately** if a token is compromised: open the remote instance's **Settings → Infrastructure → Nodes** and click **Generate Token** to mint a new one. The previous token is invalidated instantly. - Tokens are **encrypted at rest** in the local SQLite database. -- Tokens cannot be used to open interactive terminals (Host Console or container exec). Interactive shell access always requires a real browser session on that specific instance. +- Tokens cannot be used to open interactive terminals (Host Console or container exec). Interactive shell access requires a signed-in browser session. From the control plane you can open Host Console on a compatible Distributed API Proxy remote; Pilot Agent remotes do not offer Host Console. ### Transport encryption diff --git a/docs/features/node-compatibility.mdx b/docs/features/node-compatibility.mdx index c7d345c4..388375ff 100644 --- a/docs/features/node-compatibility.mdx +++ b/docs/features/node-compatibility.mdx @@ -67,7 +67,8 @@ Every Sencho release ships with a static list of capabilities. The current list | `notifications` | Alert notifications | | `notification-routing` | Notification routing rules | | `notification-suppression` | Mute rules | -| `host-console` | Host Console | +| `host-console` | Host Console (legacy advertisement retained for mixed-version fleets) | +| `host-console-community` | Host Console without a paid-license requirement on this node | | `container-exec` | Container exec terminal | | `audit-log` | Audit log | | `scheduled-ops` | Scheduled operations | @@ -94,7 +95,7 @@ Every Sencho release ships with a static list of capabilities. The current list - A node connected in [Pilot Agent](/features/multi-node#add-a-remote-node-pilot-agent) mode never advertises `host-console`, even after upgrading, because that feature's control-to-agent path is not yet wired through the enrollment tunnel. This is the one capability gap that upgrading the node cannot close; it only closes on a Distributed API Proxy connection. + A node connected in [Pilot Agent](/features/multi-node#add-a-remote-node-pilot-agent) mode never advertises `host-console` or `host-console-community`, even after upgrading, because that feature's control-to-agent path is not yet wired through the enrollment tunnel. This is the one capability gap that upgrading the node cannot close; it only closes on a Distributed API Proxy connection. A handful of capabilities gate a smaller piece of behavior rather than a whole panel, so a missing one falls back to an older behavior instead of a lock card: diff --git a/docs/features/overview.mdx b/docs/features/overview.mdx index a7b43368..681c3f68 100644 --- a/docs/features/overview.mdx +++ b/docs/features/overview.mdx @@ -210,7 +210,7 @@ When you manage nodes running different Sencho versions, the dashboard detects e ### Host console -Open an interactive terminal on the host OS directly in the browser with full xterm.js emulation and color support. No SSH client required. Limited-availability surface when present; admin role required. [Learn more →](/features/host-console) +Open an interactive terminal on the host OS directly in the browser with full xterm.js emulation and color support. No SSH client required. Admin role required. [Learn more →](/features/host-console) ## Security and access diff --git a/docs/getting-started/introduction.mdx b/docs/getting-started/introduction.mdx index d22f083e..cd1a11d8 100644 --- a/docs/getting-started/introduction.mdx +++ b/docs/getting-started/introduction.mdx @@ -34,7 +34,7 @@ The **Home** view is the default landing page. It is designed for a fast operati - The activity panel shows **Fleet Heartbeat** when remote nodes exist, or **Stack Restarts (7d)** on a local-only install. - **Recent Alerts** shows the latest notification feed and includes **Clear All Notifications** when there is anything to clear. -The top navigation starts with **Home**, **Resources**, **Networking**, **Security**, and **App Store**. **Fleet** appears when your role can read nodes. Additional operator views (**Logs**, **Update**, **Schedules**, and **Audit**) appear based on your role and license tier. **Console** is a limited-availability operator surface documented on its own page when enabled on an instance. Fleet-wide views describe the control instance, so they are hidden while a remote node is active. Choose Classic, Smart, or Compact desktop navigation under **Settings → Appearance → Navigation**; phone navigation stays on its own layout. +The top navigation starts with **Home**, **Resources**, **Networking**, **Security**, and **App Store**. **Fleet** appears when your role can read nodes. Additional operator views (**Logs**, **Update**, **Schedules**, and **Console**) appear for admins. **Audit** appears based on your role and license tier. Fleet-wide views describe the control instance, so they are hidden while a remote node is active. Choose Classic, Smart, or Compact desktop navigation under **Settings → Appearance → Navigation**; phone navigation stays on its own layout. ## Stack workspace diff --git a/docs/getting-started/quickstart.mdx b/docs/getting-started/quickstart.mdx index 2da8778d..fc099426 100644 --- a/docs/getting-started/quickstart.mdx +++ b/docs/getting-started/quickstart.mdx @@ -107,7 +107,7 @@ You land on **Home**, the default operational view. The health masthead reports Below the stack table, **Configuration Status** summarizes notifications, alerts, automation, security, backups, thresholds, and crash detection. The neighboring activity card shows **Fleet Heartbeat** when remote nodes exist, or **Stack Restarts (7d)** on a local-only install. **Recent Alerts** shows the latest notification feed and includes **Clear All Notifications** when there is anything to clear. -On the local node, baseline top navigation includes **Home**, **Resources**, **Networking**, **Security**, and **App Store**. **Fleet** appears when your role can read nodes. **Logs**, **Update**, and **Schedules** appear for admins. **Console** and **Audit** depend on license, role, and whether those surfaces are enabled; hub-only views are hidden when a remote node is active. Desktop presentation (Classic bar, Smart bar, or Compact launcher) is chosen under **Settings → Appearance → Navigation**. The right side of the top bar holds global search, notifications, and the profile menu entries **Settings**, **Billing** (when a paid license is active), **Documentation**, **Open New Issue**, and **Log Out**. +On the local node, baseline top navigation includes **Home**, **Resources**, **Networking**, **Security**, and **App Store**. **Fleet** appears when your role can read nodes. **Logs**, **Update**, **Schedules**, and **Console** appear for admins. **Audit** depends on license and role; hub-only views are hidden when a remote node is active. Desktop presentation (Classic bar, Smart bar, or Compact launcher) is chosen under **Settings → Appearance → Navigation**. The right side of the top bar holds global search, notifications, and the profile menu entries **Settings**, **Billing** (when a paid license is active), **Documentation**, **Open New Issue**, and **Log Out**. The left sidebar is the stack workspace. Below the Sencho brand, it starts with the node switcher, then **Create Stack**, a bulk-mode toggle, and **Scan stacks folder** for re-indexing compose projects added outside Sencho. Use **Search stacks...** with the **All**, **Up**, **Down**, and **Updates** chips to narrow the list. On a fresh install with an empty stack list, Sencho scans your mounted compose directory automatically and shows what it found, including compose files that still need to be adopted into their own subfolder. Once stacks carry Docker Compose labels, the list groups them under those labels, with pinned stacks always floating to the top and unlabeled stacks collected at the bottom. diff --git a/frontend/src/components/AuditLogView.tsx b/frontend/src/components/AuditLogView.tsx index 4c99d92b..cd8741d7 100644 --- a/frontend/src/components/AuditLogView.tsx +++ b/frontend/src/components/AuditLogView.tsx @@ -27,6 +27,7 @@ interface AuditEntry { node_id: number | null; ip_address: string; summary: string; + acting_as?: string | null; flags?: AnomalyFlag[]; } @@ -464,6 +465,11 @@ export function AuditLogView({ headerActions }: AuditLogViewProps = {}) { {entry.username} + {entry.acting_as ? ( + + acting as {entry.acting_as} + + ) : null} @@ -492,6 +498,10 @@ export function AuditLogView({ headerActions }: AuditLogViewProps = {}) { IP Address {entry.ip_address || '-'} +
+ Acting as + {entry.acting_as || '-'} +
Node ID {entry.node_id ?? 'Local'} @@ -673,6 +683,9 @@ function StreamRow({ entry, now }: StreamRowProps) {
{entry.username || 'system'} + {entry.acting_as ? ( + (acting as {entry.acting_as}) + ) : null} {verb.toLowerCase()} {target || entry.path}
diff --git a/frontend/src/components/EditorLayout/ViewRouter.tsx b/frontend/src/components/EditorLayout/ViewRouter.tsx index a27f4602..45301ba5 100644 --- a/frontend/src/components/EditorLayout/ViewRouter.tsx +++ b/frontend/src/components/EditorLayout/ViewRouter.tsx @@ -1,8 +1,11 @@ import { Suspense, lazy, type ReactNode } from 'react'; +import { Unplug } from 'lucide-react'; import { Skeleton } from '@/components/ui/skeleton'; import { useAuth } from '@/context/AuthContext'; -import { useExperimental } from '@/hooks/useExperimental'; -import { PaidGate } from '../PaidGate'; +import { useLicense } from '@/context/LicenseContext'; +import { useNodes } from '@/context/NodeContext'; +import { resolveHostConsoleCapability } from '@/lib/routing/hostConsoleCapability'; +import { LockCard } from '../ui/LockCard'; import { CapabilityGate } from '../CapabilityGate'; import { HubOnlyGate } from '../HubOnlyGate'; import LazyBoundary from '../LazyBoundary'; @@ -17,7 +20,7 @@ import type { MuteRuleDraft } from '@/lib/muteRules'; import type { ActiveView } from './hooks/useViewNavigationState'; import type { StackUpdateInfo } from '@/types/imageUpdates'; import type { SecurityTab, FleetTab } from '@/lib/events'; -import { isStackEditorDeepLink } from '@/lib/router/readUrlRouteState'; +import { isStackEditorDeepLink, isHostConsoleStackDeepLink } from '@/lib/router/readUrlRouteState'; import type { NavDestination } from '@/lib/navigation/appNavRegistry'; // Paid-tier views are loaded on demand. Their internal PaidGate / @@ -147,7 +150,8 @@ export function ViewRouter({ quickLinkCandidates, }: ViewRouterProps): ReactNode { const { can } = useAuth(); - const { experimental, experimentalReady } = useExperimental(); + const { isPaid, licenseReady } = useLicense(); + const { activeNode, activeNodeMeta } = useNodes(); if (activeView === 'settings') { return ( ; + } + if (activeNode == null) return ; + const capState = resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: activeNode.type === 'remote', + isPaid, + licenseReady, + activeNodeMeta, + }); + if (capState === 'loading') return ; + if (capState === 'locked') { + const nodeName = activeNode.name; + const version = activeNodeMeta?.version; + let versionHint = `${nodeName} does not advertise this capability.`; + if (version && version !== 'unknown' && version !== '0.0.0-dev') { + versionHint = `${nodeName} is running v${version}.`; + } + return ( + + ); + } + const nodeId = activeNode.id; return ( - - - - - - - + + + ); } // Stack workspace: keep a loading shell while the stack URL hydrates. diff --git a/frontend/src/components/EditorLayout/__tests__/ViewRouter.test.tsx b/frontend/src/components/EditorLayout/__tests__/ViewRouter.test.tsx new file mode 100644 index 00000000..bbe6660a --- /dev/null +++ b/frontend/src/components/EditorLayout/__tests__/ViewRouter.test.tsx @@ -0,0 +1,179 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, screen } from '@testing-library/react'; +import type { ReactNode } from 'react'; +import * as AuthContext from '@/context/AuthContext'; +import * as LicenseContext from '@/context/LicenseContext'; +import * as NodeContext from '@/context/NodeContext'; +import { ViewRouter } from '../ViewRouter'; + +vi.mock('@/context/AuthContext'); +vi.mock('@/context/LicenseContext'); +vi.mock('@/context/NodeContext'); + +vi.mock('../../HostConsole', () => ({ + default: ({ nodeId, stackName }: { nodeId: number; stackName?: string | null }) => ( +
+ Host Console +
+ ), +})); + +vi.mock('../../LazyBoundary', () => ({ + default: ({ children }: { children: ReactNode }) => <>{children}, +})); + +const baseProps = { + activeView: 'host-console' as const, + selectedFile: null as string | null, + isLoading: false, + settingsSection: 'appearance' as const, + onSettingsSectionChange: vi.fn(), + onTemplateDeploySuccess: vi.fn(), + onHostConsoleClose: vi.fn(), + onFleetNavigateToNode: vi.fn(), + onOpenNodeNetworking: vi.fn(), + filterNodeId: null, + onClearScheduledOpsFilter: vi.fn(), + schedulePrefill: null, + onPrefillConsumed: vi.fn(), + muteRulePrefill: null, + onMutePrefillConsumed: vi.fn(), + notifications: [] as [], + onNavigateToStack: vi.fn(), + onOpenSettingsSection: vi.fn(), + onClearNotifications: vi.fn(), + securityTab: 'overview' as const, + onSecurityTabChange: vi.fn(), + renderEditor: () => null, + stackUpdates: {}, + urlHydratingStack: null as string | null, + isFileLoading: false, + quickLinkCandidates: [], +}; + +describe('ViewRouter host-console', () => { + beforeEach(() => { + window.history.replaceState({}, '', '/nodes/local/host-console'); + vi.mocked(AuthContext.useAuth).mockReturnValue({ + can: (p: string) => p === 'system:console', + } as unknown as ReturnType); + vi.mocked(LicenseContext.useLicense).mockReturnValue({ + isPaid: false, + licenseReady: true, + } as unknown as ReturnType); + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: { id: 1, name: 'Local', type: 'local' }, + activeNodeMeta: { version: '0.96.0', capabilities: ['host-console', 'host-console-community'], fetchedAt: 1 }, + } as unknown as ReturnType); + }); + + afterEach(() => { + window.history.replaceState({}, '', '/'); + }); + + it('renders Host Console for a Community admin on the local node', async () => { + render(); + const el = await screen.findByTestId('host-console'); + expect(el.getAttribute('data-node-id')).toBe('1'); + }); + + it('does not mount HostConsole while the active node is unresolved', () => { + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: null, + activeNodeMeta: null, + } as unknown as ReturnType); + render(); + expect(screen.queryByTestId('host-console')).toBeNull(); + }); + + it('does not mount HostConsole while a stack deep link is still hydrating', () => { + render(); + expect(screen.queryByTestId('host-console')).toBeNull(); + }); + + it('does not mount a root shell while the URL targets a stack-scoped Console', () => { + window.history.replaceState({}, '', '/nodes/local/host-console/radarr'); + render(); + expect(screen.queryByTestId('host-console')).toBeNull(); + }); + + it('mounts stack-scoped Console only after selectedFile matches the route', async () => { + window.history.replaceState({}, '', '/nodes/local/host-console/radarr'); + render(); + const el = await screen.findByTestId('host-console'); + expect(el.getAttribute('data-stack')).toBe('radarr'); + expect(el.getAttribute('data-node-id')).toBe('1'); + }); + + it('renders nothing without system:console', () => { + vi.mocked(AuthContext.useAuth).mockReturnValue({ + can: () => false, + } as unknown as ReturnType); + const { container } = render(); + expect(container.querySelector('[data-testid="host-console"]')).toBeNull(); + }); + + it('shows a skeleton while remote metadata is loading (does not mount HostConsole)', () => { + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: { id: 2, name: 'Legacy', type: 'remote' }, + activeNodeMeta: null, + } as unknown as ReturnType); + render(); + expect(screen.queryByTestId('host-console')).toBeNull(); + }); + + it('shows a lock card for Community + legacy remote without mounting HostConsole', () => { + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: { id: 2, name: 'Legacy', type: 'remote' }, + activeNodeMeta: { version: '0.95.0', capabilities: ['host-console'], fetchedAt: 1 }, + } as unknown as ReturnType); + render(); + expect(screen.queryByTestId('host-console')).toBeNull(); + expect(screen.getByText(/Host Console is not available on this node/i)).toBeTruthy(); + }); + + it('mounts Host Console for Admiral + legacy remote after meta resolves', async () => { + vi.mocked(LicenseContext.useLicense).mockReturnValue({ + isPaid: true, + licenseReady: true, + } as unknown as ReturnType); + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: { id: 2, name: 'Legacy', type: 'remote' }, + activeNodeMeta: { version: '0.95.0', capabilities: ['host-console'], fetchedAt: 1 }, + } as unknown as ReturnType); + render(); + const el = await screen.findByTestId('host-console'); + expect(el.getAttribute('data-node-id')).toBe('2'); + }); + + it('shows a skeleton for legacy-only remote while license is still loading', () => { + vi.mocked(LicenseContext.useLicense).mockReturnValue({ + isPaid: false, + licenseReady: false, + } as unknown as ReturnType); + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: { id: 2, name: 'Legacy', type: 'remote' }, + activeNodeMeta: { version: '0.95.0', capabilities: ['host-console'], fetchedAt: 1 }, + } as unknown as ReturnType); + render(); + expect(screen.queryByTestId('host-console')).toBeNull(); + expect(screen.queryByText(/Host Console is not available on this node/i)).toBeNull(); + }); + + it('mounts Host Console for Community + host-console-community remote', async () => { + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: { id: 3, name: 'NewPeer', type: 'remote' }, + activeNodeMeta: { + version: '0.96.0', + capabilities: ['host-console', 'host-console-community'], + fetchedAt: 1, + }, + } as unknown as ReturnType); + render(); + expect(await screen.findByTestId('host-console')).toBeTruthy(); + }); +}); diff --git a/frontend/src/components/EditorLayout/__tests__/useViewNavigationState.test.tsx b/frontend/src/components/EditorLayout/__tests__/useViewNavigationState.test.tsx index 81303274..25c21452 100644 --- a/frontend/src/components/EditorLayout/__tests__/useViewNavigationState.test.tsx +++ b/frontend/src/components/EditorLayout/__tests__/useViewNavigationState.test.tsx @@ -51,7 +51,7 @@ function mockDeployer() { function mockPaidAdmin() { vi.mocked(AuthContext.useAuth).mockReturnValue({ isAdmin: true, - can: (p: string) => p === 'system:audit' || p === 'node:read', + can: (p: string) => p === 'system:audit' || p === 'system:console' || p === 'node:read', permissionsStatus: 'ready', } as unknown as ReturnType); vi.mocked(LicenseContext.useLicense).mockReturnValue({ @@ -63,7 +63,7 @@ function mockPaidAdmin() { function mockCommunityAdmin() { vi.mocked(AuthContext.useAuth).mockReturnValue({ isAdmin: true, - can: (p: string) => p === 'node:read', + can: (p: string) => p === 'system:console' || p === 'node:read', permissionsStatus: 'ready', } as unknown as ReturnType); vi.mocked(LicenseContext.useLicense).mockReturnValue({ @@ -269,13 +269,13 @@ describe('useViewNavigationState', () => { expect(result.current.navItems.map(i => i.value)).toContain('global-observability'); }); - it('shows Update and Schedules for a community admin (now free) but hides paid Console and Audit', () => { + it('shows Update, Schedules, and Console for a community admin; Audit stays paid', () => { mockCommunityAdmin(); const { result } = renderHook(() => useViewNavigationState()); const values = result.current.navItems.map(i => i.value); expect(values).toContain('auto-updates'); expect(values).toContain('scheduled-ops'); - expect(values).not.toContain('host-console'); + expect(values).toContain('host-console'); expect(values).not.toContain('audit-log'); // The auto-updates nav item surfaces under the short label "Update". expect(result.current.navItems.find(i => i.value === 'auto-updates')?.label).toBe('Update'); @@ -429,53 +429,29 @@ describe('useViewNavigationState', () => { expect(result.current.securityTab).toBe('overview'); }); - // ── experimental discovery ───────────────────────────────────────────────── + // ── host-console discovery (no longer experimental) ──────────────────────── - it('hides Console from nav for a paid admin when experimental discovery is off', () => { + it('keeps Console in nav for a paid admin when experimental discovery is off', () => { mockPaidAdmin(); useExperimentalMock.mockReturnValue({ experimental: false, experimentalReady: true }); const { result } = renderHook(() => useViewNavigationState()); - expect(result.current.navItems.map(i => i.value)).not.toContain('host-console'); + expect(result.current.navItems.map(i => i.value)).toContain('host-console'); }); - it('hides Console from nav while experimental metadata is still loading', () => { + it('keeps Console in nav while experimental metadata is still loading', () => { mockPaidAdmin(); useExperimentalMock.mockReturnValue({ experimental: false, experimentalReady: false }); const { result } = renderHook(() => useViewNavigationState()); - expect(result.current.navItems.map(i => i.value)).not.toContain('host-console'); + expect(result.current.navItems.map(i => i.value)).toContain('host-console'); }); - it('does not normalize a host-console deep link before experimental readiness', () => { + it('keeps a host-console deep link selected when experimental is off', () => { mockPaidAdmin(); - useExperimentalMock.mockReturnValue({ experimental: false, experimentalReady: false }); + useExperimentalMock.mockReturnValue({ experimental: false, experimentalReady: true }); const onNavigateToDashboard = vi.fn(); const { result } = renderHook(() => useViewNavigationState({ onNavigateToDashboard })); act(() => result.current.setActiveView('host-console')); expect(result.current.activeView).toBe('host-console'); expect(onNavigateToDashboard).not.toHaveBeenCalled(); }); - - it('keeps host-console selected when delayed experimental resolves enabled', () => { - mockPaidAdmin(); - useExperimentalMock.mockReturnValue({ experimental: false, experimentalReady: false }); - const onNavigateToDashboard = vi.fn(); - const { result, rerender } = renderHook(() => useViewNavigationState({ onNavigateToDashboard })); - act(() => result.current.setActiveView('host-console')); - useExperimentalMock.mockReturnValue({ experimental: true, experimentalReady: true }); - rerender(); - expect(result.current.activeView).toBe('host-console'); - expect(onNavigateToDashboard).not.toHaveBeenCalled(); - }); - - it('normalizes host-console once when experimental resolves disabled', () => { - mockPaidAdmin(); - useExperimentalMock.mockReturnValue({ experimental: false, experimentalReady: false }); - const onNavigateToDashboard = vi.fn(); - const { result, rerender } = renderHook(() => useViewNavigationState({ onNavigateToDashboard })); - act(() => result.current.setActiveView('host-console')); - useExperimentalMock.mockReturnValue({ experimental: false, experimentalReady: true }); - rerender(); - expect(result.current.activeView).toBe('dashboard'); - expect(onNavigateToDashboard).toHaveBeenCalled(); - }); }); diff --git a/frontend/src/components/HostConsole.tsx b/frontend/src/components/HostConsole.tsx index 0761a337..21e62b55 100644 --- a/frontend/src/components/HostConsole.tsx +++ b/frontend/src/components/HostConsole.tsx @@ -8,6 +8,8 @@ import { useNodes } from '@/context/NodeContext'; import { copyToClipboard } from '@/lib/clipboard'; interface HostConsoleProps { + /** Resolved active node id; WebSocket must target this id, not localStorage. */ + nodeId: number; stackName?: string | null; onClose: () => void; } @@ -27,7 +29,7 @@ function formatUptime(ms: number): string { type ConnState = 'reconnecting' | 'connected' | 'disconnected'; -export default function HostConsole({ stackName, onClose }: HostConsoleProps) { +export default function HostConsole({ nodeId, stackName, onClose }: HostConsoleProps) { const { activeNode } = useNodes(); const terminalRef = useRef(null); const xtermRef = useRef(null); @@ -93,12 +95,12 @@ export default function HostConsole({ stackName, onClose }: HostConsoleProps) { }); const wsProtocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; - const activeNodeId = localStorage.getItem('sencho-active-node') || ''; - const nodeParam = activeNodeId ? `nodeId=${activeNodeId}` : ''; - const stackParam = stackName ? `stack=${encodeURIComponent(stackName)}` : ''; - const queryString = [nodeParam, stackParam].filter(Boolean).join('&'); - const wsUrl = `${wsProtocol}//${window.location.host}/api/system/host-console${queryString ? `?${queryString}` : ''}`; - const ws = new WebSocket(wsUrl); + const qs = stackName + ? `nodeId=${nodeId}&stack=${encodeURIComponent(stackName)}` + : `nodeId=${nodeId}`; + const ws = new WebSocket( + `${wsProtocol}//${window.location.host}/api/system/host-console?${qs}`, + ); wsRef.current = ws; ws.onopen = () => { @@ -194,7 +196,7 @@ export default function HostConsole({ stackName, onClose }: HostConsoleProps) { fitAddonRef.current = null; serializeRef.current = null; }; - }, [stackName, reconnectNonce]); + }, [nodeId, stackName, reconnectNonce]); const handleCopy = useCallback(() => { const term = xtermRef.current; @@ -236,7 +238,10 @@ export default function HostConsole({ stackName, onClose }: HostConsoleProps) { const stateWord = connState === 'disconnected' ? 'Disconnected' : connState === 'reconnecting' ? 'Reconnecting' : 'Connected'; - const nodeLabel = activeNode ? (activeNode.type === 'local' ? 'LOCAL' : activeNode.name.toUpperCase()) : 'LOCAL'; + let nodeLabel = `NODE ${nodeId}`; + if (activeNode?.id === nodeId) { + nodeLabel = activeNode.type === 'local' ? 'LOCAL' : activeNode.name.toUpperCase(); + } const kicker = `HOST CONSOLE · ${nodeLabel}`; const uptime = mountedAt != null ? formatUptime(tick - mountedAt) : '—'; diff --git a/frontend/src/components/__tests__/HostConsole.test.tsx b/frontend/src/components/__tests__/HostConsole.test.tsx new file mode 100644 index 00000000..bb883b89 --- /dev/null +++ b/frontend/src/components/__tests__/HostConsole.test.tsx @@ -0,0 +1,141 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, waitFor, act } from '@testing-library/react'; +import type { ReactNode } from 'react'; +import * as NodeContext from '@/context/NodeContext'; +import HostConsole from '../HostConsole'; + +vi.mock('@/context/NodeContext'); + +vi.mock('@/lib/xtermLoader', () => { + class FakeTerminal { + cols = 80; + rows = 24; + open = vi.fn(); + focus = vi.fn(); + write = vi.fn(); + clear = vi.fn(); + dispose = vi.fn(); + getSelection = vi.fn(() => ''); + loadAddon = vi.fn(); + onData = vi.fn(); + } + class FakeFitAddon { + fit = vi.fn(); + } + class FakeSerializeAddon { + serialize = vi.fn(() => ''); + } + return { + loadXtermModules: async () => ({ + Terminal: FakeTerminal, + FitAddon: FakeFitAddon, + SerializeAddon: FakeSerializeAddon, + }), + }; +}); + +vi.mock('../ui/PageMasthead', () => ({ + PageMasthead: ({ children }: { children?: ReactNode }) =>
{children}
, +})); + +vi.mock('../ui/button', () => ({ + Button: ({ children, ...props }: { children?: ReactNode } & Record) => ( + + ), +})); + +vi.mock('@/lib/clipboard', () => ({ + copyToClipboard: vi.fn(async () => undefined), +})); + +type FakeWs = { + url: string; + readyState: number; + close: ReturnType; + send: ReturnType; + onopen: ((ev?: unknown) => void) | null; + onmessage: ((ev: { data: string }) => void) | null; + onerror: ((ev?: unknown) => void) | null; + onclose: ((ev?: unknown) => void) | null; +}; + +describe('HostConsole socket targeting', () => { + const sockets: FakeWs[] = []; + let OriginalWebSocket: typeof WebSocket; + + beforeEach(() => { + sockets.length = 0; + OriginalWebSocket = globalThis.WebSocket; + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: { id: 1, name: 'Local', type: 'local' }, + } as unknown as ReturnType); + + globalThis.WebSocket = class { + static OPEN = 1; + static CLOSED = 3; + url: string; + readyState = 0; + close = vi.fn(() => { this.readyState = 3; }); + send = vi.fn(); + onopen: FakeWs['onopen'] = null; + onmessage: FakeWs['onmessage'] = null; + onerror: FakeWs['onerror'] = null; + onclose: FakeWs['onclose'] = null; + constructor(url: string) { + this.url = url; + sockets.push(this as unknown as FakeWs); + queueMicrotask(() => { + this.readyState = 1; + this.onopen?.(undefined); + }); + } + } as unknown as typeof WebSocket; + }); + + afterEach(() => { + globalThis.WebSocket = OriginalWebSocket; + localStorage.removeItem('sencho-active-node'); + vi.clearAllMocks(); + }); + + it('opens the WebSocket with the explicit nodeId (not localStorage)', async () => { + localStorage.setItem('sencho-active-node', '99'); + render(); + await waitFor(() => expect(sockets.length).toBe(1)); + expect(sockets[0].url).toContain('nodeId=7'); + expect(sockets[0].url).not.toContain('nodeId=99'); + }); + + it('includes the stack parameter when provided', async () => { + render(); + await waitFor(() => expect(sockets.length).toBe(1)); + expect(sockets[0].url).toContain('stack=radarr'); + }); + + it('closes the prior socket and opens a new one when nodeId changes', async () => { + const { rerender } = render(); + await waitFor(() => expect(sockets.length).toBe(1)); + const first = sockets[0]; + + await act(async () => { + rerender(); + }); + await waitFor(() => expect(sockets.length).toBe(2)); + expect(first.close).toHaveBeenCalled(); + expect(sockets[1].url).toContain('nodeId=2'); + }); + + it('reconnects when stackName changes so a root shell is not retained', async () => { + const { rerender } = render(); + await waitFor(() => expect(sockets.length).toBe(1)); + const first = sockets[0]; + expect(first.url).not.toContain('stack='); + + await act(async () => { + rerender(); + }); + await waitFor(() => expect(sockets.length).toBe(2)); + expect(first.close).toHaveBeenCalled(); + expect(sockets[1].url).toContain('stack=radarr'); + }); +}); diff --git a/frontend/src/lib/capabilities.ts b/frontend/src/lib/capabilities.ts index 850581e5..76292bae 100644 --- a/frontend/src/lib/capabilities.ts +++ b/frontend/src/lib/capabilities.ts @@ -16,6 +16,7 @@ export const CAPABILITIES = [ 'notification-suppression', 'notification-suppression-schedule', 'host-console', + 'host-console-community', 'container-exec', 'audit-log', 'scheduled-ops', @@ -40,6 +41,12 @@ export const CAPABILITIES = [ export type Capability = (typeof CAPABILITIES)[number]; +/** Legacy Host Console advertisement (Admiral hubs still accept this on remotes). */ +export const HOST_CONSOLE_CAPABILITY = 'host-console' as const satisfies Capability; + +/** Host Console works without a paid license on this node. */ +export const HOST_CONSOLE_COMMUNITY_CAPABILITY = 'host-console-community' as const satisfies Capability; + export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes' as const satisfies Capability; export const GUIDED_EXTERNAL_NETWORK_PREFLIGHT_CAPABILITY = 'guided-external-network-preflight' as const satisfies Capability; export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability; diff --git a/frontend/src/lib/navigation/buildNavigationModel.test.ts b/frontend/src/lib/navigation/buildNavigationModel.test.ts index d6ffba00..f1a24944 100644 --- a/frontend/src/lib/navigation/buildNavigationModel.test.ts +++ b/frontend/src/lib/navigation/buildNavigationModel.test.ts @@ -77,19 +77,31 @@ describe('buildNavigationModel', () => { expect(values).not.toContain('audit-log'); }); - it('omits Console until experimental discovery is ready and enabled via reachCtx only', () => { + it('includes Console for system:console regardless of experimental discovery', () => { expect( - buildNavigationModel(makeCtx({ experimentalReady: false, experimental: false })) - .allPageItems.map((i) => i.value), - ).not.toContain('host-console'); - expect( - buildNavigationModel(makeCtx({ experimentalReady: true, experimental: false })) - .allPageItems.map((i) => i.value), - ).not.toContain('host-console'); - expect( - buildNavigationModel(makeCtx({ experimentalReady: true, experimental: true })) + buildNavigationModel(makeCtx({ + experimentalReady: true, + experimental: false, + isPaid: false, + can: (a) => a === 'system:console' || a === 'node:read', + })) .allPageItems.map((i) => i.value), ).toContain('host-console'); + expect( + buildNavigationModel(makeCtx({ + experimentalReady: false, + experimental: false, + can: (a) => a === 'system:console' || a === 'node:read', + })) + .allPageItems.map((i) => i.value), + ).toContain('host-console'); + }); + + it('omits Console without system:console', () => { + expect( + buildNavigationModel(makeCtx({ can: () => false, isAdmin: false })) + .allPageItems.map((i) => i.value), + ).not.toContain('host-console'); }); it('excludes hidden views from quick-link candidates', () => { diff --git a/frontend/src/lib/navigation/buildNavigationModel.ts b/frontend/src/lib/navigation/buildNavigationModel.ts index 82eb59d4..faf73f17 100644 --- a/frontend/src/lib/navigation/buildNavigationModel.ts +++ b/frontend/src/lib/navigation/buildNavigationModel.ts @@ -28,12 +28,6 @@ function isVisuallyDiscoverable(item: AppNavItem, reachCtx: ReachabilityContext) // Settings is always discoverable in the launcher when the operator can open Settings. return true; } - // Console: fail-closed visual discovery until /meta settles and the flag is on. - // URL normalization still uses isViewHidden cold-load deferral separately. - if (item.value === 'host-console') { - if (!reachCtx.experimentalReady || !reachCtx.experimental) return false; - return !isViewHidden(item.value, reachCtx); - } return !isViewHidden(item.value, reachCtx); } diff --git a/frontend/src/lib/router/readUrlRouteState.ts b/frontend/src/lib/router/readUrlRouteState.ts index 4d190751..6a53d161 100644 --- a/frontend/src/lib/router/readUrlRouteState.ts +++ b/frontend/src/lib/router/readUrlRouteState.ts @@ -19,11 +19,21 @@ const DEFAULT: UrlRouteState = { filterNodeId: null, }; -/** True when the current URL is a stack workspace deep link (detail or editor). */ -export function isStackEditorDeepLink(): boolean { +/** True when the URL is a stack-scoped deep link for the given view. */ +function isStackScopedDeepLink(view: ActiveView): boolean { if (typeof window === 'undefined') return false; const parsed = parsePath(window.location.pathname, window.location.search); - return parsed.view === 'editor' && parsed.stackName != null; + return parsed.view === view && parsed.stackName != null; +} + +/** True when the current URL is a stack workspace deep link (detail or editor). */ +export function isStackEditorDeepLink(): boolean { + return isStackScopedDeepLink('editor'); +} + +/** True when the URL targets Host Console rooted in a stack directory. */ +export function isHostConsoleStackDeepLink(): boolean { + return isStackScopedDeepLink('host-console'); } /** Read shell navigation fields from the current browser URL (cold-load bootstrap). */ diff --git a/frontend/src/lib/router/senchoRoute.test.ts b/frontend/src/lib/router/senchoRoute.test.ts index 42a07c6a..f744382b 100644 --- a/frontend/src/lib/router/senchoRoute.test.ts +++ b/frontend/src/lib/router/senchoRoute.test.ts @@ -165,4 +165,21 @@ describe('senchoRoute', () => { expect(parsed.view).toBe('networking'); expect(parsed.nodeSlug).toBe('local'); }); + + it('round-trips Host Console without a stack', () => { + const path = buildPath({ ...base, activeView: 'host-console', stackName: null }); + expect(path).toBe('/nodes/local/host-console'); + const parsed = parsePath(path, ''); + expect(parsed.view).toBe('host-console'); + expect(parsed.nodeSlug).toBe('local'); + expect(parsed.stackName).toBeNull(); + }); + + it('round-trips Host Console rooted in a stack directory', () => { + const path = buildPath({ ...base, activeView: 'host-console', stackName: 'radarr' }); + expect(path).toBe('/nodes/local/host-console/radarr'); + const parsed = parsePath(path, ''); + expect(parsed.view).toBe('host-console'); + expect(parsed.stackName).toBe('radarr'); + }); }); diff --git a/frontend/src/lib/routing/hostConsoleCapability.test.ts b/frontend/src/lib/routing/hostConsoleCapability.test.ts new file mode 100644 index 00000000..f43141f1 --- /dev/null +++ b/frontend/src/lib/routing/hostConsoleCapability.test.ts @@ -0,0 +1,94 @@ +import { describe, it, expect } from 'vitest'; +import { resolveHostConsoleCapability } from './hostConsoleCapability'; + +describe('resolveHostConsoleCapability', () => { + it('returns loading when the active node is unresolved', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: false, + isRemote: false, + isPaid: false, + licenseReady: true, + activeNodeMeta: null, + })).toBe('loading'); + }); + + it('allows local nodes without waiting for meta', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: false, + isPaid: false, + licenseReady: true, + activeNodeMeta: null, + })).toBe('allowed'); + }); + + it('returns loading when remote meta is absent', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: true, + isPaid: false, + licenseReady: true, + activeNodeMeta: null, + })).toBe('loading'); + }); + + it('allows Community when remote advertises host-console-community', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: true, + isPaid: false, + licenseReady: true, + activeNodeMeta: { capabilities: ['host-console', 'host-console-community'] }, + })).toBe('allowed'); + }); + + it('locks Community when remote only has legacy host-console', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: true, + isPaid: false, + licenseReady: true, + activeNodeMeta: { capabilities: ['host-console'] }, + })).toBe('locked'); + }); + + it('allows Admiral when remote only has legacy host-console', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: true, + isPaid: true, + licenseReady: true, + activeNodeMeta: { capabilities: ['host-console'] }, + })).toBe('allowed'); + }); + + it('returns loading for legacy-only remote while license is not ready', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: true, + isPaid: false, + licenseReady: false, + activeNodeMeta: { capabilities: ['host-console'] }, + })).toBe('loading'); + }); + + it('allows community-capable remote without waiting on license', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: true, + isPaid: false, + licenseReady: false, + activeNodeMeta: { capabilities: ['host-console-community'] }, + })).toBe('allowed'); + }); + + it('locks Pilot / empty capability lists', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: true, + isPaid: true, + licenseReady: true, + activeNodeMeta: { capabilities: ['stacks', 'fleet'] }, + })).toBe('locked'); + }); +}); diff --git a/frontend/src/lib/routing/hostConsoleCapability.ts b/frontend/src/lib/routing/hostConsoleCapability.ts new file mode 100644 index 00000000..71004993 --- /dev/null +++ b/frontend/src/lib/routing/hostConsoleCapability.ts @@ -0,0 +1,49 @@ +import { + HOST_CONSOLE_CAPABILITY, + HOST_CONSOLE_COMMUNITY_CAPABILITY, +} from '@/lib/capabilities'; + +export type HostConsoleCapabilityState = 'loading' | 'allowed' | 'locked'; + +export interface HostConsoleCapabilityInput { + /** False until NodeContext resolves an active node (cold load). Never treat as local. */ + nodeResolved: boolean; + /** True when the resolved active node is a remote Distributed API Proxy or Pilot node. */ + isRemote: boolean; + /** Hub license: Admiral may accept legacy `host-console` on remotes. */ + isPaid: boolean; + /** + * False while LicenseContext is still loading. Legacy-remote allowance + * must wait so a cold load does not flash LockCard as Community. + */ + licenseReady: boolean; + /** + * Cached `/api/meta` for the active node. Null means metadata has not been + * fetched yet (or the node is unresolved). Must not be confused with + * optimistic `hasCapability()` which returns true while meta is absent. + */ + activeNodeMeta: { capabilities: readonly string[] } | null; +} + +/** + * Whether Host Console content may mount for the active node. + * + * Unresolved nodes stay in `loading`. Local nodes are treated as compatible + * once RBAC passed (same build). Remote nodes wait for metadata, then require + * `host-console-community`, or (Admiral only) legacy `host-console`. + */ +export function resolveHostConsoleCapability( + input: HostConsoleCapabilityInput, +): HostConsoleCapabilityState { + const { nodeResolved, isRemote, isPaid, licenseReady, activeNodeMeta } = input; + if (!nodeResolved) return 'loading'; + if (!isRemote) return 'allowed'; + if (!activeNodeMeta) return 'loading'; + + const caps = activeNodeMeta.capabilities; + if (caps.includes(HOST_CONSOLE_COMMUNITY_CAPABILITY)) return 'allowed'; + if (!caps.includes(HOST_CONSOLE_CAPABILITY)) return 'locked'; + // Legacy host-console only: Admiral hubs may open it; wait for license first. + if (!licenseReady) return 'loading'; + return isPaid ? 'allowed' : 'locked'; +} diff --git a/frontend/src/lib/routing/reachability.test.ts b/frontend/src/lib/routing/reachability.test.ts index 24058316..11362710 100644 --- a/frontend/src/lib/routing/reachability.test.ts +++ b/frontend/src/lib/routing/reachability.test.ts @@ -49,25 +49,25 @@ describe('reachability', () => { expect(isViewHidden('fleet', noFleet)).toBe(true); }); - it('preserves paid views when license metadata failed', () => { - const licenseError = ctx({ licenseStatus: 'error', experimental: true }); + it('preserves host-console when authz is not ready', () => { + const licenseError = ctx({ licenseStatus: 'error', can: (a) => a === 'system:console' }); expect(isViewHidden('host-console', licenseError)).toBe(false); }); - it('does not apply experimental hide to host-console until experimentalReady', () => { - const loading = ctx({ experimental: false, experimentalReady: false, isPaid: true, isAdmin: true }); - expect(isViewHidden('host-console', loading)).toBe(false); + it('hides host-console without system:console when ready', () => { + const noConsole = ctx({ can: () => false, isPaid: false, experimental: false }); + expect(isViewHidden('host-console', noConsole)).toBe(true); + expect(normalizeHiddenView('host-console', noConsole)).toBe('dashboard'); }); - it('hides host-console when experimental is ready and off even for paid admin', () => { - const off = ctx({ experimental: false, experimentalReady: true, isPaid: true, isAdmin: true }); - expect(isViewHidden('host-console', off)).toBe(true); - expect(normalizeHiddenView('host-console', off)).toBe('dashboard'); - }); - - it('keeps host-console when experimental is on for paid admin', () => { - const on = ctx({ experimental: true, experimentalReady: true, isPaid: true, isAdmin: true }); - expect(isViewHidden('host-console', on)).toBe(false); + it('keeps host-console for system:console regardless of tier or experimental', () => { + const community = ctx({ + isPaid: false, + experimental: false, + experimentalReady: true, + can: (a) => a === 'system:console', + }); + expect(isViewHidden('host-console', community)).toBe(false); }); it('hides routing and secrets fleet tabs only after experimentalReady when off', () => { diff --git a/frontend/src/lib/routing/reachability.ts b/frontend/src/lib/routing/reachability.ts index df3836eb..0349e9f8 100644 --- a/frontend/src/lib/routing/reachability.ts +++ b/frontend/src/lib/routing/reachability.ts @@ -43,11 +43,7 @@ export function isViewHidden(view: ActiveView, ctx: ReachabilityContext): boolea if (!ctx.isAdmin && (view === 'auto-updates' || view === 'scheduled-ops')) return true; if (!ctx.can('node:read') && view === 'fleet') return true; if (view === 'host-console') { - // Defer experimental hide until ready so enabled deep links survive cold load. - if (experimentalDiscoveryReady(ctx) && !ctx.experimental) return true; - if (!ctx.isPaid) return true; - if (!ctx.isAdmin) return true; - return false; + return !ctx.can('system:console'); } if (!ctx.isPaid) { if (view === 'audit-log') return true; @@ -67,7 +63,7 @@ export function isViewCapabilityLocked(view: ActiveView, ctx: ReachabilityContex export function isFleetTabHidden(tab: FleetTab, ctx: ReachabilityContext): boolean { if (!authzReady(ctx)) return false; if (tab === 'container-labels' && !ctx.containerLabelsEnabled) return true; - // Defer experimental hide until ready (same cold-load contract as host-console). + // Defer experimental hide until ready so deep links survive cold load. if ((tab === 'routing' || tab === 'secrets') && experimentalDiscoveryReady(ctx) && !ctx.experimental) { return true; } From 85842cc54788b453071890fb57764ab608de78a0 Mon Sep 17 00:00:00 2001 From: Anso Date: Thu, 23 Jul 2026 17:57:04 -0400 Subject: [PATCH 07/13] feat: add service-scoped stack alert rules (#1681) * feat: add service-scoped stack alert rules Stack alerts can target one Compose service or all services. Breach timers are per container and cooldowns are per service so a healthy sibling no longer clears another container's timer or silences a different service. * fix: gate remote scoped alert creates without losing the body Remote hops skip JSON parsing so the proxy stream stays pipeable, which left service_name invisible to the capability gate. Buffer POST /alerts bodies for inspection, fail closed when the remote lacks the capability, and rewrite the buffered bytes on forward. Restore alert-panel alt text to match the unchanged screenshot. * fix: bound remote alert body buffer and reject encoded JSON Cap proxied POST /alerts buffering at the local 100KB JSON limit with structured 413 cleanup, reject non-identity Content-Encoding with 415 so compressed scoped bodies cannot bypass the mixed-version gate, and cover oversized, chunked, and gzip regressions. * fix: harden service-scoped alert delete, cooldown, and proxy gates Reject non-digit alert ids, dual-write last_fired_at for rollback safety, gate cooldown on persisted notification history, fail-fast oversized proxy bodies with 413, and clarify Not in compose UI semantics. * test: expect dispatchAlert persisted result in crash-safety cases Update notification-routing assertions for the new { persisted } return shape so CI matches the cooldown-gating contract. --- .../AutoHealService.evaluate.test.ts | 2 +- backend/src/__tests__/alerts-api.test.ts | 148 +++++- .../__tests__/atomic-deploy-hardening.test.ts | 4 +- backend/src/__tests__/blueprints.test.ts | 8 +- .../src/__tests__/database-metrics.test.ts | 63 ++- .../__tests__/docker-event-service.test.ts | 2 +- .../fleet-sync-retry-service.test.ts | 2 +- .../src/__tests__/fleet-sync-service.test.ts | 2 +- .../__tests__/image-update-service.test.ts | 2 +- backend/src/__tests__/monitor-service.test.ts | 461 +++++++++++++++++- .../__tests__/notification-routing.test.ts | 8 +- .../proxy-service-scoped-alert-gate.test.ts | 256 ++++++++++ .../src/__tests__/scheduler-policy.test.ts | 2 +- .../src/__tests__/scheduler-service.test.ts | 2 +- .../service-scoped-update-routes.test.ts | 2 +- .../src/__tests__/stack-bulk-routes.test.ts | 2 +- .../__tests__/stack-docker-disconnect.test.ts | 2 +- .../stack-down-remove-volumes.test.ts | 2 +- .../__tests__/stack-op-lock-routes.test.ts | 2 +- .../stack-self-protected-routes.test.ts | 2 +- .../stacks-failure-notifications.test.ts | 2 +- backend/src/proxy/remoteNodeProxy.ts | 211 +++++++- backend/src/routes/alerts.ts | 43 +- backend/src/services/CapabilityRegistry.ts | 5 + backend/src/services/DatabaseService.ts | 81 ++- backend/src/services/DockerEventService.ts | 6 +- backend/src/services/MonitorService.ts | 165 +++++-- backend/src/services/NotificationService.ts | 16 +- docs/features/alerts-notifications.mdx | 21 +- .../src/components/StackAlertSheet.test.tsx | 248 ++++++++++ frontend/src/components/StackAlertSheet.tsx | 97 +++- frontend/src/lib/capabilities.ts | 2 + 32 files changed, 1736 insertions(+), 135 deletions(-) create mode 100644 backend/src/__tests__/proxy-service-scoped-alert-gate.test.ts create mode 100644 frontend/src/components/StackAlertSheet.test.tsx diff --git a/backend/src/__tests__/AutoHealService.evaluate.test.ts b/backend/src/__tests__/AutoHealService.evaluate.test.ts index a6085940..e9e530a8 100644 --- a/backend/src/__tests__/AutoHealService.evaluate.test.ts +++ b/backend/src/__tests__/AutoHealService.evaluate.test.ts @@ -55,7 +55,7 @@ beforeEach(() => { const db = DatabaseService.getInstance(); db.getDb().prepare('DELETE FROM auto_heal_history').run(); db.getDb().prepare('DELETE FROM auto_heal_policies').run(); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/alerts-api.test.ts b/backend/src/__tests__/alerts-api.test.ts index 53a3a600..693da057 100644 --- a/backend/src/__tests__/alerts-api.test.ts +++ b/backend/src/__tests__/alerts-api.test.ts @@ -57,8 +57,8 @@ describe('GET /api/alerts', () => { it('filters alerts by stackName query param', async () => { // Seed two alerts for different stacks const db = DatabaseService.getInstance(); - db.addStackAlert({ stack_name: 'web', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 }); - db.addStackAlert({ stack_name: 'api', metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 }); + db.addStackAlert({ stack_name: 'web', service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 }); + db.addStackAlert({ stack_name: 'api', service_name: null, metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 }); const res = await request(app) .get('/api/alerts?stackName=web') @@ -105,10 +105,101 @@ describe('POST /api/alerts', () => { expect(res.status).toBe(201); expect(res.body.id).toBeDefined(); expect(res.body.stack_name).toBe('new-stack'); + expect(res.body.service_name).toBeNull(); expect(res.body.metric).toBe('memory_percent'); expect(res.body.threshold).toBe(85); }); + it('persists a valid dotted service_name', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'svc-stack', + service_name: 'api.web', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(201); + expect(res.body.service_name).toBe('api.web'); + }); + + it('normalizes empty service_name to null', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'empty-svc', + service_name: '', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(201); + expect(res.body.service_name).toBeNull(); + }); + + it('rejects whitespace-only service_name', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'bad-svc', + service_name: ' ', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(400); + }); + + it('rejects reserved _unlabeled service_name', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'bad-svc', + service_name: '_unlabeled', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(400); + }); + + it('rejects service_name when the capability is disabled', async () => { + const { disableCapability, enableCapability, SERVICE_SCOPED_STACK_ALERT_CAPABILITY } = + await import('../services/CapabilityRegistry'); + disableCapability(SERVICE_SCOPED_STACK_ALERT_CAPABILITY); + try { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'cap-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(400); + expect(res.body.code).toBe('capability_unavailable'); + } finally { + enableCapability(SERVICE_SCOPED_STACK_ALERT_CAPABILITY); + } + }); + it('validates required fields and returns 400 for missing data', async () => { const res = await request(app) .post('/api/alerts') @@ -202,6 +293,7 @@ describe('DELETE /api/alerts/:id', () => { // Create an alert to delete const created = DatabaseService.getInstance().addStackAlert({ stack_name: 'delete-me', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 90, @@ -216,6 +308,58 @@ describe('DELETE /api/alerts/:id', () => { expect(res.status).toBe(200); expect(res.body.success).toBe(true); }); + + it('rejects leading-junk ids like 1abc without deleting alert 1', async () => { + const created = DatabaseService.getInstance().addStackAlert({ + stack_name: 'strict-id-junk', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 90, + duration_mins: 0, + cooldown_mins: 0, + }); + expect(created.id).toBeDefined(); + + const res = await request(app) + .delete(`/api/alerts/${created.id}abc`) + .set('Cookie', authCookie); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Invalid alert id'); + expect(DatabaseService.getInstance().getStackAlerts('strict-id-junk').some((a) => a.id === created.id)).toBe(true); + }); + + it('rejects fractional ids like 2.5 without deleting alert 2', async () => { + const first = DatabaseService.getInstance().addStackAlert({ + stack_name: 'strict-id-fraction', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 0, + }); + const second = DatabaseService.getInstance().addStackAlert({ + stack_name: 'strict-id-fraction', + service_name: null, + metric: 'memory_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 0, + }); + expect(second.id).toBeDefined(); + + const res = await request(app) + .delete(`/api/alerts/${second.id}.5`) + .set('Cookie', authCookie); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Invalid alert id'); + const remaining = DatabaseService.getInstance().getStackAlerts('strict-id-fraction').map((a) => a.id); + expect(remaining).toEqual(expect.arrayContaining([first.id, second.id])); + }); }); // --- POST /api/notifications/test --- diff --git a/backend/src/__tests__/atomic-deploy-hardening.test.ts b/backend/src/__tests__/atomic-deploy-hardening.test.ts index 7814549a..9008e501 100644 --- a/backend/src/__tests__/atomic-deploy-hardening.test.ts +++ b/backend/src/__tests__/atomic-deploy-hardening.test.ts @@ -168,7 +168,7 @@ describe('Rollback notifications (M-2)', () => { mockTier('paid'); mockDeployStack.mockResolvedValue(undefined); const { NotificationService } = await import('../services/NotificationService'); - const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(res.status).toBe(200); @@ -186,7 +186,7 @@ describe('Rollback notifications (M-2)', () => { mockTier('paid'); mockDeployStack.mockRejectedValueOnce(new Error('restore failed')); const { NotificationService } = await import('../services/NotificationService'); - const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(res.status).toBe(500); diff --git a/backend/src/__tests__/blueprints.test.ts b/backend/src/__tests__/blueprints.test.ts index 935230af..337298e4 100644 --- a/backend/src/__tests__/blueprints.test.ts +++ b/backend/src/__tests__/blueprints.test.ts @@ -600,7 +600,7 @@ describe('BlueprintReconciler drift alert node wording', () => { it('suggest-mode local drift uses on this node', async () => { const { NotificationService } = await import('../services/NotificationService'); - const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); const nodeId = seedNode(); const bp = seedBlueprint({ name: 'drift-local', drift_mode: 'suggest', nodeIds: [nodeId] }); const node = DatabaseService.getInstance().getNode(nodeId)!; @@ -618,7 +618,7 @@ describe('BlueprintReconciler drift alert node wording', () => { it('suggest-mode remote drift keeps the authoritative node name', async () => { const { NotificationService } = await import('../services/NotificationService'); - const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); const nodeId = seedRemoteNode('sencho-test-02'); const bp = seedBlueprint({ name: 'drift-remote', drift_mode: 'suggest', nodeIds: [nodeId] }); const node = DatabaseService.getInstance().getNode(nodeId)!; @@ -636,7 +636,7 @@ describe('BlueprintReconciler drift alert node wording', () => { it('stateful marker-loss on local uses on this node', async () => { const { NotificationService } = await import('../services/NotificationService'); - const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue(null); const nodeId = seedNode(); const bp = seedBlueprint({ @@ -660,7 +660,7 @@ describe('BlueprintReconciler drift alert node wording', () => { it('enforce correction-failure on local uses on this node', async () => { const { NotificationService } = await import('../services/NotificationService'); - const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); vi.spyOn(BlueprintService.getInstance(), 'deployToNode').mockResolvedValue({ status: 'failed', error: 'compose up failed', diff --git a/backend/src/__tests__/database-metrics.test.ts b/backend/src/__tests__/database-metrics.test.ts index 522e0d87..0e4b37c5 100644 --- a/backend/src/__tests__/database-metrics.test.ts +++ b/backend/src/__tests__/database-metrics.test.ts @@ -324,6 +324,7 @@ describe('DatabaseService - stack alerts CRUD', () => { it('adds and retrieves stack alerts', () => { db.addStackAlert({ stack_name: 'alert-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -339,11 +340,27 @@ describe('DatabaseService - stack alerts CRUD', () => { expect(found.threshold).toBe(80); expect(found.duration_mins).toBe(5); expect(found.cooldown_mins).toBe(15); + expect(found.service_name).toBeNull(); + }); + + it('persists a named service_name', () => { + const created = db.addStackAlert({ + stack_name: 'scoped-stack', + service_name: 'api.web', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 15, + }); + expect(created.service_name).toBe('api.web'); + expect(db.getStackAlerts('scoped-stack')[0].service_name).toBe('api.web'); }); it('filters alerts by stack name', () => { db.addStackAlert({ stack_name: 'filter-stack-a', + service_name: null, metric: 'memory_mb', operator: '>=', threshold: 512, @@ -352,6 +369,7 @@ describe('DatabaseService - stack alerts CRUD', () => { }); db.addStackAlert({ stack_name: 'filter-stack-b', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 90, @@ -371,6 +389,7 @@ describe('DatabaseService - stack alerts CRUD', () => { it('updates last_fired_at timestamp', () => { db.addStackAlert({ stack_name: 'fired-stack', + service_name: null, metric: 'net_rx', operator: '>', threshold: 100, @@ -381,29 +400,55 @@ describe('DatabaseService - stack alerts CRUD', () => { const alerts = db.getStackAlerts('fired-stack'); const alert = alerts[0]; const fireTime = Date.now(); - db.updateStackAlertLastFired(alert.id, fireTime); + db.updateStackAlertLastFired(alert.id!, fireTime); const updated = db.getStackAlerts('fired-stack'); expect(updated[0].last_fired_at).toBe(fireTime); }); - it('deletes an alert by id', () => { - db.addStackAlert({ + it('upserts and reads distinct per-service cooldowns', () => { + const alert = db.addStackAlert({ + stack_name: 'cooldown-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 1, + cooldown_mins: 10, + }); + const id = alert.id!; + expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(false); + + db.upsertStackAlertServiceCooldown(id, 'api', 1000); + db.upsertStackAlertServiceCooldown(id, 'database', 2000); + + expect(db.getStackAlertServiceCooldown(id, 'api')).toBe(1000); + expect(db.getStackAlertServiceCooldown(id, 'database')).toBe(2000); + expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(true); + + db.upsertStackAlertServiceCooldown(id, 'api', 3000); + expect(db.getStackAlertServiceCooldown(id, 'api')).toBe(3000); + }); + + it('deletes an alert by id and removes child cooldown rows', () => { + const alert = db.addStackAlert({ stack_name: 'delete-stack', + service_name: null, metric: 'memory_percent', operator: '>', threshold: 95, duration_mins: 1, cooldown_mins: 5, }); + const id = alert.id!; + db.upsertStackAlertServiceCooldown(id, 'api', Date.now()); + expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(true); - const before = db.getStackAlerts('delete-stack'); - expect(before.length).toBe(1); + db.deleteStackAlert(id); - db.deleteStackAlert(before[0].id); - - const after = db.getStackAlerts('delete-stack'); - expect(after.length).toBe(0); + expect(db.getStackAlerts('delete-stack').length).toBe(0); + expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(false); + expect(db.getStackAlertServiceCooldown(id, 'api')).toBeNull(); }); }); diff --git a/backend/src/__tests__/docker-event-service.test.ts b/backend/src/__tests__/docker-event-service.test.ts index 2fb418e3..e06a97a8 100644 --- a/backend/src/__tests__/docker-event-service.test.ts +++ b/backend/src/__tests__/docker-event-service.test.ts @@ -27,7 +27,7 @@ const { mockGetDocker, mockIsOwnContainer, } = vi.hoisted(() => ({ - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockBroadcastEvent: vi.fn(), mockGetGlobalSettings: vi.fn().mockReturnValue({ global_crash: '1' }), mockGetEvents: vi.fn(), diff --git a/backend/src/__tests__/fleet-sync-retry-service.test.ts b/backend/src/__tests__/fleet-sync-retry-service.test.ts index b86c06c3..109c7acc 100644 --- a/backend/src/__tests__/fleet-sync-retry-service.test.ts +++ b/backend/src/__tests__/fleet-sync-retry-service.test.ts @@ -18,7 +18,7 @@ const { mockGetFleetSyncStatuses: vi.fn().mockReturnValue([]), mockGetSystemState: vi.fn().mockReturnValue(null), mockPushResourceToNode: vi.fn().mockResolvedValue(undefined), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), })); vi.mock('../services/DatabaseService', () => ({ diff --git a/backend/src/__tests__/fleet-sync-service.test.ts b/backend/src/__tests__/fleet-sync-service.test.ts index f30430d5..1787687c 100644 --- a/backend/src/__tests__/fleet-sync-service.test.ts +++ b/backend/src/__tests__/fleet-sync-service.test.ts @@ -40,7 +40,7 @@ const { mockGetSystemState: vi.fn().mockReturnValue(null), mockSetSystemState: vi.fn(), mockTransaction: vi.fn().mockImplementation((fn: () => unknown) => fn()), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockAxiosPost: vi.fn().mockResolvedValue({ data: { success: true } }), })); diff --git a/backend/src/__tests__/image-update-service.test.ts b/backend/src/__tests__/image-update-service.test.ts index 65a0b828..3645ba22 100644 --- a/backend/src/__tests__/image-update-service.test.ts +++ b/backend/src/__tests__/image-update-service.test.ts @@ -25,7 +25,7 @@ const { mockGetSystemState: vi.fn().mockReturnValue('1'), // default: backfilled mockSetSystemState: vi.fn(), mockAddNotificationHistory: vi.fn(), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockGetStacks: vi.fn().mockResolvedValue([]), mockGetStackContent: vi.fn().mockResolvedValue(''), mockGetEnvContent: vi.fn().mockRejectedValue(new Error('no env')), diff --git a/backend/src/__tests__/monitor-service.test.ts b/backend/src/__tests__/monitor-service.test.ts index 4d51b855..39003a15 100644 --- a/backend/src/__tests__/monitor-service.test.ts +++ b/backend/src/__tests__/monitor-service.test.ts @@ -9,7 +9,9 @@ import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsM const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContainerMetric, mockCleanupOldMetrics, mockCleanupOldNotifications, mockCleanupOldAuditLogs, - mockUpdateStackAlertLastFired, mockGetSystemState, mockSetSystemState, + mockUpdateStackAlertLastFired, mockGetStackAlertServiceCooldown, + mockHasAnyStackAlertServiceCooldown, mockUpsertStackAlertServiceCooldown, + mockGetSystemState, mockSetSystemState, mockPruneScanHistoryPerImage, mockDeleteScansByImageRef, mockGetRunningContainers, mockGetAllContainers, mockGetContainerStatsStream, mockGetContainerRestartCount, mockGetDiskUsage, mockGetImages, mockGetStacks, @@ -29,6 +31,9 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine mockCleanupOldNotifications: vi.fn(), mockCleanupOldAuditLogs: vi.fn(), mockUpdateStackAlertLastFired: vi.fn(), + mockGetStackAlertServiceCooldown: vi.fn().mockReturnValue(null), + mockHasAnyStackAlertServiceCooldown: vi.fn().mockReturnValue(false), + mockUpsertStackAlertServiceCooldown: vi.fn(), mockGetSystemState: vi.fn().mockReturnValue(null), mockSetSystemState: vi.fn(), mockPruneScanHistoryPerImage: vi.fn().mockReturnValue(0), @@ -43,7 +48,7 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine }), mockGetImages: vi.fn().mockResolvedValue([]), mockGetStacks: vi.fn().mockResolvedValue([]), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockCurrentLoad: vi.fn().mockResolvedValue({ currentLoad: 10 }), mockMem: vi.fn().mockResolvedValue({ total: 16e9, used: 4e9, active: 4e9, available: 12e9, free: 12e9, buffcache: 0 }), mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]), @@ -65,6 +70,9 @@ vi.mock('../services/DatabaseService', () => ({ cleanupOldNotifications: mockCleanupOldNotifications, cleanupOldAuditLogs: mockCleanupOldAuditLogs, updateStackAlertLastFired: mockUpdateStackAlertLastFired, + getStackAlertServiceCooldown: mockGetStackAlertServiceCooldown, + hasAnyStackAlertServiceCooldown: mockHasAnyStackAlertServiceCooldown, + upsertStackAlertServiceCooldown: mockUpsertStackAlertServiceCooldown, getSystemState: mockGetSystemState, setSystemState: mockSetSystemState, pruneScanHistoryPerImage: mockPruneScanHistoryPerImage, @@ -868,8 +876,12 @@ describe('MonitorService - breach state machine', () => { function setupAlertScenario(cpuPercent: number) { mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); mockGetRunningContainers.mockResolvedValue([{ - Id: 'c1', - Labels: { 'com.docker.compose.project': 'my-stack' }, + Id: 'c1abcdefghijk', + Names: ['/my-stack-api-1'], + Labels: { + 'com.docker.compose.project': 'my-stack', + 'com.docker.compose.service': 'api', + }, }]); mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ cpu_stats: { cpu_usage: { total_usage: 1000 + cpuPercent * 50 }, system_cpu_usage: 10000, online_cpus: 1 }, @@ -879,6 +891,7 @@ describe('MonitorService - breach state machine', () => { mockGetStackAlerts.mockReturnValue([{ id: 1, stack_name: 'my-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -887,6 +900,8 @@ describe('MonitorService - breach state machine', () => { last_fired_at: 0, }]); mockGetGlobalSettings.mockReturnValue({}); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); } it('fires alert when condition met and duration is 0', async () => { @@ -898,9 +913,10 @@ describe('MonitorService - breach state machine', () => { expect(mockDispatchAlert).toHaveBeenCalledWith( 'warning', 'monitor_alert', - 'The **CPU usage** for **my-stack** has exceeded your threshold of **80%** (Currently: 90%).', - { stackName: 'my-stack', actor: 'system:monitor' }, + 'The **CPU usage** for **api** in **my-stack** (container **my-stack-api-1**) has exceeded your threshold of **80%** (Currently: 90%).', + { stackName: 'my-stack', containerName: 'my-stack-api-1', actor: 'system:monitor' }, ); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number)); expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number)); }); @@ -913,12 +929,22 @@ describe('MonitorService - breach state machine', () => { expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU')); }); - it('respects cooldown after firing', async () => { + it('respects cooldown after firing via persisted service cooldown', async () => { + setupAlertScenario(90); + mockGetStackAlertServiceCooldown.mockReturnValue(Date.now() - 30 * 60 * 1000); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled(); + }); + + it('respects legacy last_fired_at floor when no child cooldown rows exist', async () => { setupAlertScenario(90); - // Simulate that alert was fired 30 minutes ago (within 60-min cooldown) mockGetStackAlerts.mockReturnValue([{ id: 1, stack_name: 'my-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -926,14 +952,94 @@ describe('MonitorService - breach state machine', () => { cooldown_mins: 60, last_fired_at: Date.now() - 30 * 60 * 1000, }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); const svc = MonitorService.getInstance(); await (svc as any).evaluate(); - expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled(); + expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled(); }); - it('resets breach state when condition clears', async () => { + it('fires after legacy last_fired_at cooldown expires and writes a child cooldown row', async () => { + setupAlertScenario(90); + mockGetStackAlerts.mockReturnValue([{ + id: 1, + stack_name: 'my-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 60, + last_fired_at: Date.now() - 90 * 60 * 1000, + }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number)); + expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number)); + }); + + it('does not advance cooldown when notification history is not persisted', async () => { + setupAlertScenario(90); + mockDispatchAlert.mockResolvedValueOnce({ persisted: false }); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled(); + expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled(); + + mockDispatchAlert.mockResolvedValueOnce({ persisted: true }); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number)); + expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number)); + }); + + it('drops in-memory breach timers when a rule is deleted', async () => { + const svc = MonitorService.getInstance(); + (svc as any).activeBreaches.set('55:gone-container', { breachStartedAt: Date.now() }); + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetRunningContainers.mockResolvedValue([]); + mockGetStackAlerts.mockReturnValue([]); + mockGetGlobalSettings.mockReturnValue({}); + + await (svc as any).evaluate(); + + expect((svc as any).activeBreaches.has('55:gone-container')).toBe(false); + }); + + it('names unlabeled containers as unknown service in the alert body', async () => { + setupAlertScenario(90); + mockGetRunningContainers.mockResolvedValue([{ + Id: 'orphan123456789', + Names: ['/orphan'], + Labels: { + 'com.docker.compose.project': 'my-stack', + }, + }]); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'warning', + 'monitor_alert', + expect.stringContaining('**unknown service**'), + expect.objectContaining({ stackName: 'my-stack', containerName: 'orphan' }), + ); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, '_unlabeled', expect.any(Number)); + }); + + it('resets breach state when condition clears for that container only', async () => { const svc = MonitorService.getInstance(); // First: breach starts @@ -941,6 +1047,7 @@ describe('MonitorService - breach state machine', () => { mockGetStackAlerts.mockReturnValue([{ id: 42, stack_name: 'my-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -949,13 +1056,14 @@ describe('MonitorService - breach state machine', () => { last_fired_at: 0, }]); await (svc as any).evaluate(); - expect((svc as any).activeBreaches.has(42)).toBe(true); + expect((svc as any).activeBreaches.has('42:c1abcdefghijk')).toBe(true); // Second: condition clears setupAlertScenario(10); mockGetStackAlerts.mockReturnValue([{ id: 42, stack_name: 'my-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -964,7 +1072,28 @@ describe('MonitorService - breach state machine', () => { last_fired_at: 0, }]); await (svc as any).evaluate(); - expect((svc as any).activeBreaches.has(42)).toBe(false); + expect((svc as any).activeBreaches.has('42:c1abcdefghijk')).toBe(false); + }); + + it('falls back to short container id when Names is absent', async () => { + setupAlertScenario(90); + mockGetRunningContainers.mockResolvedValue([{ + Id: 'abcdef1234567890', + Labels: { + 'com.docker.compose.project': 'my-stack', + 'com.docker.compose.service': 'api', + }, + }]); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'warning', + 'monitor_alert', + 'The **CPU usage** for **api** in **my-stack** (container **abcdef123456**) has exceeded your threshold of **80%** (Currently: 90%).', + { stackName: 'my-stack', containerName: 'abcdef123456', actor: 'system:monitor' }, + ); }); }); @@ -1074,7 +1203,12 @@ describe('MonitorService - restart_count metric', () => { expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1'); // restart_count=5 > threshold=3, should fire - expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Restart count'), { stackName: 'my-stack', actor: 'system:monitor' }); + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'warning', + 'monitor_alert', + expect.stringContaining('Restart count'), + { stackName: 'my-stack', containerName: 'c1', actor: 'system:monitor' }, + ); }); it('skips Docker inspect when no restart_count rules exist', async () => { @@ -1261,6 +1395,8 @@ describe('MonitorService - parallel container processing', () => { mockGetGlobalSettings.mockReturnValue({}); mockGetStackAlerts.mockReturnValue([]); mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); }); it('fans out per-container stats fetches in parallel (wall time ~ max not sum)', async () => { @@ -1311,12 +1447,15 @@ describe('MonitorService - parallel container processing', () => { }); it('dispatches a stack alert exactly once per cycle even when multiple containers in the stack breach', async () => { - // 5 containers in the same stack, all breaching the same rule. Without - // the per-cycle dedup, parallel workers race past the cooldown check - // and each fire dispatchAlert before any DB write lands. + // 5 replicas of the same Compose service, all breaching. Dedup is per + // rule+service, so only one fire should land this cycle. const containers = Array.from({ length: 5 }, (_, i) => ({ Id: `container-${i}`, - Labels: { 'com.docker.compose.project': 'shared-stack' }, + Names: [`/shared-stack-web-${i}`], + Labels: { + 'com.docker.compose.project': 'shared-stack', + 'com.docker.compose.service': 'web', + }, })); mockGetRunningContainers.mockResolvedValue(containers); mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ @@ -1328,6 +1467,7 @@ describe('MonitorService - parallel container processing', () => { mockGetStackAlerts.mockReturnValue([{ id: 7, stack_name: 'shared-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -1335,6 +1475,8 @@ describe('MonitorService - parallel container processing', () => { cooldown_mins: 60, last_fired_at: 0, }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); const svc = MonitorService.getInstance(); await (svc as any).evaluate(); @@ -1343,7 +1485,290 @@ describe('MonitorService - parallel container processing', () => { (args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'), ); expect(cpuDispatches).toHaveLength(1); - expect(mockUpdateStackAlertLastFired).toHaveBeenCalledTimes(1); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledTimes(1); + }); + + it('allows different services on an All-services rule to fire in the same cycle', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'api-1', + Names: ['/stack-api-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'db-1', + Names: ['/stack-db-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + mockGetStackAlerts.mockReturnValue([{ + id: 8, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 60, + last_fired_at: 0, + }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); + mockGetGlobalSettings.mockReturnValue({}); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + const cpuDispatches = mockDispatchAlert.mock.calls.filter( + (args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'), + ); + expect(cpuDispatches).toHaveLength(2); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(8, 'api', expect.any(Number)); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(8, 'database', expect.any(Number)); + }); + + it('does not let a healthy sibling clear another container breach timer', async () => { + const svc = MonitorService.getInstance(); + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetStackAlerts.mockReturnValue([{ + id: 9, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 999, + cooldown_mins: 0, + last_fired_at: 0, + }]); + + const highCpu = JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + }); + const lowCpu = JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 1100 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + }); + + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'api-hot', + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'db-cool', + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockImplementation(async (id: string) => (id === 'api-hot' ? highCpu : lowCpu)); + + await (svc as any).evaluate(); + + expect((svc as any).activeBreaches.has('9:api-hot')).toBe(true); + expect((svc as any).activeBreaches.has('9:db-cool')).toBe(false); + }); + + it('ignores sibling services for a service-scoped rule', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'api-1', + Names: ['/stack-api-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'db-1', + Names: ['/stack-db-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + mockGetStackAlerts.mockReturnValue([{ + id: 10, + stack_name: 'shared-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 0, + last_fired_at: 0, + }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + expect(mockDispatchAlert.mock.calls[0][2]).toContain('**api**'); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(10, 'api', expect.any(Number)); + }); + + it('does not block service B after service A fires under new-schema cooldown', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'db-1', + Names: ['/stack-db-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + mockGetStackAlerts.mockReturnValue([{ + id: 11, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 60, + last_fired_at: Date.now() - 5 * 60 * 1000, // would block under old rule-wide semantics + }]); + // api already has a cooldown row; database does not, so database may still fire. + mockGetStackAlertServiceCooldown.mockImplementation((_id: number, service: string) => + service === 'api' ? Date.now() - 5 * 60 * 1000 : null, + ); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(true); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(11, 'database', expect.any(Number)); + }); + + it('honors persisted per-service cooldown after a fresh MonitorService instance', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'api-1', + Names: ['/stack-api-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'api-2', + Names: ['/stack-api-2'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'db-1', + Names: ['/stack-db-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + mockGetStackAlerts.mockReturnValue([{ + id: 13, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 60, + last_fired_at: 0, + }]); + mockGetStackAlertServiceCooldown.mockImplementation((_id: number, service: string) => + service === 'api' ? Date.now() - 5 * 60 * 1000 : null, + ); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(true); + + // beforeEach already cleared the singleton; this instance has empty maps. + const svc = MonitorService.getInstance(); + expect((svc as any).activeBreaches.size).toBe(0); + expect((svc as any).firedThisCycle.size).toBe(0); + + await (svc as any).evaluate(); + + const cpuDispatches = mockDispatchAlert.mock.calls.filter( + (args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'), + ); + expect(cpuDispatches).toHaveLength(1); + expect(cpuDispatches[0][2]).toContain('**database**'); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(13, 'database', expect.any(Number)); + expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalledWith(13, 'api', expect.any(Number)); + }); + + it('preserves breach timers when container enumeration fails', async () => { + const svc = MonitorService.getInstance(); + (svc as any).activeBreaches.set('12:still-running', { breachStartedAt: Date.now() }); + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetStackAlerts.mockReturnValue([{ + id: 12, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 999, + cooldown_mins: 0, + last_fired_at: 0, + }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockRejectedValue(new Error('docker down')); + + await (svc as any).evaluate(); + + expect((svc as any).activeBreaches.has('12:still-running')).toBe(true); + }); + + it('removes breach timers for stopped containers after successful enumeration', async () => { + const svc = MonitorService.getInstance(); + (svc as any).activeBreaches.set('13:gone', { breachStartedAt: Date.now() }); + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetStackAlerts.mockReturnValue([{ + id: 13, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 999, + cooldown_mins: 0, + last_fired_at: 0, + }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'still-here', + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 1100 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + + await (svc as any).evaluate(); + + expect((svc as any).activeBreaches.has('13:gone')).toBe(false); }); it('caps simultaneous Docker calls at MAX_CONTAINER_CONCURRENCY', async () => { diff --git a/backend/src/__tests__/notification-routing.test.ts b/backend/src/__tests__/notification-routing.test.ts index 1bb7dc6a..08125b96 100644 --- a/backend/src/__tests__/notification-routing.test.ts +++ b/backend/src/__tests__/notification-routing.test.ts @@ -217,7 +217,7 @@ describe('NotificationService - routing logic', () => { mockFetch.mockRejectedValueOnce(new Error('Network timeout')); // Should not throw - await expect(svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' })).resolves.toBeUndefined(); + await expect(svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' })).resolves.toEqual({ persisted: true }); }); it('does not dispatch to global agents when routes array is empty and no stackName', async () => { @@ -553,7 +553,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', () await expect( svc.dispatchAlert('error', 'monitor_alert', 'Container crashed', { stackName: 'my-app' }), - ).resolves.toBeUndefined(); + ).resolves.toEqual({ persisted: false }); // No external dispatch and no routing read when there is no persisted row. expect(mockFetch).not.toHaveBeenCalled(); @@ -580,7 +580,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', () await expect( svc.dispatchAlert('error', 'monitor_alert', 'Container crashed', { stackName: 'my-app' }), - ).resolves.toBeUndefined(); + ).resolves.toEqual({ persisted: true }); // The history row was still written; only routing failed, and it was logged. expect(mockAddNotificationHistory).toHaveBeenCalledTimes(1); @@ -598,7 +598,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', () await expect( svc.dispatchAlert('info', 'system', 'Host rebooted'), - ).resolves.toBeUndefined(); + ).resolves.toEqual({ persisted: true }); expect(mockAddNotificationHistory).toHaveBeenCalledTimes(1); expect(consoleErrorSpy).toHaveBeenCalledWith('[Notify] dispatchAlert failed:', expect.any(Error)); diff --git a/backend/src/__tests__/proxy-service-scoped-alert-gate.test.ts b/backend/src/__tests__/proxy-service-scoped-alert-gate.test.ts new file mode 100644 index 00000000..614ab340 --- /dev/null +++ b/backend/src/__tests__/proxy-service-scoped-alert-gate.test.ts @@ -0,0 +1,256 @@ +/** + * Mixed-version gate for POST /api/alerts with a non-empty service_name. + * + * Remote-targeted requests skip express.json() so the raw stream stays + * pipeable. The hub must still buffer that body under the local JSON size + * cap, reject compressed bodies (cannot inspect safely), reject scoped + * creates when the remote lacks service-scoped-stack-alert, and forward + * unscoped identity-encoded JSON intact when allowed. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import http from 'http'; +import type { AddressInfo } from 'net'; +import zlib from 'zlib'; +import request from 'supertest'; +import jwt from 'jsonwebtoken'; +import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; + +let tmpDir: string; +let app: import('express').Express; +let listenServer: http.Server; +let listenPort: number; +let adminBearer: string; +let capServer: http.Server; +let noCapServer: http.Server; +let capNodeId: number; +let noCapNodeId: number; + +const noCapPaths: string[] = []; +const capPaths: string[] = []; +let lastCapBody: Buffer | null = null; +let lastNoCapBody: Buffer | null = null; + +function metaServer( + capabilities: string[], + seen: string[], + onBody: (body: Buffer) => void, +): http.Server { + return http.createServer((req, res) => { + if (req.url) seen.push(req.url); + const chunks: Buffer[] = []; + req.on('data', (chunk: Buffer) => chunks.push(chunk)); + req.on('end', () => { + if (!req.url?.startsWith('/api/meta')) { + onBody(Buffer.concat(chunks)); + } + res.writeHead(200, { 'Content-Type': 'application/json' }); + if (req.url?.startsWith('/api/meta')) { + res.end(JSON.stringify({ version: '0.93.0', capabilities })); + } else { + res.end(JSON.stringify({ id: 1, ok: true })); + } + }); + }); +} + +async function listen(server: http.Server): Promise { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return (server.address() as AddressInfo).port; +} + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance(); + + adminBearer = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' }); + + capServer = metaServer( + ['cross-node-rbac', 'service-scoped-stack-alert'], + capPaths, + (body) => { lastCapBody = body; }, + ); + noCapServer = metaServer( + ['cross-node-rbac'], + noCapPaths, + (body) => { lastNoCapBody = body; }, + ); + const capPort = await listen(capServer); + const noCapPort = await listen(noCapServer); + + capNodeId = db.addNode({ + name: 'alert-cap-remote', type: 'remote', mode: 'proxy', compose_dir: '/tmp', + is_default: false, api_url: `http://127.0.0.1:${capPort}`, api_token: 'cap-token', + }); + noCapNodeId = db.addNode({ + name: 'alert-nocap-remote', type: 'remote', mode: 'proxy', compose_dir: '/tmp', + is_default: false, api_url: `http://127.0.0.1:${noCapPort}`, api_token: 'nocap-token', + }); + + listenServer = http.createServer(app); + listenPort = await listen(listenServer); +}); + +afterAll(async () => { + await new Promise((resolve) => listenServer.close(() => resolve())); + await new Promise((resolve) => capServer.close(() => resolve())); + await new Promise((resolve) => noCapServer.close(() => resolve())); + cleanupTestDb(tmpDir); +}); + +beforeEach(() => { + noCapPaths.length = 0; + capPaths.length = 0; + lastNoCapBody = null; + lastCapBody = null; +}); + +const scopedPayload = { + stack_name: 'web', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, +}; + +const unscopedPayload = { + stack_name: 'web', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, +}; + +describe('remote proxy service-scoped alert gate', () => { + it('returns 400 for scoped alert create when remote lacks service-scoped-stack-alert', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(noCapNodeId)) + .set('Content-Type', 'application/json') + .send(scopedPayload); + + expect(res.status).toBe(400); + expect(res.body.code).toBe('capability_unavailable'); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false); + expect(noCapPaths.some((p) => p.startsWith('/api/meta'))).toBe(true); + expect(lastNoCapBody).toBeNull(); + }); + + it('forwards unscoped alert JSON intact when remote lacks the capability', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(noCapNodeId)) + .set('Content-Type', 'application/json') + .send(unscopedPayload); + + expect(res.status).toBe(200); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(true); + expect(lastNoCapBody).not.toBeNull(); + expect(JSON.parse(lastNoCapBody!.toString('utf-8'))).toEqual(unscopedPayload); + }); + + it('forwards scoped alert JSON intact when remote advertises the capability', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(capNodeId)) + .set('Content-Type', 'application/json') + .send(scopedPayload); + + expect(res.status).toBe(200); + expect(capPaths.some((p) => p.includes('/api/alerts'))).toBe(true); + expect(lastCapBody).not.toBeNull(); + expect(JSON.parse(lastCapBody!.toString('utf-8'))).toEqual(scopedPayload); + }); + + it('rejects oversized identity JSON via Content-Length before upstream', async () => { + const huge = { + ...unscopedPayload, + stack_name: 'x'.repeat(120 * 1024), + }; + + const started = Date.now(); + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(noCapNodeId)) + .set('Content-Type', 'application/json') + .send(huge); + const elapsedMs = Date.now() - started; + + expect(res.status).toBe(413); + expect(res.body.code).toBe('entity_too_large'); + expect(elapsedMs).toBeLessThan(2000); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false); + expect(lastNoCapBody).toBeNull(); + }); + + it('rejects chunked bodies once streamed bytes exceed the JSON limit', async () => { + const oversized = Buffer.from(JSON.stringify({ + ...unscopedPayload, + stack_name: 'y'.repeat(120 * 1024), + })); + + const result = await new Promise<{ status: number; body: { code?: string } }>((resolve, reject) => { + const req = http.request({ + hostname: '127.0.0.1', + port: listenPort, + method: 'POST', + path: '/api/alerts', + headers: { + Authorization: `Bearer ${adminBearer}`, + 'x-node-id': String(noCapNodeId), + 'Content-Type': 'application/json', + 'Transfer-Encoding': 'chunked', + }, + }, (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => { + const text = Buffer.concat(chunks).toString('utf-8'); + try { + resolve({ status: res.statusCode ?? 0, body: JSON.parse(text) as { code?: string } }); + } catch { + resolve({ status: res.statusCode ?? 0, body: {} }); + } + }); + }); + req.on('error', reject); + // Write in small chunks so the hub crosses the limit mid-stream. + const step = 16 * 1024; + for (let offset = 0; offset < oversized.length; offset += step) { + req.write(oversized.subarray(offset, Math.min(offset + step, oversized.length))); + } + req.end(); + }); + + expect(result.status).toBe(413); + expect(result.body.code).toBe('entity_too_large'); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false); + expect(lastNoCapBody).toBeNull(); + }); + + it('rejects gzip-encoded scoped JSON instead of forwarding as All-services', async () => { + const compressed = zlib.gzipSync(Buffer.from(JSON.stringify(scopedPayload), 'utf-8')); + + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(noCapNodeId)) + .set('Content-Type', 'application/json') + .set('Content-Encoding', 'gzip') + .send(compressed); + + expect(res.status).toBe(415); + expect(res.body.code).toBe('encoding_unsupported'); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false); + expect(lastNoCapBody).toBeNull(); + }); +}); diff --git a/backend/src/__tests__/scheduler-policy.test.ts b/backend/src/__tests__/scheduler-policy.test.ts index 0edca7c4..0739c6af 100644 --- a/backend/src/__tests__/scheduler-policy.test.ts +++ b/backend/src/__tests__/scheduler-policy.test.ts @@ -34,7 +34,7 @@ const { mockMarkStaleRunsAsFailed: vi.fn().mockReturnValue(0), mockDeleteOldScans: vi.fn().mockReturnValue(0), mockGetTier: vi.fn().mockReturnValue('paid'), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockGetProxyTarget: vi.fn().mockReturnValue(null), mockIsTrivyAvailable: vi.fn().mockReturnValue(true), mockScanAllNodeImages: vi.fn(), diff --git a/backend/src/__tests__/scheduler-service.test.ts b/backend/src/__tests__/scheduler-service.test.ts index 18f8ae4f..f629f717 100644 --- a/backend/src/__tests__/scheduler-service.test.ts +++ b/backend/src/__tests__/scheduler-service.test.ts @@ -59,7 +59,7 @@ const { mockGetStackContent: vi.fn().mockResolvedValue(''), mockGetEnvContent: vi.fn().mockResolvedValue(''), mockCheckImage: vi.fn().mockResolvedValue({ hasUpdate: false }), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockGetProxyTarget: vi.fn().mockReturnValue(null), mockIsTrivyAvailable: vi.fn().mockReturnValue(true), mockScanAllNodeImages: vi.fn().mockResolvedValue({ diff --git a/backend/src/__tests__/service-scoped-update-routes.test.ts b/backend/src/__tests__/service-scoped-update-routes.test.ts index f2564257..5811e4e9 100644 --- a/backend/src/__tests__/service-scoped-update-routes.test.ts +++ b/backend/src/__tests__/service-scoped-update-routes.test.ts @@ -45,7 +45,7 @@ beforeAll(async () => { writeStack('web'); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-bulk-routes.test.ts b/backend/src/__tests__/stack-bulk-routes.test.ts index 3d9fe675..f62454c8 100644 --- a/backend/src/__tests__/stack-bulk-routes.test.ts +++ b/backend/src/__tests__/stack-bulk-routes.test.ts @@ -78,7 +78,7 @@ beforeAll(async () => { authCookie = await loginAsTestAdmin(app); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-docker-disconnect.test.ts b/backend/src/__tests__/stack-docker-disconnect.test.ts index 90dfcf8f..f1150bfa 100644 --- a/backend/src/__tests__/stack-docker-disconnect.test.ts +++ b/backend/src/__tests__/stack-docker-disconnect.test.ts @@ -90,7 +90,7 @@ beforeAll(async () => { authCookie = await loginAsTestAdmin(app); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-down-remove-volumes.test.ts b/backend/src/__tests__/stack-down-remove-volumes.test.ts index e390a3d5..117b8bb7 100644 --- a/backend/src/__tests__/stack-down-remove-volumes.test.ts +++ b/backend/src/__tests__/stack-down-remove-volumes.test.ts @@ -68,7 +68,7 @@ beforeAll(async () => { writeStack('web'); const { NotificationService } = await import('../services/NotificationService'); - dispatchAlertSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + dispatchAlertSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-op-lock-routes.test.ts b/backend/src/__tests__/stack-op-lock-routes.test.ts index 08a1fb45..4d76ea95 100644 --- a/backend/src/__tests__/stack-op-lock-routes.test.ts +++ b/backend/src/__tests__/stack-op-lock-routes.test.ts @@ -89,7 +89,7 @@ beforeAll(async () => { authCookie = await loginAsTestAdmin(app); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-self-protected-routes.test.ts b/backend/src/__tests__/stack-self-protected-routes.test.ts index 315a33e8..db7330d5 100644 --- a/backend/src/__tests__/stack-self-protected-routes.test.ts +++ b/backend/src/__tests__/stack-self-protected-routes.test.ts @@ -102,7 +102,7 @@ beforeAll(async () => { writeStack('web'); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stacks-failure-notifications.test.ts b/backend/src/__tests__/stacks-failure-notifications.test.ts index 168f0877..588e311b 100644 --- a/backend/src/__tests__/stacks-failure-notifications.test.ts +++ b/backend/src/__tests__/stacks-failure-notifications.test.ts @@ -132,7 +132,7 @@ beforeAll(async () => { const { NotificationService } = await import('../services/NotificationService'); dispatchAlertSpy = vi .spyOn(NotificationService.getInstance(), 'dispatchAlert') - .mockResolvedValue(undefined); + .mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/proxy/remoteNodeProxy.ts b/backend/src/proxy/remoteNodeProxy.ts index b359dd0c..e3a35185 100644 --- a/backend/src/proxy/remoteNodeProxy.ts +++ b/backend/src/proxy/remoteNodeProxy.ts @@ -5,7 +5,7 @@ import { PROXY_TIER_HEADER, PROXY_ROLE_HEADER, PROXY_DEPLOY_SOURCE_HEADER, PROXY import { LicenseService } from '../services/LicenseService'; import { isProxyExemptPath } from '../helpers/proxyExemptPaths'; import { remoteSupportsCrossNodeRbac, remoteAdvertisesCapability } from '../helpers/remoteCapabilities'; -import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry'; +import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY, SERVICE_SCOPED_STACK_ALERT_CAPABILITY } from '../services/CapabilityRegistry'; import { getErrorMessage } from '../utils/errors'; import { DatabaseService } from '../services/DatabaseService'; import { redactSensitiveText } from '../utils/safeLog'; @@ -145,8 +145,19 @@ export function createRemoteProxyMiddleware(): RequestHandler { } // Body forwarding: conditionalJsonParser skips parsing for remote // requests (see middleware/jsonParser.ts), so req's raw stream is - // intact and http-proxy's req.pipe(proxyReq) forwards the body - // automatically. + // usually intact and http-proxy's req.pipe(proxyReq) forwards it. + // When a gate must inspect JSON (POST /alerts), we buffer into + // req.rawBody first; rewrite that buffer here because the stream is + // already consumed. + if (req.rawBody) { + proxyReq.removeHeader('Transfer-Encoding'); + proxyReq.removeHeader('Content-Length'); + if (!proxyReq.getHeader('Content-Type')) { + proxyReq.setHeader('Content-Type', 'application/json'); + } + proxyReq.setHeader('Content-Length', req.rawBody.length); + proxyReq.write(req.rawBody); + } }, proxyRes: (proxyRes, req) => { // Mark every response forwarded from a remote node with a sentinel @@ -246,7 +257,8 @@ export function createRemoteProxyMiddleware(): RequestHandler { } } - // Mixed-version RBAC gate (non-admin only). + // Mixed-version RBAC gate (non-admin only). Runs before alert body + // buffering so an unauthorized client cannot force unbounded memory use. if (req.user?.role !== 'admin') { const rbacSupported = await remoteSupportsCrossNodeRbac(req.nodeId); if (!rbacSupported) { @@ -257,6 +269,49 @@ export function createRemoteProxyMiddleware(): RequestHandler { } } + // POST /alerts bodies are not on req.body for remote hops (JSON parsing + // is skipped so the stream can be piped). Buffer once under the same + // 100 KB cap as express.json(), gate on service_name, then rewrite + // rawBody in on.proxyReq. Compressed bodies are rejected: the hub cannot + // inspect them, and treating parse failure as unscoped would bypass the + // mixed-version gate. + if (isAlertCreateRoute(req)) { + if (hasNonIdentityContentEncoding(req)) { + await drainRequestBody(req); + res.status(415).json({ + error: 'Compressed request bodies are not supported for remote alert creates', + code: 'encoding_unsupported', + }); + return; + } + try { + req.rawBody = await bufferRequestBody(req, ALERT_PROXY_BODY_LIMIT); + } catch (err) { + const status = Number((err as { status?: number }).status); + if (status === 413) { + console.error('[remoteNodeProxy] alert body rejected as too large:', err); + res.status(413).json({ error: 'Alert payload too large', code: 'entity_too_large' }); + return; + } + if (status === 400) { + console.error('[remoteNodeProxy] alert body incomplete:', err); + res.status(400).json({ error: 'Incomplete request body' }); + return; + } + throw err; + } + if (alertCreateHasScopedService(req.rawBody)) { + const supported = await remoteAdvertisesCapability(req.nodeId, SERVICE_SCOPED_STACK_ALERT_CAPABILITY); + if (!supported) { + res.status(400).json({ + error: 'Service-scoped alert rules are not supported on this node', + code: 'capability_unavailable', + }); + return; + } + } + } + req.proxyTarget = target; beginProxyTiming(req, res); proxy(req, res, next); @@ -283,3 +338,151 @@ function isServiceScopedUpdateRoute(req: Request): boolean { } return false; } + +/** POST /alerts (path is post-/api strip). */ +function isAlertCreateRoute(req: Request): boolean { + return req.method === 'POST' && /^\/alerts\/?$/.test(req.path); +} + +/** Same default as express.json(); remote alert creates must not exceed it. */ +const ALERT_PROXY_BODY_LIMIT = 100 * 1024; + +/** Max time to wait for leftover body bytes after a size/encoding reject. */ +const DRAIN_TIMEOUT_MS = 5_000; + +/** Error with HTTP status for the alert-body gate catch mapper. */ +function alertBodyError(message: string, status: number): Error { + return Object.assign(new Error(message), { status, expose: true }); +} + +/** True when Content-Encoding is present and not identity (gzip/deflate/br/…). */ +function hasNonIdentityContentEncoding(req: Request): boolean { + const raw = req.headers['content-encoding']; + if (raw == null) return false; + const value = Array.isArray(raw) ? raw.join(',') : raw; + return value.split(',').some((part) => { + const encoding = part.trim().toLowerCase(); + return encoding.length > 0 && encoding !== 'identity'; + }); +} + +/** True when the buffered JSON alert body targets a specific Compose service. */ +function alertCreateHasScopedService(rawBody: Buffer): boolean { + if (rawBody.length === 0) return false; + try { + const parsed = JSON.parse(rawBody.toString('utf-8')) as { service_name?: unknown }; + return typeof parsed.service_name === 'string' && parsed.service_name.trim() !== ''; + } catch { + // Identity-encoded non-JSON is forwarded as-is; the remote rejects it. + // Encoded bodies never reach here (rejected earlier). + return false; + } +} + +/** + * Consume remaining request bytes (or wait for abort/close) so the response + * can flush without leaving the socket half-open. Does not buffer into memory. + * Caps wait time so a stalled or endless chunked stream cannot hang the gate. + */ +function drainRequestBody(req: Request): Promise { + return new Promise((resolve) => { + if (req.readableEnded || req.destroyed) { + resolve(); + return; + } + let settled = false; + const done = () => { + if (settled) return; + settled = true; + clearTimeout(timer); + req.off('end', done); + req.off('error', done); + req.off('aborted', done); + req.off('close', done); + resolve(); + }; + // Bound hang time: destroy after timeout so mid-stream rejects still settle. + const timer = setTimeout(() => { + if (!req.destroyed) req.destroy(); + done(); + }, DRAIN_TIMEOUT_MS); + req.resume(); + req.once('end', done); + req.once('error', done); + req.once('aborted', done); + req.once('close', done); + }); +} + +/** + * Buffer the request so a capability gate can inspect JSON without leaving + * http-proxy with an already-ended empty stream. Enforces `limit` on both + * declared Content-Length and streamed accumulation. + */ +async function bufferRequestBody(req: Request, limit: number): Promise { + if (req.rawBody) { + if (req.rawBody.length > limit) { + throw alertBodyError('Alert payload too large', 413); + } + return req.rawBody; + } + if (req.readableEnded) return Buffer.alloc(0); + + const declared = Number.parseInt(String(req.headers['content-length'] ?? ''), 10); + if (Number.isFinite(declared) && declared > limit) { + // Reject immediately so the client gets a structured 413; drain leftover + // bytes in the background so the socket can close without holding the gate. + void drainRequestBody(req); + throw alertBodyError('Alert payload too large', 413); + } + + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let total = 0; + let settled = false; + + const settle = (fn: () => void) => { + if (settled) return; + settled = true; + cleanup(); + fn(); + }; + const finish = (buf: Buffer) => settle(() => resolve(buf)); + const fail = (err: Error) => settle(() => reject(err)); + + const onData = (chunk: Buffer) => { + const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + total += buf.length; + if (total > limit) { + chunks.length = 0; + // fail() removes listeners; drain leftover bytes so the socket can close. + fail(alertBodyError('Alert payload too large', 413)); + void drainRequestBody(req); + return; + } + chunks.push(buf); + }; + const onEnd = () => finish(Buffer.concat(chunks)); + const onError = (err: Error) => fail(err); + const onAborted = () => fail(alertBodyError('Client aborted request body', 400)); + const onClose = () => { + if (!settled && !req.readableEnded) { + fail(alertBodyError('Client closed request before body finished', 400)); + } + }; + + function cleanup(): void { + req.off('data', onData); + req.off('end', onEnd); + req.off('error', onError); + req.off('aborted', onAborted); + req.off('close', onClose); + } + + req.on('data', onData); + req.on('end', onEnd); + req.on('error', onError); + req.on('aborted', onAborted); + req.on('close', onClose); + }); +} diff --git a/backend/src/routes/alerts.ts b/backend/src/routes/alerts.ts index b6bea853..1b707c1a 100644 --- a/backend/src/routes/alerts.ts +++ b/backend/src/routes/alerts.ts @@ -3,9 +3,21 @@ import { z } from 'zod'; import { DatabaseService } from '../services/DatabaseService'; import { authMiddleware } from '../middleware/auth'; import { requireAdmin } from '../middleware/tierGates'; +import { isValidServiceName } from '../utils/validation'; +import { + getActiveCapabilities, + SERVICE_SCOPED_STACK_ALERT_CAPABILITY, +} from '../services/CapabilityRegistry'; const AlertCreateSchema = z.object({ stack_name: z.string().min(1).max(255), + service_name: z.preprocess( + (val) => (val === '' ? null : val), + z.string().max(255).nullable().optional().refine( + (val) => val == null || isValidServiceName(val), + { message: 'Invalid service name' }, + ), + ), metric: z.enum(['cpu_percent', 'memory_percent', 'memory_mb', 'net_rx', 'net_tx', 'restart_count']), operator: z.enum(['>', '>=', '<', '<=', '==']), threshold: z.number().min(0), @@ -22,7 +34,8 @@ alertsRouter.get('/', authMiddleware, async (req: Request, res: Response) => { const alerts = DatabaseService.getInstance().getStackAlerts(stackName); res.json(alerts); - } catch { + } catch (error) { + console.error('Failed to fetch alerts:', error); res.status(500).json({ error: 'Failed to fetch alerts' }); } }); @@ -34,8 +47,23 @@ alertsRouter.post('/', authMiddleware, async (req: Request, res: Response) => { res.status(400).json({ error: 'Invalid alert data', details: parsed.error.flatten().fieldErrors }); return; } + const { service_name, ...alertFields } = parsed.data; + const serviceName = service_name ?? null; + if ( + serviceName != null + && !getActiveCapabilities().includes(SERVICE_SCOPED_STACK_ALERT_CAPABILITY) + ) { + res.status(400).json({ + error: 'This node does not support service-scoped alert rules', + code: 'capability_unavailable', + }); + return; + } try { - const created = DatabaseService.getInstance().addStackAlert(parsed.data); + const created = DatabaseService.getInstance().addStackAlert({ + ...alertFields, + service_name: serviceName, + }); res.status(201).json(created); } catch (error) { console.error('Failed to add alert:', error); @@ -45,11 +73,18 @@ alertsRouter.post('/', authMiddleware, async (req: Request, res: Response) => { alertsRouter.delete('/:id', authMiddleware, async (req: Request, res: Response) => { if (!requireAdmin(req, res)) return; + // Reject leading-junk / fractional ids (parseInt("1abc") === 1, parseInt("2.5") === 2). + const rawId = String(req.params.id ?? ''); + const id = /^\d+$/.test(rawId) ? Number.parseInt(rawId, 10) : NaN; + if (!Number.isInteger(id) || id <= 0) { + res.status(400).json({ error: 'Invalid alert id' }); + return; + } try { - const id = parseInt(req.params.id as string, 10); DatabaseService.getInstance().deleteStackAlert(id); res.json({ success: true }); - } catch { + } catch (error) { + console.error('Failed to delete alert:', error); res.status(500).json({ error: 'Failed to delete alert' }); } }); diff --git a/backend/src/services/CapabilityRegistry.ts b/backend/src/services/CapabilityRegistry.ts index 578155a9..ba806c5d 100644 --- a/backend/src/services/CapabilityRegistry.ts +++ b/backend/src/services/CapabilityRegistry.ts @@ -59,6 +59,7 @@ export const CAPABILITIES = [ 'stack-down-remove-volumes', 'guided-external-network-preflight', 'service-scoped-update', + 'service-scoped-stack-alert', ] as const; /** @@ -88,6 +89,10 @@ export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes' /** Capability for the nested per-service update/restore routes and the `effective-services` model they read. */ export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability; +/** Capability for nullable `service_name` on stack alert rules and per-service cooldown evaluation. */ +export const SERVICE_SCOPED_STACK_ALERT_CAPABILITY = + 'service-scoped-stack-alert' as const satisfies Capability; + /** Returns true when the string is a usable semver version. */ export function isValidVersion(v: string | null | undefined): v is string { return !!v && v !== 'unknown' && v !== '0.0.0-dev' && !!semver.valid(v); diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 2d36dd49..9f66c424 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -101,6 +101,7 @@ function stringifyServicesJson(services: StackServiceStatus[], generation: numbe export interface StackAlert { id?: number; stack_name: string; + service_name: string | null; metric: string; operator: string; threshold: number; @@ -1062,6 +1063,7 @@ export class DatabaseService { this.migrateStackDossierHashes(); this.migrateGitSourceMultiFile(); this.migrateNodeUpdateSkips(); + this.migrateStackAlertServiceScope(); // Reset the cache once at end of constructor in case any migration // populated it via getGlobalSettings() and a subsequent migration @@ -1098,6 +1100,7 @@ export class DatabaseService { CREATE TABLE IF NOT EXISTS stack_alerts ( id INTEGER PRIMARY KEY AUTOINCREMENT, stack_name TEXT NOT NULL, + service_name TEXT, metric TEXT NOT NULL, operator TEXT NOT NULL, threshold REAL NOT NULL, @@ -1106,6 +1109,17 @@ export class DatabaseService { last_fired_at INTEGER DEFAULT 0 ); + -- FK cascade is declarative only: PRAGMA foreign_keys is never enabled + -- on this connection, so parent deletes must remove children explicitly + -- (see deleteStackAlert). + CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns ( + alert_id INTEGER NOT NULL, + service_name TEXT NOT NULL, + last_fired_at INTEGER NOT NULL, + PRIMARY KEY (alert_id, service_name), + FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE + ); + CREATE TABLE IF NOT EXISTS notification_history ( id INTEGER PRIMARY KEY AUTOINCREMENT, node_id INTEGER NOT NULL DEFAULT 0, @@ -2280,6 +2294,25 @@ export class DatabaseService { } } + private migrateStackAlertServiceScope(): void { + this.tryAddColumn('stack_alerts', 'service_name', 'TEXT'); + try { + // FK is not enforced (foreign_keys pragma off); deleteStackAlert removes children. + this.db.prepare(` + CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns ( + alert_id INTEGER NOT NULL, + service_name TEXT NOT NULL, + last_fired_at INTEGER NOT NULL, + PRIMARY KEY (alert_id, service_name), + FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE + ) + `).run(); + } catch (e) { + console.error('[DatabaseService] stack_alert_service_cooldowns migration failed:', (e as Error).message); + throw e; + } + } + private migrateScanPolicyFleetColumns(): void { this.tryAddColumn('scan_policies', 'node_identity', "TEXT NOT NULL DEFAULT ''"); this.tryAddColumn('scan_policies', 'replicated_from_control', 'INTEGER NOT NULL DEFAULT 0'); @@ -3075,22 +3108,19 @@ export class DatabaseService { // --- Stack Alerts --- public getStackAlerts(stackName?: string): StackAlert[] { - let stmt; if (stackName) { - stmt = this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?'); - return stmt.all(stackName) as StackAlert[]; - } else { - stmt = this.db.prepare('SELECT * FROM stack_alerts'); - return stmt.all() as StackAlert[]; + return this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?').all(stackName) as StackAlert[]; } + return this.db.prepare('SELECT * FROM stack_alerts').all() as StackAlert[]; } public addStackAlert(alert: StackAlert): StackAlert { const stmt = this.db.prepare( - 'INSERT INTO stack_alerts (stack_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?)' + 'INSERT INTO stack_alerts (stack_name, service_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)' ); const result = stmt.run( alert.stack_name, + alert.service_name ?? null, alert.metric, alert.operator, alert.threshold, @@ -3101,9 +3131,16 @@ export class DatabaseService { return this.db.prepare('SELECT * FROM stack_alerts WHERE id = ?').get(result.lastInsertRowid) as StackAlert; } + /** + * Delete an alert and its per-service cooldown rows. + * SQLite foreign_keys is not enabled here, so child rows are removed + * explicitly rather than relying on ON DELETE CASCADE. + */ public deleteStackAlert(id: number): void { - const stmt = this.db.prepare('DELETE FROM stack_alerts WHERE id = ?'); - stmt.run(id); + this.transaction(() => { + this.deleteStackAlertServiceCooldowns(id); + this.db.prepare('DELETE FROM stack_alerts WHERE id = ?').run(id); + }); } public updateStackAlertLastFired(id: number, timestamp: number): void { @@ -3111,6 +3148,32 @@ export class DatabaseService { stmt.run(timestamp, id); } + public getStackAlertServiceCooldown(alertId: number, serviceName: string): number | null { + const row = this.db.prepare( + 'SELECT last_fired_at FROM stack_alert_service_cooldowns WHERE alert_id = ? AND service_name = ?' + ).get(alertId, serviceName) as { last_fired_at: number } | undefined; + return row?.last_fired_at ?? null; + } + + public hasAnyStackAlertServiceCooldown(alertId: number): boolean { + const row = this.db.prepare( + 'SELECT 1 AS present FROM stack_alert_service_cooldowns WHERE alert_id = ? LIMIT 1' + ).get(alertId) as { present: number } | undefined; + return !!row; + } + + public upsertStackAlertServiceCooldown(alertId: number, serviceName: string, timestamp: number): void { + this.db.prepare(` + INSERT INTO stack_alert_service_cooldowns (alert_id, service_name, last_fired_at) + VALUES (?, ?, ?) + ON CONFLICT(alert_id, service_name) DO UPDATE SET last_fired_at = excluded.last_fired_at + `).run(alertId, serviceName, timestamp); + } + + public deleteStackAlertServiceCooldowns(alertId: number): void { + this.db.prepare('DELETE FROM stack_alert_service_cooldowns WHERE alert_id = ?').run(alertId); + } + // --- Auto-Heal Policies --- public getAutoHealPolicies(stackName?: string, nodeId?: number): AutoHealPolicy[] { diff --git a/backend/src/services/DockerEventService.ts b/backend/src/services/DockerEventService.ts index 1bcc9a3f..638b00ca 100644 --- a/backend/src/services/DockerEventService.ts +++ b/backend/src/services/DockerEventService.ts @@ -735,15 +735,15 @@ export class DockerEventService { } private async emitError(category: NotificationCategory, message: string, stackName?: string, containerName?: string, systemOnly = false): Promise { - return this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly)); + await this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly)); } private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise { - return this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName)); + await this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName)); } private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise { - return this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName)); + await this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName)); } // ======================================================================== diff --git a/backend/src/services/MonitorService.ts b/backend/src/services/MonitorService.ts index 4840b08c..4df2a575 100644 --- a/backend/src/services/MonitorService.ts +++ b/backend/src/services/MonitorService.ts @@ -32,6 +32,39 @@ const getOperatorPhrase = (operator: string): string => { return `triggered the operator ${operator}`; }; +/** Sentinel for containers without a Compose service label. Not a valid API target. */ +const UNLABELED_SERVICE_KEY = '_unlabeled'; + +function displayServiceName(serviceName: string): string { + return serviceName === UNLABELED_SERVICE_KEY ? 'unknown service' : serviceName; +} + +function parseAlertBreachKey(key: string): { ruleId: number; containerId: string } | null { + const sep = key.indexOf(':'); + if (sep < 0) return null; + const ruleId = Number(key.slice(0, sep)); + if (!Number.isFinite(ruleId)) return null; + return { ruleId, containerId: key.slice(sep + 1) }; +} + +/** Per-service cooldown timestamp, with pre-migration fallback to rule.last_fired_at. */ +function resolveStackAlertLastFired(rule: StackAlert, serviceName: string, db: DatabaseService): number { + const ruleId = rule.id!; + let lastFired = db.getStackAlertServiceCooldown(ruleId, serviceName); + if (lastFired == null && !db.hasAnyStackAlertServiceCooldown(ruleId) && (rule.last_fired_at || 0) > 0) { + lastFired = rule.last_fired_at || 0; + } + return lastFired || 0; +} + +function normalizeContainerName(container: { Id: string; Names?: string[] }): string { + const raw = container.Names?.[0]; + if (raw && raw.length > 0) { + return raw.startsWith('/') ? raw.slice(1) : raw; + } + return container.Id.slice(0, 12); +} + /** Shape of the JSON returned by Docker container stats (stream: false). */ interface DockerContainerStats { cpu_stats?: { @@ -153,21 +186,18 @@ export class MonitorService { private janitorConsecutiveTimeouts = 0; private janitorBreakerUntil = 0; - // Track the duration a specific stack alert rule has been in breach state - // key: rule_id, value: AlertState - private activeBreaches = new Map(); + // Track the duration a specific stack alert rule+container has been in breach + // key: `${ruleId}:${containerId}`, value: AlertState + private activeBreaches = new Map(); // Track previous network counters per container for rate calculation. // key: container_id, value: { rx bytes, tx bytes, sample timestamp } private previousNetworkStats = new Map(); - // Per-cycle dispatch dedup. Stack rules are shared across every container - // in the stack, so parallel processContainer calls can race past the - // cooldown check (DB write happens after the awaited dispatch) and fire - // the same alert N times on first breach. The synchronous check-and-add - // below is atomic in JS between awaits, so only the first worker wins. + // Per-cycle dispatch dedup keyed by rule+service so replicas of the same + // Compose service do not fire N times, while different services can. // Reset at the start of each evaluateStackAlerts call. - private firedThisCycle = new Set(); + private firedThisCycle = new Set(); // Crash and healthcheck detection live in DockerEventService (event-driven, // causal classification). MonitorService no longer polls for container @@ -604,6 +634,10 @@ export class MonitorService { else alertsByStack.set(a.stack_name, [a]); } + const activeRuleIds = new Set(alerts.map(a => a.id!)); + const allCurrentIds = new Set(); + let enumerationFailed = false; + for (const node of nodes) { if (!node.id) continue; // Remote nodes are self-monitoring - skip direct Docker access @@ -623,22 +657,39 @@ export class MonitorService { (container) => this.processContainer(node, container, alertsByStack, docker, db), ); + for (const c of containers) allCurrentIds.add(c.Id); + // Clean up stale network stats for containers on this node that no longer run if (this.previousNetworkStats.size > containers.length * 2) { - const currentIds = new Set(containers.map((c: { Id: string }) => c.Id)); + const nodeIds = new Set(containers.map((c: { Id: string }) => c.Id)); for (const key of this.previousNetworkStats.keys()) { - if (!currentIds.has(key)) this.previousNetworkStats.delete(key); + if (!nodeIds.has(key)) this.previousNetworkStats.delete(key); } } } catch (err) { + // Enumeration failed: preserve existing breach timers. + enumerationFailed = true; console.error(`Error fetching containers for node ${node.name}`, err); } } - // Clean up stale breach trackers for rules that have been deleted - const activeRuleIds = new Set(alerts.map(a => a.id!)); - for (const key of this.activeBreaches.keys()) { - if (!activeRuleIds.has(key)) { + // After successful enumeration(s), drop breach timers for containers + // that are no longer running. Skip when any local enumeration failed + // so a transient Docker error cannot reset duration timers. + if (!enumerationFailed) { + for (const key of [...this.activeBreaches.keys()]) { + const parsed = parseAlertBreachKey(key); + if (parsed && activeRuleIds.has(parsed.ruleId) && !allCurrentIds.has(parsed.containerId)) { + this.activeBreaches.delete(key); + } + } + } + + // Clean up in-memory breach trackers for rules that have been deleted. + // Persisted cooldowns are removed only via transactional deleteStackAlert. + for (const key of [...this.activeBreaches.keys()]) { + const parsed = parseAlertBreachKey(key); + if (parsed && !activeRuleIds.has(parsed.ruleId)) { this.activeBreaches.delete(key); } } @@ -666,12 +717,14 @@ export class MonitorService { */ private async processContainer( node: Node, - container: { Id: string; Labels?: Record }, + container: { Id: string; Names?: string[]; Labels?: Record }, alertsByStack: Map, docker: DockerController, db: DatabaseService, ): Promise { const stackName = container.Labels?.['com.docker.compose.project'] || 'system'; + const serviceName = container.Labels?.['com.docker.compose.service'] || UNLABELED_SERVICE_KEY; + const containerName = normalizeContainerName(container); try { const rawStats = await withTimeout( @@ -727,7 +780,11 @@ export class MonitorService { }); for (const rule of stackAlerts) { + if (rule.service_name && rule.service_name !== serviceName) continue; + const ruleId = rule.id!; + const breachKey = `${ruleId}:${container.Id}`; + const cooldownKey = `${ruleId}:${serviceName}`; const currentValue = metrics[rule.metric as keyof typeof metrics]; if (currentValue === undefined) continue; @@ -735,57 +792,75 @@ export class MonitorService { const isBreaching = this.evaluateCondition(currentValue, rule.operator, rule.threshold); if (isBreaching) { - if (!this.activeBreaches.has(ruleId)) { - this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() }); - if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}"`); + if (!this.activeBreaches.has(breachKey)) { + this.activeBreaches.set(breachKey, { breachStartedAt: Date.now() }); + if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}" service "${serviceName}" container "${containerName}"`); } - const breachState = this.activeBreaches.get(ruleId)!; + const breachState = this.activeBreaches.get(breachKey)!; const durationMs = Date.now() - breachState.breachStartedAt; const requiredDurationMs = rule.duration_mins * 60 * 1000; if (durationMs >= requiredDurationMs) { - const timeSinceLastFired = Date.now() - (rule.last_fired_at || 0); + const timeSinceLastFired = Date.now() - resolveStackAlertLastFired(rule, serviceName, db); const requiredCooldownMs = rule.cooldown_mins * 60 * 1000; if (timeSinceLastFired >= requiredCooldownMs) { - // Claim this rule for the cycle before awaiting - // dispatch. The check-and-add is synchronous, so - // sibling workers evaluating the same shared rule - // see the claim and skip — preventing N-fire when - // multiple containers in one stack all breach. - if (this.firedThisCycle.has(ruleId)) { - if (isDebugEnabled()) console.log(`[Monitor:diag] Skipping duplicate dispatch for rule ${ruleId} (already fired this cycle by sibling container)`); - } else { - this.firedThisCycle.add(ruleId); + // Claim this rule+service for the cycle before awaiting + // dispatch so replicas of the same service skip. + if (this.firedThisCycle.has(cooldownKey)) { + if (isDebugEnabled()) console.log(`[Monitor:diag] Skipping duplicate dispatch for rule ${ruleId} service "${serviceName}" (already fired this cycle by sibling replica)`); + continue; + } - const { name: metricName, unit } = getMetricDetails(rule.metric); - const operatorPhrase = getOperatorPhrase(rule.operator); + this.firedThisCycle.add(cooldownKey); - const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue; - const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold; + const { name: metricName, unit } = getMetricDetails(rule.metric); + const operatorPhrase = getOperatorPhrase(rule.operator); - // Node-neutral body: the hub bell badge attributes remotes. - const message = `The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`; + const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue; + const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold; + const serviceLabel = displayServiceName(serviceName); - console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`); - await NotificationService.getInstance().dispatchAlert( + // Node-neutral body: the hub bell badge attributes remotes. + const message = `The **${metricName}** for **${serviceLabel}** in **${rule.stack_name}** (container **${containerName}**) ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`; + + try { + const { persisted } = await NotificationService.getInstance().dispatchAlert( 'warning', 'monitor_alert', message, - { stackName: rule.stack_name, actor: 'system:monitor' }, + { stackName: rule.stack_name, containerName, actor: 'system:monitor' }, + ); + if (!persisted) { + // History was not written; do not advance cooldown or we silence retries. + this.firedThisCycle.delete(cooldownKey); + console.error( + `[MonitorService] Alert history not persisted for rule ${ruleId} service "${serviceName}"; cooldown not advanced`, + ); + continue; + } + const firedAt = Date.now(); + // Dual-write: per-service row for current code, parent last_fired_at + // so a downgrade that only reads the parent column still has a floor. + db.upsertStackAlertServiceCooldown(ruleId, serviceName, firedAt); + db.updateStackAlertLastFired(ruleId, firedAt); + console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}" service "${serviceName}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`); + } catch (fireErr) { + this.firedThisCycle.delete(cooldownKey); + console.error( + `[MonitorService] Failed to fire alert rule ${ruleId} service "${serviceName}" container "${containerName}":`, + fireErr, ); - - db.updateStackAlertLastFired(ruleId, Date.now()); } } else if (isDebugEnabled()) { - console.log(`[Monitor:diag] Cooldown active for rule ${ruleId}: ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`); + console.log(`[Monitor:diag] Cooldown active for rule ${ruleId} service "${serviceName}": ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`); } } } else { - if (this.activeBreaches.has(ruleId)) { - if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}"`); - this.activeBreaches.delete(ruleId); + if (this.activeBreaches.has(breachKey)) { + if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}" container "${containerName}"`); + this.activeBreaches.delete(breachKey); } } } diff --git a/backend/src/services/NotificationService.ts b/backend/src/services/NotificationService.ts index 2a6ebbee..ba414ce0 100644 --- a/backend/src/services/NotificationService.ts +++ b/backend/src/services/NotificationService.ts @@ -181,7 +181,7 @@ export class NotificationService { category: NotificationCategory, message: string, options?: { stackName?: string; containerName?: string; actor?: string }, - ) { + ): Promise<{ persisted: boolean }> { const t0 = Date.now(); const { stackName, containerName, actor } = options ?? {}; @@ -191,6 +191,8 @@ export class NotificationService { // WebSocket broadcast can all throw on an unhealthy DB, which would // otherwise surface as an unhandledRejection and take the process down. // The whole body is wrapped so the worst case is a dropped notification. + // Callers that gate cooldowns on history use `persisted` (true only after the row write). + let wroteHistory = false; try { // Internal writes use the middleware default so they share a row key // with user-initiated requests; otherwise the UI and monitors split @@ -216,11 +218,12 @@ export class NotificationService { container_name: containerName, actor_username: actor ?? null, }); + wroteHistory = true; StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, true); } catch (err) { StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, false); console.error('[Notify] Failed to persist notification:', err); - return; + return { persisted: false }; } // Separate [StackActivity:diag] namespace from the [Notify:diag] lines // below so a single grep can pull every per-stack timeline write across @@ -273,7 +276,7 @@ export class NotificationService { } if (suppressExternal) { - return; + return { persisted: wroteHistory }; } // Resolve retry extras once for this dispatch (shared by all destinations). @@ -298,14 +301,14 @@ export class NotificationService { ) ); this.recordDispatchErrors(notification.id!, errors); - return; + return { persisted: wroteHistory }; } // 4. Fall back to this instance's agents (keyed by this instance's default node id). const agents = this.dbService.getEnabledAgents(localNodeId); if (agents.length === 0) { if (isDebugEnabled()) console.log('[Notify:diag] No routes or agents matched; skipping external dispatch'); - return; + return { persisted: wroteHistory }; } if (isDebugEnabled()) console.log(`[Notify:diag] Falling back to ${agents.length} global agent(s)`); @@ -322,8 +325,11 @@ export class NotificationService { ) ); this.recordDispatchErrors(notification.id!, errors); + return { persisted: wroteHistory }; } catch (err) { console.error('[Notify] dispatchAlert failed:', err); + // History may already be written; callers must not treat that as a miss. + return { persisted: wroteHistory }; } } diff --git a/docs/features/alerts-notifications.mdx b/docs/features/alerts-notifications.mdx index efa0acc8..ee67c6b3 100644 --- a/docs/features/alerts-notifications.mdx +++ b/docs/features/alerts-notifications.mdx @@ -1,10 +1,10 @@ --- title: Alerts & Notifications sidebarTitle: Alerts and notifications -description: Threshold and event alerts for your fleet, dispatched to Discord, Slack, Apprise, or any webhook, with per-stack rules and channel routing. +description: Threshold and event alerts for your fleet, dispatched to Discord, Slack, Apprise, or any webhook, with stack and service rules and channel routing. --- -Sencho watches each node it manages for container crashes, host pressure, scheduled-task results, and update availability, then surfaces every signal in two places: the in-app notification bell at the top of the shell and an external channel you configure. This page covers everything from configuring channels to writing per-stack threshold rules, routing alerts to dedicated channels with routing rules, and tuning retention. +Sencho watches each node it manages for container crashes, host pressure, scheduled-task results, and update availability, then surfaces every signal in two places: the in-app notification bell at the top of the shell and an external channel you configure. This page covers everything from configuring channels to writing stack and service threshold rules, routing alerts to dedicated channels with routing rules, and tuning retention. Settings · Notifications · Channels panel with NODE Local in the header, a Delivery retries row showing Extra attempts 0 and a Save retries button, Discord Slack Webhook and Apprise tabs with Apprise selected, Enabled off, an empty Apprise endpoint placeholder, and Test beside Save. @@ -37,7 +37,7 @@ Use the Generic Webhook tab when you have your own receiver: a Mattermost or Tea ```json { "level": "warning", - "message": "The **CPU usage** for **plex** has exceeded your threshold of **80%** (Currently: 91%).", + "message": "The **CPU usage** for **api** in **plex** (container **plex-api-1**) has exceeded your threshold of **80%** (Currently: 91%).", "timestamp": "2026-05-08T22:14:09.812Z", "source": "sencho" } @@ -161,9 +161,11 @@ Every alert Sencho dispatches carries a category that you can filter on in the b The four `blueprint_*` categories are accepted by routing rules but render as raw category strings in the bell because the frontend label map omits them. -## Per-stack alert rules +## Stack alert rules -Each stack carries its own set of threshold rules that fire when a metric stays above (or below) a value for a configurable window. Rules live on the node where the stack runs and are evaluated locally on a 30-second tick. +Each stack carries its own set of threshold rules that fire when a metric stays above (or below) a value for a configurable window. A rule can target **All services** in the stack or one Compose service. Metrics are evaluated per running container (not as a stack aggregate): each container has its own duration timer, and cooldown is tracked per Compose service so replicas of the same service share one silence window while different services can alert independently. + +Rules live on the node where the stack runs and are evaluated locally on a 30-second tick. Open the rules editor by right-clicking a stack in the sidebar and choosing **Alerts**, or by pressing `A` while focused on a stack. @@ -175,13 +177,14 @@ Open the rules editor by right-clicking a stack in the sidebar and choosing **Al | Field | Purpose | |-------|---------| +| **Service** | **All services** (default) or one Compose service from the stack. All services evaluates every matching container independently. | | **Metric** | The system resource or metric to monitor. | | **Operator** | Comparison: `Greater than`, `Greater or eq`, `Less than`, `Less or eq`, `Equals`. | | **Threshold** | A number ≥ 0. The unit follows the chosen metric. | -| **Duration (mins)** | How long the breach must persist before firing. Default `5`, range 0 to 1440. | -| **Cooldown (mins)** | Silence window between fires after a rule triggers. Default `60`, range 0 to 10080. | +| **Duration (mins)** | How long a container's breach must persist before firing. Default `5`, range 0 to 1440. | +| **Cooldown (mins)** | Silence window between fires for the same Compose service after a rule triggers. Default `60`, range 0 to 10080. | -Sencho tracks the start of each breach in memory; the rule fires only after the breach has lasted for the full **Duration**, and only if the previous fire is older than **Cooldown**. +Sencho tracks the start of each breach per container in memory; the rule fires only after that container's breach has lasted for the full **Duration**, and only if the previous fire for the same Compose service is older than **Cooldown**. The notification names the triggering service and container. Containers without a Compose service label are grouped under one shared cooldown key and appear as **unknown service** in the message; that label is not selectable as a rule target. A scoped rule whose service is no longer listed in the compose file shows **Not in compose** in the rules list; evaluation still matches running containers that carry that Compose service label. ### Available metrics @@ -441,7 +444,7 @@ Switching the active node tears down per-stack rule editors and reloads channel Check three things in order. First, the channel toggle in **Settings · Notifications · Channels** must be on; the kicker on each tab reads `enabled` or `off`. Second, Discord, Slack, and webhook URLs must use HTTPS (the form rejects plain `http://` for those channels); Apprise endpoints may use HTTP or HTTPS. Third, a routing rule with unconstrained Node plus empty Stacks, Labels, Categories, and Severity matchers will intercept every alert and skip the global channels. Use the per-channel **Test** button to issue a one-shot dispatch and watch your endpoint for the literal message `🔌 Test Notification from Sencho!` Sencho records the failure reason in `notification_history.dispatch_error` when delivery throws, so a row that appears in the bell with no follow-up at the endpoint usually means a 4xx or timeout at the receiver. - Three causes account for almost every case. First, the rule's **Duration** has not elapsed yet: the breach must persist for the full duration before the rule fires. Second, the rule is still in cooldown after a previous fire. Third, the panel's banner is not green: a remote-node banner means the rule was saved on a remote whose channels you may not have configured, and an amber `No notification channels configured` banner means the rule evaluates fine but Sencho has nowhere to send the alert. The evaluator runs on a 30-second tick, so expect up to 30 seconds of latency between the breach starting and the timer engaging. + Three causes account for almost every case. First, the rule's **Duration** has not elapsed yet: that container's breach must persist for the full duration before the rule fires (a healthy sibling service does not clear another container's timer). Second, the same Compose service is still in cooldown after a previous fire. Third, the panel's banner is not green: a remote-node banner means the rule was saved on a remote whose channels you may not have configured, and an amber `No notification channels configured` banner means the rule evaluates fine but Sencho has nowhere to send the alert. The evaluator runs on a 30-second tick, so expect up to 30 seconds of latency between the breach starting and the timer engaging. The Docker event service defers `die` classification by 500 ms to absorb out-of-order `kill` events from the daemon, then asks the lifecycle classifier whether the exit was intentional, clean (exit 0), a crash, or an OOM kill. If your stop happened far enough outside that window, or the daemon emitted the events without the kill marker the classifier looks for, the exit can be classified as a crash. The classifier favors avoiding silent crashes over avoiding noisy false positives. Compare the alert timestamp against your `docker compose down` time; entries within a second of each other are usually the same event seen from two angles. diff --git a/frontend/src/components/StackAlertSheet.test.tsx b/frontend/src/components/StackAlertSheet.test.tsx new file mode 100644 index 00000000..46f413cf --- /dev/null +++ b/frontend/src/components/StackAlertSheet.test.tsx @@ -0,0 +1,248 @@ +/** + * StackAlertSheet Alerts tab: service targeting, services-state machine, + * capability gating, and active-node reset. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, waitFor, fireEvent } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +const { nodeState } = vi.hoisted(() => ({ + nodeState: { + activeNode: { id: 1, type: 'local', name: 'Local' } as { id: number; type: string; name: string } | null, + activeNodeMeta: { + version: '1.0.0', + capabilities: ['service-scoped-stack-alert'], + } as { version: string; capabilities: string[] } | null, + }, +})); + +vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() })); +vi.mock('@/components/ui/toast-store', () => ({ + toast: { + error: vi.fn(), + success: vi.fn(), + warning: vi.fn(), + info: vi.fn(), + loading: vi.fn(), + dismiss: vi.fn(), + }, +})); +vi.mock('@/context/NodeContext', () => ({ + useNodes: () => ({ + activeNode: nodeState.activeNode, + activeNodeMeta: nodeState.activeNodeMeta, + hasCapability: (cap: string) => nodeState.activeNodeMeta?.capabilities.includes(cap) === true, + }), +})); +vi.mock('@/context/AuthContext', () => ({ + useAuth: () => ({ isAdmin: true }), +})); + +import { apiFetch } from '@/lib/api'; +import { toast } from '@/components/ui/toast-store'; +import { StackAlertSheet } from './StackAlertSheet'; + +const mockedFetch = apiFetch as unknown as ReturnType; + +function jsonRes(body: unknown, ok = true, status = ok ? 200 : 500) { + return { + ok, + status, + json: async () => body, + text: async () => '', + } as unknown as Response; +} + +beforeEach(() => { + nodeState.activeNode = { id: 1, type: 'local', name: 'Local' }; + nodeState.activeNodeMeta = { + version: '1.0.0', + capabilities: ['service-scoped-stack-alert'], + }; + mockedFetch.mockReset(); + vi.mocked(toast.success).mockReset(); + vi.mocked(toast.error).mockReset(); +}); + +function mockHappyPath(services: string[] = ['api', 'database'], alerts: unknown[] = []) { + mockedFetch.mockImplementation(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.includes('/agents')) { + return jsonRes([{ type: 'discord', enabled: true }]); + } + if (url.includes('/services')) { + return jsonRes(services); + } + if (url.startsWith('/alerts') && (!init || !init.method || init.method === 'GET')) { + return jsonRes(alerts); + } + if (url === '/alerts' && init?.method === 'POST') { + const body = JSON.parse(String(init.body)); + return jsonRes({ id: 99, ...body }, true, 201); + } + return jsonRes(null, false); + }); +} + +describe('StackAlertSheet Alerts tab', () => { + it('POSTs selected service_name when capability is present', async () => { + mockHappyPath(); + const user = userEvent.setup(); + render( {}} stackName="my-stack" />); + + await waitFor(() => expect(screen.getByText('Add Rule')).toBeInTheDocument()); + await waitFor(() => { + expect(mockedFetch.mock.calls.some(([url]) => String(url).includes('/services'))).toBe(true); + }); + + // Service combobox is the first one in the Add new rule form. + await waitFor(() => { + const serviceBox = screen.getAllByRole('combobox')[0]; + expect(serviceBox).not.toBeDisabled(); + }); + await user.click(screen.getAllByRole('combobox')[0]); + await user.click(await screen.findByRole('button', { name: 'api' })); + + fireEvent.change(screen.getByPlaceholderText('e.g. 90'), { target: { value: '80' } }); + await user.click(screen.getByText('Add Rule')); + + await waitFor(() => { + const post = mockedFetch.mock.calls.find( + ([url, init]) => String(url) === '/alerts' && (init as RequestInit | undefined)?.method === 'POST', + ); + expect(post).toBeDefined(); + const body = JSON.parse(String((post![1] as RequestInit).body)); + expect(body.service_name).toBe('api'); + expect(body.stack_name).toBe('my-stack'); + }); + }); + + it('POSTs service_name null for All services', async () => { + mockHappyPath(); + const user = userEvent.setup(); + render( {}} stackName="my-stack" />); + await waitFor(() => expect(screen.getByText('Add Rule')).toBeInTheDocument()); + + fireEvent.change(screen.getByPlaceholderText('e.g. 90'), { target: { value: '80' } }); + await user.click(screen.getByText('Add Rule')); + + await waitFor(() => { + const post = mockedFetch.mock.calls.find( + ([url, init]) => String(url) === '/alerts' && (init as RequestInit | undefined)?.method === 'POST', + ); + const body = JSON.parse(String((post![1] as RequestInit).body)); + expect(body.service_name).toBeNull(); + }); + }); + + it('shows Not in compose only after a successful services list', async () => { + mockHappyPath(['database'], [{ + id: 1, + stack_name: 'my-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }]); + + render( {}} stackName="my-stack" />); + + await waitFor(() => { + expect(screen.getByText(/Not in compose/i)).toBeInTheDocument(); + }); + }); + + it('does not show Not in compose when services fetch fails', async () => { + mockedFetch.mockImplementation(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes('/agents')) return jsonRes([{ type: 'discord', enabled: true }]); + if (url.includes('/services')) return jsonRes(null, false, 500); + if (url.includes('/alerts')) { + return jsonRes([{ + id: 1, + stack_name: 'my-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }]); + } + return jsonRes(null, false); + }); + + render( {}} stackName="my-stack" />); + + await waitFor(() => expect(screen.getByText('api')).toBeInTheDocument()); + expect(screen.queryByText(/Not in compose/i)).not.toBeInTheDocument(); + }); + + it('hides service selector when capability is missing and posts null', async () => { + nodeState.activeNodeMeta = { version: '0.90.0', capabilities: [] }; + mockHappyPath(); + const user = userEvent.setup(); + + render( {}} stackName="my-stack" />); + + await waitFor(() => expect(screen.getByText('Add Rule')).toBeInTheDocument()); + expect(screen.queryByText(/does not support service-scoped/i)).toBeInTheDocument(); + expect(screen.queryByRole('combobox', { name: /all services/i })).not.toBeInTheDocument(); + + fireEvent.change(screen.getByPlaceholderText('e.g. 90'), { target: { value: '80' } }); + await user.click(screen.getByText('Add Rule')); + + await waitFor(() => { + const post = mockedFetch.mock.calls.find( + ([url, init]) => String(url) === '/alerts' && (init as RequestInit | undefined)?.method === 'POST', + ); + const body = JSON.parse(String((post![1] as RequestInit).body)); + expect(body.service_name).toBeNull(); + }); + }); + + it('resets services state when active node changes', async () => { + let servicesCalls = 0; + mockedFetch.mockImplementation(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes('/agents')) return jsonRes([{ type: 'discord', enabled: true }]); + if (url.includes('/services')) { + servicesCalls += 1; + return jsonRes(servicesCalls === 1 ? ['api'] : ['worker']); + } + if (url.includes('/alerts')) { + return jsonRes([{ + id: 1, + stack_name: 'my-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }]); + } + return jsonRes(null, false); + }); + + const { rerender } = render( + {}} stackName="my-stack" />, + ); + + await waitFor(() => expect(servicesCalls).toBe(1)); + await waitFor(() => expect(screen.queryByText(/Not in compose/i)).not.toBeInTheDocument()); + + nodeState.activeNode = { id: 2, type: 'remote', name: 'Remote' }; + nodeState.activeNodeMeta = { + version: '1.0.0', + capabilities: ['service-scoped-stack-alert'], + }; + rerender( {}} stackName="my-stack" />); + + await waitFor(() => expect(servicesCalls).toBe(2)); + // New node's list has only worker, so the api-targeted rule is missing. + await waitFor(() => expect(screen.getByText(/Not in compose/i)).toBeInTheDocument()); + }); +}); diff --git a/frontend/src/components/StackAlertSheet.tsx b/frontend/src/components/StackAlertSheet.tsx index ee665527..fbe70db0 100644 --- a/frontend/src/components/StackAlertSheet.tsx +++ b/frontend/src/components/StackAlertSheet.tsx @@ -20,12 +20,14 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/comp import { Trash2, HelpCircle, AlertTriangle, Info, CheckCircle2, Loader2, ChevronDown, ChevronUp } from 'lucide-react'; import { toast } from '@/components/ui/toast-store'; import { apiFetch } from '@/lib/api'; +import { SERVICE_SCOPED_STACK_ALERT_CAPABILITY } from '@/lib/capabilities'; import { useNodes } from '@/context/NodeContext'; import { useAuth } from '@/context/AuthContext'; interface StackAlert { id?: number; stack_name: string; + service_name: string | null; metric: string; operator: string; threshold: number; @@ -33,6 +35,11 @@ interface StackAlert { cooldown_mins: number; } +type ServicesState = + | { status: 'idle' | 'loading' } + | { status: 'success'; options: string[] } + | { status: 'error' }; + interface AutoHealPolicy { id?: number; node_id: number; @@ -161,8 +168,9 @@ export function StackAlertSheet({ open, onOpenChange, stackName, initialTab = 'a function AlertsTab({ stackName }: { stackName: string }) { const { isAdmin } = useAuth(); - const { activeNode } = useNodes(); + const { activeNode, activeNodeMeta } = useNodes(); const isRemote = activeNode?.type === 'remote'; + const canScopeService = activeNodeMeta?.capabilities.includes(SERVICE_SCOPED_STACK_ALERT_CAPABILITY) === true; const [alerts, setAlerts] = useState([]); const [isLoading, setIsLoading] = useState(false); @@ -172,7 +180,9 @@ function AlertsTab({ stackName }: { stackName: string }) { hasEnabled: false, enabledTypes: [], }); + const [servicesState, setServicesState] = useState({ status: 'idle' }); + const [service, setService] = useState(''); const [metric, setMetric] = useState('cpu_percent'); const [operator, setOperator] = useState('>'); const [threshold, setThreshold] = useState(''); @@ -183,7 +193,33 @@ function AlertsTab({ stackName }: { stackName: string }) { if (!stackName) return; fetchAlerts(); fetchAgentStatus(); - }, [stackName]); // eslint-disable-line react-hooks/exhaustive-deps + }, [stackName, activeNode?.id]); // eslint-disable-line react-hooks/exhaustive-deps + + useEffect(() => { + if (!stackName || !canScopeService) { + setServicesState({ status: 'idle' }); + setService(''); + return; + } + + let cancelled = false; + setServicesState({ status: 'loading' }); + setService(''); + + void (async () => { + try { + const res = await apiFetch(`/stacks/${encodeURIComponent(stackName)}/services`); + if (!res.ok) throw new Error(`services ${res.status}`); + const names = await res.json() as string[]; + if (!cancelled) setServicesState({ status: 'success', options: names }); + } catch (e) { + console.error('[StackAlertSheet] Failed to fetch stack services', e); + if (!cancelled) setServicesState({ status: 'error' }); + } + })(); + + return () => { cancelled = true; }; + }, [stackName, activeNode?.id, canScopeService]); const fetchAlerts = async () => { try { @@ -224,8 +260,10 @@ function AlertsTab({ stackName }: { stackName: string }) { return; } setIsLoading(true); + const scopedServiceName = canScopeService && service !== '' ? service : null; const newAlert = { stack_name: stackName, + service_name: scopedServiceName, metric, operator, threshold: parseFloat(threshold), @@ -240,6 +278,7 @@ function AlertsTab({ stackName }: { stackName: string }) { if (res.ok) { toast.success('Alert rule added.'); setThreshold(''); + setService(''); fetchAlerts(); } else { const err = await res.json().catch(() => ({})); @@ -272,6 +311,32 @@ function AlertsTab({ stackName }: { stackName: string }) { } }; + const serviceComboOptions = [ + { value: '', label: 'All services' }, + ...(servicesState.status === 'success' + ? servicesState.options.map(n => ({ value: n, label: n })) + : []), + ]; + + const renderTargetLabel = (alert: StackAlert) => { + if (!alert.service_name) { + return All services; + } + const missing = servicesState.status === 'success' + && !servicesState.options.includes(alert.service_name); + if (missing) { + return ( + + {alert.service_name} (Not in compose) + + ); + } + return {alert.service_name}; + }; + const renderAgentStatusBanner = () => { if (agentStatus.loading) { return ( @@ -355,7 +420,8 @@ function AlertsTab({ stackName }: { stackName: string }) { {metricLabels[alert.metric] || alert.metric} {alert.operator} {alert.threshold}
- Trigger after {alert.duration_mins}m • Cooldown {alert.cooldown_mins}m + {renderTargetLabel(alert)} + {' '}• Trigger after {alert.duration_mins}m • Cooldown {alert.cooldown_mins}m
{isAdmin && ( @@ -378,6 +444,31 @@ function AlertsTab({ stackName }: { stackName: string }) { {isAdmin && (
+ {canScopeService && ( +
+ + + {servicesState.status === 'error' && ( +

+ Could not load Compose services. The rule will target all services. +

+ )} +
+ )} + {!canScopeService && activeNodeMeta && ( +

+ This node does not support service-scoped alert rules yet. Update the node to target a specific service. +

+ )} +
diff --git a/frontend/src/lib/capabilities.ts b/frontend/src/lib/capabilities.ts index 76292bae..887ccedd 100644 --- a/frontend/src/lib/capabilities.ts +++ b/frontend/src/lib/capabilities.ts @@ -37,6 +37,7 @@ export const CAPABILITIES = [ 'stack-down-remove-volumes', 'guided-external-network-preflight', 'service-scoped-update', + 'service-scoped-stack-alert', ] as const; export type Capability = (typeof CAPABILITIES)[number]; @@ -50,3 +51,4 @@ export const HOST_CONSOLE_COMMUNITY_CAPABILITY = 'host-console-community' as con export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes' as const satisfies Capability; export const GUIDED_EXTERNAL_NETWORK_PREFLIGHT_CAPABILITY = 'guided-external-network-preflight' as const satisfies Capability; export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability; +export const SERVICE_SCOPED_STACK_ALERT_CAPABILITY = 'service-scoped-stack-alert' as const satisfies Capability; From a89498ae5b4bb8b88178fe8088e1acb46e3821b9 Mon Sep 17 00:00:00 2001 From: Anso Date: Thu, 23 Jul 2026 20:22:57 -0400 Subject: [PATCH 08/13] fix(ui): hide log service chips on single-service stacks (#1689) Service chips only differentiate multi-service or multi-container log streams. Gate rendering with the same layout criterion already used in stack details, while keeping parsed prefixes and download attribution intact. --- docs/features/appearance.mdx | 2 +- docs/features/stack-management.mdx | 4 +- docs/images/stack-view/logs-viewer.png | Bin 76733 -> 118044 bytes docs/reference/settings.mdx | 2 +- .../components/EditorLayout/EditorView.tsx | 38 ++++----- .../EditorLayout/MobileStackDetail.test.tsx | 74 +++++++++++++++- .../EditorLayout/MobileStackDetail.tsx | 10 ++- .../__tests__/EditorView.test.tsx | 72 +++++++++++++++- .../__tests__/StackLogsSection.test.tsx | 77 +++++++++++++++++ .../EditorLayout/editor-view-blocks.tsx | 16 ++-- .../src/components/StructuredLogViewer.tsx | 68 +++++++++------ .../__tests__/StructuredLogViewer.test.tsx | 79 ++++++++++++------ .../components/settings/AppearanceSection.tsx | 2 +- .../__tests__/AppearanceSection.test.tsx | 7 ++ 14 files changed, 359 insertions(+), 92 deletions(-) create mode 100644 frontend/src/components/EditorLayout/__tests__/StackLogsSection.test.tsx diff --git a/docs/features/appearance.mdx b/docs/features/appearance.mdx index c2f5e32b..f66047e6 100644 --- a/docs/features/appearance.mdx +++ b/docs/features/appearance.mdx @@ -89,7 +89,7 @@ Sencho uses three type contexts. The interface and data faces are yours to chang The **Display** group holds layout and log-chip preferences for this browser: - **Density** switches between **Comfortable** (roomy rows, the default) and **Compact** (tighter rows and tiles that fit more on screen for dense dashboards). -- **Log chip color** controls how service chips are colored in log views. **Unified** uses the accent color for all service chips. **Per service** assigns each service a stable label color for faster visual scanning when following multiple services at once. +- **Log chip color** controls how service chips are colored in log views on multi-service or multi-container stacks. Single-service, single-container stacks do not show chips. **Unified** uses the accent color for all service chips. **Per service** assigns each service a stable label color for faster visual scanning when following multiple services at once. ## Navigation diff --git a/docs/features/stack-management.mdx b/docs/features/stack-management.mdx index 0774066f..be87bb04 100644 --- a/docs/features/stack-management.mdx +++ b/docs/features/stack-management.mdx @@ -235,10 +235,10 @@ Single-service stacks keep the existing flat container layout; service headers d The logs area at the bottom of the stack view has two modes. Toggle between them with the segmented control in the top-right of the panel; your choice is remembered for next time. - Logs panel in Structured mode for the plex stack with toolbar, level filter pills, and timestamped INFO log rows + Logs panel in Structured mode with toolbar, level filter pills, and timestamped INFO log rows without service chips on a single-service stack -**Structured** (default) parses every line into a DOM row with three columns: local time, level badge, and message. Level detection is automatic: +**Structured** (default) parses every line into a DOM row with three columns: local time, level badge, and message. On multi-service or multi-container stacks, each row also shows a service chip so you can tell which service wrote the line; single-service, single-container stacks omit the chip. Level detection is automatic: - `err` rows get a red left rail and a subtle rose tint. - `warn` rows get a warm tint. diff --git a/docs/images/stack-view/logs-viewer.png b/docs/images/stack-view/logs-viewer.png index bf7f7482f821d73b88d7f71150754d6f809c5015..0c64b5931b7f656565816e2aa4484e4de658858e 100644 GIT binary patch literal 118044 zcmY(rWl&ww(zS~RcXxujyF+kycXu`(2<{dL65QQg5+Jy{I|P^D?tCjb=iGYl{IQ{+ zYP0r~o}-^0O{B7-6cRi>JQx@ll8p2xRWL9}LNG80aaai8C&(kA!(d>jU^1UX)x9!L z3_g6p?0?%FFS_3#EeuzN2r(2xAu0=krIU#Im`n4KRA`D?b>o^$`%UR*K-5JN6{$g) zDu%8YLK#$oh&R)%lsD8yn+= zRg5u3H8AvQ%p3NCwrG==kRqbR%}zNA$&hTc`N`}Kes{Htln$t>_qTqNix3)h#W#g= zX|krCjyf%@b6*_R|` zb{5>T!WfsxVnIVoyow?nq0TRVSX9yX@nOaM61oj5Dp3<@Ly=uppA9$aw za=3F?%^?<0d zaf;%)%sXRFQvWpVcJu4&j~{7ic~<4AvL`V^6G^LhkS8(427w^HVSi%*1*?fY|N9F z&3D|t&+`;ooK`Hg++cV89MfSblY$lg!jlUC6|?Z;21#s->myRRP>L|Yoa>B zc6#3F>i)#I+TLApccZo)j-x5Ka$43D@Ms};KyK?Rti(>_<@JTa*Y3quLAppuO>AV3 zk8=)BSeAd7SV?zvx5yDNDN^0X3$&Yh)vyC~si%5842Nqdj7M?p$WtU(=H-csiq_ZH z>#?$4!ol@9I+{!uyW5eF(IOsub;?8;ob9uzuFCbhM>?(e@uI;D->Tu~-gR&I+1FW0k8^!43r z2>wC(vzwhAS`*Wt+leet+2|WS-7A`UhneXLzD?tqS|_WXnctHwA0n@7~9H$ylAO@Ft4O%Z1HKMq-{)uOa2; zLMVie{w?A7&R_U--Ya<;6`Jd9o*%4OSXse8@VMO2Y}b1JS!r_rv*2@o>ho}smy=`k ze0S>q{_4rj-Y_#WlP4Z-`0ERAAS#SXws;ugrOleQscjSwn~b6iuiwkV604cLgTtN+ zZw$BNTC2r@wGDm)!zGz0;=IVBFxqz;?j{y4jK!3*)9x zL>f-|(wtjgt;x`(jb(9KnT@ox^i7kUmp~M!<&CwL*5b|0%|ex~`^gk|kud10u(1}k z$=$<6R!+{<%^8>1w69+-Wg?Tem)2!*Noqkob1x}pZrlD5S)m3vHgAtBAW-Sz#CJOs ztW_kX$Dq(caJuH`?HDn%vLWEG$Lmo7`*T}4Co7Dj(Bu^v&qAL%S7o@Pow3E9tHYCV<(U9dnct$a++lQlm+umNz`=);B zGEv9P%_f73GB`X)>}X834cA`sXH(Z?lo7EWhga?GyYC0`nM6JHcnZpu7rSHzmePw` zZh3Tc`SbO8oicb`NVnaGNQ~m*;#}~+w>QtTb;wXKD`^=So4HanX$=oW-@!FqoAKK({-{jjb1{kyT^2NWU*19BT<@cSkI-y)it*c_zJoV!M< zs-|Xkc2;%>s-ZWnL?QM&#R|S5mC9Uee-3w(5)*fJcg-Cf7^~6d@>@Tkpyp#|n%vIg z!WCDl25PPHvdKFuqjFFyw|e&Kf_yF-O)()6uvWgd97nyp zxNvrH6L`Phl#W{<6m%(cN~F?qurK`RtD5OQ-mkL`S^)FG+Nqpq<(adyLTVz&zLJ3d z|AjhmAl~rTPJlSND%Nd$0u?iAR1Rk{Po}x z!M?J5Y1K?Yl@Ri&Z`^T>B{tgZ`U-bPbgm4ZG_s9Z#)rXfMV+Lqyp-q@nZHMdNoH2o z`1p9uX8STCUV8e71)DW1DF^ocJ}g=a!rpWWbpa7hiYVp*6g0w-i>phQOxJ2Xlofr2 ziVOH}J>}Y$*H;91_y{7X(eCc8k!1SQl_v2_E9Ma)PPg_;S*+<2TuI7d)Bu&Bvp@;7 zXW)Q}W_KkrFfc?#M{9VrzNd$UkS1hh8K~>8(J>(IK}N6Dxl)U(@M=@GRh`gPc;t(> z0GpSrMfM~gm?JBzKf}x9!6%a{nYe#9Z{d5nWzANri%WTw0VYl-8K(s|!$kSH=AYQC z3TS>Ui?**xgj&#T544g?c8n7=J28sVMVfy)nTfds-bv(i%I2<8aul4_XVPjL6}*;H zD!w%xjb~UMOpK;KSX2ByImL_oH2vNA4zIVAhxHEcLTB5`2<~uq$M^%E~D_+sF z(rH-!#~&FP8*4W$a01q3-QB7Fzx5%-1Nuit{|wKTM8d%_rgOJ6Gz2%RYie3LIlb== zV}wNDAhfF)t-?>L^>uf_pb$#0`7b)HwR+!{CnwKW)U*fvF=mLb9D#B&v(&`qiTM!< zO(s8*k}(}o;B|4f{>wFdX?=Y?Ix6aSg#C0r^!5uHl?574?*O%0cN!gAG}Eq)g zO+`@==_?_MYiGr+X6?RIp|Eb@Rn;Y@aYMRXVcxRsez<@fZBmX`5! zqr-Yk`%jFM!pv+FMtXmrHX$FWrwbGQld6t$tqka!r6L}lyiMA+&HjFo!>VYFDf|^4 zWS5Yy?GLLKIPKnxts;Xq-IL|WFS~U*xlrI~YPH6xP$9xw5tPb|7Ge zwmACk9;z&fF6uVhrzNN96wh?cU6YBhMVJ$xo%Nl~wz3ks&6X&L^ENcpt1GeA+EZ|- z)YI(raY!TCX@Y?MMmf_#lc#_oYDGP1r5 zt(on>@4(;RY%M6EF?YsvE>!G3V~6nsOV6aA^92N z3<@ggvyDC13RH`{adYBqbkUyaNca`-dNoZ=O<7slKy`TP^sJ7KqU8C7)7AIq*L!Nk z%!}LGJ+EX1%k$qCH%DqA5d<^knb{@1y|9v@)PUH~r6FjRHn!O<)r5Zhz3~hJ`ThD8 zk&%`W3~{#6@L}&$L@VCA)1!u=QLHkkcw}J#Tk+HUE2$FdS=Y1o?aAk>Nnw+tg(^K0 z6BAVzHq7SQ?~H9yp90+TJ49;DAu!zBK}X9C)Y6AJ{;&N9!*mlc6{pM+$*cl~%hjQr zT@&9|o9;#`r-4A)Mx)q^j{rnZm_Fb|BLrfe3#hU^5M2Zf0RPc-xdQW%Gwl8pzMd$uq?+q-3W`ID^i+A#S z(Vkw?iu!=Kn>%yzB5{Dik(wi@wotC|R_=eP7KSH1TY5OiJ>Z_2)i&w*xay;FYMSuIti!d??m2K*eAD_K4xM(QV>8EM}Az=c% zB#hb?3ViN899GlejRHQ|eGFfH`phID2w7oOLkWar`wrfnB4#G!+hvFWVJcqAGPiz@A3rmJ~{D}+iZhouP&BT$cH>aDz^H8~U-;j(J` z2y2CiI_0`iu{K~2$>*pjH8pjIS}Dj(Qc|)93$}_u;=xw~Zay&iK0q%7+Ztq=EgH_H`o zZ8}&+8Uv5icB?XPD>i+3hHcV}rC+ncyv0ISRPqckwdjyEfGEtgaiqTE`J*qd?r*}C z7aGGhF~mC%Vv+4L^aH1TkRBIKwkS%3m8o&t6xQQ`a|^rgCOs!bOLLjo@Za+*3)Iv$ zH!{#N_6N~;{@;X)Ry+^+gll|!?2{~3AWbFJ0u)aq``jmqwv*g#YJRlp z?=cNc?l3UsxYo%|T+UaH-`VGauF}>0ip%lyTt3+?!8OiCoNB~GCv$x>K=g$tg#J51 zFIm_*lT%ZZ;Dz~z(=uF}JtnD}AhPJ_APP)FH;)Uaz`_FMnXFDl%Au5tZKU-HkL&HS zn^Z$|t;*e&u&DDt@kTQY^o>rcm-YAO3GpN4D3mI5(lthJmouthop2K3oW~&Y*!|s; zu}wgLy(MjItc@StvtV0B*g$5sx26NH;i9Eus{SpXECel$S(5vAcx?p|g;H|ReI*Z% z(P*=XIv}VNN$e^`rKO`4?bP~GESr;+ms@IOd>bQI2V9MI=O-z@S}&3SnZl!&Xw_q_ z-C*KA@sm@eP3*SvmjUk3lWJ5=`7(VRy`%4U55);!$D4sTfnlOkzo05`{~N07S`VI$ z5iv?vQ0=x->{Lr@P)?6nt670_dtUjW^hyfI`b0%9^#k5p-g!jJDpAPP*gYoxG6fCc z2|VE1{vmtvQw<*nxI_R=4}M2CfS2VVIc`Xd6YhN~f4X9$ElvjP#Q+{PBp;&wG~2(S zvFSe)0wGVas9cm*5c9v|77X3h_g$Y0y8k-h$#r2Qo?YYVq{)5ZSb~S((~dhaH4KqQ zJr*~G1>=$(nIX!Ai;54?ZBzHo=??+5B4!Q|n+gJGMa1$hm4CZY0B6d#IBL2c*Rf_0 z1UaG3>h7`-wc}Qz@FE!8#KHRTMqt_8XB5o=EF2PAh!n-pYS_sSR7Ad@x8?7qnGrr zi3YAykB3S{)){52^zRh{eH8L~ROoLwi)ghyC}A_3tOfrTJJ;)5Hz=xrj!k4DnCwIv zGuCtQ-8lJXFd1}=}^&wDNVhM!_&UW230$AyT$KWf)6j&JvZF%!@0wWW7YzxHRD$xUKg4BbKZh zw)q1@OE7ucwrx7~szlv%arRb!9gzk97b%%qIFK^R_j~lJEtF;r*tGM3UU!iC;BMf& zvFHg~wv*`)TwSFxKegLRMDZV$O_J)YWh{=3y)rrm2pw`Wu%i<&q=FRXum zI*IG<6?TG`!#ImmGm*yb`!^}AZ3b3;%M}O0vbfm7h4;_)F?^VYU)s55d^3d37^QLaTCP}OR${+FBP5J+}BnvFIrR6{Z9q``b z%HkukjE-72{dUsBEat1?E#gRYyC)|+^z;K`m0z^fh)7pwRGVw#$47m>=_Q9vs#%*j zDoSh6X061){rS{c_CTlM2ppATMD)nD|q=9`*vv3l9Lke{D}APP8Fo-9j1w>;(q{c%x?_HVad0;#eT#qzx(oS9S_ zF^fSB!C_5UY`6O-T10fw>Y&0>Y;DNX%qj;)HaZAb*9Yw7moH5v4V7oA^H%(^i;7c_!3*|AH%MpIfmvLXA`L%OqVIVWb=#X=T?R zfhrw0Gji{SP36T`_O!W{c8r?kbgJI=4)#Ai&%Y|w)z!g8M)J*;4&KrT+8faDSr-E< z(Ad~$H$6R*pN~J;xe@lzsbN-cVDTEsU~8s-(OX-$*Oim1WkwrpqV|5&+W&k3l` zW!+jWY7Ta@W3XLX20b4#O;&(20kfJgGqd8iaE;cJW?^SXpE!Kwg*N>_2y2^9vH9iM zp(b0ORb=tK*$eo%|4!tg8f}nR;F5$c~w!wdzgJi!rHF-JLP>7Y|e^C z8OCulV=t)^ZF>?$E7-j`7RTloJWe9V@6`f=?pjJ2(a%BC4mDeVV#%6D^7R1rG+(gS zqe}SNEdP6lr-G`Q!(w&!AR9?e52$9P(#R?yt4cOe_}Sq4`e2ON$dt}E%FA!cb=Pd@ zceg@12RJt!Bcp<>ET8j*@%cuFcu=tC*-EI5|KU^~x6|6t@VM{i8|%nVJ#cv0e7^X) zV4nhWySmoa^r@*!$fe?_D5;}~2iXQP^DeJ%?KXd{zis++b8~+X_OTg?A5CF4Q)p~y zA-TR9wK+QKe1GGgX|Y{q`}h$e28uE=E>76{BsD8*9pHCo8*QFfyD||>jJ-V%Nif1% zt3uv$G+PI4k80Djw6uks>7ijgrlzJYI|H%oR+G)}2(vVjI%)D&Z><(0?{DzyAr=Oi zS>ecd2{}1VW@Z_Q8Lq#(z~44uxtcQ()%HBFSAA}eZ(sf2`OO9pIns5rHEg-R8EA-s z(qdyPx~JGRiYO_$oz}$)0qKp)w=HMw?eFh@v;5lH+TJc;+Si4I2pe#@)f@ix>sOTR zooN!5r{~AKgKc7gE`+W*uqDaZ_Gown1AYC>%q$W??~p~rh48X6FGo#FfKg;sF;u}v z!^_wT;Gbiu77O~tq{uR*tZM6VU*7gB)6>zhadL+KN={4+{*tuZpqn*VKG&4{<8=+D zQAA)TF5|wFPqP}1b~ZINU7HhBhji(6b&wTPz*@BYg;hH;(k>2_-Y8|v_5S+2X!I)T z%nvlhwIq1aA>_BY;@3MRoorwe=@(bgVX zUz2BJ3(=AKfJhXV?)j(nyem&!@#|MorCf5I<(^N}C;5;suQldY4I}bw4fV*x0@|?L zp}c25H;{_Koz63py1+V=1m^x^a2QUjyEhdd3 zGeM1Ll$(=?%U*j>F>7pmOvL8{5+SS^NJhrl*?F3Y zk*if6#npV@6uVB#xAQx9>m7`nEwDeN_wh<$x(_`3%??2&Aq>RFbDXH2fo7a~LUh)K z{%Z8Dn3HDCxL0rnTr;j-1g=_b`|sYoUem^sH+=`JyNI=ac=^O9b?3)O{*Ol0{mVpb zyQ|TTrF0rBi&(ZI2~npE3ZJ;H3# z-oz1ED_3-K21|^g>tj$TVl=W?-{-1`oiDF1+uPfuA|iq_Obz-GBFo-SH)TLoAUVg; ziyj1ldwZAi&Hq@7{&l&*s?(@bh?i2rBG4RK5_%n67gDIlK{u35y_3m$Lk5HTD&`pA z77s8FcdNGh%SEmShJxM^ajYfgwK%$gI~#801U#<&)Yvp&B8aS4yP2N3SULfN*lS@ zcvFEU4e#p$Ks*aJyxlS;gEqrXKS3ipjY1V?&o}>gy~~fMZ>KFmJv{(}3X0;ior6us zPFey&i|5G_8r0=vB6U${=pJLPuv@q!WzciicTh!~Z6WO_?U#pZ%2kp3pb%=>)wi&I zs;>E!$-Rk^X9N;UoMQ7IDAs9jgNznvc$zo*b^R+w*VS`bDOvo`OD%VxSlEfqWIB2V zM%m$!&R5*~l_vW>nhHi{28A_eDr!C*LpsPkF$h;`y9A}ng1d?Jpt+}{3Z&2hRLU?j zw=*Z+@>T&7VYd)+@Axs1V!1|HI3Y;+U>xf|7j^&#kJGIstL8Wy`1hlzA<`Vbv|qvT0^YU+sH zGD&^w=t(>V`=jqS2)ZxPyJk(-+(1V8)}XntFa=%ZH6z{()P#bHjn_n4Whug!dC!Iu zsg2KC5&iX_Jx&e%Uz>me9>3>VQrnW-l%7zCR-Impgy~ zJbYI(6>LZ)@-}u>;rShRD|TY*I_+i`X8G#QI@!Yhe!m02jm;v1?v*zi8b^k$n-R6( zOt?Mk`rj>rq?9zZKi08u*0HAGBqeF-sWT@_JXhz(=xd8ii!7e3J`5F7T6~tkr|f=j z1?u6VtlL*?TO~sj4B@`AaGhldSV=4{l5HU)CCBvuEXK9M5ue)Wb@(3(hyxVlcAEJ9 zANuR7k(_H{Lp2$X?`s3!XUR42o`_u~e_hbwh3%ZJccOF>_tNbKjE#*!en7#dUH9Kb z#QqU7+^I~*_|eF?93373Av!ABV112kW)|%uGJ-xUfN5)Dh!9Y8Jv_LSR(Q_3gFoOf z8HO}4=j(BoTDf-#!)bF@q6P*NR(H##~aGm&3merC&^TB+cCjKS4~}4aDGW0#(Ad*XK*nU8T-L z>4QnAZGmp58uS3UUnBz~BWpH`p{QDAWmV+#zyh`(_XW3gww~VuxZv@@xp1!k`$I@b zCtanunCW)Qrtcswm_X8}i{@PE(9qEB3c4PKlwp?CT00d3I?}ahlc3-r)DHz(x26`x zvnZQa_;}S%nV^U7S2jN}g(%?@*;SdtM1U<1z%-IR; zwoGyevQFlBN&komly#~eV62EG2fNHFCw#XvrMo1?%J$$HFjJP3*sPlzhokeRk{6i= zkcd@`8qNF(qM-N!sWSG;*vpJWg^`Cd7Z2s_Rc%FLE1X?@XXD84)> zQ;RPqRVGtZw9{x|PAbJN8v)8$VsH?r+A`Vn0I2GWNzCU4TYyCA z26BIOxIY6nmFMY7GLIuYeGkuN z(LZ1bePOPV0TV&$bEAxwS7J@GK{| z7&T*>ZBP2rX(}C6m~oR%91H)>o2`bP7?WRUtqZMXwM2kl8Y{T~OE>sQ ze#n^yQgT4V=e-60Kij^Nj!?#QS9FU2Z9umauOt{1Addc#b5!hMAwCk!km%SjDszSO}{z>$%pNUyE07v$&1 zhgq@c=m1rl{+=GG5mH75s0#8A9x7m%O+fMXLQJy+F-BQ5LTQ%;)jayj8${R_%GNEV z<5!uwBh8xRj`O57^kMB!QJy%UhNo(UgoKU&#k+sPAmN963nvRD{7T3RQ73!N z8rFV%d@0o7ZGsv=N9MmRq~!DBRmvhn0b!gZTLgFxdIKD1i;*(Ny#&1OkJzo0p{)zO zL?HhD`OdEZE-Q-U zVBqyYx@w`ZKECWTWpUU}Q*NcQG6&62N5`m?-)Z2EysBE-3REeICVZ>`w?uz!LsO`F z-Vmer04XdOsltJW3MrZEcHB!fL&+@0BjzeBzgc-92k-G#=o|5igeB zNZHQlX4Cl+@n=H1wYfnY6`Iv!6T^?&(afoI;2PC4c-w$fQ@$PJ>9IHY`c3oD?5! zV&S{1v#~9ny&|$4XW;?D0G$Rv0pMd~FEftjoCx_Hk|FXig=fOX1>#uj6K3L+;^ zvUF0hNgN-)RDU5p|1kz@YGvuyLQ)nhEsZf|XN7)46oZ^nzY2%IAznC8Spa~=zmZ># zN~^)lcXP;h!an2Fhh09!(O!#@qgZBeY<;|1oi9y2c6h$mIlGeI@)R zAH1j)sL{C!qZxp#%rqD=s}8kaY&M(OZmqhGH9RU}00)5cLViLWkBxJ2JV_Rd=f9r{ zzpH@{Amrw5ga4^d{ya5RmelC{&y(F$VjIz(6{*d8FN`8EV=Z3zPy zHSGUe7X05b8X%d-Ep9|2w!Y!zM_o5JXXjeLH&E00rmL;a;oJp=f)Z!9SgoV0+w8dh z1b8x1YV@Z+`-CGCtu#6kVG%K=498yqse#m2Vu!iXZ{cnHGR7t*(?Cev8H}T*qSR1E zB(VFVQXr|Lr+2>AL{CpY+9Hz($f3c$K1^)vkE;aSj)(Lj(6C=R{rA&+bZ~aYiK+-< z&(&{Ogs5`lM>y}{JKtr9`9!@X3%v&IvcU!LN%U>Z%3vkGinu7OhP;SR4?~6 z3HG#V2p9MFMptqExy9BReFU72fLHJBzAF%N4YcH+cZwiB69)L2oU9CNC zwdgwtVkcP}m2HD<2p*|~DfWka)tg{vv6)8Kr{J2HWFe7R%xzyK-kjR~pk&)W`fN(*18Su8C{<$&xsa#PI z3NZy0RRZlFB0diw`kG+Lu?6=SWAXajF;vS^p#K6(VI~k*r!TRg z$)GA`^gpWz;bwC~av!c++129j~oe2n}5eWT|yd zo4*btN=r*$pYO>PKD9Wat+y3a)xzeH2nq_;7BX86?$ZTPJGpZ+yN19-Ic)L4?8mow1zrV+6GXwPuH3Q2uF+naB z;|y|l2Z3x>o2!%(sc?_~EL4$C;`4hjsG9bjt+t?e%pr`2G69x0TEP7TkF3d&qRH!v z&#h_x=a_Edd*j?_fmDm_JRK(|mN`O4z*|q)?!`B98j9^I-4@B<`@;<)W))-QsVsg^ z;J&Hj?H-j#N2?jg)M;RGF*P&2BjE`kbc;i>@W;Zzq5m1Mqi!ZnmmKsKKeK@z0E<;24{>^2CITyBbm1Ja^DG4*Gl~$yz z#puJq4_?elliJ`E1oq~Ff&%F1=<1%KEjGct2(y$*<=S^ZM4~7PEu#D`$dZ?z|54+u z{mH`S^TUNnGV~v9K=1Fm@Ghm&O~w~1Tw*e`XirB%6KoE|;`8$GiIw#TJOf@9NIQy< zd~9V1l9BpCN~NpQbRRhk_RkoDfhwZuAK}bHDVw{AXCA1V->U1;b!-&#bmcTzC#-`k z#DaoFY2t|Z!+0$3;A%P~tK~De9Oj0;fj}VbqT1H{h1AQsv#^j5V=@^fC4Y+(?Z%m? zWcqLv5+R4lCateg3jg9cB?DB z{Ja`x%k@sa?mzvR?Q^Lx|nEafuX`Iq+n?UM!163+dGg~a}8FL zt1V6i7gH?XYqSG3WZMj-*SOixhcEsns!63v=NmXo`cr9)Gt36{Q1l!_p8z(`>h<>c zcn~AP1_P^0rP=CvFczpBb$LChBn)lQ?)U2LtgP(i^)MKX6C9X_#MZ07FH5e=Y z4B#sy1#n3VN6WBzw3iwGdKlXOv4E8k_|NEQX!bA#-hB~hq(Q*3qa5EmK8`99MQCBa7jd+lSB$TfR?s{n24`q{ybR*di*^k;cU9 zxGXYatp#Z=CBPAanegTfVXij>1 zdOn&me>RgD+cfUAghZUGn#sHUk*e}Xb?{(p`H0OfLjEw0)GWW3N2{KAXN_s(%EMT! zk=z2L$;|0{2FRA0zby>}=Guzoh`;ib>5@VE6}7d0xE+nHVvdxGZejqdv7D}$=|^?? z`9{e1T<4l}>$v%9_X^F_2EW%+?)lj|RwTP+^!x9n*koVD9XI{%4L-x+4JBTTe;X^MbSwoQWT+Pe(Nj;ij`6?BZA`qIn$QPV<-n%@&?=Cj;N z9)SXthwJfgr(@%^dhs9pHPFTla-;5Ts;mZyWPe8NNCi3z1 z+uDx{`alHRMZ4Mk@%nnU(!yM1ZSrq0=@?>J%RyeEmU<1ANRqHoP;XEqsMxw3J4dr6 zDlCY^hZ@~vWWTWq2e*!_TYxKNJdOQB`iE~#fCUSONf!z}nyz3Q)$;d)+5>7F(JJg@ z3RpyhhldBfRux32sNGm5uWR4M))3M-r~lg*0nho)H}|6%$|`SqRG;VO3H!&ul&sf1 z|2IgwTIa1Ez^Pxp$`i{F@-2?*%l*3NWR@_B-vUrBFqL`^AkT-l24`6Wd|JnZ1T^nJ ze%bkWr{~5o1n@>xi2)e$2-P4(`UTu_{{?~iGK{QXqsWA^3}t` zPF#}|+6rpSarE9O>#j6Ge&ZpgphIbyUs`Hvz+Q=v_a}x!wUPFjrjyU}7;2fskw`4=3$#Lz7u4Jt zR)5`XpCfLYdnv58k%or=n%?pG+xa_7udTTqEZf)qZ6$k|hck+v8T+GKPk*ZKZS=^I z7?^8vogHpDp&14sNn$O{w)c-sK&3`3B53K08O-1S5iz{96e*rREHX3)LMerg?=-nN z3^OM37t*xE0`d^l1oSnC@t{i5m-w+fh;qv>43N&NqBK2haai~|TOKZv8gFQaW7vL# z3@&wI6GIqcasj`CR4lt9>HF;hi;9#&C{b-G`H->AOAVOIz=pssFneKn!MUkui26oO zFfRvU32<<*x{1PgS|_|BOO*=)pgg0j`WJyEmT7%hD}bdj*bGJ zN{uNRf(iq_+ZjM8yN;!@cAhK+!h!dFy@&WyJIO*20VO?j zl{(6#-!6R?8V%)$%&oN;KD1EljmI5ItB0sxPQ%&OOa&1dDkG;9j)0Z%TcK-ze6;fc zVt!B_$2`}Xi6O@5k)}fBg)G{2@whi)JZ}AvH`~R|Jdy+$0hc>@f9xN(J~%l! zDQkcn3Dp#%forI4xV)jEN?263b-6=^)kxfbf2BEq0#@@lf4k$8ccYyX*-?Yx1+g~=8ne1wBeufaO3jdKuVV$Ic;hx+jM-H3Gm<+OO&bG4 zaa~bpd#2uE^mnKL=!(L;T>=3o%5KnlUdsxO+kVaKr4O$4lttF+*DIH<``$=0VnN%l zy{ziS2n!on31f;cbiu2r>@YBh0)>4LHnt=Ko&z2zuLjF)tLPWRJ zrY2!p5jaIq2-(lqlfv)yfGRL(LnahR1&Y*gg4Fy36}iHn(8!SByPP(!uaRm}vXzxa zOPg>DRt;ZUdwN9RYU`Vcg@9xZ8IP^dLK}B1^=29Y92{IltET4>qRbe`5Nw%X&_tN0 zcxZ=*fo$Ul2anfrjpzF`lCvhq(fjLtr+*e7v{vPh!xk=)ARW=%a?;h-=lkpHtCbd~ zICJNYJg*qP+T$6Vy2A=BTU$DF7F5!FKT+2iEgcS=C?Y^`Jw%xF3Z-(___leJzw}Sq z0;=l>q2O?k#NU1+ADVEv9O?}m993T>dLoM-I8_v|Y%sg5i0t{_(I_aXbP=M6GIdaA zihn&8%cjki>OxOVXclCaTZt-zJYB^zGv#N7q;?{`&Yy11#La~dRb{r%XXu)+r8AK4 z?yM%V4S<@THorNSW>RrGn67vdRzJ7n(#7=4BWdvkSpFz!5Z&w_FAMEDRZ1`>Md!;p zvw^O;0x1+ZdV0|#b56U$tP)hipI`8h?RdELIGnzYAQQ5H4QOvcVHL0r+_d*hFIJnd z!+hXy46ng+7iGm2J?@=AmE=hx%F5JMLEctPZrKV$+H)xN{P|lPaUv7}8Nk4U&H8$8_Q^h82R(oh6)!Gu<}7D7>W|0I68 zTisjU-xVS;0t*v{#_)UxgxuXJ!aUSWZdI2(m*d()_w)HXMP@+|yIV&s=OUanT(t0; z*LXVT@AOA#XaX*Kk(!RTr$0a~DH@4S!1JPsl9C!An?wlmB@t~0)KU#0ySMH zr7S+Iv`nn88XfGe`!_bTq!x_Yt;+m*YslY>SLVumJkBMZifJi7W+|(oC1LyCdOuu{ zsSnRd#nsLQQeJFg%?ASo-O%vx;JrA&R-h5Qde$Q7(>Iu)KKT<11CkMA8{RfV6jpfW z0*!n-^jzDx-S=i%YPTKvkYBY2_7d?5cP$iE3L`OLN}u2F5&Fh%B#9Pq0DUq=jSdb+ z&dUI71EJ$!Pz}uA8GWvHhetFjs_8?y9jvV?)~O0#KOhtGe7-V|h?M@Wbcd*WS_a{^ zEfJ0F(&K5b0zI!K6H*w(h-J9i+R7vF1R(Xx$xXX_c6N$AlQ&qsqK-|SRCtYLvJgH$uZpCe)@PiOAp21v9pX#g{AVb z{Jw3;qfxSWxc*AyYB_1x=D}D$Xb51a+rt{H(G)ZGFvM;lpt|O}8lgbj%P+71QMv!(8i^~v815rwf@vy7#h0y?5y`kdP$Bg?)6BVrY#R=Be~BFElt2&|GUTD0k{GULrturDKKg(1m?>-fFWA zN=h}D6#CYbmELxHIHh~oc1 ztqF@rdD43>n_iAVa*QK?@1bJvoMGm)15Pkoc^J6wYacPcpZsOTWlXk7*W4XT1rN|0 zw0KR2XZN@7RvLiGgy}t-lZoZ^Y@v~}%$m{SbX1FuL)NserU=*ao$+JJRbT%ROECgc zjh>cw?Mq>C#(aayZ2zbOz~-0=hR%9@L7KX z?sA~-18+Syrre|}{%w|B0KZFz{+q#RvPo>iKSL)OQj>V7ssw;%p_;FlrGci`BCB@% z7)kB-Nk0{ZZ-1XU-vg|uoUT*tpEdWAfplX5z~%Gz&nBEsH9VBDvtE#KdZ`)o>;8S8 zdTm3!bycfE0q625kD0S<)Bo|$Jwa4spF9=k0=B@+1i&7fHyHcc@@}ndQd&YeubVpU zu=tf7rbIpZKMyQabATs#$5&z1himFT0BW{8v8`dzV+VQw;=?9AwL91~OdG&II%DmK zIVLYjT+)Ruyg612il2xHURv$jt0iRrnJJ(>DdG)9uEGZ_Jr*;j+Jw5?k`Sw;_cLo^ zI-+%IQvT;ATQO;cG?dZIxLQ?g0yzJvMf|nQrYgT@(FMQVP-u>Z05(9NOTEjG9cs3& zs^&)pk&tvHV^>|fWY|uE*$4l&{f&0Yk3)Sc`tu&y;(9;skjHu47pt_?&RI_@o^K8) z%n@rRjbGRP#{$^5xoXyc4t{{c7Z3mo%^^_hZoPOtbJV&R0Q3!%fbeU|K+uN$9|+A_ zXUr@u7xH|cySlo7W>>WmB^+$*9!Fctg<^$_si~=i?QNj%xy5m#k~AF1(Oayid)9ik z?@m{zRa7k4bbzxN6cpTVlV5G9ATL|qN^y2}R;Aa(=lLe^u2kxxUoa5c=5`iiUn3z^ zfGAnUFcU&O^MLqA#lj;Z5uu?So3m&TP-} zWDliJ{ETGhU}{D-$y_ayCF$1m`Fg*m^Gca|>gvOS#2J8S5Ds-$y)guwVvhY8|r6yVi)IHRWr*> zW0lEaZq7+?G2KpAIQ8^arvd7(quWEg+`?QsGfu#rkdW~4r@P&zVU5cy&@WwW&?XcS z7x%-RzS3e;$oHu$GdU|O2^Kv$goTlDe0f+#h3}QJuC7^91_A(Bh)!MXqQZ zvEWZ4g4H@RKrFA|8Im`}t2BNfPdwOEa7A>=uTM>t(f=6K9@YXP!L-~vwKjFsz2U*} z9EO+EiHs~uD-3hT-_!8-e}5$LgT?mQI`34_z6(YX!euj$xa0t$mX;5;8u(Pdu;b>} zcvjpqM@OJQEH4-k8;c|vQ4x)xh7{24wA}s}yJ%!(i@CC_*}3lA*3!~qVzV4|T0Z)N zmlp-_G_tXEu@A%%lzgr+dHw-lY=s=I@t&S-Kx159UP6bwsyq2VPVk=xj2p*!Uhj{e zp`pFM+^CUXZ}t9gJ*d`eQCzLoB&jnvCr&{t0pRyz6c)YPW(C)&x=nrl-~aZoDORLf zA~5O)V_k>2rVf|JZuVfU2`MT$}C|kZ$QN>68>H=?3X8 z>5`TP0YPb`yGy!3De3O+M(SNS@t^mcoYDw_?xjJ>qe8=T^Uo53`G!J8mc3o|CwO3M{zivaQP&QXMV!jR9!(bCe)jQhcqJ}JM)c%IBN zxnI-qq}TVZI^DUgZ}a4Ha&pXuQ+VI+VlzJLiIx)m@#E(xKRW2JwrGAyNB0EdxZjeJ zVkQw`VY97vq>Rfge+K(T@7|3@r=_K-m+1*QZ)2#im8)uqcx8skh5f_}IV|9bJOZ)HsGbZzdSeQOo@|r!C40M9QsC3PSC@pqqKI zH&q7S$8G%*<&z5}i&<=i5Z%!=Hnspb)OEt8-rk9ci?wLs9z_AC-{;9yW=dMl%!gw5 zr?96XW~ih>MB|YtX~)OMdV-!}4_z=DTSw6_)8nFKR1&YOIzhYrotH>0=uv~VhaVe0 z5q7obFiFcQ(G@tx51O!N>gl*Qn}Y)h_q?KLp~^3BleS7_Wv#Fa6e>6vguPOLTI?Up zRlFiJ(*$US0(x{{AdG6-ip{LX+_Xrr|5tR<3h@_o724g{(9|j_D&J&@h_Ik`_KsIN z>5<2SY!@2)Hyrad3ylH;1DEOSg^VP&-c~Z+8ATFuIodl1@?)K?fg;ib!;7%0-)mXX zp$VokuhZ7&xULeZP2;E;hWfS^e4+mD9Dz@FLh4;4bt}Nd$>MvXh{cJVP6DU&C@(F3 z<${0$%NnOhCHind$?C_DhU+L#(tL%(zbQfmmfTl994vzmDtvpXuH|sW54#NNFY7Wg z$k5Tzi#at+M{L0ug^E$99AXa%SJ$2rd~d#Zoyfz(BSI4}4Xsoz^NAXy2tlOWghECz z{kX4%a5>h4t%dcfX+eYDO4UFFg56gx(OCBa$_mnb7qGWP476TNwKf$6t&SUEF3mAS z5g7js6kv8UQaSSb$6mZH7whfmIzL<>4A@Ff_hr^G7R6O!RqOFsSxBJ) z8riUlDxCak-n5+CLRMCLl8cLhA{qY)51#996A>Z{{YGHH)PjhB`~k}R6Xv+=+;T6l z&J8pz=<>O<(N2oEu9~mfZtWHtgNPd6A2j-O1~+?Nu{SQy1%_B+#}0_C=zVu%i`aHp zwbK5*lh-(>_+0EOcTH7YNT8(9!|5OkIGZGWcAi8&2o^=I^Z~P8HABN?WjsQ?w6tw+ zIh`bTjCM{50dHC+Y5H2PX^s>@S7uEJQ;zRK0tFpO`@i6`cr+N}RwRFfnA;1mNoJ&V zh2kSRylOj8q+1WRL2Sg)T+tn2P_?B@sG#$9W<5`H@5FDPiW8BY#+Ge9C zOonZ(ldimKJh&1|cQ_W~WN(}-C?%u1keY37FBv|wbdrlHrwNBFT)Sgr;B%NRV=1`9 z#+wbYo(U~i5p!8$%OV&qFX{UrQr};E;o5#%Gx@B5%UqVXHxpy7&Sno6^8};}padYJ z@Oo?S;%X_~`!cyu?<==mVnPC!mSITX+)@(YQg#hOeXOLL1tCA@Mn*NkYZm4V{yTXXm031p(yk_RsYd6aDD=aHaFTdS+tbr@B$lDo=u9Wf zjR&^@5C3~}&Te!RkChgsua^Wef=t$-QI?t8$jbTZaKW9_CzRxIeJz@ehSkIG)I6G^ znjCN9`QYaOg-_hCEJo?zYRShHkE_7^qpueQ^7rlw6bv|QckpkT$3$-q!yojV`$Gns zW$sYFNn>PYqD-0G1?SlT1Q`@khoi`^FEc6q*9u_-l(|#~W1wx^S~2`BpHY56n59~G z$A-vGHSPmf6alHHX|%d?`5OF1yDhpA9QBAmd($gsKXOB#z%U5d2T&P_cp&xTbw6DP zpY4f;w={hZ_IvRNbBflenAC_Qklq841ozo@a6802QWib4veAWf6x2=zzD6cCQ znqBQ0#1u7?92jwZBM=amQeSC&VQPLj9PV;-?Iw3%LORS@(rX;<_*u$B8O`ts|1_RH zMDYTnuy%a9zB3$tCm)>Qd=coOT>3NOj4yr;4nk*(1wNudc8bh8I64Pnh$;JPXVk@G8@h@>eY4tFRc)e#3$FrH~-#^R)j4YbieA{a~!fq3HCvu70AMjJg*0wVU|p>uC= zdf}b%@A^W;OJ4Q1ukLL0Pccx4>41nr8f|*J6Aus1kMLPMnvqhP7evTw2c=O^2*7}! z!WTA#qR@C7>sw3alGU=TEFF$}ec5zW-e)uO9UrH?su<=V5OdOWHCa|p+K*VCVO7&~ zT!P@a+mw!ukZxnV3IYfzPEL)9Ld@8LMa9)5mN47JnYWt?VT#p^#$gu%QQ?Yg)m0(i zKn`qENmJiVV+d#lbs#{3?{5HWo?o6;*3`V9!M?ze%fmb@XM5LkSNRT}8U~{Y6g)U^ z)ZS*NY_mj|vC?<_i6o|_-3flg{0Suxg=Hr)tMsHJ%wDvAWQ>TsySwX=Me5rf){g+} zmv{yg>FMQVu$Vhg%^(%#COix235kdGM*dA?s${QYZa`oWuXLg|DedVUHMR3hGxs!o zEz`wv%B)OzewsryNJ7G1IWQ(L0UW9TSzGCN$?y{L>r0l$Rb&;? z68L{Zi@XoufG#vZ%So;r<$vdOj+}_GKi)6F`S4S_4*Mp2j55&9q2u~wJb~hk>gyn zKgImKj2rm;w5z{cOp5iNS-_{56I>>pK8qJ)yj-D?gxt0QKC7<)oy7LP6eEHpKav9S z1w|`~vu!!4{5W-jyd2;k*bJ7*UyzbNf0HbL4`NFD0}ltWVWUN|_fk~qyiyLoks0Co z^u@%WzEPj^r)++wC~dXmbMi+*&MZ%&6b+rSDJSKma{FlRH4{x7t6BXW?6@g zx<{1z)WY!)bhaRqt@UpLUI;U!AmBi7!@3h9fAcktlO!!o`mLM_MItH!iZuZNQV^m8 z2Xc6(cyKki6(_`DUPgs>W{OAonG0Zcpj-yoZ^uNt9S-xm-2Gqi##fw{7kLATOD)e3 zBTG(}ezKolRd%17Bm8p%O-p-^UnxU*IUE7~! zy%IJsJP0sK!MOLk^SoqaB9y)k8ZA}oiGiw<3?fB(1qB5$?IM1mzhQrV%y*dO&2CWK z4-G}=l0qSUX#egWQ=oDao*tx%I`&+0G+Ov!8U#`B@bIeP8HOt^ubB8;cc7X%$jM21 zl+7&|UyrlLDJUCd)nrV!C&QB4?0&WT&5+P=+GFzmJd_7*;!SKxzBPgCCo|D%a;d!j zeySQTyjVGk>61lKj@sG{lc&^Uwu1zGX$IatfRZ%0TEJVW_gk-;RYjnIkIYMJKhlHk zSjwls<9p!eMyT@LI`NoJA)Eaqw$Q8X(ax~}dbNVgO1TX`GYqsndH>6R2Lc`gh=?RF z&fRW$j;<=Q_mo0L)2#4Pz1X8dFgiN7uRqV4`0>GBAfsS?`E*@Js394m98?$F_k&*K zG;m8%R4!kG!k64(uBPEzyx(P#&ABl5USvOyx=Xk<#^nd)Rkk29MDKES}@8$~)2bR#x^b4}MM#CpTxlS4T@_srK9mA|SaT_uVImC68xCmFNYT zhaktNVv+C9qzdHPVxeEaIc|*FgvRCEyHZ1aUTX0DR zGb}6B8QW>UIoJCm-(R`%3ss&kJviRbVeT-h-VEt|liSXpdVB~$cqNq|R}f9|!6o1Y zEtj-pCiKVEnADEW&XsnUY_W-Dy_sfDg(_3Ns27I3zDZw@9ALRR$O6+Hu0t`+^(5^v zeLxO^_X)2$<7j;Z1$Q)m+q@o`Jc^q7h2j0&$skLH{@Yx+WF7|!EO7&VFgIwl0F5-o zopNd43hNor6ICnKdaq0O-D2&FS}nl_6_4k`9XJ<|cYfzKgo^v1Z@J!Cpw7zj(iPOt z%}teqG<1cl0;g~V@S=SzwiUm77?ipNks@w=V&zu!Q4~Z-1SfsT=xi1^gm?+-F4k)R ze}EfWX6tcr)~XQ-bgr0r{Mpn(Q-HXbDahR4@2XUj=c1vwWzS+XF^;C=pIer!JPmO0e8T?BF%1c8`_l5TqOpnYfNwxJ6~2K(%Ivl z(=T2+IzE_`j}4|K-o_h(Dkh^%#Cm{BO^Tv3)XdBqg2Tz#k$wF$Vw3YuTxtT2)pQxV zr5SQNgdBla5Yg{a0+h1P^7Ty(HRhufu?a3s}#z()pa*nb!q zn@N|4Lbr&uP?OaZrNI9~KRpv3twe(F6_&~I*+ujFJ+*{6jDX3G`Ma@=!9p$bDdah5 z>*kibVP8PA7V-*3O5G&leEkHbo=$++ovXD|pBvbfDYL~*UmU4IEgd$i+TElZmPr8& zwwuEbF{pyVW4vb)Z`%6kly5DGo3nsFu6WF%9F9QDO=ONyqkI{*psX~8P8dFSYG_-C z!L+y;ZB5ba_HySeeWGYkfXG|5p!RR5TNHMJniti)Uc{=YwX zh&h;qL(u9e-cXv$F#F4R3*wYQWRKjiEM=87DTF;*`QgCP^0W}iGzpGL`Ha+D?){1A zDQjC!`LzqiQk6o>-H8$-onZ%gHXe_j@W0yWS!rUr1*<4L5gn|JzgQ=tIY7gG{7Z)E z!Pf4GXQ`5kTo!1Hz{tLw3Day zg3Gl=T+Ya5WtOAM%9!U}2cOGvoz-X`b|UBvS~;n9x}(`_%<`Z27c|*6fANU6C>wZ{ zVmPs8i7TlfE9m0T#}3{j>uK_LDXE6ed2C$|5o>Z>`w_NyR!>qORy@h{0(Op}UFaBw2_^Cc`rQ8Xr)ByElP7H@t&`oiL3Fey(2FLaet zNl6JbZg4(#Ep%ps^4``^I!oVhZ!Z`JgBls^j9z$DX+6YjOpn>}Fkc(fm89%s5WJ~8 zceetv9Mc;cae?bWq9u`2WzEgx+aMVQcjUp$M#R$w(&u&VXQo|E&If!Trl>vy z7hRvl3h}VdrLyj5cs%94tv<}m%s{5RRHp69&t3bQ>2h}xsyb~bKJ?O`jVqpN?(5o zw9|z>&JmoP!Gvxu9NAzzy|9Jd{b7N#_n4YK{9|$#FGZi#tNPmv)*?xH>djnXq?c;uzb%$B-8cGXXvXS8cB~~f21C;%A)7BjLEYfm z1zPMt)>)|E;cEJmfZr75((R?);ywxH2`Kvp5j*cL-rs_ee3j`>u_N^}xE>JYON@;iZ^ziZW8s0Q`T`|RVl z?}nN`iFw!S`dYl-8NG|ml%wYN*-w&H!ebRF(YuF7F(<&PG&@>c|3>9_bdJwaSykzB zIGe*^_R{sr5HOi2-|}*ZdE8hZ+M$(^@{>r1zJ{E96nF&m!nf+$nr@E>Hk68IbnnRQ zk!+H_exB;;?d|;a$*xnk2}nmIqlge4y1OT8009G>F71l?bv}abxc|>|L$tl!%jy0) z+lANr3nxCW5zOhU9&niHmQqZ7zvQa`*Zvf7&@(b#jAs9dI6$hnHumc){nHv*no$H$6LsPY7r;v{f=qcWm2kh zbq9wN@C|q4sfl~@se+t|F_1^r<5*1&icPIK#MqXB$gc3P7gJZp0#rNLs=3!Pa ztndBdXQN?j%e;K;bNoEI9!Gr}hb)>}jQz;*nQ_KY!W)*oB0IWLj|bzT?1SiRI4;?SslkMD?Jy zef>Y-MF%1R#-uI_OWE{X-HFs`MOD?=`7zKzD{kN~=A!1(NGk#lgpg5)tg=yEmRN_gS~WULe2&NyHs%*c@`Sqe;VA-oSFGQvWwh@zYqvqTwJ3xZ{WXJNuUpL{}v5ULyjT!~RPdb%3WO}FZAM?lc zWBvpFiX{9lC@1Uj@lPPcK4=PoXoPAS+`&iyzcZN>T5(7rh|O<3uaD1O7_nz zpxS6-0K8ZQkY;@1ncUPo`VJk}_`9e~XJLNXv3osghBTaL_CU$fvdQHDmstZMEFcI5 zxSKLSn9;WM;jqVLX?~u_3#!Pi;JDMYQ!0XX*kzLOuOtIO`Nib7Ha`gCQjNH7rfigU zEs->j%Y8%^RwVSwO0VuP2K7?&sc_886Y1b+jb{0DzKJ~7BV?%eBSZM(#yfE@G`QZq zGxE4PjBGyxuG~+0-LoJ44R<_Thlm>F{v;W%XF-w?5a9oGv2h6S^5=$;3SkW>Z057o zk+weu2AUCUygQ)x)F7^S&=k+;Mk2;2UnhMmudD0L5B7nk;Bh_F8Pfr7f@bh7Gc**8 z*ZJ~uDK0SGDj8eZ>ZO&i-W_Y967XOw2paccbd4e#z&#-mNRj^{5&H%tiss61-hoU;44?=sApUQ%aX5eT z+SwU<^F>^+Doe$XnITb@;o0fwP>NtUPtT7h&vTN{Q_ zY6mvF*nFi#csKYB3x;)n@eRpAl1!CviRJq%eXoIZTjS3A&U@OXfPyw8>W7)eLu zUtjmF{>~oD@fKA2y045LI)1>$wbQI&3%f>XVj1wC1;-Et2-*P*-t8N zA7*)7s87B;y@>R0|0Ee?!EZNn#&i6t#b17r+us~-Um9PeFzJ<$WHz&epWeqxKqDYI zN6{!x=;aIfE5`7?#AgJ09)HS&r2!5UwVyLin<9Z4VAigtHjf~Ql^(vz0^r2ou%D6ww^WLH2@z?_;1-S7D-}RDyGnk$6 zN68F~VR!)d@BkU#W*4ScjNZiUq;EeB0y446dyZQ%9UM4X-+bjBNvz(hvG_$@%o$1W z67GH5X)xzVN*jvvCd%r@`u=xt2te?p9cwyjnQH)fMLz96c|{t!@EyjvpdckQag+D0 zOSN{{V)GV=EL6GmG@{R*U5*D3dTN# zTB#1A&$F4bB`~Q1gQTeW5uG#P-}$dUGt&xFVR(B`)l!%nV`!Sx5~?skQ=dF>6C2hm zSrR|R#8oL{LbEoqV+~nPM}zY>8_u@chs%FFxm2M7{t1ja?WLW|k zqhU&ck)I6U$)S@WCfdTIF@yfZ7#lX%Wxl#lZ;aoki}(|SRX_?xYbaVd7wuZ!T#q~! zqmT<0V7l*}z$0S(UW(H95tSKHOlQ_D9H48>SO(9x@AfAY&c#!DzQphRFoh)$$Aa}s z^h(@XzbWekT|Ik79shv3H?@YD#44J{s>rrnqaFO@+V4ajOrQ_2Yr8}rhxfqgLWwC9 zSc&0jKnf0&z~(Rax&KYfW1r#gC7&j6GnMWuw%x;wVN0AwBHC!)L24kY-01d-JezMi zbb$lcz?w9Z!oYi@3`wE)8C#tX;Bgc+0hkeJ&6d;dOO>X~5Nbx$n(Dx)W~~6oiwAv> zZwPJC7_~O7#wcP*5Y$82u`HOan5w75_9pbdx?<;g;G(ccSVw1r_D50^wVuz;%C4vZ zuJ2n?sO}Gz2=p_>njBpfTbpL~i9ttjq$?|qj?P`pR4?i!fbIBm-j)poGt!UXZ(HjA z&7QHt`AXU4Ki&leKQF!$2rJa;keaTk3OWRj|Bk^#w?-AD%olA-t=~}j+hpy50^6PdIXF$D&dN8}+Gt7!$oLi5@Vx3*j_=NnFYE5aUA+@muF^U{6j ztC7NC=N1-Hsh+_x>hz{qhK5;|lBnlCuj2?rMR9DY_HQZj3D**GL-HW9!zfA<)`#Vp z(!hN)TBXk7tadYPfXB{lkF)R8)fWp!J!gM{kM5q_JBL{l+-R5_SgfHWws$@=70}P1 zLKi@ti0hZt62m|h7~J6v-*HD6+TVWr_)Img9or`KAgpCRkQ2}GxBMIYQ$}_uO(hSt znCxYjbt%lw49YYC_pAXr8va}V4A>q)H$Ml>LqrqjtIE}TEJi%XZR~QkK6Q+ur)*&U z!7g4HXy~SY!Hp%<}q%Mk6x?YOWt@NkU{g?^fHg%+0LA@pbs&<>r%0 z$~z33+pD2I5D#JnI_SW@^P4Nx`K{KZZfL;>VT@LaB=DT*X%+}{=g`rea#Dv9wf0V6 z^b_l(W!W$xqJz!E#31|AsM;`4QjK?P?AMA|kWn=lY~3;Gw-8PAY9SA}{Y_^W13F_l zdjr-~M&(O-w9kilC|G}uk!#?}!^%@q(zwP2j4&xNapVrkUYODI(&82kGV=dl#<3lL zI;UCt9Y3gE+JjE`b8gjtQjUPC>8WA#lmPumJmN5f9td*`gqLb8<}`RT(Z9yc zKinSOoW&9_QI=kuJy+LAvb=-R6fEB>Laq^p_Pi&c*=>ERswn_aj$-#2US2wMQn5>T z@eX<;D>HZ-Xo28wC&%Znz8E>e`I|sPL4OSn{?dWkRcVTg3ob26UN2)--m-U`Mnq%0 z>%)vW4MFk+oE(n7c!*b*lkZ~18gPOE!Q=bo<@S`6lrEzvbovx8er~%nM{jshZm4JE4CSqDx!U|z2cBOo;r$iDecz+nRYC*U|8NU3xo zuqk4Jp;0tuu*9EeF62pr9fi||?E(Quu4aqmMU7k~Uu6E8aUbz@5aVo-!S=N>i zW7OZ3{tI>#{g}nANi^1UV-o(oot-Te)dn5Pe1@fG>3v~RGk{2pa9a&JlnKo z{iuu1c6Q0X7ujpL?oL;LhX3}Hqn)**Hq6lZ+<(YS(B~ZR^fBtE8n~VUKq8U2-sX@Y zB#roDxU*9{;n}O7CuJ>R(txcmFKBYTVuqt>ECcfsH)6Mf%I}fm?Ul`*7l4al@h}G6 zzsIZFql)JBudivfZ*Ft+HUUvNoGdserFhbP2}h)^fhR~u7^;gRXlhx;@bqZ|c5eEJ z00^K%_y-8dWt86D>3I22s?#tg1+i+NA{9lXlp4N@UZ|}d;F)luQgjU}AYe*iN0WjI z6{Bq{k49;J-G?EY)blX+l#hH8CfB1T`lOD7cnQK$Yl`7iet2|)?KwLu<69zw#};pJ zbQm=UwYzROJ|S|*F{s_B?tM@by_UI7#7N29>sMbv80WX`3`Pxq&Jvsrufx%rVbBT4 zgc>kdSxcyRS5jQO+;F=#Lhh!^<}Ib)1jk-&u_h^ikka4L0Yxkbh62XWkI#+$Tup&u z%!c4wF)T8!xfwG5D}Rs*9LzKmHYZC`QOF-9)62gu0LW@+lt!B}sUXxr&zCN%7!QE$ z;gFCBS*}a7(yJWo&r|`db2Ivdg&@P`WC;*^VyG)-19{9fJ^;l@SdM8T7%*TXWcutf zj_LyKVD(yEQ1#U7kwco#v3fYZSb?F`|H6>9QFwHceL{R*uv;kKIfku8qZTDCJE2Hz zMNi^J<4yM=h(nAd6mg_((TRBM0WtCnn84VNU)$u!dcD)LJq)Gf257~A&f1U>Wk~#RpF5N77 zo|1gx{YBl9|DROkgJCdt*Z2p>vO3iF8#kaGJq)hQAi1Z7$n}4}H}zWXjqiL5@mqK} zh+tGlzoxUVb}gefjO+GX^hizw)EMKKUy-nnIsh@^vkXCy2?Rxb+}h_%$;HKt&Q|~d zmqtsnEdQ`Sk_oK`BcPju+Y0a=%OiD1_TZU*;m239`_JIeNh0P(qo<$6X_>d^E26^k z69rAz7~!1_&C9AP{At&78#_PF`6dL$q?VVyOQSH$3?)wK8(Ji}EfHW}P)~wZrVvy< zm69yfgOq$f85Won^o9znp4iC*>$>pSH_2*6P~4B$IvnYb9)B@I=m>wI*H|l~6vv!KNrHRfPYI z&Geq{9scJ)_f0d~{AO~n$3#7jUaz3=le$X8F5Sz2q&ye!WU8xnk8B$bxszba0vqu6 zjB}pT#((7})cj`_FrmbpUXf*CtrHDpS*%O6`&RCctDnh#IbIR?qTetECTD3(n;U(*QC4f%@L%bN13}_=E)Y-BnPZ zx!A3L@syq2LsTR#EGZa=smt~t%9%osx@>) zYp4TT0hSFlPqPDu(;3>q96TIcZeCtGpLe4o{f1Ml^z8p~l#G|8QC8$_|KmGMOu*0^ z1qlhj1|P%1z>pn#DTlbx)2-TeVt4-L4EE-RmGkuSa4ZM8vKg1Sgx2rDlS|CP3+|y9 zdfw6Qo3XB&!KU%syXv;_v2pG$@dU}g0goLz3(>jUGzIy^p{C@Hg#NZ7x#FK2u-(DJ zqxt3T;kf??-tmdXYK_iAOtMb5L;*kbvj1%^3@n@v80CN2d2;VrqNCbyo1Cr#w)J49 zDg-m&-7fj~Tx5xV90;f@!Qn`?=t1?T4c5O&%8BEf6i!6-KfK4V@Y$oSL&7N$tG&Ua z&%KV04j7BK=+_;(SQoQL#@-s3Y;SG`57~g0Fc|1>FsM(K>8Ho0m9FX4w%Bex)u4r5 zl6NN2Qqu)=f{O+`94xR`{G6P08P8V&oCvFwq-3r6;@xH>p^S_SCMM>=Y~9pnaX<=y z5!-JvA(HPje`6g{J2)??sZXTAYB?8|(rPN8AEA{I5%FtC#%FA4X#wH2&-pgy=5`b@ zf2|Z`MDb@++W*2F!2A4NGO!69smgNQ_%E1a$~Y*e82F>2RH)u=C_dr5zu{qiJRXae zPXm#}T%d{Y^>P5Yc*tpy7_2MZ(S%LenEp~6S1L~KcY!-0CbiGZv za1|zA>+uQt5Bau>2yRrcTz#GY=$S;!Q&NGQ=?=i|@Txll6GeA$v4())xcBld(`k`A zL_$E}plKHb zDk@+!igD`*L`2A>hP%EEXb_uR#eofm9EEw3WQ8|+ya%jpx!N6BJwUM=5JX4o>)#rQ6G1JChmWsgx=tY-0MNk76kxFP1J~_l<1@Kn7QY-7Rw@^T zNNue5ZBJJZg?O@@%iVMZIE=-;Cb-xYHW2w0gApp%56;Z(=R3bZE~g}r%G=b?gigr) zqfX{oXaN9Q+g9HU7}(lAtbBLJOYi%X?>))Q4Gfaq_T9Ta;P!zhy8MdJUxd?QtgEv# zfYU#`>J0pS>%vsq%uNDx1?2*)gN!wc1XeD(iWk#$cBK2?{slBZgi*LT2H^qpI^UYX ze#b`DA~jHr0?IIPmW(Do=WYAzK1^&Z2OTk>UixoVVLqNe8rvBn!%@k|Z+2-Nkup6! zec$>RMRXpCI;P+K;zc2|VkD}C;YxOIw9H-s_BKdB_?qfNn^b{*NuIpkiAkbU+&ffZ^w8{&UL!6u5H|9&YT@Lj*vXL53~ z5Z54{@~JifCG&oovY{80~KQiEK~T@!!4aJ{^tW=ZgDnG z=->ud9cc0LSAbY%ogM%5X3H#{%Wm-w2)3RYrvbviL4<;06S|W6_0vDu0;@SHt`CY& zHN-T{bq*JdR$`Dtzwm>hogMlD>~GZ2WlyIqBaBrS;LHF#jsMMu(%4O-YCqI~A?sM| zvWX1QfRP2o7tPR7bedPghXg`gFwmjx>Bx&FfvEv3SBzx8=7ts2 z5AP#E^PXjGA$j?QMi<>y!%mR((+`#W{q_D&74F*SC%qZA97{g8{m-lsITisFuE0`e z*K#AdRE1KeUei8Vs#~%!4L0#%mCfpcstPKV?RqkOFkQSXSg!o9R0&5t!SpFS7;DSV z&?-2*2gNviu4B52*&5pkq=?nc7Q=Wri!o$z46e-36}=PSLldQ>(H+r8dFsn4?C$ z{vQT$z@+S=W4!K|oo4Qa(C(M-JaU|2*5+R#Fv+{Ex;32s_?K1562}d0M(^E3il7*P$O6juxxTZfk8a zgusx+2Y?=9zsCNG4?@5Ei4P!@LejmRn{WF~GT) z_ke&k+W=31)aROf+tWi5uGazh$fTMsb{7cckQqGg0^s|oLH%x}BM75M(L{wv7Dp=^ zoKeoUhQZd3=^j+VzF%Y%M$ao8H)L4_s(YTY12m8w45jnGe9VqM)`@?N6LD~c`ihx4 zlO?3F&;)%lv{`B4>_5q{wrs7gPbwuS@JYph58r$>dr34*QA@3nz}InwBK z3^-;5k`fYyoCYxXX^Wgf;!7B|ab4{%{v;Jz;|W#%LQM06n5Z*fS~!0UWLR6?*W|Rs z+n5@urXD6lDLCKN*uubCJeR&K1C?bjJjZXSN?It0Ke>%->f9M;(rkcyXLq~+kF~+w zeLFPW=Z;S30kReek7iv0;tjl^za%UhicJe}H>|J6Zw$7!`Jmq%2Q=MjJ~le<7`?bK z(tNs&a*h+zODC|sG@wM?g>FVNf|FKFvzJcMffetdKtjh+vwreh=&hR}rvyUx9ZC$b7%+y78rjHnVUS{2c{UFdBTaF=%Fw zS{mZPWvHd{a>X*Vs6GFkk{-Ho7$!bYbmxVpzI>2#%cx@$p1$ssV-gk@uh1m-zV#W6 z){l(cb~H9h9$JJVGVWhWN~1ARV2RE{h=@Z5fxj%+Y#c!Y1Ac>}Vn|vDy#lKPf;;|~ z2GG=AuOk|8QwgqaBd>EOkHE z6tvN9xt06(eSqX}>Ca}xCL0AhiSe$L?Zhg%&j8ebC7d zyG{+D6Q`uH<3mFP4+publ$R+yj@y>MLbSAFhmU66@qvsjJj z3(n6{7aNZFv}=GnE7NV2j{|rE5{(KcI~^cuTMgjRw-72aPV5#NbA~+0fBiEHILmyd z=V=2lhtbhdVAXG*n+QA)Ic9ZJ5_<w+GJqdiGH!E zVV-h{9OK0Wr*zUXGgFg-dHzaLCtoi@h%wk@1T`Z9u!Tf3NuF17e`5>Z#pKs-{ZIn7 z9=^ZpWodmv7vw_?d3kx0t{p3@G*|q>>9s4p)s01Izl}Pm)#(zkzZdnaco*!?4(9oi zJaj-(_4?H^L;q0!2|7Y(qsPTzD*v=HIvD8F-+d2qiCl&jTl+>u)A=~@G|p!zEBCNO zZ49FZXXJ+~pe^<%vB(``(!iaEVk5M&N(eFzKL%g?UbSmI3(?_&f1wNOY}a3bQ-D7I zb3W`K)9>AcFJE?#w%|46^Y!sQD4#sbS=_CE|wKw|Z=rdIZ#+AQD z`kirzJ7Vu~V7CGAXbH)NBS)+!Km9s0r=l1*M0Yi1K7O3^H-b0At$!GQxpYo5kfI@H z#Bd{~gWs=)u~L1c+I^B`sM6HXz~-Mj_*! zJn66b7huFWWj75mIW%Z(Y4Q#~Qj7M;L;~33UUs;;ocCe64J@++AQUF+-Rg}VfU#b= z0wGYst_pgbmseL($mT_Hk6%2;Ax_m-9BEQ7+cc17+*>)cHMI<5hHe+r_j3eXy1e_@ z&$=ZK;@>*X8%i_zAR52WF~@EcJscjF%rO^$cgWN$<0LS?1G!Cl?Mt@`6Xx zU$DYK9UG=7j>7YnlaQM7Xo$aH1?4R4lmYZYWcZZKV}KawcNc*Xye%^YH5MW}BUQ+x zguN26%J8~E_?di4G2?{lv=S9YJ(zMBq7(2rNQcnJb?fHb2q<4Ugi7(nb>=rB`kM+3!+9I|r7w zfmy~EZkyS(g{IN3w5qTdSTz}6B+Q!0_{0)`!x_`u^(r(JiL(z_ifd8P;Ri|Cj9>~~ z09Xp1_h-Fv)oD0Rdy}2xfRHlFlEh;==I{@U1%f7>r$K;pCY~L?>jA|Q(s!Q&f<)lr zl|muB#F{jnuX)#tHd00h;Di(&%#Y;d#C&e$cE;<#sZI9Dfbyif(nKLmK(oOT=Wt?* ze@eQRQWC{eCbmRA2kAiAe7Oyzf8w^Hm0qhZQ}P9`{x^^(!=H_AalCQV(IK(~OYHtq zHSH`AMn=5GBcv71CxLm5H@dbR4hapnjKy||8-WMcdVe`rz4L2S-woIK6_74;^P=*i z)`h^p4=1rfcY@nQ0W`Fp9ib(pb|q=8wJ41bD3HK0jRmZBUL36)Vqz2B&6uV4!yG*O zjxURo5&?Enxvn1@p0g}wgF;oP0+5DJ+%W21C}WexFH8=hi8E+sqE^W@-@~4HU{Bei zwr8=J9L#9eKr?ylXL%MhL(WAWTv*ra=>Fpajls!qz|+j?|_O43+91_xyye&VwSB1zN@O$9N=Mc~0jf2(ZtZeqyi zL>RHh;Vu@t<~-t(B~v`n-q{hVU?MhH1pjC(2WV0fcSjP2I172O(WE1`5~QN=+6!Q` zfXeBb33>s*+rEgph-ofPKN5=6@oYkmCZK8gVkm&u>4seeLE>hVQT7Qd{y#?W#lScN z4Cvpic8a;sDhb<3dSbd_EEQo{YyFXm3=Aq|k>O6{<|8B_pLhn?qk^Y7JW%$OsLZdC z{TeA@(QPR%`~s9DweDf<77&G+PrbkANVuDu6w089-_FVqn9VU_@HA}(e*(bcTWLue zr)Z;>%rw?fO)9IAe5`_`KQ81Y1Aqt#fG9#GP^QeVtI;UoGcznN5T1dh_NDs_@$!8e8e! zk@A*6xn^~pe1EYgf?$+PG=culD*>;Pz>f~BlSf)p8Z>JybW!il%K4Hjtq?F8Q9%O`{;6U}3e?FEBz?wB4m&AFfPDpP3PeAkcn_ZL4qrfo+tt-|I=t!?@8F6a4Hnv`^82=s36qk7;-Bl^-3OWCa4$N|2csZWBEdInUm4nkWF|uK1 zp`AP_{(=CcN+-Xq%D7Nbz3;A1z!2|Iz`c??O?hnvTsZ=upoJf<*ZABivgeq|I60kx zR1&~qbHGD14TjL0Grcql$;ykiV7v9mJH5-BzvkU)Y!gou!C`MVI2L}yeCS^N_=!4` z>NB(c11MMKsNPR~hDRZqZq*ll{`@oGC61;_nurMpRnTn@54Y#RSGk6UhE9v42@sFt z@9dyFeXgtg5Z_V#M#qDY1S|NSyhTsj3bZua@$m6yvLw#Hf&!`+FC<>QdSWz}4#`dcKe-Ff{+K3AeSyA6*d=EV9crB#;$#eXr#d2F49wDhlO(1PcK*6YrPr zuR>7pJ5-20UZHR?1y0odQ~lqSE~7d8Q~ZDD_IqHp*tUtsSR>VikdQC}LtR~c!4)63 z)^fpPFp;UO^+YnkX#~F(x6&JWhMp z{FlH8l|TLV>ZrH3_c?~CTC?M7Wa#$dJupIiX3;+!YdG_d7Ao{fdyi9DV`ZkfU}4#5 z>y}dDg7(DtaN8CuPvk36qPt!JV85^8F|9@m7y z&;3l0A3lKXyg-1i`0bdDf&cb@Q*KBmPg*}r>@v{+_+CM`jJSfoB}Eu44l->iPc!a> ze`nl6&;G9S7e=J)(WM1+@-&E{#{4ti{^_BqM0fOwjq6LC8su`6L9!Kyd#6XUH5k>4 zHAaE^92y>twd4Pi5`n1O+|-t2`REZu9F@3A*dzZRWp5dlW!i@A(w#~PNDD|wNq0(! zbax2SA>9w%AR$PDNOyOKpnyns3QDIm-}T_kyz#wjTWj%aHpfvw=f2PDxb|~DbeF{y zALW+mFSPp#10j)SIL!kYYx&6d>gxRbO#sURAq9$>HD~};P*5nEGkU~l8k(C+9qA36 z@gP_c^AbQw8n*t%114qmkJ4{L!X%XSF_{ICW~9z`Ngk3!V1RD%Lz8|M{gxMHR^#Z* zz&?aPrQ+zw-spOH@)MLx((AhqhhJZe0u~(LRBbOmh{#Kd8y(VWfaWUU>jUw~SL-t4 zP-O{#J&YTSorOV*Dh7f=wD|vwwyov}-m{KRmFh@mJa_p*^sy(&5Nya*fYlDt|0)+q z3jguFp^^b)M01z))lZ(gyPs^SmulyI`eeIMhuP}4_fNqxQY7cfXh70`@Aicgk1aJ&+`{PqVZJ3yYM|@5YM$iMY3O z(gR+%#{zH0*O8Ht!9j*++=QTI;6CW?Cxv)J?qhBoVj5a1i&&_{mwd^6GaZaGCm=Cu zdY;VrGZ%HOpSowaK>fw{PmR31^rc1phb5u53l-$(c5?*cWnQF?X+N%>L}Cj2-c^k$ z%e$6#b+H=U_$-THeCxGRH!6J!wHvCCxOA_nHAesqA7xJzwOaCxoUB}Rb(M%QYqp`~ z+0jurzQ_Y`-|Oz-MngkWV2cW>GUW(A$%z&1Ej3H;5F{mK32Az;y(xMCHlKC0w8?JQ zu>Tc->&h3ga;!d0{&9J>E5{T=E@(Ml5El?2Ze&E6+z%qL({VpDj7vf0+x9W0H=pCZ zfI@E!fs#8KK3BQ%4yp$d-2+#v|Ri zmzc)oeOVAk-KkzFdkV@QYHz{%NCn{Q)lm+@H2R_?8T|t>!8{8DrLi#RAIE^2sq;G& zsi&BufD4brM*M1QF#J@xk)k14@|fSl^j^0YGv=Bq1(uk@Q>`S+qH zb^uO&b7RxlubGEmup_vabc=5BgGN1^O&KF$TGYX^He47f`&)WYGr;3}g+hv_CPKO! z3@sS|ME!lqfZI0_S`<2wR>kjNjS9$;{GGiIdkOja`UVq6oev&lBYKfO=6BihBMbZ! zd>cDK$Pn^)K=7UO#nKK%PCpzhZbMEXb{9On&fBxjH^G>|45B7i9=Yb-f35^&>;KNa z!Jn1H6pP-dH-2q4)hlHbq+BYn^8pwvFx+3%1CRf%>r&_K#IFg81VEOWBRG>%E@H8= zwxAc_zNV~D#?gwzj1&)@f1vWc*>x|AaCt+w`SsVzwSKm`+NEcs>Ia^zZv?R7F&2sO z)aKct^A1m>i#EP&9(yBcGw0TBR7S zmD#}v^b91yRGWkwbdIg-{yReu7=YkU-GMNAuI5dAM$#v}azN9Ij$4m6uob|E!i{zK zBD~JxFfVbzSp?=qbR%p@51{z3l$Vm0%zgxn!ch>T1CeqxEL541fn^?BU@5lXwjeRo zwwFJyBIp*8)dH=sJea`wBVH(G)!_{K1g?QE*k-=MzXoOi!37uWry%^*IItdyJFt7* z=mg+-lo82`XL?N^C);yyYF?SwH6Z=r=21=maP!XNs;DKG7UHVMfQR@G=MRX#)A?M+ z*|>PH?o~gjc4Dc|b(&03(csG(&41fovq9+djvLdqsPA)cHki@1ENMgM%SFTlX-_;+ z0>oUn&F)xgn`wYNV}ymzRTO~=RoaLt0#9~ zpESAmI0!JV92|)96bIVwZas4lOf>8_haac!6H-~^#1KA{iVNkE0%rlr;BOz%_JK7Mdtjc@9B)DzZzd;QQSlQIC=@9Ld#N`KPtsGe{8HgEiw-My`~sh^9OVn zGnHFUNqYzPD*ov8%a_qpa6^O3(ytL-;WWkI{{o!@stCqJH1u+Eazxqwop)Wu*~T`n zQiK~c9F%b9jVk=MBSR|s9RNHheQU_z<;_dhfXwU%3r}1FY6k=eN2>5@9)9%i=x!#~ z;-X~3b>S%;&fwS3ch8$$XL8#Ux;atl?1PYsP^wH|y^4!xs800uRGO$8^rZARj6KeK zBJA2_nVzf!_ewOzBp^FG=|Js;jA-B{GT455FUiKGsa}*2GF74lPYZMv_YwD^L2Xsx zl!o~myTuk>AV4qxrs`j`0CD%e!YF^*1@bNvfdC~+tY?dWZ^zH?#IWt==$NTCygML* zC~fG9Cduj@warrJ>!)0N=3|Y1k?b!iIbF$`@3^#PT*U9OzoJgt58hGay<_CC7XW<)ELCpOrDK04ki;y8O@M|j9irvN0MuE`McwU}#3|Xto z4WA4WZCpV_jBJok=L5Ju(RO(I<3gDBGkLe&S}?{)OH12<+JO?Ao2yesgZOby&QBD= z2K+kG$RT@z9)^K9=zNk1t)FbO^M;<*Fz$Nz6Bru0{b%(6@Nmir^1hEo0M!4)A z9{04QPowJNnTsuw5l`;Uk=4};B*J3nUyrEknP~;tp|h~^h`V!rQu7IHK)4UJbEenP z+PR!QjIh(J*Jyuw=hcIgKVH>M7s?&v7`cI}w3)*IpYkw(uYHCg1kshN0?FXWKwDbuF*TaK|3eln+W2Rs4zY;eqD zL@fn#=Fw^k%pN6Ii3%Hvr!?!=-obpP@9m?xRRE@w@z_()(IG$dP;lX5B(+yEldId0#2s~F zfS5Vw8(BRod(-H?_6bzzfjfU3z+Rx3ib+&dkNkU@ZK6*Gt+6dUI3}3o4bT@6mz7Zn zzAH8`U2o9PH#!w~{1*S^B`6Kv2kJ5W$*MfDdYm$d2*qm?o6@RSYh1CXz&Xc_m634W z(gGB4;{6cMClnM}*H5*xwQUj!zIsF{jWdFlubb5OnXks5MT+D!SZsQ&{dgJqq@*?! zDCDmL+>g%>X+3fnuted2hX;3_FAMH^g=j+!4~EQg9Il8?Q@bTATvVf*wK~wUz)M}( zmxD{zfr7QpCeVJ%dJ-HthJ0N1rp&lAD{mC-waNU|qa*U-MKnvM;26 zP&-zK;^aw~ZEx5ODrs)sNB-ORN-xh!(9TI;tIZzedu@KyfhH!~tu z3uU5jiA5eOTmr?Zj@F>p+W4f)!tBjVqa0xkzV?@q5$ry2$TTKJqVHSLo8VE!k4(sF z5nw_m=MxwZYMC#4o$q@RbK5H73tDPoa~0D0o?DD&fte{4fBo|93W={K`W1=@Y2+_( z5G}$iSk3Y5`YR3j5LJ4@~5TGfi2+Tdu_XdJJqQ3?&z>yeq*LEJQOo&YkX9zZf@$+?8$Ech7mr@;U z6%|m75k_oO)6pIfl;qV9w`SQXO)mPm!)>n7itxf?>eOngA~1%O_rWSXJ=WfAr3v!n z&OgDnHR(UWcGj5De+AoEe+An=pS&w|NsT=t=d@o2&TE5IR@l!+L8U0U8Oj2hm4%u$ z7Pw8ps+8*2PKtw-4jV^iJnf3P@AW-U%gI7tWG=RT8)yjp7i$i!A|PUgc^_I?ZwMSb zz)|$+WoGnkS9KP(vFer@dxGF_nD#P&HUe}7P5Lk}IDmvF8mtj$j0NWiR6e;{2_YX1 z1AUmiS&|<15)HSNPe4xye4hX-h#aJa6aJl=!k(J{hSLS z*&|B<|3#j$#$#FusA2~rE;RPR&p71`?n2yg<+@YYSP{6OefKxlnpD~O*KrAhFvwJ# z__4FHDlWkP&&WU5VPG~sOu+9mN_>!Ublla~ecwFL>Vj{I@XQ}c9b2pL-=?DNfGSYu zkAxvztrT8SEv=o?Zfu(Tugp+T6_r%z*+}!Lx#7E9cIQ%4Q_?|nRkY#94s#&OI6Hby z+61)eGGTWYjHAY#kk&L~QU+!~-Ovimf_MItw?c{D%N_nWDU;rJf}Gmc(|CL-1^Or# zc8jXDW>6@#jfc`jSB+8)4{wA`9Yq?F)yaxCLYOstw)=fkxLD&j=lui4-mE*D*_Mlj zgW@O9*i;Sai5eyflgh<2>z|dvYYkYvlJRTX%8C{*x|`5a%r9*k&~`@*TB0)swr;05 z3e#tc&B|$Pjci%`hvaFC7Yq=<&FKAyTbq%kHr?GQ8X^GPi~GO)QZNanAg_gyLKFa8y+ z{|aDG!y6$jA*-fzQLq2|6T5P%)X0SvEp!ude3ZraugxsBD*xn?t$)tpRL3kc;{0LWd?P3r{H7$bD%5!!Hg?PJNcui2jJbho*}Q( znzkp43x6k{VIcN8A*#pBK)+Xiz7T8YosvHQ{RkIAxp8D*N{SY!mW6|d|Ni|us7+DR z)Eq$BDFM$bV3kSymk}p5b*asXj*lVy`r#oM`2U9vC!G=J_Ve5y9ZoOczCp8X(4yD7 zzq5c|#L-~R71aNQbJrE-GeothCb|Fc0rj%C@I&`zoMYQh_Gat=Q-5jAw2oMNT7JW6 z_igobO5ci@_!5}0!DMjjenj&fht+*BA*-VUsO^&7Cp!kx*xlKK2Laia>=E;;e(o;; zPN{aU>SNN+n?AbpKVGI1tz^z*@Vt4@?eJHOjej@%*4^y_xQyM=z3EsD79`B6rx-KDp#;yU!pC9)X%%NqwBnr_V+i{lqff`=oTKy zPYFEs@8RvOZ%}w!N=gbsx6tB14RqpERFJ#;g;SWo67XOHuJs1|hY!JpL8g$mVb4i8 z$-{WJk9m2#{QMf)7$hXAP!t>i!7ptkx4e9RxibK~EdqI)K(Mqsr=i35?(^~fC*ELg zrAP8#Xohh`^{KlSkn3uWTsYadnLQG^=@ODE9z0ZW%KcgB52|`IfuqOb_%CML?e2zB zJebuvx-Bja+ze2Qb=!d^qvOLvqau_)fB&hcZAwbn#B}xaob$Pv7K@y>q3W93J`%Nt z`W98~vq`Z!I|`jAc)OqKuGnnr1Ls;>Tdho|^1?TN{16I_7;-x@1=sm&2Hv%n?scH% z_p;{A^u;{LZH0s~tV4>l6voFpJ7KS)!1@e`k~YCxwA3f#{{Lys#8J&WG|SkK=BrNe zkPK5(zfm53-iwP1aM6ohG?vJy0i43IeS@>JBU8TzH-^>;XaX8QQ!26~8hLe9HFyJx zsnTm{JyOS$V=NAVBN9C%=Cn|p4?t!}83Y%f%`RRDhY#L=6z`AbotdE>v|N6gl9B=z zlq7Yj;tUa6OPEB|Iu6q~HnpoZW7X4iMa4GEjv&o%fMq}QJ$@n;BBNu5qr465JT8;d zAGbbU1emHBfv3WKxdT=g)aDw3RYqAkSw)9};0pOg80 zz-T~Q{KaNVz1?R4N~5SKXGfQ_lam&YjX_|b$qL8ZT3Y)2`KPFtfGXQO)+1<3R&;bU zD|qszUIt?bK}3r#j89qOAxG zEb{9ak|e!PjTn)Xl>96sQvrBlqs-;KJ$*p4I`Yjaz8VHdtHd-EEjw&WYpg1){26U6Rj2<898%Iq-rq4$A&ds$`#&;! zw!(Wsf9Dv0-wljA$CnBmOsq94G5p*G+%FRfu@K7CKj>**(=4?C56sF_yA>W*hM+6# z$O>SA8PL0==%xi07+v)b%x0Fzl$4YO{6zSxhWMs$5CAOA1)#nPwHl&H>~Z2^tVz9( zA;uvaD~THVV6|Ac(YO~__2Y={&GleS(M=mv24K!0S>s#7pwsgp%s2ee);Hi};Nh?{ z{nFe`V$y=BOS;a@PWvVJy~-jNT~mZnt5a60)0~Snl9!jyisfym*@i_EFzkTA=>eQ6 zH5QP&TsSH^oq9OaX%iOc0QG+XE#9C?{wwk27W)-QoArAZy>NVDlPcH!MIvB9x^|MH zcm$23ZU2g+LGSH|Q1_+x&80Jj0EQ|a89Ku(4(VeX$X{v~R*R_ic3~u8ZJ3>h*JIi^ z7Df_R({pp;uKqsX3r+fgq~5urUfQYg?*RI36z6-ctkYj#2m(fihR~bQ-*QhQt-7J5 zwHG|+q2GdAwcxIS2@ijbe7r@5>%Z3>F*z`o8p|+_PD6B+_9uLvFbs< z@gy?_$JPcBHqgV9Na39@W57gWvKtCGoIWMtCPwXQ{x!DsEw}Qczmn1Lzu8*8H>okQG90SyJcTCAo*g4 zYl}{>#=87bw3L|ML)HiOCBJ7+XKUVoWo(d_1@QUNQ(#_8qd5OST}526`8r#hmHkK1 z0u{>tBWSTlzi+YyMtMb=db_1)m+gsW?qmW2?Hpy13z$ZPe9=TD2YNLNOx9sgJx>m` z$p>SuYf(I$mMqb2d~l6a%I5UrOB}Tn1$Tn)tu*;cBmEut4Nla^p|LWNjNt==+f8B3 zTTqHS&cs;*;p!QXCdv5_Yu+)k3`hN!M_$fCnKNj((*Gy`-W$|=Z*AqtC1tTvsF_-` zLg;%{krbh|wZ<@6{h~2hNk8SnpxSsSI1Ma6h{2?9x-(imj9w5;2NS<|CD(n{(KSyh zz5#fH{np=P)EEk*OcnLu_2jQwYqvx;BK9)ws2uF`(p_F$*#XuUnB`vtNKuf+6&l$; zH`&CfbOHl;l-q(pP~V$tt=?jhqKY+aP>Jj@5zxRRpQY)e`o1GJYr0!}yxjK=zWq*L zl$P#QTE|-?=?7e^$!&571c)nWMi2H^zoGABfGD~B-=gH-bNg-GzlO~D-B{I$;a%%R z`YJWA!eEdTus}8M$`Ob|Vlzqej*cu$Q-2*&<2sVro57KS# z$cnNd#;eAM0#BZCW0PR0FR-XmBI4SkkK!Fw(f1(@$tE5he0XlvaW92G3wC_{y6oD= z3el5Y?G54;&_MP_J~{^S9FYYCtChz#NJ|lS0gNUB=yR+D^{+ng#0g`5vPRCv8MY`JHpm+tD= zwIbGqc(V23r57CH-Q)EooSaFj_yqW!GO&WEmI%V`P$NoA7zR4!YbhqmWwwJKhDtrN z&wl7IOwZ1AE`{J@g;idPIlDSIdnbGdmovlw^G<0p>T?1Zc4Q6240KA_me)NPQjRJkQeItFdd5hV>~ zA51QJ8ojC(C>bfHMhbpk&+SIC^}f8EoJU&qZOyPX8;FPfK-%#f_`-T`H3Tf#9A1I5vcP}3R4DpcHz6oN8lbR9lB6x9g6yRqmyB4 zfAe0>zMbbn){iH!6+;%K3rn&kDin&#af}LZlh-w&3#K9?fS;P-O+bpC?q%wd5UDp>V z*jJ?h863%8YW;jr2Bs-kX&0;!^GFanDNR`{^`aOo_-&4B$KEVgy&5}(e`3#+GYK+6 zt#%)8i|?NpQc9IWGZFhAUB8u%A@zTcsfUC+e&Fd3iuepyz!_4qMf&$%mP8T!hv0fj zSP5x)37juJmp|!HQka7>ZAH;LR#M~BU-k?!zJF#yBmQz^@QVQ0x2l+L_MDQS@)rnJ zka(8$5G)eDWbZ-oS8)S?jJzNmSRM!QjLxykZZWtK7*UxAC%Nc7KgYkb0LB@xu!@99 zd;+{~CQtlyGbu@JI<=?&(TbodOt&8)Pdt418FmxA>Y+Sy?J#A%u2BHW9WHy|gHMMN zQ31ag=&)dSV&#{wL}V(;LKt3U7F2m-!T6$Gdsx(f!^S)EwMzVCOacy8q!RMb{JcCP zz47<(HG#bmgynkGNdF_?&E8lckdc+`S_PMM&qq;iG2-QsE+Nm69=~kMjV2)DG3?;q zZwWAsts-F3%u4^aWOqM95G1>K5~{E~r47R#{$Iuau*%$;Vu-}+mATS$!67CNK^o|w zN!Av;L)JCl^;It$z-t6FK%i0X2+9MgmT<*q1YRe&&+HNscjs#(+*Zqg!o(eek-JAJ z()^xKK4||Z$o+RhIo%ipxr7D+pfh%(NUcG`z`y*B+0ai*@|wFWfB1a)jO*Dz)E7z5 ztR+usX{g;U37EA5%P7?p+3?v8X8U@3SH3+1QWAZ^8ma#XZ#%gRC7|+j45AJcEOsJK zbSaDgut;YE);PDG383?>B5izT^&X|Cv&BZm3f{*23r=2~=0bSi8=T$HKooTW)NY0! zx+bmm<{LrM@g%_MrpdV{Gr}b_EUZ($)^%V1C;%&&5vWE)mVf~z5-#hd%k>fZX9A{b zQ2Q`V7i;G+iT(G4U^mq!3yie_y`q&ZEiK^@QCAbPU*0O_e^f4f(PYj_-njsuWD+V7 z_@6%Xq^ORptdOTqiHoVT^H=SVw&0T=(ZK#Iuf3`(Gt(qFkmD;fN4J5gL28jc)j?A8 zySI9{x}xA4zDP_GZN4_}KE|Jz zD1J^(4qBj-Y7V!-+j5FS21|y-qY+Z%=?GN^%gpKYI!fQ_fkgM&ePf7=gBH z!Scdm^X@jazBtp9{?)TUuDZJETS2d6V)8o?=Df3NXaGB=_{swC#P|R-J|r~>vZq1$ zR}UeHcNWzY2KZJl;T zN+2bT|7`iYp7F0HR{9LPH=&6KnOL!%KrM?MUu~ zYSGiLpi2Hk{tq$?py5-|k$-sLag4~T`nwDb&8tSfnnTLNoRgL`M%3X2NFFx(N7PrO z+E)ob@CXD4gL}U$K0s|?cc08VUp*1XkG@sdy{^iMuNv+@Y*&5-@<#A+2nWK}4SVZt zt36BP{eqs^u3GcZ%>ny!+5guN^!|$gbaigxf3dvIMHLmXxYwcJ&4P-F$^7(b&l~WE`{3b2MwT!Dy=_mH0)LV8ZSBQB{_kd&(p5MM z_;9>`k2o##z*&5J@C%4IsSx`=%bBH^?-9}pZU|Q~u?X_I^Y6b#)1W;jo+S2?V_`P3 zM232`!9pvzExJ5+m>o`UzXpu0ym-g@Qx(WPy=(GAi@HXu8f+Yx6aWWNNa61SpJtHB zG11YPPZX)ipZa<2Eyc{WxId)6=9qg(_p&Pj6cguVy6zO5ETv3_{CB#EaqZg7*a4!= zLr8&;kk@&IRIu*LI-@@+DHL&skz!N>o?@6>IETvG+Q-j5YVs2CBS8hivJAKFqWoW~ zZcs1^FE$CabYqjvhlin}qi=uD$BsZx=K-KwP;7q!8d5L=yE4$ z77cIpZU!$Re7p{d3MRVEBamhWaw}>5&oncDk^K!SBU_wiGFvUYudgo?3CJ>^=^DPA z28$_%1F@9y>A7HU14O~0qX#;GtM&5oO7?&OYf1M=?k8f4iKbI95$OA_VSbVlt@0L6 zDD|wJ!>sr8^yGLBxA9xs(b0*#r`OEXG?IvIbwb0ytSIoWGEop|gCrJOCVFupUkgj% zynpMxvrS4$ic|QO<26O3Ww$ml2^1N}>U*lD7$7je0FvEv;A+Al210G$`;t&2DzM#) zMyva=va0S3)FwDyus0%prmzJy6QXg(2)A$72XNUBfwhL>xqp)2AC)(@n;YPH`>)Uge z@(5xdR^=$u7;Ud7`vM6>C;ODo%_(>zv3)DotjqyNuL20+#=m{T?Q8SCe7iT#ODP1lUE*w(+Ft+op#<<_YybBKK99zz zo5ZQ&R29pAFl$TPmV}7|# zeYAkpeO!6d@vpd&OGZY4g#kSOu+*QwM;c8nvIcD0cJ7xX6)qE?i3JwldfPcL2)P=G zJN~C)Ld{TdG<~jq_c7^#2N|zdGAH#=6*#$f4v60E-QBpZXAzP7_yK!+>$SNz4`ir6 zp(L&vA$?!41Qj!r*Y!EuQ|1i0S#$8Pp|5D5n&sU>@c0*p>nyPri`82Pdxd~9!Z*wM zB(I>N0*H)5Z;H#?-VJRG3Eia;ovI~~licUIIm9GxlYX~1lqpA4EYQ6A)jLg=?ChF(AScV;%54Ze4_2h?$}K!K z6H_Io<9nfj3QYTW{P&5k3HX;@ULA{nh}udD zs4jC{w^=_jE1vR^P!3VPLS}G>`$`vL`V7?#SZ}HnGm$_0Q?uD67QxcHrpT%&UXtLL zg0TjQH@%St-TsL$fkGQ60T0BNqVGgKA(Muk-OJMGSu&tu6VabQswM_nuX!4}(#`L@ zbG|s=h;?@6#y_vC=Xtt=`UAO1>Vq5!Kxbb^REA8U{heLPejimFCnm+)7W|*#`l5SD0{@@FOKcEQkny=G=B}?An5FQ$vbYJ8?)@!|@VNot zu=6D0K7U^fv}_a7?^ShQwpn9z&&>wcXA4=V65wk!8bhNfhtObX?_Qx;3z+^u#QWr< zY%`{?b_IpEY?eTwQSXJ#6mSqqSpjFk(H}OEKy{F#NP>s@IRsc1R_ zu#MKEV)s8N_!k=M;A*rhN_-xJ1JZ8v#Q5rCkovHSDSmWa@~)TL90u#$fB7La(X%xW zLjoy~Pmb>UFo8$t!|`6Tz5K%s7&)eaCp=#PtV@w93fNdbhc0YTe*6vW&b~AEmp3Hz zcgBUrm&fU@&<5ps#oG&V@4(HR>sXPKS{U`oBkP#Qa}7oTMP?l|(C(m$nrDCH5QZC^ z;;=*)7SJ|YUQh@>h>9CDt;BB+j$F)sw2~^I36@@%XB5;61yI6Lqrodn_90P+D)S-N zO(PT`5I|u4Kmlg_E7rdYFbgEIsQ*eb#S;k9jSWfEVcNj6Swvr~RU0L7f|pAfAu3H} zA_8nXyO7Kle<;?fh6bdKBI1%JYtwyE;R>o&;!h5II_a<;qcc!X(>Qj>zj!T&fwb;7 z;ClDl7XaACQ2RM;B(2)?VLb+VfIcD{{5EM4`NR`enyuXfoYU>4JJ4965&QXbJl);p z_A{P$6}bKHMC!6%dU@m0KGhRwimhk&!nHuDn zG_)R46bONoGKi=+pVLSmSBB6&?PL~_io*{if5)4QDKmn?2zvy}-E|%P)%8z`nS=A| zPL(HX!u$bD+^Lk_F!^A6%_|7OjG%`yJhu^ zFcH?-pZjHHC(ewTT7&nd!iat=Y0!H~6KfcQQZ0C@P_soKCKyafX>5G0RO5@K^B4@X zf$%3_Cg}{c2jfa2p(c0DV2o|A8*L9<3(kkw`p_h`e9VhMG%S8KGv{c`*w<1et!mT3 ziJTfwUiOk_ibQ2SWry(N*I-Z3z0aWUJ$<)o4Ag)*on1W2L5*aWTu_1YIf^$Kzb9~H z49q4!V!QG0H!SVS39U8wQm@(}FQKkT)AtKK2?m**mQWWB3k@0PZ{>21A8I8@mUt=rf2a-I@K(Dz(90UE%wf zd@SW`gNncoD=FnwDetA|w1ZhIQ2riL2A|bNJvR+ipUsbFQ|z^vJ&~Kdx&5J;a0jGQFcFqf&NlAHr8@keQ0DPL^dHg zwnNT#@Tg)CoE>Co-Wa39thifqWfO7B=^wSGAn0M{eyHfw?a=hlBh;qoL+N zHiO4u^(2KOf`Fw&>=tKQfmArU6iezS&w}>O?IP)7;$0dV1Or z3m~;pENa&;#)1iesq(|nG+aWGBUN;0$#^gUIG=WhI7pXP_II`Tg~2FLmDnWXRU~e< zBk@?`xnanN598oH{jCGG;q1Q{v6uup<%)-31zBgwhQ(e}z_G-wp%cHVsjapG20z(^ zE+^mH3v=rTFo=RHfB&>J($VIfG}02tt&dNU^~G*8i|iDR!*L%bBV+MOcQ7RuV)VId zMYZ&^b7TYwy;ii%p}T5VGO(9#MIxNQtO_gYfuAR+qVxXwT@;LUXb>5%ai}V%wNDJYA$;scYI3ej7>teYchmRwBQo zfc^I$yd$12u1C2sIVSu*gEvX+9S`8J3kxc67&`|F3Vd&mGxZjf#7lGJ+An_<3r9aD zS^rt+MvY~YP3cP?_{yl4fB?UjTv*6)eZW`f)=k0IR?lWPmQp+xv+~`Z4FjfvXv@6p z&muh{H=g!dXXh_BVq}YY+c1$2nZAJQwdM71NubR5h#Ya(o=JzFU((=ks=O2bK0?a`HPs88$UF z3|Mnb{3=OC?NsdfiM6v7gF>N@k{1;8>k4ubiBWLSPv(g+N7>oArGE?iGO;K}!Nx8+ zDvQGU6_Kz$7Z(yX3nO!oGI!Dv>T(;_VA#k~HuelM27YtI$kW}o4-y^0K*wxjLlEA( zZ2HR8xgA~X_*u(A7yQvvZ3x9)sD}bR0!~^OP^SoA?NldAET0)93eQ4nS;esE+DYaB z0b|Fra;tWh<>4db*})6$;`#VKOsUbAe)@z&qW$3meTC9WwzE_S@`O_8YU|@Mr6)u` z3Q#AVHO@}L#4532L-QLN8?(PtQB&h$K%S%E4}B)&C_zi*9?FSF2{z8zZ`ds5zkO~s z*>dqC6&zB!igYy$+CfJ&BLf4t>OVVQixDrQbPUlXIf4CT=cbx`Xg0!HTPN0L7}pU7 z^51-=0boQimWuKKRE}1ihJkk?{*|!0wcca>PZ|8bp(5Z6K@ua0w68}xTz<-Y0_|N3 zj_3(>=%1){Kaah8SDcurxzcnIf9|f-NX6NIQb!wo$$>yM?_Cv-lq&fSF9DWFp45AHVPqphO$! z=;9tB&9$|9Qs?IU5WEx6b8Jz*ms0G#_Cgm}`8cc^4BMmcnb& zH%^x+QP-qkAk}dh?;DmH4i=>zKW4J(dL`_v+p@vyF!STbkJZr_fg_Zwk75#T4PUry z?fR!HSE}ZG0z`O+($?CJ<89W$<;U)>|tHP^Wk9!rC$^ zO?262Mo^%v!isi}jVb56#v|uIoIJe@M(R}r=&C$Kk+(=wVj%2#U}g=OcF(*L8}2~i za!7b{VrA&TjAPFEXTLHL7~J|boLbi}kBEpg)$~eN>)Bmc2wcxJo3cF5_u02#i|XZ` z+P|&vpd@nb=X+ZwOz`F!9W$KOYH+67^*TvzrgDB^r9pRhb>>1}qSUrCSt$>d;dQe~ zuOVC^ds9a#dRBKSQ<p%?s~^TlRt!usUxWqWJ- zM#cSWRDxIh$}pw|1IZD?mD`-G9Vb_!I<@4eH1eZao!O#y>z5hfGx&70MgxQX&2|p0q6BBFmiG61!j0kI+<~nlaw67l(pfR}~cj4(yzRa7jZ|m6YG5q_Q$D{;sDgCMKr1*viSt zsVU7?_8^uT53RbY3S6;Ijg77T{P}ZdrzCk392lTT+|)+^i!3fK(!I(sZMU|xv3X2H z6l$!ZqS9P0V!r%h<<->Zk{@15YGY>pab+K?6m+Dsgh#JmN1vT>z1co}*cgR4W%}$n z=ZPK@LyYN!KE37pI?d?c-^TIlo^v|=Msjui4VDBO8yXT46K?=6N&s=u@q+7;!x5k_ zK@%?*0MP2ezI2SQx4Wvm{K){o*3L?cinMROax>i8x^lN14Kg-}oWHL!)+q0rAN$G@ zzWa+!kYw@hV#F7?c&A-dRK5e-;)Q_&7Ah(zXT0#~4sPCm2>w>JE6x>$M@I+eKmvF7cpH;BxSPK^7c*j#vmwD`s*!P4T5Wf;PIYbQv>hLA1=*q+LWt2hCd z9vGH3_W0wJltAp;)n2p@H(_OEH6#eL_4B7uAPD6hUH#wJ{{}r|pFoYHZuu(nvoiy< z-ZKu4<{_W%Jph;LmS<&m-j){@cA9yDNCWjDtpJ-D+7vY_$CngA9r_2N-@$VzudOV) z*=lr5$oo=vFwP+izc_rkd7_NWWHNUJ^D%AQ3r(AEnOkZKi_D`R8p<5jKIRt>;UXNq zmlVuLvyE+RyZ~o;Mn*-SgM&r>< zy#iUGIg6^56{9*{rKT=ZNIbtjP%=opUi<`k+szsHep;!L7Gz)CMa9qUzDDv4o^DUQ zO5>qql#oMb+yYThBiZ-G=ysCAAOe%ttTIhm9XXwwUPl>8<~kkX)Hq3TxwSgIJxs;Q ztxd4E;|p@<9-6kdKgUK#N1;nRc2Qt@0@u9Cg?N}DR-p-?5$;`lbJ!M+{7gkHF)?v0 zTsBY&hrQMG^E}%T9Yo>P*4739XkT9+SQP+>F4+0(ekmL-iPp}q4A35PQ`5knfzVJR z@sJ-I8={6!nF$0y#X|*|10Uz}=J*|Y0&M|mY7@W`5TOE`8f2Dup2gzLqwt#6hV1wR z1XUOy_bV(blR8aEvVQ%VfP^H14Tb~%agbV4p15!O_0bSfCVhOPcT-bSjNlnjFk_O@ zx`de~xp5)z=yofbr^D!&t@ea@Z(y7qSA&jw#k(J=cGH!LaE`{IdyB0kKTDcN=>Wr6 zLB;BEvG)sRnAV^qGc$9y)Oc>g1a)C+V?#|@`3Q6luQ4r~t^}`xhAHVMl~MJhD+gTm zTB7z~ASWRvUOnpR?QK#L7hek^)bAMRypU8#7@e3l+(c&-^hGnJrsmOwb-f7;J|Z$!vjF3B$bc31oS`L1JTaJu3=m&hv-$k3(zNJcR}KKGR(@W?Xlv|&H0 zt*s?*xoa~Xw4pm3Oay|(xKpD|m!}yL^WU;{rUj75+}CjpgoWQYark;WpmpoW8MzIy z1QyWy*VU~CjXnyclXnH;YiP}wF&_&R=S)Y3$XZz*nJCU(djTClEvY*+)bygOWel=n@ei|B~H8;H+T$^2q8=V!n*JNSh zkzlN^2gl!8fVX#EZ%!FV6dRg5eg@BWa7C}K{t8z&H_btOG%eMhjjX8l*3(Ey$=;rI zozSJes_u0a(k=ugSjg>HVg4mQ|Fer__&gJnLy(V1O64Gr+TFs!GHR5F)DQGY-Q6~H zI3VRQGsn(8E-tG$Yd>F&P;4F5p@(g~0pG;E*;=_bo|H~(KIcjw8Mzx51O*m7j$hKO zs-tj|gF4?zUk!uhzw%kN9GdvMvJ-LSvT2Z4nHgomw4!onDZ~P=m8EQI;M` zpP4}m$~Fo-_Nf-wESC+bA43;?Q;LCsnT@AZ39iW|KRaLVx8DT6B`EMVJ2aj*v_}xV zPE@)|VJK;{DJ^YK>XV4%H<9PJQS~7< z_F%7h-|)azV98$?^Vq-f%TCO9XMT7{u0*Dv};hT+KY>~6?nn^dygzU zx_=6R=EQQj^>hWSIcx;lPznVyoCQDu+#>ephI_>@ItTpqfL5QK=^ztS?hU_rj~Q?= zHCbn}ISf|w4B32uDAUW!yK>RhmQX!wGKm0OF@pmi z;Fs&_=>X)l$&!^N$UDv3pZP#)RNi{qlv>QVT^4;e*f}6gKJ2>^6pxBpKqrrv6g??Q zorAgs`}S#**w1NJPF6pp%j@G2hn%?3K@ru;%FfR(3;D6EU*v(kLt~ib$oA7oo#cm2 zgJG8gQr|d6=%aI#zJ23KjeSn!Sc0vjlZJ<5hFcd--kdB%?^0}X~W*q$bcchCR9+}`}Y3(K*MLKhM>FA_# zsf%}#60K=%Vgiq8`Nr(|{>7$IF~`&e?FpqZOFQg-c_H+zk@UlE>7aU zeQWrAvpV#w^0Lafa*RH_dau!g^`US0Q+ETh&i1ii!8L?j)d>;gdum@4eG93!(-){X z;6G2yo+YX8=~Ll_9Xyj4C1TL*by$n0hyUapicy!8r9<&#VSb+HLc0r3IpCTDVW2O^ zlZmbd(St_tLIA_=Cth8#L?JJ9XyZ0@SuH2EO|Av673|% zZOZ&`7H0!u;*lcyylk2fUKHFZUv04{)(M{&-HSjOzc1mo1yrm|PhP%62{9tk-VCG5 zSY9_G+=0vMnnh<0GRa2Hn!@!15Y*%72$M==1l~q#v$(>RYcCrcW`t5r)0Bx+;iNDD zM`V+L=g6xtSTkRBHOV*nS5?K=&TQtJOb1MC8vGk~8^Z4fMRjmqif>xYXAE=0>fsU- z^syvinZS#9i^)*`urM~pdaXEH!4)`wSdsB`R4I2)FA)E zFSgxT=qtrq3-8Q|epBi;oi6!~98JwPe9J~YGNV(vs*5iG!5zG*r}fb~uBa$c6A?y3 zLsmR17Nc!1EO1l=E;!H?c4F^edT_OvO}&p(3PJB2>k=z0fb`a$Y<-fySJo{f6)EV@@TzL&MeAjP&d# z9uV1%DpdyGHY304WAHp|`h8)uF7gHO{A#VCK_xOOD#*qpt2_ojp^zRgf>m0WgX1i1 zvK3ytsIcZkbH9iX6+0sJN^`B%VTg`7CwXfK1*vX4An=l2QpLT!Zuxj!Rxj$KKgxS8 zPPuh`f=-w~wk^YZ;9OB-m>BdhS5v!rzSX4sS0a8yrN2syuNGP#%+&Le8kZn|tTnnm z*&SyYfWyF$=&aB+I_MzoiGBgN9A zb0W4iTPC;hkMlF6S24NHxTeJuKm&&|!S;jWe38m>)ok&F1Z&q7)rBM#Ej~U`74h8G zZk?m|Zri}A+4003l&P=v$54xW`WRRi?HmD;8$&L?&ml^e^G+k#8b%8adJb3gaN$1H zAo|*IqN`t2YsF?e^J(8uJv^XODn6E3=jn|DOA=xOw)3T(LjCPv(N!vE$010^eN?i_ z6Q&SPaK^~ZwWW=d)80sbYxNQW8_d4_%XH+MiC2zDTTSY7s{C+@+tq^KKik1Av^+v` z1`Z9RGO!V;{%FUsg3k*Mr3_`J^sTEb9SsC z+}KL!B@&X5kWkChpf%f>D#ZF;;hU3(dk41j4gRRDmwlM>?iDiU#%A^VgER{UxbmZX ze0<<5@8=@rAz1267aWG<#;bsU0i~^jL8SjESOuBYI~Vljgh%)HcS~Gv0;kLbCYv?c z+n*qmLL|q{)|6bz!EX{3UwgGCt5wO{*f>fr9*ohs%MYQ5)|*phPu|bde25uUHjBsz z*I!gxY;p#3^mbzUn2;rF_D@hLKjZP)7w_ck|FHI!VOh3qx2|+|clS+8cO#9Ybayuh z(k&$|4HD8I4bojAQWDaLq;v}F)C$LCB|m znOEk-Tv zpt|A6lZ1m$-f4g3bUqF8ce_9UFL61U=+Vb53*8StL`8R9QeZwu2|drYEbhB&LZMeZHV?Fko2jgg&^;d(Dk`W4gqjL9;ybmvG(Xc!{ZAYWIqUwy7RL@RuA#ZjYhKJ>sUHXW_Vz#$piRgD(vn;>9XnCbe z(9R5H)C+pUZo0H87A?I;Jbj`+fz-f;A&Nj!H?*shD1%xC^_O3ANO(QU4v0$Tv>|wA zNr()@dQjon!miX-Gi5*X0?EIdSbhdFc24ZT5eVEq1G2m4~HV4U*FtHZcC{b*hXC+EOh3;$qJF8sAquHO$Lh%mMjYX zSCIShb0qI$Z_AT9sS5&R6OSR->1pqgU!(qqH0k2u@o#N1ejaF(uJ1vg2LcHUANJqU zWSDz$TH2gexDY|bbCTOY52f*OpdyIg(AG2EaD^-h;qc)(cRUEHpxH|f*`x?Fuq}Zq zInvuCIr5}{mjxeasf9^%-ZpdaTAZlW5k}BqyAHat0rD)o85f5WwF!7gxZ}^l%_Wj5 z)qS{>kaDt{4wstHNI6WvZHv%qS4xSD@HfyG&T<1EnndrfHyiYKEUoix*Ba^$fl_;r z9F(I#!I7*5}P8R132ZH3BJiR#R+UgvbZV67jmu|zX@rCKa>hs@ zK8Z;Ilk#q3S}(E26?EtKNr-VN^m_fnUSQ&6GUH>pQH!~iKTJh4FBh3nz}7mPdwWHw z&zpK$oNAsVLm95`?N~Wj^ap-3;RA7;+FCjl*JxqjGY{y>IppRYxj8-iaxu1SmX(mZ{JF${gLfsNW(adp zMTHT#8*rf~FBrT6^~vkfuO=IT#R(fi2fp`lY^$l|j*MV9C3rn3IKQ41n}^3_m&08@|gX9BL6K%K2w(wD^vPMhfFM^ z{{Jp(svS2!l52aADcxN!-;Yd}oW3Vvx}opOA*t5n-O{=|X-fTJ8&k^3A z`7VB_npfT8#WzU)3`1-Q8V)jcIFXRT+IIPJZU?ospTjzGe1+6)jz= zn)d)`chJ^3x&lM5lz~Y^)arX?by+de)YJqR_aV)`+n=J{D#5yt9SsiOhtn~3z*#+M zK~@pDfAV>%ajC!kU`Lguwey!*z9a^{E%2 zyo8;Rk&x4@f33$*kPp+MFA}56|Avdc=x<$=62z`PuyRObWMnMY5d)>FEIK*)|D}m? zbAw!*EaWoz*Y^4=1E->$Uj8v9;$yUBUE8p$Z#Z)WwPoK+9+pg#$UFZnh$0{WY;ElT0oH8kM-SAan~0>BCqKUl^wq`1?CdwGE>ok!lL5COdo(v> zp61%x3}8k?ANn&;_wL(lJK#2RL4Ox&)J#J|Vj||ScD&Dmx+-!v8W$hmYP&akwS|F? znwah>xQ)YddhCo7d4-l06>YAq1*JO>ScJBwW@W`@F{@PPT^_G~F({jx)7Di^v>b5b zZ{6haJGZgeab%X~TW;K>^U|zDO2IiA;Nju9DKhAK^QS)=rxJ4j`8?xqZFOvg$neLC|iTIqdlajEo)rV_M`%X_!DQi>kQOE<%)pZm2$@Jc@yzs(BVa-iS3Wnyy z29j$3YLk}@f!{MWGADl&6qHx+kn!2s*|OZ}cczB@gM(&PLk1D?y&DtLGLq}SkoNW@ znk#cD5)Z5Snvhj@B?%|hG(cBFBlH4WD#sJrAr(YqU8cgk3Toz<>AKk=@q(lTWfK+x z&!ep1KHK>Vb#<(x2hP8>)SbM6+nev-7k@ysAN-xw24!WRf?)<>;E}53j^=*LK603-YE+s{0QWDplLNr~v!$8V?5_ ziWg-YBeR`_hK9M0xzw-zfl27aa9o40gj6;J{VXKRm2nDkkdwy{ze40>TxkC(K0?(g zHo}*Ln3-<6@Zg;Iz!>lOeI=oqts0iT+{Jxhn$xR}-}KjjaIf{CR@~Srh?m;bP*PTQ zEe1+7Qw(#8PcnfQm!R>`#Ol2%4Mc2v{56 zP#Cnk0>?#>PR{Ms{QS32rYMbwGsM!_^4ar~2R68c35usFNQ)&~6VbEsb>-qO+~Pe z=Oti;yZy%BnE>mpC;zh3i;43jkpW3IrE3yc|5 zS)O|m#2z?s=W81qpTRw2N};&T_SCUHKfg%Dk8P90Sv!I*CHchB#Hegc2JQ5*q4yTPsuz&3jyAm{C(_C3Z2R@h2xI-T!y+bRCp2LO?9Jc$cYq z50P|pZCx-5skPeTbjYtx^I4vUjvOCgeX0}e^^rx|b(CJkoKs&ks>CN0yJs|u^St`) zvp>O?6s6B}PUYZ`EjoLL8Cu%ZgwKXD2ygcgPO8Jlm-4OE{A=bHktH^&yEY= zx2SjnAw_gv=f%)N26h6Isp&;MJ?%MViXUD9scF3#uj(z3m?L_k4P9g(fZki*h(m+l zhZngktLYgX0blYB`2N=+Yq(P9Cba1~n&O_OCg$C>DC=PPa=h0Xb$RKf#HB_%MPWmu zi94#f$oBTL9JwznYe>A@AI@Mqwx-r(OUV2Wz!%0Y!&M zI%AsatBx)vw-al8#kR{SzuQkQQ>)NF(Xlzo283TI0?x5^`hYKPv-eA<4;EfGe7lHv zdq&&FRu4tI^lWI@q6A3ag~+!!ar$S*udnuM#o=Jc6z^g?E;UuTQxQJ@U6x;lyE<_3 zHi+_4D|<$b*y_bV0A~irTS_j<<7?^h;g*1GPILGsb;!{yzb8km+o!h|vzRs#R74o= z%m#t)idXq{7Wiy}rValmG5YSD!itfev2OJC=Ef9e9&b*-U=Q>hK6REcB#8LcK%wIuuohb_jdwWctW!1SNS zErCu?uxz`|+UYDrfWirD{_M&fIDPuKK(0Vu9f`trL$f*MA;5n@m1xF^vEMH`&^0<9 zkh9e2b*Q&1%nzH0p3$3#5KH%N9v-PUc4o?=b{CVJme^?k;gGV~O1=I&#-&(M7 zxBwQN2`gCpI_s*f@0-wBC5z7S{-r__p+$0r{^(ALN!Cu;_hx9P4<`?nv@_a&AmQnlG#JX zswlT#Tb+;OgJKv{ttQB;7HC6RNx}bBUfQzOrT$59X4;0)r^0&q3v{;-KI7kX{(OPI z6UO#_tUxqJ?rYhuO}4T;8mhotWErbUOiTbGxbn7_=Di_?f#1hb=?8;t`<^a+BPZ_^ zt1emHIdu}6vsQdKu_T{~hXkj%SNF3%#OjKw!x9W317jZ)9Qy1h4#_0nthfI&0*MBZEOTDu)?C2mf+!@7ByyTewY$&%*jCbT3(ze#(VW+m@nU zV{qh9@XeJ|d@=vd%C@-?lfsZ}JNQ-zxX!(~UPquU6d&HG9u3n<$cmhPhixTQDE3tm z*n1R~;9eocA?CoQ$LFwDzDv9=LmK3uXWQy|yI5K(VYJa%05tKikZ6(+cq@2%HHg@~ zzM8zQw5vO}+m<|j-d+riTs(N=Q~cKCgfkiD!V6!52LD5CY9hWxqJK6-k$?@_icgeG zPMRvwn6Qy7Mk~Amf%~L13Mv zul+g5u`PkTHrk&GA4%%{8Ix#yR;pQCp>0&hI2+3=wLXwjj4Lv8JFKa@^;2!b76s;D zWzQGwmKk;Q9fe9x-GVxWUreIs`7Z9!Mc!=qov{%m^iLXi34^YvVUvOs48v~uvPQoi z9XA)xb2XEnUOdBfb&Zx7zfEPOtYr8!vEIH=y3NS1i=(?ly7?xii&3kn<;i33+z#=o z2qp(BFbm-D5F{lS+Ms}n02(+GX}C=CD~nc`sjFQ&-inR3Zrk3lvF7-R;p`a3A1(Ga zYjfcP?0UzV(D5PMZ0$WQf9CR>CAGc0Y;-e{?#+I2@OF=CRZzlmruI1mArvXW+Itu; z0pU)V_aE6Csy99X+I#+G_==;014r(6$AytF*lW^2(Skxfiovm-H3~j(u|u{hz4{R- z7OI1RJQQ*tG8U3b0)rQe^Lk6X%G|WS(2v#4<&tx0W^UGVH8N(NKnXn?S32HAy&L?P zySBB%rN2}}X-_f9ts_6Ow~J}r$h6K4j*$@@yvz&LAW~c36;vW1nEqy|Q5PXQQ?VU+1}r-ujb{0I2A2yi_kR&ZXs zQpFzbm10Bw=H5N9+vZXQ<~IVPql?5#4iOS)HC{M)U#F&Krh9QmUr#404=1cIwi;%% zP;#%&)|q|HYk7V^Mq104)$&Z(7F!ELbT68D{NnetsYXxS+7?F28us+eOn>xHco09j7}G#1(l`wcW1>hvk+l*R&6W1X^NywE#q2!(+diAHkc8WH+#ZbLv%!(O+8nq21EzkeGLbK49jsHnVA*7LwfS zDa8OTsxDyJx5JK#%A%+!571&=fEF88$r8e3yyTQKzvx8{c0IL5JiRnu_rm-~dtbM; zYUO3vWv)c6&6*O@DC84r>-{6N4-}X{@dVWxDbv6C_q65QojNXtI zT9B>^ELKmE>4vtgD-S=zpoxOG?{Rr?~uKXZNRk$9{zamqYp%-Q{K#a$#d$1rD6 zA=UQjz6>L#2Pw%Yo9<;KN^sYnr-z3HgZi(F*@14SGe}I*UE+@V2V&JQRJhW18iDDH40DYuntR0kLp&5h< zN+-B$v+svOS872)B0C>Jh>81Yw^;k4t962Q5$FhJLhJ-14H4PW>;&l+oEvZG63M@` zfDL0Mq742AKz(_+@c4j%e5&}#knJxWW`=j%s=w{s9WywaH!wK#$qZyqaMYj?alty_ zY$(UCENCt(4@qu%MebO&XJ_YU;x2`g@ROocKXdO&EUg&q!cQS(pc zd9L{f6UHfA2zRL<*MSD0V?~QfPQ0}Y$%6PE?J!|RlJeo--(;i<5qFL83ID&*z2qX@$XY%E~Pm(zc!#7t9yy zti8Q`-iJTpxKE$PcrdyT(vEl&$kK#pBf`6i1R2U5v@|z8X$)Dg&9a2qyRfJ#!!1su zK?C8x2)8HpeZ zfj-X&&%paY&7aniW$r0xW}Om`hGyxGOOp3{n*Rp1Pn)xtGUe?)x~b)5-6#N~0dVBS z!)=l~KRst<>W4eln`CQi8?``veKwfoObtCu{;H}s*k__D+6TlQbTqQbZD#fgL`Ej20nW2o^@HDWWRY~eQ_ zs1p1<_|5*psS6#0JZ!j@4+`mZ=Uq>{pEeZUDj6+bZCe`h9Zb+9ZQM==P-CF5lat$D zW!$wfn@+svd76rMTq{R)gMXje$qY2uej^XOcU219_3dADIoY&R67hCgm?}6Rq8}zO zXy*T#5wDKu$4^fW&-<*t2gB!LRtH=%U;hH{4-iawS)C)~%Bx;+L_By@Dzws3?**!7 zU)33^o#{ecS0hd$cs`O~oUR4Kyy@&zyUECydLDL&iD;ET$3%{@^m{=uCf2k%Uxtu1 zidzK)%&0z)!Vk_?#1hz#QbY&&TQ?!l^F!7VIlD;*dYYxnK!O>;I}Y8l&(`_Sj-ztYI96|Re#0v6Q< zVmAN@RC!Rs$Od|Du5j68b&7s!~0xuHH9a6{8-Z}kE$k=%JttB))MH`&Xb3s!3QShxCb{+QTli&C}@u9Vh;<-bv6XF=;xTS zDAkmjsO^l71XdU}qeyOvZ*I?Rc@f^6jxc1HQkG6tWK3ju2W1(_e|fCZgQMx{BM{tp zgN*_U6!cX3Bkp>amJcXb^(+>7{$J;he{&KUG-qURP;1piCF0y9X~adUXs(gt-BAYF zBh3rEZ9_Ls8{YcJyh;HzfTSvuT7%GEg-2T%vpTxas%vAT1eLMbc7u3DrvBMbp(mgz zvef#G(GRlysn&-BDBXS!OsjA#pCR`BWU{e77`~LNT_FZM^$L9sD+^`**G&aWHKA>p z%He5V#+(_}B6p`Rv=Lq-AV?R#UXCPe@VmWjJ+0~5qbEUyu{mld&CY3^jRbVz(fx#e z8O#T*SH8a52*s~4TE;HR`NIgSot%K`L^c>S-Cx}@rvBj;w-mSWc5g>3FUhqrR&|^) zq=yMRT7`~bwT{G8rbzeAhGeRCl|i)U zRA#=XA>qq}alQj{O>5FSXRhVebp=Z)BTEWr{aZ1_0-{Ag%2fpRe3EUy%P zKT{@y%KL71K>5vOz9P^fE4*D#L@FDozO^U!N8-GBN`ox|5iMMBqjh za(8{*($>by$5%S`s{|H9K(c@XMBRdFELK)ldEnUx^L1TqZ3`&ze))2mEKp#myW3qsc6uB|G_Z0BofZ&wB7QUe3Um6Zoz#uW5B zXS~ue8GK7LIXUUvaRPt!h$ahGGTfkl6D&Yfj)H$4-5mmC0zl=1U_(5ZC#Za;46svQ|AERs`!&XNEv=w|-3c;VBGAi~$N-`&21wmU z{Jr}H^Mv(xzj_i53J(E)*xK3(isOcdhu6qa+iW2cm>AG@LRuM^1!a^5pkD6rBq1HQ_Mn-}mM=Y>2Ki{N33M&-B``5B~A~2cXpKgy$iKxk7VPVNjQ!ygD9X^rLjuZfJ zEo-P&5VtZnH+Ob+2H-UKlc(7@JBxt=4iEzg@*IMKf}EK1sSm3hJqgC{t`|?3-b1q{ z=Rc(3_}>lq?CBEOA~$9!I8%OZR4*?m{z%uM6Ri0hhnoqkPr%AV%KX20@}#7Hcyco< zE35wk%1vxs0Z*Pj4|wu?r&gn4_d0JN>#dBA#_Vh5PrH0K`t0Y~8-@hBP}lun*~Wq8 zW^EEd53N4Gvoja)<1hPwP};Tr>(>qN5P$KzwXNm@8_+MG{fVp*@OH=vE#i2d-+!w6 zWaSDgAuM0U0JjF8nU!_C(%xKOKMfwC{(e}+2as0C0VRQ9+Pl4#n;aSnV^nXLh-Qbx z=Rcm*d&uNen@F+LvKqo0A|E0_2fqB-{K4L1V`EeaCeo%PzLYC2^YQlwbS`P3mM=BV zRAr)`)*57VE^bRobg9DD4R}A8D%GvyNHKfDP9pMEJAbY9S zya0BqA9DrN`Aom^E>Hzq+4t|?b7=oXh+F9l)I0UZ82xX!a1L+RzqxSps)6m1c7mx@ zU~hw#MW1AMD-5e1gcmaY9dV=4oq%T1eLJz2Ooc#Fri$RZ_+L19W8>lB;V!9{BxmHB zXu%T-wJRlS=*Y?q%;&3*y$Fi@zSHyb1VfU@P7Ra@H`zInd?fHKNx<*YHwO4?6AqfB z9K_`>`L;1bl6eSlaGr!`jPAg0vCG+Vz%N&cA(Y3iWN+A5an`tOX}-Vr;BZi;&av)- z=tXB1X58Xp3Q{wCyn*)|>@9*O1_g=AKZ9*@mMAS^p^loWrYV<95XLS18CL@eJw0Ub zX@f9MEZ?Kla~$#=%^kl)5o{15N2gu?nB(N-cZbRsVaq1tZqgAO-sA1-YjLSp8c-rN+JWsO=Gi^`U>i!1QxIS zIYI`X!N(c*=Z>ij^3KoQt|EeA5DE{~G#3W%htp&^USB#ayxvvyV5j$MAm#>_&aELy z=)VYY^SZ~~qJ+T&iiB~sibV9YorylpPO|}I4+w7}6u0PeSV7W#zO>7Eqt|HIpb49W zPP-j!JV-vrd_(g@356;}wT=llNX5Sc4?1YUGfx{*Na($xYvnHENP|U`h7GEXlL`du zrNrKxrGb$EuD6^WVHtz(U_x;!baV#(UK=Ed2@8{k{sc?t%`Iw}4b*70NHr2dbw&hd z(^O?e#lF+f;Fv~aWMF^0EL5F(uFJKnI*jN62QTLLtdE2)-znPwEk7{dN=^ToPZyT} zqTAhu5-I?JppIhX^vRV81vN3U;#2+(lffz_v4dISN4B8Tn@B-x`s%z3rHQu^@38Bh zEqZ{?5IREKfHDmQz5TH?x3?ACVxy<)>U>VjNr$9-6@-uBZZ56`&VU>PuCXIdJA|s*TQiA}k{;=C57fuG z4=<8A$i&r|+N7U?X$%mny~b_SG!G1In3P0x&DPZ=)YmW36=JugZJ#0!ycsxX%y-`Z zt;e^&(j@!D$Ty~Ij24WK`|gCJwD*GH)oq-6ct9vT<4Uprs&D%*i9slp#fZW}3YC?1 z)yMVAbR>n!`V6VM;Ey=_q$Re?AIwC=N5NtImMB}fUzZ2WnStgXyEnozAPg9jk}Q;1 zOR@N@y^F_tdumCzsJ-`4cIN1|e5LH$ccw;}8Yq&dZNczj-dRYtT6ITccZ3sn`Bqw*WI zWQABbT~|=c8XpTi8x%UNOi>^CW@ z-yjhP)sPRs#RTh0p{B379#4V8_W|Px1lN7@1cGx|_h=FdxY5R{dJcdnB~UJm(I4PK z6K_x6Jdm-`3=(OxSeJiFxKX79&E{wjkz!r>CmQPp+?Z)~w8QCOSZ%0s(jAY{T~9EZXOc*IU<%eci_FZCct z8Jxlyj?rA41Lm!JC5HSG5DQeaiML9)3$2D|Zj|K_VXuLWIEq)9306(H3P$uY%Zw6xG^r#X`KF_3)fy7jk6Z zw_%Q^^F0`n!VPWs!Qv}gRhtNV=i$ip5Xt@Ymj9U*WMwuHfi~z#O!BOpSR_b|0K%4$ zv=VDloGlhe4mCv)c#nkx9Y#ihrGb^L{npxxBw}T~f=Lvu#EGI_4tha%54E4-XmyRF z$3o_c`U5%|nmvM?sTr}zpwoB?v@@x>=-r0QCHAJjj~XT|7Ceb8q`~)%5qV|3&{a@G zgju_VxshtkX5+z5T59IV$LWQ+Ij~YgUsseDF$kK90(oTF_qdy#=_8$h{pm5(5R5RVC=C7vnNqxw~|%D(WU2%HV^4utSM{u;c)$ghnvaCB0<;9 z)YAZsG+^fCva`Rp=WPY)5$Se2wuJ_QV6z9)uWlRSM>ihhUj7#ZwSqHIzUhx0|LxLFlkR zMo%Hj`jB0tsLIqYe4lNWmaStOR4NH3Tz3Jh(;2+_x6( znAqNeud%lyLJQN|*1zuHGUqj8i_1E+mH)E7ISD#x7ANPgO-TNeNZaz5E9}c9*i@m~ zDrH2;@G#I2I|52Y?yYr(dS7){-Tw_`dpE~)!hKT0jjNa(*xA|=FG2;1Y^ht>y*h@9 za+@7N+6&r^Y_2O5-!?nQTO3W2Br+B9iT=~MSviTiX-XAXMZ5xhd8h$#-uUoh#HyhE z=$>&0R0orc6DNFzE+~_DGD%h!gIthPK>obdy-p$6pl>gBiBN@2^vH&(AYB)S(!D?; zwOj-i|Ery}%-^)NOqQ}IW%-)D*@P@VYUwy2XR1~Tj@1!-46}tjV&4pRBujn6t6k>6 z>fz|y)p`%bhnh&H25JER{O-e;df8GGDmZix6CBc?Y=4^lP;YIx*xlqK?Ti^%Lgh9~quOy$-eB~=3 zP0RKJFob8n2yYO*#MJWa06_|Wk`eH`cjoRDGmq9?i<#H+m%?+XO z)9_a#i*E4^dF-iiXGG(qAGgnlb;XH}r8ot?+ba+*29W|9XI+r#!?rU^p6?Ek40 z&e#=KHmfhWaj(5mF=k>kD2fk5-VfDl>gmP2m!k97lpq)OnF0lIu`IZF?;||nf=!AO z=%}8-R>A+32N4`U#0DvkI*EYqUF*sJ^Q$yZmlnMT7gZBtRaaLBvLH}UxaWHRrD&S+ zJ|ShLNKo`G9)u&mgV(xxTYEYUwOYVq@u}d^b!A=mw`AgBys@kr%cD3c0IWl%YeIW1 zW9`u*Wq(Hnyo5^U5F~Oob*S}Wq(%G}S3NKiXDW*IxdCK=X1ivyUj~b!!l;nd3LOpY zOy6Al?Hai&MELq<4Z4IqV{Q9hL+eY822BJ(hAE7(Y?y zik9jKxHc1l z#q`o8%M{kyzN5+Kr+T4vgD)*DPHC!juRRESdQwtz;#a>{EvH~Kozq{gVfy&^m~gh6 zlL^iT0(|V+8(8puADWOaf8{nA3fb&+;y7Zm^F$eZ%dD)%xkSbSe={9=ys)+ZtbP08 zykNSnM=nN-=u&|sHrT?{)OBx$X~0Qy-+yHy<9M$y6g@&O+z2EF(8yu-i@<+P_&XUQ zcqBM*y~hREjNnU%rSTlfK_=Oi6D~K%iIDtvzQg}{PDIeb>+)*loajKursW&&&c7IJ z%7@A4O-&vb_yYf3nneB&=SPuB-MwG&_4VK8X1cv30uH}UPjm8mm6*B4Y5L{2)GAPO zVj`Lyl6f{XFcW9JVP{M6AXBCLTarT((Bd*ci<4rlx=z1c3<>1t-?On7A|`Y|k0qWO z8dj?0{_z9K8I6Jjci#>V4<8BxJy}{}R8hG#L)Bj5r|d<)6k%0v=kz1R{a4!uBqQ28g3={^5Hpa*n~vJj(KNGW$oJC&|+G$5-4^eZ4YfUt3dBjyL9vs>(7hXU?&x zsC+6qt3`!a?C<8PhUSN*f97Qb*@Q3F{|{VPqJvXY?YGsC2; ztWTyZen%42UcJ{rrhRG^cdP0^(e3{%>!^-LjAv_Mk)4~i^Ss;l;CjK>`^e700W}Vh zZ-1U$-R(VbR)}Os68E2b-vaTV^O#e&%Glv= z3sVboKYnPzy^T{wy(6?LDgBfVNWy!4OzWO?)3mucMq@*DJX704Q;WB8GyC&%R@Ou) z;i`fdQ?qkk+*}WW+=d^D;ixVeVzY8OJH@3wG`}$Q3c^4oVO+xIKnDGar2W0tDp* z@Hkw%BzFcF8=lu^z7ms?lCpsW!1V2AsK1edXZOAOeBU8F%VK5eIG|v{uI^)KNLET; zmK9sE-(j7*?H6`z(nX~Dg_(u5Jv;f^CuHFa)YH@V%_a|D`{bG&mWQ9hP6DyO{m(^$ zG8g^NuhEG4&45<=y<%R|);|a8eOR88~+-}_LV-Eyq zRoR{SSr{Jpt?ZqTxA(cHCN9jh)6{6O5n+Ef>mtSMlI#lc3*zD7YwKv$;o^Mj9EJ}q zcfVK`5pjWtmg;4<>7lcR$K5XFrDNOJ*{!$@5Sks{{h`#ex7U~TyKQS}^hfTOe*Ib~ z?|LLgH4+XjvaHwZM{0+v*bOWRd)F;;_}LdZX_<(ya5uMv zy&^gaGnjYwbJ+qt1yo9!aPV+&afC@}@7o?hh*S-sF?8R=iip*vLxy$j)H`gPP^l1* zq4Z|)G_<+qSUf!#VAV(D}vt>;lUwT>F;gz>P^#1@M`z8Aw)5F8v$|B6_54dB+{sRNuOHJJXw`lwGW#Gyu1f~Sl%#F3( zsbz?=i1(cN6eX*2)#}w@-}Uh_8>#)w?pNxh+Ojf4Z}dr+Y**Pg%k^-LJUrYo>V%IS z*HU%YI+VElf&%Nr-yhDjb)7aDRDyog* zqbnG{zjyTJbm1guJvjT8#(JRfUs}LSV}n6;mFZHWRNKJ+1x)@EtZfw2@ zcb$qO!Pk*Tc0IimVWHKa$RcA6z%h*YhNBU{Oi*^j^lO5p+B^H{YyF=x=I}U#Il*@w-L4)W7Y)c1Lk%u(l2e z2zZO^AntBFKFK`%t|ggYZ5xGY%1H5Hp-?0jQ52nm@g$$lAr3;3Ffnb#&(E z=M`xuDw|puf_W;cs&ZUTX_KQv|3va*5UK&!!YfzL>8+8S9sI6?A91nJ)5!JX%e~O^ zDE0(kBX+V@)buOFgd$CUxst}gnhE3CiE~^QKU6FZE}j94=t(-65T5rCa@dl`a;N#- zOKq<}YwMK!kHy7B6-)CC&ZZJjq!7{aC}Bp{qly|lMa*a!(17?ov;JpZNMG;8`MEmC ziR+UJ(Rin7+~pOwkEO~lu&JtDLBhjMm$UY*pjgSX|E75(V0Pl&;*Qma-^d4|l$bVR zw+rS^ndVKJ%qy2?bCu($XlQ7Jw7-20$jFqHmH8_5hq(IGB&hqVlas$4537Zwr5$wa|JtWwm~%i3SFYb+zxfjziw^`eLo_s}u@dEW?*mjqL64roa4J*NimrRNzP5`*9-D#X%um-` z-`EWZw|Ed?KIZv~$!A`h548*43ORqq!y^;1pFEf#;j5P$eHI@t=1|!>OZB#?=~ATK z>#^2WW9|2PRHg6T_464%pSxR7(d#mMs+weaQ?4~7wSX8KHU4GHm_Jao=_k9tt-Sa% z=oC=K5qWquYF|^FEqXRGU0kW8o9uAnw|MsGEt*F~r0ts>TzQdOLwcwtc%4EjE2{`{2)Y|U8 z!XVVSeHvQO|G)}k08d@HNw5~!SV(;}O@}-A@k`RQS4=-<^KW zqlXY9LH}K~LAdYc7LJZptpcs|0m^HKoD5R5xV`t?@MEY6(zN(AqT#46__4^A@llsm ziFDWknW1!_8Z%^Q6=$b{1`xeJtfxtoQCrf{k#wzgH@RZ3GuJKlj)j%I9FKnY{U?$N z9i7C4?#e?owYA11D$?)B2^#!Zj)lowfxZiPwKGV|GOcZI_vY3dmay<8Aw`fUta@=p z1qvP_L0q&9*waNzhos-TZ#x)kErwr1oWzcFInPPZ)r*fudK;1A?1}KXnW}$?$Qt&o8~I zTVI9H7%VNzfo;F=8K;8MQ90Ki>>uPcd4RL zXF9MGRjsa}eCE%j@wB~vj9gm`l!7ba0jn+u|JSO`kN@L}`*Yw&Pw;1Zb9)w9mrv!R z{U6v14acctKtRatI8p73nZvrYH0=Ss9zM{gGKuodU5$02tm5$W#bX}=#(+x3uD~!3 zzMk(ZfqrCfXGV4wux;v}vSj}rsaoJXWjOMpbR%KaW0+E$o13ehS^C23>7sxkvKOGL z^~%;Z7NjqCekfa*o74vBgOp;%#>RmMVJG0VRWJN!d}1A0{HQh^T_xT}QSm6HJ0qm4 z*f~EhH8&WSPc>mQCg!%yPYB`86=#rSm?ruUWE|_ToU!XBij(WLpD)pflzjHGK-62J z?aiMqj{#|Ag7Cw0};qJQxvC~uxW;IRSC`{(_ z?UC#2s4{t2qjshb6<#ZJwI;5z(&wHwPnRjC8TWGHy?s^kgoG&ao0;tIE$m{ciNli2 zADOH}C8Z)FVL>y_o}k}v%tuy@7>x4G?VFkBD2ik1=0goljZ~*@Tip`0I|ie5jm?4g zBMf?a`dG~h3i4}W@C)k_p17|CIhi2L(_`}pLy{X`r6TJ%=I56Edy%MEb@>td8=8cL zjs4hgk#n`d6@vVopEq4?29uw5!6l;p*Di*s4Qm z!n11$Z!in?Fv^#N7AW}r!W?z9waG<&YAdppmEYAyMX49b^{FWd1Xx|glR?OOEo&bW zo#zID57g<-m*ox~iZ&uFLi{-c9j%)km*w8fi)0EsMA)ccJ-t^E5s}5YiX!xU+Crb) zzMlq8_>3;i6QFDKjwGX_$IIWdu{oWP)UfNfLZ1jBPI;czT=;p9U{e@V6v*QWa3Ndv z<~^N>#UNPpZN}ZQyNjLr<74OJL*uAD2RhZ7=;JL5EUBx`GX}h0wf)+GHI&yl12ibagNGDbkMV+AvU2P^GVOU#Mtb^?YoP6v_GXJtlPu zhi+iN`<@p9%RIN=ASv8%5E8J^KR1VCkP2$VqtP@#}k`r;nm;Qnb8fn!ahYXDO zD=GO-?O+IuiVPZLdsD7Li+!=hl1}M|33`vpYzS7cip|Q@(MzH2$YM=AYA}wNuwQUb zLX_&ynmftLNix1^=;$}QelBP3vKBXZb9S*O@MYZaZML?jSN0Cc`Px~|l7wS-!DnwKixRw>;J-#&7|BZnlN9O1# z2NODBst$8Ym>KOkxFobzMP*eYCM%s8)ep2$@M(rUVH=VKQ`WV4pNllMXDgx^RGyCp z=IAKuFO4lNeYebSXcb`L=4KHo%EE?JRo&nJk)c+%8~whln66tGUwu@YPw9?w7SeH_ zvb9xGQsShrD`wJ7>0AR2@2wC0(;L`nyL;+o(iLl}n+w7vEpKT55KZp)N&7!DsV0bNOlfqpUjqZbx714OfkK zG)PB`A-8ofn=XLfjRA)~+Wwq71~&LQAfS>&=&d})KRp#kJmBwFpq`|r`ODVU@7B+J zXlQSDcNm+d7on}y%1EA_8A3`-hzNX`N7}q|H0EaI73HTjTX5m9uThD^q;W$EVWFWc z(vqT20MT49-NxKAFu}tQ!9@ruk0lpb9Gpat?v#qS#vqH~`ozVxEG#0~+sn@rUEA;j z6F<63#z@r17J@7KOW35Yxh5;gb+OT2J~+{Dao|v_GZc>aJ)CFb@*m07?(V@k{Frq& z74$n6c9!|sZ;cK0l4cTbM1KUV55RE)=SpTBtF4_R$ecvAp2+6@Zt0+6FZ|+Z~ zr{A5~+V(|Y7+;T$$v@}bw+7bD?2C(wubf3cjBg7-glQ?d_MCPl>}9(5wn9$*>c|oK z%;8y0*M?ur-$OGtVwgyEcjyQud(EuvvSYB?U%zZ_Kb zR{c^K0^RZYe66521bWNX(N=~Wv2k!vb~ejv&+cZ?A=6ha@b2s-w6Zem%_nW)|9+x2 zZnr{fI;-9axE*0w@VS}QNn)jDwu|Ztd?~rn?%WmEZ=D+`O3lng3rR_Stx3UpgGs_^ zJ~;lKn}Pp1tr+;#U_gbE?AxHF=Kt%ddVX=R=Xpk9aiiNcZeF#p6t|MS!9x7FG zb@8xqN(0Hqu|iP)%dxubW8H{v5clM8f__teQ;AgxBl2b*K_DakBXStD90P*^%~IeG zoW5VVQ`Qh@wdbgKmX_l?EjdofjS57N+@S% zH-?k}ppmMejaXl=_QACEbK-ksn$e{%e4D*+DMb~fk;R(Ksqf!s&d$w%?n~45SJbV( z(#f4tp%Kkw3D$+knFD67GOvixpLbhQRuWW>GF~b;v5h5oQ~fHhsIDk36E{>G+g>DY zsv>;Q7ol$GlE}Mn)5Cw@#*XPkMCQfWpHV1Y4wa6rcd-Vcp7*qdhM&jMoi9~oZ^LV& zTuLAQ2wA7`t~w;eRw(ulGujjh#gwD38N4N-(ANVG877B=jqM92Nkc6yEpWJ*Q$#!3 zChhEO`TTpH4)e>&&MC+RUQnos08Sy4!K|89lMHty&A8aOfy;Sg|AKQDLgwpG()74Y ze5ME@2KO~B$Mdy+X#q+(w?d{t05wxAX01dj(*?0){||X@9h8N-cYiCbASEf?-Q6u9 zN_U5pbT^1}gOqejcS<)R2uMpe(kUIk3-{h9cb#~icb<7?_FrelIU{>>-`D!CwLVLz ztu4NSL4A7a?ChNW@>^D=vAnkHx#4j)s5(a7F9_+47w_(oPUCuPTb7Zbw`(_;Sv9C! z@`m9XuXTDt!q>pag@P@mM~}XHTpSCDJ6K{4m&3>CTbr0TIIx8Z24yFVVVsg;rAY#k+GF=1an9j$d!*-x)y{#Qx#$TX^! zHkGNKH;p;G_c~kHESI2YrDYYZi_bcB9Jdu`@Mw@NM{F*`ccj$5f-bBa1TMLGKtyP8 zFgCQ9!%FLYZPbjay3%H4+WRzQjrVJXQpc3Wa=OTdU?DOhR|!xsf`gT`T5h&Vw2~8f z7gs_B)QT-f8)NL>$OObOkD;v32cIf&NWDr*ifX4RyL>H}e*MiL;zcrRP&AuP{my!^ zS^_qBFEdIk98B_XgUFbrYHhfof5scpKrIT2+*ux{xY)Srs+t9z6H5!a(R4*ADaSdK zRx#l-rnZ7y~gj#iuT7vT$Nz$^CLbLBq#SxS^1iB03-Mpa=JZr-SPf~cSpdW2IKs6 zPi{tY+w{cxP@7L-oL4YHpGxr?{Pw4+qAxwQZ`#j~%K5w&r9>bgG&QwU=yedM_hKis zrMNuSqI0Cw5=wS0pH0WUi+t@Uq+Fs}v;F+gw7DkvYLVSJAQ;^zQ=Qf8KGY13)h#Uj z1YogwuI7L-qWayB86woFcQ2PEAwdf{zv`K;+2Ri$xb{Aw2U2(Cs-~y;3KPY=_=~b0 z@V!hI%8iz86ZdX;gQ_{BDYq7$=Gm;Q7KC#FsAEB*^74u^V$hseMC>&+;z;-BJc>9S#@7N@1L7>la-Wa3y;BYv$&OHGV~PYopz=m`yld(g(BSM6!Um^ zQ6zwwKHHAjW+*W!+H812R6!w{3!69tc@36b)ySEYxE$+%>gDT7RYgiW(JCuH+Nh7c zxU>nyU9`k%1^zI}0(B&h4kPgLq}3#QyZ1QCEIJn!{zhf1S)VA%qmG7-?d(z8vaj|> z-t(H|)h~zh+1XBD(X_h9HrPJ$;w?D>ue2pzTZe(1(Y-CW%x5YpnAFJhPacN=9t@VQ zbFYBPz==|hR^Z^$$#OWlNK3h;mAUMo#+`|R&kAnP`$T2V4Q{UZ073-Pi**EW(Vx!(oz0Q=3r0$@L6SP64e!Lzz_d6x z-swhDdf4ZKo1!|p8T5Q!VREGS^Fg6%-K>(DK8kB30$wy8z07q&8C8>2!&PLOnWl`! zqN4nK6p{EQ(Rf?iDBs;@(%k4?SMNn&?k6TheN`eVs09G{H_qSW_C~xSsp#aXFq|pz zHFz|%Jg(ndxqEnAgV!7n67I*tV-}8p8BLdUK1Z|;41HYgaJlq+s1 zCg!2lRQ2I&`>?!9b0ap^lg-|=3jGF>%G~()C|=*r-n7te`u&ay7Urmy+g5K%Sp^Y; zm2Y48Jvjty;cj7W*+(brepg^8em~A%nRY4-;4A|OJ_eLVjVfmw`{xJ~ZdlmZM7!O+ z1kvqO^t7}uH)O$YuR7cbl@Whh9Q#lgG~~9sI;@KHl!?-gu)YZGyokOqaJ zM&7hUm}}No+waC}*hRI&aXCMtUaF}+Phfvkat}gjNRlhwc5)XFUlCUF!Gms(7v$GR zHs7P!z*z?y8$4X972oNe05~Nt+1a_ao7OKR<_#4U zhx=_lTC3)--ujA(hCbm_SCFDvJx-YAaV)cki*LITX;#CTn4GKwAKM!2mM?>dGKeB) zB_9MOCq7T$1_xeoXo&mz>2b<&8H;-P>+%u3VPgYDnl)ECEUAYl-~BpbtPit&d`M7R zLP9jIy0`k2OIlUm+|#1CRWEglr#(3MjuI>yMUo{;e)R!-JIYU$?0*dWFvtg+i}C$- z?{ZJ4un}e%8DbZ_h~L}>J7?AizP4A@2dB}BLS;g)sSl%*0HMaujCe|Sl zbf$}kQCxF!axvW!4;K{JLns$(taf7w)HWKNpYhx1F%Rp_T|F6NuNr-_W2mLZ?GT*$ z`)!~6N*fW>uGR`mZB7x+9O06l<U^ySHwaD6K`SYa>^v2{AmY3V;3k!3Q>8na7HXjnQLRM*h>Kcc$p&z2a#j{yL| zY9bZLI>v@LM~8#G@=NK?0(`Q0?Uy7#xglL_a*;ubFA|*#go;Rfx?{Pozt494ryQj|6#c6Pbo&7$zWr8ZBOQRW~uI;ljD{vs5*B8h}HlKBZN_ zhl|2B+XVBKi*uihl1`b9n6{Y1S8T~DDjrGu$q(~cb$e&a=w-6w&J%gU>{Lga%ZNQ) zZ+|19+3U+yBM(SO91RUh|2{6R?b2V|=Z8UH;?(&RSkL|L7p5<}YXDHl<>Htrna%Bl zrnCuMDj41goFDW^Nl6Xl<~+jTcVDDoAoI?}>a`JYf{2*Rt*WL` zjykC`7fa)?X{PokmGcOJwNU@Xcb;SL6pM`o7viVg-HwPS;3GH~9)4sX)yK*2PC4W7 zH2ke}{>f_wH&P+~IpHV?Pz~W*pPZK@MZK=KQ<0Ji35bvPi&YXh@&qsCl5>hW_Z>OzhRYDoO7uR<2S=`wBS>R@}J3AL5 z7eIR>t>$WBXNzaZQ1``)u!aC}EepKh&;1_*!-EdKA63_sZ^ote6epX%ngLrqV;>29C*6l^30Zr7_PbjPwvMusVAOG#Chm*3xvE>(4Y z3B{{;n<4UVE95OElgPOD<9nAUdRL&!AI|pb*aU1#C@PMQPOk8_VjHjY&}qnOD6lA? zK^sT?I}*8#(8fAOlB}o=Zqm@$XfeW2P_ZQKgNvRge@jSiZEbisy`oB%rXo59+QQ)Y z96u><=%A1AE6bU@;^Oak%us$(No1VJkCnQ~Jpy1JS@GZxh7|N4+h2Wfa~1YH`0-1$ zsjf~xFeRQEJ|NrIt|cTG=C+=REZCak{f#JgrFQIr@o2pj&C$Y!gJhhflim!*A*2N zpMndc$z|9%YR?Q$Ej2YGl_L$;Ffbx>i?r@8>^>1w@1+b3Nba2F+%y5JU$d5#s4!|T zVf_8e(w8D`P#AfU@$X5Ao7(7je4cenwYKdhV5gtWbD`RRN<*YGEFc+ZLEjGwWdG$B zuvcgqH_FaI;-{i}df!{{@=FhX5~nF@O^*CVOcW-tTDPESm0OmTSI$6qivp8X51@dHb;?B`eI%UXEXDYbLK!QrWF#wTlGIA2K& z?}_&iu#eAdB0ts8xJ=b*@S3cmvOe2)fnRg`v##-Lko5Vq+NXEfbU0;3l{@HOVNXRf zGB*3dv;p-o>Ja_MPw0JdDU$r3C*q<7W?=SY%k^ayL)I8xzjj;Rxourlzd7hBZLC7> z5uU)~vX|(3e0N{xh0CKWD2r>TYhdHxSl9XhA(wN827eqaONonu@dOpyMt6^28V|YV z=;WxFlA(uyVnTJsV@1NWeBvYxO<|3qvog#{{w+^7uiX-slb1KV!`Vm13z+dL7+An?x5?P>5jCGglKY9@=Dad#)%WEYCcK^W-i(QjF~9PJoPLUc zSb|zexBV0-?qo{r0(hLj2*}5ScBW6SFg;kt|ywx zWEN6bfsE>;o0{hG6?b8Q5meN3SWgL+QFh;4-T6HChOJ#d0UCaDl0e)>`z&C{!rY<@ z&Qe(^0h|)6LexaTwe@)#7bJVouy}lZMox}Wx$=Hk{ig>moyWL}@a&uQ_&5Pb_`xZR zU+f|&{DuC>(EXB=tVx>D@HSeyi=bsAQs}jj(Mw0Az|Eh{p;5P@hN7w}G3qV|9eG7_ zGvmU-Qk}@O($Z1^$9Cs+Z%DON&$TscXu_6BDOd{e#+N(7mD$+k0=KX4nUK?736VT))#UY{K0le=B0{94 zrJIz3+QXpJ#>IQw@f{xoNCF#d4tuhZ*XjPeSOhAyLjBkMSh``1WE2z+yeHsON#`L5 zUt>v7MD&P@M!3HkFrSG}dnWOcD0VcaQYeE6L{-XB1s(ACoC9w!D7#?CP^_MC*brAj zwF>0E@Z+hr+hRw1W9Vt3yCBG+l1(t@_1dX`n>P-cxG38d{= z9^3wR16_wvDJ|T>84JCpBl|ttyO$1oQ5^wjEcS@C;IEkb@~v3<_43;1!2z)PHug+a z8@XX?X^ERt{^Ns?C?u4EoAml7pKI?d@kvKR!_`F}*oIGEM@b)q1Y)tdR=tl?{BAlCgO`Vz0dHVzHy&VxHdJs42md? zkR=Vq#r9)f9Q3HMjU9IM6Mx$8vCG9-opQ2T3UkFBe8kJkn2d5S%3Lycx-Z#lZhxJ33b%sM+ypgXtGOYEC_7Wxv!#%A`!pF zL)9}Motqt5jWijS?m}%6JH1#)C5~e42z(>ycr#ur;s9yqy7_90y+lQ6^%Nnn&#GCQ zZ$04y4gYOXbv4`G;p!ebKV#k}j?}`jA8IormqPldwYtvvXv<{k3n9ZF>g$hI?0%m^ z@Rw0ML78T`>%|Uv8K;=ol$!T2HSn8e5pi+h<>f5sA!gabkX?RCs6pO>$Yy(aSvg8y zCXd~0GO}YmB?9imT;TNou+;zH?bl3H{vCE_??BWZFA+CDI#(?k)lzM`|Y1@#L=_p`qYt z{46-+v2-$TotMCzE2y~A78`DsWi}G8_HW>6)Rt?dCbF@x(rS`oWLUL?Xt-5p>Q5<* zi2M336kpW;iYT;MT53L5j!o;|(#p)t%E}aflJw3#?_NDI2Zu2TjR7YGBw_A%_-81e zwq)J?q}>3pLCHjs6ivxhJu$U;Zt|cUOe0iu2JU-XmLy;fhdhK2fPoQabofV6NV7x| zeUp(zA}z_UrJ(^3{#CJI4k~0&eo&W4coyySgSP zimIwx!G(0r6EhH~#rn2Wq`=aioH(#BOE)+^qC%FJX{fvIaVq=!n6faF=0LBW7r0Q? zyVl1kD@)LR9PDXeXspv+8<8PlsR0`)A4hT{7K^@C>bUHHAG2HARd0lVwz|44Kb^(o z{vy1N3wDr^(KX@As7l~ravP0aN(odsSuc#XeVPi6ZM``x(adiki&BDhxV+jbtxH|v znY8L2sf2>ii=f%pW&YS__I;|3N+H+w?kTAQ3qG{#2#u~U@Gd|g?$a~y-9utv08&HA zQe0hrex}BQZ4JWs<>BFBDR>B^HSiUtX7DAN%`;QPZiv0@!?uwTwX&qcqgM)pkL(@1 z4ZmN`d3>~EqJe+X*WO-{o%b$U@}UQ;IE6{|W&yRdr3sMPq^X=PR6IBgAW&pIkK_W| z0L?Iams5^Y(2~}o(@COZLE5zP(tehnpHJpB9xPtu(Lx|Wn3Q0{zg=VDgE%}3r@=s=_ zrtIwOf*yzH@DZV}10HhCjZZlq?=P+>f9oMA!-o zCe!4|W6NNY5|WI6l8wj3e;^y9?U}vj{d-9CtD54|VmTp`mZs z>$CDU@1YIX9hB#gkqtqCkBzvv7~w-en#ab$iSQt;$q@s7&N85Ka7_;W8*gGqhC)ZgGkY$j%f&5M`$#c^?T{-lm)fKf6qcie(Uk?IjH(ftREaf z%Ii5f;t9bLudbk)nwnwPm}E3XQj*41g&PMwy#&oiW;=NJj|$^IzA@XxEk>YiVf*MT z$43U!YGYtupVfB1NPgY*AoyVkdfHiEcnWnqon3o7F$oUNlXr#?vg$J`I!*&rXb-O; z1Bm?J*)IXc^ka;*MWVJqeC_yvP1n}P(gSB5-8ui=_;KH}K9I}XXK=&(q7;*!L&mdb zeHQuHj2%kIa)1m=m=rb{d& zJ#W9nAEwrD1PDw=7zDg}-7oEtWo2VJo*zK%<>Qko{oQ^w_k;~hdW6F#zTTsQjRgPa zh2*2`J0bbFY9O#|DDq<@=4+m_p^p-vqGOjkn4QW8KRf=y$Qee6sExyNX0pj* z8a%F}mVTs-X&Anj|8fhsOCFzu#@FLg(Va})MC*UL@^!8`kbY(x5MwNjyS&R_oqo$Lva`_;mly_CC1?yon_M(jUpfY1%r8tq zQc{*hd0dO1#pzcpQGp=8+$7Q$jQgA)9H5Yplnk+T@q`pX2|^gUUCcmV4jfsDqZY_+ zHMyo$VjYS2r>_}}(&trcIoMf$`U=GUE{g6BM?1Hhn%WbA-z9}b3hi>PnhPTK$ZKHn zz1$D352V;=hpVd#_&Xw(yxfvKpR96qo z*2NyjfYWutn`_b8@Zt0lo0|TRh{NIh?~BrUzVYGV0#%x`r&_C>E_$wr1<*JSS`lHt znK2%1_?`e`Pep+#`K@FnTj8L`iu)yUdIENW7J8=ZPm##G=Cg{j`V?`jzBdCEbxn2U z*w_SEV*#ZH0vlrVf|Z99O8SngO2k^J;P%INyGW5o>z7 zIJ@M$PhLew&nwRVdT=z3Cve?pw^h^BaH^=H1WIj!ihh^VU|8g%6ABWVkE-|#t?AS4 zK>@;hi4Wkt!1pgBrcBziW{*?XKuCBg_!HE&2C)!K@t#iWrU0;TuwTd1A zafr}a#l?QQY5Hp*cLu4Y$=C{R=(eW_xMZL{M9T}oG(<+=wX)@OL{cc`=LeS|tdklF zBErF<3)M-&^{*zhhKViOuu;BDWB?=KDo1uO|5aY8fD11_vFnP0ozZghm7*{)e7@i+ zl%GEW(u3}2_r@HK7l<~B*Xz&;6az^As#+~xnfszL@0vl>ZUp7NKZ1Q@6fAH7JFy|; zkRg#>Soj#36Typ?g2o$ZX#u&+8adF^!o|7ySd$Wsm&@YG&19cP%PyboS){)Iz#_Z! z%~{Oz3b-n-na1<4E=O_2>JR$h^_{ddHDSdca_KQ05($qHDAqca30NP)O_vlE@w)Hp zs&FMGl7g&N*3$_8jCZyd{kwQPs@%SPy0=90MPX*<<};wqsaJ12mkAqCe({3I8U-4roE1azW@Dux%q@V z)Qa~@Ro!$!O5W(?qFo;aCnPuS%Sv>b3<{t;&Si+~q2I)U=kbU4@7F%7=q&GI@9!Y9 z$3D{_v`mj%`|Opm_Vke(e@Zl!lnUr0b^7{RK9jBeaZnWzG{6*4qCgFDJ&cuN^u9|6 z!x=#8afUm~{$fyK$uag0%wIU>S?Wq7LA2pafSjfqOy??xPHE~<{|tJX7mO3j{}FBa zeeqmR=!Zmhg|;ntS@+~gfV%nt!}n_FTWjmlFXN7^jFY$+Z0QkE39818O^|a>^o=eI zL}P*^`tQ6qC?U%98U-@6haNDOUwOEdw3`y0mqfF!CS0nkszID1MGNK`Xp}OuR1UK! zQjuO?`CZi4BU(X&x>|Am=jSqeuXJ*HHVt0-;sIm5&yMWWmV=Cv>_0Beva8PmFF{^2 z*&1N?uP+bAdG?x&P3aT@{#4?(QAtN7EhV1H)-bFS!tEx z$Tc3U6Xku2>Lw@0qkdiT@$=`eWtM!WgeW|E?yY5dd1Kdmly{dGZLs+HFf5prnORy! z%I?VmucP~RDUTCK@cTdYP5QnSpX#VxM1ubWixMMbFhx?Xp2t3-BsDdxc;_Bw{C0o% z)+a&&D+ZnUqR?H|gkb~hlZ}yyQ8Z}Kor}t?uMbv;hLbWwA~7(})^4d`=jl_WbRTwk zT~bq%6m>9>ut7{m2bZ6#HM4#E{cqyE_hw`5HioM|J&TjqRxT;bH8V9-QkR!kQd(X3 zcGeyO5AbK>C2Bsv3H}a*3Xn6EMuYf_x~)9~^w+0v&cSIkJC#R8LxbmEYsgXrxr@tk zV5&^G4D$AUhnsKnWJyg`QdW25qR2HycV@(1S!J_oyUGfPF#lQ?b(d_oAqG3t3vUF6 zaQ|41G89C<(9p16JzO2f_-ZGC5f>*beP>!$8myp%@Wm)g%0=mCHhVYwWGTq$={dwc zr)YaG?@9{^>GQt3yUK2TQqHwCzi@R(eloA++J(yd{rk8pH5&Z#lWR}p;_7NT5(8p-LJeihoUprq1 zaT<|!VXhD2-@m*?D!<>$W;T$#FoaBA0P!Qi`>PJ6s)=M!P#aMjIy$<5#W_gYIi>ft zCMG^{LW-!L;zmmX?HIhr%UkR42e537zw1vndnhud|I&;7tpG3EbpWD;nFWvQne}15 z!gg^`gtUG&HRVSRCibAq6AjV}SSS#cF@did8OXV*H%5}jPh;+C`;zfNEzxzis3%=W zJyE@7Z#v*${(~0hc;1&RF=8+(EvH}*eRR(_If|gxT*%_Z=jU>0$zs}98LC$_$PaN- zlC8tcZ(V3;Yk|^SP-pzrYAoo5BioB4L+`@75j*J8y_d=gcZ3ef<0w;w%3@jLG1x7NF$t{_nu4gvfv!e7nFg`A(H zB?h^^rF3{eZtlEpOAY}oQ!S&@FVYfbV7qI(2t>_vTG|-b-{n2Q#0l9cZ8;O(STxVz z+};)?M!AWtkzneNG%+&T9G!pA(F;}$sV($g2zC*Rha-U5)dmv_v*qRWhV86~fmBJC zjn()sIG7CDqA+>%fR-e}ZwgCP+7P!# za*?>47&wbIdiL_`X~0`=Aw*;^xJjhX0H)v@t$X-=27y`3N^l?;nONkdauR0m!V zFYW9?bR74(yP!>>)r>6ly!*VXf28U5<)-b0%8-6VTvEeIqy7+=1!z3x&V=R>N9*$9 zA77y1$6%a}O{5$-amGo|vNN$2Q zmHL9d1%tygt>xxrI6d@;z%k2wFa?QpQy9DeO4{`Z%qu8tn()=%AtaQ>uV3A7HZ>X4Qb*dk_ z!r|zlM4h{w!-Mg=lPf=lM-SixU)zGu!?J5_m!0aE5o3O}`k2^_&0R2V9Rye>m@ij@ z4f_U2)7H%TdI>CjobpteC=0}nFa=wi4o76I<9t?dUNr@(m?kD;bcd|j|JM#`6-t*5Fc(Og18`ecExo@K% zqY|BQeuY#T17M7Ftn^_;+MPeLEi1(Lrl-{H>})fhpa>XB`%6&zp11M1?@JIep`;1J zaE6HaF^h{{Dt~+{rlqk;ByEJ!E6>KNd)bC(8tJty=m`VMe4aL1X)YaB3eG6`qg`y#ok zm(i3`_@kK0`Sr+{Ty@rkVmTT;4J=<&*6N1s4=T33;Xdh+JB|hI8n=n=apiw0F1uB3 zbGDE%`QR}htV_EvX#51IF77^BH8cd77%b^+vig|l5K?;;+u{7PWe&U%BRR5PZUM}2 zL;a!sB@Y4h`tEux65*Oz@K;xJdtoK-FRrGMyZN)jjgZ5J^Iv>TdXW88`4j~S>q28_ z{aWCL&x__Ys9Jt62}kF*CB+bk@j?GG(AErI?-}QW@2FW>SwX82BnSVtaf-|1$|}IX zVA2V9+z&dZ*Oxpr+rQLK?<~E|A9+}qHV%6B@z$X-K!nX%b~iOMBbv<3TL(HZ>G#A?)%{;Xx2>_S-~Tc1qTl&Jv8u5P2n8ut<_t zd_Bcy>D)^%mdT}G#@#zxugZTB7RFO+xh%q%Jd?GwB4b$|4}K^C>hi8dE1jL+n6l(( zIyGh6n`hlzub5h4k+G~~q?7ywPHg8(-6C-Lv)^Ud)rCb1@RLQ;LhKN5Sa~|mH5wKw zw;zJMDJJGbTa>W0{gYC&KO28Njy^c}$=yUWb_eF>R}MHrCQ??YR6}`T;#_it_tGh> z=W^bHudyh7jZBhY&=o=y!&^y-Bj<_cb^~B`Mx7N+nStOKUE*LrKEJu2f1{ZRH`!QVze7PC;+^T&kgS9ZhlAz1&3kluYUR( z{k9RLaEA;5v53=boZMhf*@PeA&`eB1xu5>`FYtfT!Mqnl{ZEGIP_&hz*IC)<2%e!~ zZgIZ%&YO#=tehMJ^X=zdOHcBT1#R?VMfC}hJEwRyO-z+o*{`)VDNQ|`Jyj!r*qYkj z$<){-{@odI%2!ez7HtB!xDHj-c#ja+1H>MEGk8fpj%~H~y4L?)!VMORAu+1m58;UX zFmFfkfuC?%o99WmbgW7#c`D}@%3B9*ItvR1D=Xu8@8PwzqW=D(V+5s#k^){C%_Y)I zU2VPj!0966`cPH363)g1_H7M@(pR!cl>-X{8VZw=1S$J_o8QGHPEBQ|CdB;DR7X>D zs-NR2$Eai{b2S-kvDfq|R2upF`-OC7+EG!>cNI6O%Q5ZbZ;%KJ?<;#~c{nPMpA>6E zea!E8;|D*>J98e-QBb)2KjLQsbk0A7ToY-DU9-bdW&xfGG^w(lhE-IQ?>0XHo~2$z zB4?SfdjP@#0XOUIGi~Lk{-D1gp^BADjuJPH(@uO)`>_v}AdWhKfi;lGcL^FnVoNQU{b^u5J12*1 zePr1G5kWKf-HS3Y8Xp`QLgvcQ&oX%0)N(8WrVym@ii(O=A8a%1eTuEuKvi>)tsQK+ z3ErY=_PI9V<&z>rKBVB<`4 zO|ENx0_}+BJZ^|{=!bz0_Jk1wDUx~(gh@3u3n0_g&s|Ygg}bs$hZe-jb~|c^fq~)g zadCChSIWj=Y0Vs;Q&gQ3+XT=Gdkf#7ij}pLa#SDB6Zx1~j)E9mS-kTp=s*At*kpGz zzP7fk8<=WM{*+v=-psCHgg-ie(-DGN{=;=>= zbc9gPMa9I#Nm`egxs*#M02nEV_Sj&{b$NGDPDLX?AFwAf4fNfw-HR>EFA@{!J+IlS z4n*B~2T06wM|yLCsW0fAm)j7q4x)3xA;H`Pcm-bu=I|-%OO)T+ki9^CX%IUxJZx@m z{`z8t1_K}aJ`(z;hT`g-Z;Kthxzgpe;&imdQ19dhtGE0v#- zon7cP5=@X!)^lpTN9-&dg9H8T1bw=?x>rAy}t%r~Sz4>0Z zTN5P|6yyu@Nq=Q7iOG$5wcf)eBf1gYrxV`+O^&yBr2O&2|OG`I6?b6cF{e)DSol?&e%D-bOk-QE` zR;?t0v$dn&`7mbs58>G2#$OG`zR7HFC_}3?pQ>`}8F-x`f=R0$c(%RC&@uzgk}n^r zGdZo%HuoRHW5T#RY8TuzogT^hls~p_v$?NW10IH^*17Oa2#4&Uu8%ZAOK^6 z&8(z^tzBad|C_?|tM3ot8KDpWHZigS5cMFvyTr*uYip_vGYcTC39kYk^$qkFfynF& z_W-Bmh#MO~KO+)>eA?Z)J=+8uyP~-KM{4wML$p^hKUX~GarPxd^_i9wKHf7f&y!sX z+>aPzAup-r(XfG|N8#UE~ zSKXiL`;6BH-X~+DE+8d|(_-kArSRGLZ#`O|DBrh_&VB%cPf;7bu$WOm3t2=GNDZ(T zd4hE&&uL$onhG^6|JBS4nU3I5M!4>-Zz>n!vx>b7Yz%>MhTqYnLrgzolz!5q7R46s z+Yj{8g_iuZ19;ANM(8EB%*w{E* zBq+YyQ$tTm4E;qkS#T^CKljgn|OgDa8wzi&Smx zr-#U>6kZ>lE+YhX&Rn(zil+g%k>?eJQ-K;WHkWHv#Qf(7+zML&A^Onr2<()*V;%@H z1v+$oerwcua|P}m31$1RVOaO=4*syc$;`+~`{xt6|8afu@AycO$K!T00FXp7$7;DG zYV_c>mvqw9=p>wrJ=rNUZJs!}bYx8x;lf+b8{VLVLZx4B0rIM;h)4w;A^K^^FRQAn zMTiLn?f$|{V!|9HV!9E1@%h?qZ9|*nzraZ%v2;6IXe+U0XOjff#;0NU#rZk0|B~fd z#)tw5z$OTfkfiwHU7uSPIU222tllOb#MG8D;JybH3_M)is533inI#Twof=!{Zq%m} z|6WEHYi7o9o}cr=?+IC`(K{hKJpu=Z zA*{v6_l@0DMnE+n7UAICMLoAts^NJ96D^}GA=*E2lJ=i`#pJMu{NaRRSDgI^MkwAk zF^v85o!JDDA^9tX4!*3oRFjRw-S0rU6Px;@Ku2$164!$yvu zcW-hXV6jtizfm(YGXqbVc^Y?J*;p%HmBOEHMlv$VKg-b;idoM8_I{Xms(aQ&2OweX zQqu5Pym(}eIXZz)M~7!%U`&lMA&~oe|rV!i&)ZkQi>4_1M_CS`GeaKkU+*WggY^ z3^iG&Th9+_|C>0ZS-PMi?!4)B9j~tMXg?XGWXN!Cn%=O{LfYDe6WuQoxMP1fquhDx zifz!~__6l~;?{h7`|s3kF@IBeI#nbxTTj^9^Fy4a7MXj!* zLoof9KBo|4c|ZAk4;w-(#bETK<3oQvyQK&NP#g#Kd|7AA%8G1xs$AyWlDN{(aS5>B zYXum{hDJy0SA&^4r%o@0s>a7_LXVFfY|PHl0)tLY9QhBxY*s#EFUkxo+xQ5j-G#$ z4Rbs>kzH}H)uS_gONdk07#9a7=O9NZbjtTX!a3*Nhy?uGXkm=ET2cyIryiG#H$1iQ zJx@U2M1qu-ni@2jZ1ERDSq_)u=s=7rfhxFNaJ7pdxT^&sqxDPggNctPX|?UPI080r zB~xN-_EpdK8kZ^srsZ0`7Z$td{QPrjid>g<8upa>#Pv9DvgOoye}gYxhP6 z%x~&ky}Sh^TTON&^=vGo@^pQ$zykvftMjS(KQvX@SVvDc zMqu&AK4?kw(8~t5Wn=)t8zaQerfOi<(8}*3kFc74ZhT@*{`tn|o+^3OXim}gew3UX zQ&xH&0C2p{Dy#hw-K8Xg|d$Ng9tBh zYr!A=|9yt{SIVGcefBnXVhKE+7w77RWR~B+L!J~>Qc|vbxL+IzxAW6yVK=(rxcl^w zM=L@!Sopg>Fe%d9&~7j_Vd@bRznS1Q8Td!65!8Xg+RH#+;dca+x(SO>ft~uDxsmq= z^yM=8Q5Few4CE%1T>_)kL@N{@JyH#|7Jsrrz$2Sd@%C-lyp`^BU|XA#ezjk+-=87} zN6!A}jkd!r{vZcyyL52Bx%<%Zd*~a|-I*B|hv84GeK;6f>7itbxl~zqU1!iWWK}c` zqUk@nZl$5FnGavVgw!_teruG~%~um$w=f0`Q$)_jwgb`?4n9?0&d)$E!@|sJeTxJ+ zn02KuzBWzAVa$P31NotI%Llb~KH79Ha0T8sLU^6mzB#~yomO-<+Bcol1a0Q)_5LBo zXt_Z6QsX{S#`NS=Fv?6(0W%1;m|#{0-hFrR+rgNl=0wmth)78l>K8BIIK)0S4%^_$ z!!RFp;A2Pp50OOxsq_cTBAQk&iZ{OqT}xBDO23?(5_cCCM%GU59s!}dllI1{At@4( zjqN%ivr$Y$+V+SaGk3Gyq#W$cZ{OWBv5nsNPB;O&{QGm2@Zf9JMyy?r}=fon2szwB$S2ri?O2$Sl!r9 z=MLsed54C9zOlOHV?cE9Eu%w1{6VB)R(H6mg_(%p`bTe{84+JuN@v?C)~oXPOV3%i zm3^Wvgp>}8>-q7qmtb&Hf?m*SJUY(BjJ^1Ao5U*7YaSp;_ z-C$4sB+91FD&e%h@<*k+HA~ObRE_yu()_FWxptRFnR@HmV#3Nx2ZH2!x5<&OX}}la^X6o@ zT=KUREVBjVl!?*N#`8_H89vD7U+i0$xj${+=2z!HqqHIhMRHcNIequ8d&*U?vc0ML z9jVVrU=F^10`?RwC>z@tgZ%5c#ysNKqyCNj!s6*VZKc?G7DD=#bJy?5#%m>KSC!-A zD2SiCCWdg>{C-Ye;&l~lBu^T_y!yM;HT*A{l7F!f6bp2{L3uQ@7uk6 z=TE-}fP==YoU$W$zfIETmGY^pb9Z+SDAJZhJwrmnRF))#fe+*X` z-+XrGViN|o(5)!;TC=etR#z?QY?{H}EH30_74MknmT}R(5+opDDIailgKcE%>_6@a zqFgn-@Fhe;gA2&z5zPw_zV|RaREiSj`5DO5O}RG%Ug35y+}y8agL?$9t} zHtJX)oJE|GH#{#lxHK01xSb+n_K1jTn>SM zWdT<}{I0M2*Dam@T1geGOZ-ktRpA!4SnkjJy87d#^BMD=rA0;d*0!w7>@{{}gCAlP zG*#a@p5Qjpte|Xto1FYA#8R&Hmtb%lhPU9t(k_hEMt{D7GyM3+ymA8s0An$mQu#xu zQ8~nm=a)-@l%nPOkXl-$O4>Zf4gJMb)RbjT_CpwO_2muRcK^URD%L2$68T>-d4Hv^ zTII(t{p>jc;)9aByff22q`Bs1NG+K0(a}0LXK)KC}l3_WQ zL$B%#;a{1(G78RIpgmkd>RQtAmRt8{ zepgLE;@sSTtP_RX|0l!|V4;08w$y%?Hn_L6u~VFt1wS@635c0>E{r76CIpyUjzY;C}OK&v={>$j7uiGNuBod0qN2=@v$NAQwq4E}mt1ki& z{;=(kx?FO1WmcM98q|=`(9jAN!$AS)0Kct`4U%ILv>-sRRgg~W<@A~pu*On8fNZ&* zo677qKK^xP(r#_^sO9|8xSmbx8e#JC}Kj3K&s0*#0l_e{qR;&jejm zN(2?D_`4vkNzopP`}+C+uz^bpQ5UG&jD?RkPj$Qu9npGT#-&+_xp3WZe(2tOb(#f> zsJ|NbDJ&DiwC`TLx65JrSb{?DkV~@WiC9X)pQb32DfK>~oTm zyOD?eK?XC|(Db8!CywDo5c#^35u#+_jv#SpS$I*SR720um|OR0sAIT@r)q`cBvxJ` z#yz)W4v~l}$haSZvMb?}D03SBt5Br`kJN?V1^)hP?N&YC<-ZbtO;&z9@t8M*sPnM@ zxZb*ra)_u)7J`c&X012e8-4Y5L{?>Qmkof~4^DMX+xfknhGlh1s~|WrFs5ow9Gm$e zG9)w<9#h{hB$x7?Yj^cujdW8Lv|C(1FhZcX%_I4(J)P@*0bk4CTGNT)uOEFxM^_wN z{l;okNjjPB{d&53nVib>YV^&&f?v`zG5mY{(&e$vAMfd8SP?q?gkS3NN*wGP#B3n# z?4gvSlq$kuvFA)*xg=g=Ra5gL!<~i+5nNSO^|E-lIXd-5PHuAlEk5LO zu^ABdCHLdBy4%j_VI2As+d1DG&ZEXM+l75)GZJ~xCY5`34+d0>8jUU1p z+1gu;xwXr$3Es=}cFqGFlU3IOU|oYs&6B_6npH#H17*L%j_u%WBjNJ3*MLC? zfP9`*8hB4GsVdqDN=R(gRr1nlUz-eIr2?$6w*^*`s%_w=z;4b~To{g;zJv*!sNFy2 zn)7mE4gYkonea2wp9nVR|5LD8cVwwV@GjUadJFP7+Xd(!(;Vs9z}9AYL}&+H&)t6H z-ES%^{v#==zPG(H`h7-yr_{SXV-P~o-TpB7?Gy_ye_qb{0d4YK6UI6SM|xLjpf^_9 za_trr8CV;c_o;7G_{I6u4GXp$c%AkCEHd;8!Q!(i06!P- z&rRxa#V=v#S}{W8Z(CcH5j_BzH!(I=4x1_Q^X^XGL$uhSpdkP5&uL{oLeLHlS~p|w zUfrfMgZLk4E6ig!D>i28FM!>%U!-?IN2ceq$5dB$=}}~;3~D@c>Ml{+pZ{N7e^sHU z=D*d;&*k~_Nm@ez1DWp6$n9~AU%0;i!{%~5Rog(D9zgP>rfP#vJ6nBJnzkYxfq5d4 z=NE}F@v&)=&k8n9i~-gNi}B-hVR11Z7gc#xoiACvYh6SAhX=mx$=e}7S~y)Pts%ag zP%P~ocsV`vT+r3XO?H+z6b^K^6tm+};w@`AO{&52I|zLsQ8&66!zrHc@v@#+1~V z_;PkEjLuSuJ;@X+^`i@p z-SzMN9d>K|2`uaVp{b{h0>*37cJ_`>$)VIHJA)ilUNI;to}FEGb@#Av6IVG+VyC?} zG!*=Q#l2-zoL#yt8l2#SV8NZ>?h+(GaEIUy!QI^*65K7gySsaE3U`;_!S74DzwX_- zPwzACK7WBxRMn_j&zjFP6lL1mhkV|?($@&rl@&Cto_L%n^6;(hPpG&vkp zXbK8ItqNse@`0(N&j|phs|c8(ldc>b=I`?P=`-4`iQq_RW5nh9mX^#rvp*KkwfTWgxKn~)MTH5C)fa}~keM$eVi49GSLV^yC_6!GB-S<$C zK-gtA^mE6DlDoTm_Srdov%Zkaz&qF2eWmMcgz0o)VrN>a7r_&uq-1bJl>RjSjQZh|pYZwRK0F3}WJp+PacQrqfxZ3Z z_fOXMp1+>%vvBeXeR})n*j*-wUs`m}ZSC~+^nt{+wv`J*9qWqx1d1h5ZT@ykIrgH` zk^(VtX(XiAj@I@)wOSVmwR`%?M&xCHl)KBfE4741G1&3$?v?=PoGYxXdea^2*xfma^dh-$1Hli|I$YCG16kwL zW~l_m#`)J~$L7T^V=@SL`O+sh7aiQ4K}=&=QU0}lMQNI*(l!*0IhcZi!n`!1gF`aV z^n-vHk=aBlo-3F&54cN{rGQLeS65eAMNM|As^m}F+N*=T9}5*7i@bzH9fYY5Lnh=r zjF~{-))kQ80_tYeNGxN+XtVGlh6u$#rHRq7J~gSPaX(_(Tl76;Dl_4F{m6y&c8i#e zzU1UNhp_pd8BnWhk9%0^K3QUqQz6L6Lq1(;t*xy;V*rq8W@?#{5qf@c)-`yd-+xVC z61{rzHK9JiTAa9Mf-s1 z@!6Az0I1FF>*;lHbnJqS9o#ZEJCpRij~U+HF)*%XVq`J*?QgIFCjGI++|EdvnKAoa zvry2_DEODwT+2(70fjp4mqVaCYAg?sbO);Qlq4j?fr=qMBugeuSD6kGWlb zf_S9%arSis4g%j`qPOTNuQ}sbJl-KBD)2v1zFCee&#J3It~Wgm`}$_Zd))bXEOIP9 z&u9+ekS*IS zlehB_-u;_=PtB{N>C8cL4%%nO>~8dm${(V-fav4RjU7M{Qf=3dRL6jTZXkHn)Z9$K zWt~}@s{|?rI*6#HX-wCw73c84G-7)a5e2Lp?ZsWuBgdp+qVRR6DqbTW zEz-w{Bja(vHV<@m>dx(EJi2U)Q&Nds6@E4vKUogCCV+$In*r$*-2mL~VGY7nnuCO7_x`LV(%YaolTB~kj z`}O*|*2+o>yQTkez3cJ9MpyQgRomd~{_!ERPCL-_{)2vCND7DTE&5wiwc!Ga`cCd? zORTpuLx#gdYfpCv{A;!r?)S$t^yM9u)(6tL`N$W7FSjjZ2(B=y@NL-jPOYL=Tc~Vt zHw+-JDq}pTDntt9$@y|#hc&#e9+OE7&NR(@vCs-G`5hPMJK0m7NW7@)fj5 za88UuUkF+sc^+k9gZ?{1qX+i;%cR$6-T9;x1ibcG5e9+%h4Fdb9@=KRG{b`iXyxdb zOK}&1vtM>P!7&dETwQx_ZAHkCT#FK}84S}9AovJ&VBVPPIs>!`*oFcnp!BX3 z*bcTn^{4qdSb$3tSK6y4T%^Iv%i(1$+H;Tq9VlaFcTqO(RQ2U|mrrRx!fUx)%{Q#o zogknxz95N_f3;{7w2}RKUAMvo5*%X5+&C;W@#`zkH#q6>L*E&GV%5F`S4kPdcL2mU z6BCo~3-KNL5h=(I2cbCGW&5bDdmr<}n#c1&uiJWMeu@tDaHsUm%X^5^6hie21q+Z2 zK8_n0=%7P?^02_``=YO}@6XBL|Dm6->kTSign-&6+ZVsDG`gm5x&mB4pk$(;wBxZR zDXeV5X$B5N$|44qm8M>QQ4x*8!bY3Nz~U4)Z7X^tQ$17SuVSczf@uY~YG7T(wYdQ< zq#qL2cfX8ay^;%<(A+dAS-q70phMXFdN$Zd8l9$K>Up$-b3#%u2n(IC0W_g&Xc%FF zX@*KBlks%)u&Wz`z~S%(Hg-tsSh$29g7NZXl>mARXPqUqujQ~?=~rws1^8zMIzZfRvEG*}cWP6Bk8r>8TFTwYz~ zNv2@DwY`rM`o;chc+BpZK!7IRMEWZ9ox1jM^khf%4!Gu+W|Nw zB{lwyR%?`T=;&uL0l_|HZug~Y+&41%LJ%+2J)oqdt{zB8t)K#f3eQLJSU9q>U@(v0 zCS~>jOKJ32@#`c>0N$&&z>aU->1&}P=fh}OWX z4N?;@o6)nf2_o^Sso&S)SUlfDT~vmCK)08_$~CR)&_tUN(2|bKt({4IzJZ(^jfj}X zgjUkE)N9IF?_GBaIB4aV{5?!2HQ}DbWT>(14E0Zd1Dp59EI2*$wc{r_BQX8W# zoOyaa9QV)nOGVOis@SSolw2EM1V*%bJvDZ$>2OGLfZM_ZEJCdr&(x9z9Z@fM&#{VTyz`gfCj4 zsaJR;cMdzz!lMlE$dt>%EIEeOfwM3-=t+{1)bKGSM`LqYf!+?TO&^z=`EbgH+Sv)| z_!pWi&fq23I$j+t}Dy@jOx-SyhojUiFJ;j5^mPB~sk!dgyu{Y-#OMkJK*afNqdI|JQ_uc; zQSn7gkvJRT=rmEnX<3`S5P3kyIRkU?!I3#?n4EWU`Lm~Dc z7?O~PkP0{co`?1BT@h1246jEE^>_l1(I^=n?tk#k+^PL+GvJd#nYJ0filRAC5y@i* zt@tjZA7^v#2f&ldYDaCGN|>#ds{74&lL$)wf({Sj*tE3@6Fhn`j#ua8oQPZ4n(KU7lkBI5y{#dHo3YP>FKaGrx$X4a&Q@nRMz_EJj@12R82w+6gpMN)>dp#6) za$+eW`lIfq&U0Xix-*MtoL-}2^X#+{44GmR0=&*b#?5wx=jG$}>o_w-0SdFnak}90 zGzvH?o12CWk(IX2208H4o_(Rw{slrhulx?qtFWo5zjb)Ewc!&crr5b^i$!NQCOD z7`!|ELV=m!y+%mc5(b2&9y=_5e5x&8ELPkR7$4u2IJ>;!JZMyY_rHRSfgE3c_vgnS zh*Q%YKptUAiwWuobHPf>(o*7lVqINb`Hed^7Vgph_UnnG!`(UB1Cd%_j(!>(1Y)+3 zA|rv2Gqa~E43CkakshVMV?|Zj#F)4MxCv=({%0UmlZ}<_nx9KCr#cK?*s)$L*TzsD zJG3%l*ZPv|4dZlvo^A}9|4&)n;Onbv!&IKnnQaiANBz4MrA6iC)m1g(_rLTF0t15j z;R7`A>n}A1fntKM7dg9 z#xV$$tzurcA8I&PR@NILGr|0dVeLG82S=Zkfz6uoEC>JCnDp`7 z0!aBc{mt354;;Ewz0xZC{4mm?6 zN0nuJ?QvCW6w?^cn4X@lK^L^2`%*};)bJCiM8d&?m8+EdID=YRLLnl0l{PpXTr_lU zOVjrB6A2L}ck@^+X~6y!Ylv^YQ>NAphs=;w>P=jYdNq(Awy-F=jEz?2nZR!~q&PRV-R zMM7T3QcpMyh-Cm-8Rur@TaD-8nHkTgC!#DOt0%X#WfN{EWMninGz~t|%tm@;P46=d z+&6G`k?2ay3i8n_cTe{a_+zg_yFu7s+7C*U=nbP-aS+Cve6x5?zG-P`f>{R5-hpJ2 zNpJAb>L(U^S#51?>+wQrquue5h?mnI%8AxXY;*HOj*WixFOLumObc*bv4FTYMp~H1vFpVD5YuMD2Z<3$LWl%kzUF zO?XD(1YcFLdG*MZlU^bQnbuIBvY;95^e}c<*p}0l1?~u}W7CGdA&v|cnxtSy`U=^Nqm2Fcdrpdo)E1@g)8* zZiUw`HMhr~X@1DK^Lfd;Ync1~?9b_kF4nDSZDsfZm#xwth5O};sfe_l{w8~KJ4&G+ z2kXqtl!E**9&WUbJy>AS>94Z@m)iAZZERC;5@jm#TcAm`PL8dhO;kYVjY*ORalBTy8GfFa}y0Jp}#!BLD1Haf>< z!jALHt*TWG6N@X(hY3TSRCezt#3e5s9c|`Y9n3@Z^DFMD!?4B37H=js4LlGe2FlA# zjYh`nP3`I$Oaz^=%<+9XUa>ENvWbmIy^oCihUJsqI>F}D0e0svvFtByXA%;U7q@eX z>XtPJQ_{bHz!F$5x5xT$%WOD4f75Rb?axW!2qj&Fh2^E)92Vy0;S4Z=D8aPCs>Inj zjAGTq`*aN7$Rwo%8yM)h)tR?+2IfX2PY;z0;UKx(t3Q;<>imL3c)fqr0A>s?2o^WE z&9oy2w(OIU=O7+u6EBBs)FiW|(w=Lpy^Cvz%QWJ*7?L+pNLe!V;~_{n?CNHi!e zjx}4e#}Q(%jb8k%@`1u)LU=kuiypdZv3ShL(9~45{qZ7H0Fo!AYQ0D!25f4`K|C zgM%Z}p21Fz4oRHUGm3bha9J0SCd%rH>+`WvfGRW?NYw%%F+VVUxH@ccEz0r^ANPmH zg~bKeCbz_A>>j;O&kB&egbZ&73~hU!y1Kfvk{Eu9$|OS~mov1l?Ka7%d}q zuD2ac{=Q{kv<1X(eOQW+lZojHurx8zxv;^pZp4MWuYm_HMXs(c>(JEHw7Mra9O6KM z?t=l5UgqJ7CpE{`HFYsiX8ID`Y-DT%hlYmYjFwy0HOv>xV}%0@Ha5j1tJTUE8AZ{S*I<~5WScE5 z^QExpkJgu}0H_7-G8E~a;-JN_1|N^||9lTDyVATx#&#Y<|2S z78gfVO6AgbwbCX4=gRlw>EsdodE1ho;Q|fc?0PgXp+GTJ3xA1R&v>w}KsDI4>3P+Q zNl&NQq&sZ%(mcdHIeB9HUG@z|4!JPLL&yQ(apE^h)tfp#KBkaCa9zli|J)+W;jqBs zft24+vG?QY(VWj;!erh*sxJQ%={Ku(P_ywI9UC(n9mX98Jw?LASBl-y6y_Tn{HP$T zf=8_ThQTwkU2*B+h?ytzm5Q^QaehoJT!f(a6x@q5Gl)qUdAX&xhuUoBEB-IVLr7P~ ztyK9WUI?d`&Qg ztqadq_nXD*P8h!{@NJ3lxbGw4X=n42JpMEu>mOfkrp|j4)%EpO>Gv%1k#4{ujSmY* z=lUAlyRb?l3k4=6$zUFK!U_&P*t>awDsrT>vKiHp33lh|N(|SDAS`S^Ds3ev;8TW{ z@5u{f(mJtQ1g4ul}I`}@D}}& zhicWcg#e)x`GlG&30DP!{4*%6TnG{D@X|i4$;pFFBhJg}gNA2~!{P>Jyt(V?GC)lM zJsRV+T3EPanBxuyziW;L?CAd}A$lek`Y*O-lE0|C|8KCZx7%&@T~1JI<-|1On1%=M z_d_>fY+BkWyfh0)U%G&e{=7h>q>c#(S3u%4qn|yw?M zxE07EqO&|Xo-5lnGW;~%J~q5=3jq$n+rooxdmAt`1iy_gG`)6oz{N-T1E^81&CMlb zM`r!?>$Wi>oQ_4~k6nPn9G!w=+4rnA&tP#$DJhSK$1x0yFP_t&p#>@yo~4DpyxK-G zdJ56(&2eZI2M14iC6{I@2!SAtw&uG@#H6?2!`&k#Mb*_GX+*%v+Me#^7_UV{=6iTh zh>1xuT#eFw)+n$OT<+{WEmwSd!$xRnfci&ZF=AU)CG+qTg01zs+zZ5AFs-{dQ@4@b z_2|}a=;UJA`Oxrd5E|UiuD3TM&!T;H_Iku_TskzuP?0}zUcq6#tF6`T>)>A3IXbob zgF9<6J2&?f-<>lDe-e8YDLEM#xm0CMRZ}q-2YLd%%IVS>_65UNCTdn5V(?PepA;`E zj)v8}BgI}a@(*%x3Gwezu*Sut7+F{p4dE!8;F760xvf&#=y)cy9K?$K+|W z&w7>>%nuDwx$Gbm)QpWGiBURs{)hxSpIt-jKqVq~hv*YZwP~#Xn3Y>plq8D zWOcB&_0=mVEHg?dE>(#Lk2vZN7clbB#H)MSA0OVO*9h(s8~!dI%gD~IUSTqEP+LxT z5jC7J>ah_LAHM_0`GX4z(2G@wre+;ulajE-uP-l2$z(h>N1q!SnwXjx`g9T0A)2SP z8YfHdk=1f6PAwjz8&!OTgwnMe0A^*?4gz1g4+heFG5vb0#RUPl1D!1ju;;_k1s@m4 z?XMC5t0N}|r|OCtdq;=DHpS)XAF{45O79mAn2U>B=&vO3RI%c@GY`EsL4)ka+uM1_ zN0#5X_5vV&Xu=oH#8bIy>+BBc!VOg9d8kVXnaIFsBdDsXC4F*O-WEE7`ScS;S5MH~9XIE?U`6eQ zfb?6wjCO^yjiiRTrUk7sNls1-Se!CEx8I9XiZ#k8Ih`<&p>aLN&h-1O zpKnfq4qITfK|pxw>*pvEUt#U~P@m7F^O}@K+_7`4)0qA}_Z!ylHw~&T0Sa77N^Cq+ z)6y2oYpY9tBOqJ*DVX?p<=+Bt2Zc{$;hlAkj!JQu6%|k@aX5y?bFwP=pk96kplSlP zNFrO2Sh4=JxSTa*gmwhz^Qg&179kW8>MSJide{$Eo<+;axf?Su6;?8I;l$AP@@n^f z#Rem9UeoGfW;Q&H&5QzB(&&C`E|4W13TICZT<_@EO7Wf++1^{06uM~E2O~az{)xB} z62NAMrk7~?j>e2kn*f$%)T(T z+}+*N16c(7+TdsMq(ZWdjD3u4&FqyEA~BL%1bDs1G4d;10b!O4@;{*_R^QD1ytLdZG3#|eW15J ztY7$kd@KW*6G*5}Xt^Xd1+E4dzkKo2(}A^0jGjK1rG0QOFVHZ~AqVjK?nmIcSZJ7Y zAkb?PS@M|b(q86}53i)gMhC}?4E0RrA){P?zu6`OQ#^Dbpp)q=mts4 zdlS+5=Iy|&2@WDilfn!U_Fcz7J_LejuMb{D8%>RXsu?&0diY`kf(%rr$uFG4}z)}Tw!Z-JsRS$-|%(uLqh{}WpBVobVEbkuc%nPUJp;s zL4 zJYJ`?Vp36Ois}MkUN$y%K*2MpQ6EMfD-e!}$Z>gj8Q^SxEKC+1DQ(a-K_1aW%_*~W~j$i%!y35A-?=Obp8ZhRI^81fK zt@l5kAlA_+=(HP&+4wvGKH&(MDF%qHl?G4XQfb?}<59?1Q8-^>mnp@PMy8CRx$n%p zp5651CxMaIHeVZ-CJ$|GPb21TyX#+Zd9}>t{_{sdvy-UEM_SD`8U{+SB@!eAByoNl zz2QD5Ssfr~;D?Y{4?G%Bx2Q~`9(Y-WPA>zC++SJ?1X9m?7SEOS^42|#vV*uD?w~$5e(4U(_7#*e!@k;La}HWS^vhOjH0Oj|($;>2(I|_sYHz*UM(4ngUM{@6 z)YW>qN^IzUFqR9b1T@@x=?g%YNKuV(a&QF4Y z*C>eD-$&ma#*X9IYt`_;4#f3;e`tX9^Ok`>S_i(Hgb@W6B|{L5BP`QaZX3lH^>qq)7d(PeIz8^rPw-8DbkHSAw7Nf(j~Vn;r|IL|cddvGsi zGY$(Q_YdmaL4`%Iu1=$+(h~vy^~u~>jVh+ALw|Y}m4}BVj`pSgc4%M#mI{UL9W@3g zZlol6caf16XY;A`T>v+36hZquLn`}qjYQ9z-&-S1 z25G=^Lo(t+1gQs)8Xvr8f-p%u@v6F!@h@ohzKp2f;%FX_9`C)Y)1S`4R0J21azvL# zuw}qyih`@RWc+!-VuX<6IfI(h+&8ESj4AJq4~}RhsH3S=sC**v(I}TMB9H~ zZFC=X6BOWBdqvd)XXn+hus{_jZhcr)%#j` z(8Td^`)oW$uAmQCTVv(mAZT!)=X+b(SWk~ZUx^~Jj7;4KB{%mT&H%mR%*--k3Wrnb zH}7bjBY-<~Nyy$fa|S9O05Qo4S9VBYh}Q@204{$PEDq)VD;yw(^9aWaklF|;?wd9} z|T+5F+z8V88Q%>72gbE2Z{ezq~0i&JMFxE;XULPC2?CkjWVi()G1-Il!W@lT)X*v%j-#8aQmRUYOV5ebByn{anb5pAr%jolD;` z@LPw%>w0~CJ!+RBGGsHp+r0#n1LaHprcyQQ3Obbv)s?JjV zq3fH}Y3t(RqPn`mMdK{IU-#`sqx7FwM<>07FV`r5L0UC{{cEQ5|A1osXQ?#@xo0l=pkYKAEQ$^AJaJKYkffx{j zDg~A#2|Re|lnj{iZ*V%8mzBh|b-p~4U^(y|MHS!3NGUCiR*w`58ytSEM3(Kl zTE}1_YU<*xB<4&Y@Ylll`>`_(n9Rk+DZn_bs8T&VGg}N)hhFRV5mvXAA*nxK?rvXT(h2K+ zHNY%f3Lw7!z~v)bgwN=FoShxB58a>w6NU zdId64-Q*Oo>sf-4H~&Lk_DrW^Vi%X+5#h$`*>hc0AzzA_VU3-QO`J?V$_BrISo;{{ zH@i^a<%c$wE%K11_ON~CVtI1%i@oYJrM1CF!Far?s;;i8s_)ju(b2ShcC7AqiSEFJ z6aj$ih+ax`XQasD7@!vyKuStl9bFgg(hDLkuw5d9P^{HEnQI2pbD~?Kj5^z1dKSg%tprAxD zHim$J_**je%!HepJ3w&I5e!ixk%^3pd`EufBMe1aS!hp)14EhHh(1$%q!- zt4Z@@PB#o_3QDR5<>ZmmYl1!P5TLhUDXilMiZDX6<%<6l!`SfRz`W|xl;0~1=Q-cDLga;v$?0QbKZ4l9LS6_`3JTUMSh<2K(y6?G_p-zM9yG ztjrVXmU0C;w88Cx3CaqXS5f^Q4^HhVyv9hAvZmPL?DU|C zF#+0Nn;lXDMfKFUMH35A1WEj65uSzN<2 zWk(1#1=3$pti3$02&@~~+8(bD5>(byIUMpQr-EEwZWqBpU}+!G`)vJx>Hu#F5J{B| z_>LhK>G?Ft#X3#4T9OLg0g?5~ z>vREo^O4aMc4UwH=`!<-XThom=1>wq*kKnDs(AZMQ@_B)e2} z1aKv#YuU7=%7yr?49_H#mXs1oex~U59uK?E%2>Fd=&1q+e z<+l&^JN5jziSf@Ly`O5>*iW1EaRsrvmGso%>Ct}kvX-)oLANe0vdsFPZAvNCy&uuK zx^*rHi^5Wl&E-&;-*m|6bx+AB)u`~UBK>vB52dw@7WzqIcB@T)g)y`?f=lX#X^ zN9u*pHs2#Jn&(^t1S7^wOcii`-`nbtHft&?8M|bVGt8`v_^@AQFcE4LM7A zdGY8Z8ZITNDD+gSu3$Wd;(&MnvA(x1_Z5n>v0|v^V!e5Oeqm>Khbap8ulV-!n}3II zkw|3!Gn%%*gwKVT*KpSS`Ru9UM_HX#fU52ZGEjqx3 zR6d}6vvY9jCLhDgCnY4sY6f^yQ~n;9pulYC!lBFqL9l^&tD4j z6EH0+swUs;wE5&HDl7KLU=Tw$ogCNu3R$*SEzJM$MxnYibTrCiUfC!l=WKhV~6n?6abIH*crLsAJu)P2M>VVG%Fn)0|P94Y@YyH!ZMn@i?#9bJ_uJWkgGgC<9xr*$?FUJr4y&(2HbZND~pb0yf#K?uX}Lk^;9 z3!0H{#W(!T{&RpEBSs3Gvrgk>n&w948Z`dST9IRIn}CmPH0hors$U3U+B4i^Y3tWF zJF@(lK2t9?tNCB|yBY8ZNP*q4F=@vK2Z19pQO!eX{t1dwj$^+LmDE)f8Hnv1*Vf55{AAO$t}sHrs_$!M}S z&)@Eci+^@iS5w>B*#b~k*`KB**vQOk|GtcpT=iKqlX=sSs#h{Y*HHoN;?>4 zdeKwJ=NH_`Bx{&Z7=Ax-6g7Z4JiC3!{DFeWesgU$XIouGdZ4|#ZQTe6QSi0W;Tj(V z4Yvc%Ly1g(USC{YpRJ$W&M09@NQk?}@i+1Te4^8G3rJR*zvsjcZ+Jyla=r%^KLe=i zVY7#T{(mhZ7P*QE<5j|>J6deUObVx?(XQ<3A6Kb%t*&N1&WQ@Th;bp2(36+fn|v!l zUEKW6Tfyem_G%0-gAp0+eYJcy3Gt627_aP?Iei~7mJOiT&aSR10LFTx_J>c$lE5H| zlYZk^H&~R*8Il-=4D=28`W4(~O+)U?yBS6%agBiC$^BDdl?o>x!;h}X^XU*YfG-Bh zeSwn}5NAt_r{4!aBiPf38t)E51V-3kb}1i&r=n85xeYUxGPPd~^j5%fP}iy=x7G z2v(59H%G$eZXfKd0Mh~wyH<*ItKViHnO{c^(gsNY}DBXs{ar8R5g@<6=ts)vtA$F%XP4hqJU&J9kp2!os2gibVer zV0|rC1{H?c-_u)_O~u5;1-Nq(ga1)PtZL%Y*vMGP2tWYs(_1D6j`^wK>f++zx$$!K z=?GImj0M0-{{2N4~ z>>fH_z650AY|vPX68?LvDlV3dq4eU9z0f5WxU+X$@}3Box!mN}u}|f6yIvNpsY=Jf zn{gtKL06JBg1|pqm?^fllpGx~kS}b_5$eA>43GTph}E}nJGz2}xH>29R#x;6q$DJp z$;pfPt$=>L#k%T#zVZ>CdaK@TYx|fi^&70kWmxtBU1qdVUS0>kA31DP;{4>2uPkN@ z%b(1&lyRZ|L@$gL+i)VbHl$V3B&)50`Oo#jppK4LUg+Xbmdpl#@wy?(A?R0Kb+rOz ztU2b73jg;CK!FyWk|vN$hXD6_<>#zA;PKPes)aM&+~^IXd^OWMl+Z7)kew5kaqLcOyp zJw0``3XC^PiqTklK<8^JQEva6-(K3Jw**1&0A7v5_UBr(sJ^WFLXi6X?y5t{LNc{} ze}vlsXq6b8W*}D)lb+$G?aGZK`AACV#twdQB z^?E%J)Axi@XBR9I?LAX_7MK9NJ?AJp!lO2gAy<%D_x4iP7!Y?SA;+ON$WI z^MUY#I~>#+4I5hlF6v2IG}+(Efftcaf>j?EI)8q#uw1#im1c{;!qp(FtNqDDR8%98 zZN#>dl92$z%Dl6yUdySj!9v7?z!vpE$NR6aQ>YJ+14HBgmIG%MnuDo|q9}=oh{*a_ zPpDQ7tXmr}VJov<41qj?rsyAryEgmj*${Rkc#q#y`oCay&lH`GF0cEg^@sLyFg{42 zGui8z85>(QKtNtwJwMsV@^`wdCX^QSiN$vJ`gK-+|D3Wj0K+!1k1oA$l3o zOqe+z2!!5$Hd@nYY-p-d2c|wGtfiy9>gUfS3xH{Oa4Mgl``Kll6hd0_9e8e)O0&|> z2x#h%=7>uXAD7p%T#3=A&+wrbR1UW!DlxSzVoY?wcj)BP!|lIT)Wf zG@)E>lZ-974Nx-U(Bu4+m>xV|+#gYFSbUeKuIKpg3JCt5R7LVN`Y)X7?*gQv;$|E+ zn`!CSQT-`Q7HYC$u+6VU;GaU>Tmw0Xg%kfosD4+&+f&7YWvt_DZZatfP&-29C8qa9 zgsD``=KT0B6ALmByiqc8UR%|fg9Q!x3E2PY&|&OgKDgiDr*=$S=YW(TPnF9)$Hb)VMn*tik~yEH09}HgPKM)p)ug-(J^hl7VJ_K-2ZU&0NkMQ1jWB}! z@}d>gY9lQU&N~#rfVPpbk+)c1S zoN0`u4HZ-Q$1hU~BD)@=pyZUIV{j4@UsdL6(q0sF{0}UyiSGYsam96~N>QW=!-UJ1 zPRScze~XEMx6qz38f5(%G3DpPHA{%_IgmL3qK74Xk^Tzu{8#DkfYbky!4(zg<^4Lm zNkqVYT!+kuRf0*W?OH}^sC*FKZ?w_4ozBns+r5CLm@HhVD3!PG>iK#W@5Y^PgqKl` z(1iF;9@om%y2071*5~0R_O(SK^CDSFBFoJ9c)ELqxluYtPp%yK_?7thcwo;bG@WVo zr|rutyQ>?U8XuZYJ7*7%M;WWYSZMnU%qG_-4xty+2PuHypA^BW_%_8z zTjfA$YH8_V3YiKBpKsKVdP%N-xoy$a@XtejoXHm>Ag zc-lKTtu{Hp+rqhqToqIdwc+1X(-#T37iz(7*%FmTd!PH-d;}Bt)1x^G-qBvU*@F83 z>dH!d^tb5u2kEBLGv3VtY68AQ0)PD3bw1M?7a|+X<1@soSI^H~9a5iPy?O=xav6ts ZC2Ke)_%vVd2K?mJr;pOYWr8}s{~y2oZ`J?+ literal 76733 zcmZ5{1yodD^zP6d0+P~5NvFclp&%t7sdRUDBi#r{r+_p#boWSu)X*`~Al>yY{{H{B z-g@V*b)4lI?wz~OIeYK#+ux2*QF@DmNsb8ufpFyBNvncDNOT|&LL)jN@IL~Iy(l0A z5J*m1Lfs>C{~qHL#cY#kY!2Co1W3FWaVYx{m>kP*G2Fr|E z#oq!=2W|Ima?ZCca^KFD&Flq=<-Wp{UQ|VVBjs|l;X-Y#3GOVH445Gsxf3fu;GVBW zCP05Z7M9uS=dwAR<{m3+o_(6_jIV9MOp1gE60xHTD6;Qf>8+=9CCoM+iz1T17YYDd z$Ic0CHLWr2*RW02*gm#kjMX%;_Y-z-+1LqbdZbF?iQ_3uB9Vr!fvJpot~7CsM1$)C zt99fO&!$-8@tb_=_b?C;*7O~O)!R=*+wByO`E7)%%(Fc>ZR~bm;i+vSj8Tb$@K3J! z6u%OgdcUgcK5R`KR#7=PCm)zDyNimudWgI_A!x`x?aBFU4038jGLz$)SiML@ij(!Xc7@Cc zIfnvx>JXYuLLYVOqJni~m5Sr9tVn9xfdk-99Pkl|7*2+a!-*io5gVSzY*@$BQI3ef zo;-nwAilNKNGn5c#>>MKjo(WOZ>j3UhC!y39H_?RKOoBq9iuY|L}79I`Q07y@-F4X zfS{6ZjwB2qy??;a&3<(hF0&!5h0D|mgYb_hy}+*o0aw5tLXEh1JOeLs@0MEWkteKc zJt%+TLDZn8Qy}1Vp8+j;tRX6dtlvyRoK0(ZDV1GWAbKNcaAV(FoQ+5aJqR^G^~;3O zv(!WJXhxS@e>Y#;AWj+3D!j4F#ECHDIWjzh_bVc&M=Va!lf`0jT2~D*MYvq?#wS03QWy!T#;c|2|O7i{2n$$S%(mTxMO))VKL7B;?R~?`N6G zURlBx`CuOQzcFxnM4~Un3>3%2QEzs_(eJJt{bWMc9xr`a>4G*-1E(XkDqyo)Dit%W zP1v+@G~1A{scvzTOt4@9<>wjxgZsmU~;AfgJLRCPQw*Hw&^)UxZ=_d@%Qxv)V7F9%{rM>*T? zPW}oXCdn5P$sm9@J~z9xH{Vo#Zuzdt0t%oBFM!6@)itctIWxt=A0MvZ;YUZ53k^9O zoYXH~u+7!VF$4QVK^DGN@D><{O}BroXRG|LH;%veekm_6R~EY`qIqZLb&&GkOc zIBF8i5ch-mLe&y4Z*S+H5KT7ySoqa9XdFd!f6HO>D}}u6orC1C&}GNl1M{&g;m>+X z!mMk9!)6R~(}Odz|9Mv0THcL7*F^oNkWK1TTcpL0sSL`mIaT+MnfA z`L7rEMkZTiK-4APe^05~LpkgSo1MaMYD)Q80c(QTMXkT!eSj>_zN$Zm4ZAPfvc4jG zJX}El1xcaJN$s!JHOM{OdRohu9=G34%xLW2@Hdd~M+kE0x8L=B(>Ee@gB~;#GcfA4 z`B82itFwMIH#fJjDVHrE`@I8F3=0dhUu-;v)ffj9Y4?|8BB9}qgBEo@VS}zthBz%j z(vq|i_&L;I*zftldZ`2z3oddo5eM|Ztba}h zdCf~lv-`MgW+~kneoT{;mf@6_4t$VqEUCI%R0t#iLjkX*RvYO3&#^IrJ3lJ=Y?y1B z9)HCou$V?D8b=B|>hxG}TcDRGym9B4|Le3;QTJ1(pYM zRUHl}K)?A5KR+!!J=J@7%1-YbQMp{Fwbh^V#pWFOIv z`brLZcAOr^L90H;PkAcnS3V(tTaP$;o-rM&`7&z1rFf>(1}-L{slZo#{-# z#}8Tff5%{KmAeAEKKKj9n2W$hH@g>oBPQeBD#cCqlo*T=9%hXw`mXI*9*xQeGl_Ds zM&+o-gUWEZLa5NcpfPGf0=j6tHmElbgk}BDbHcl5?q;dV+m~%+>6pIVJ?eqbJ9l@F zBIV13mRXP}2@pM6Yqr9sy-r1m-;U(>(4RdXTW_Z2QV=|@Jl1?~a)E5rD_%%Qg&`?2 zvVs?lOS0P%5)x`WCg_kQDnEBD!(i`_@F|~9i*b$5%r`Uesxc;$N6RKMNJ&cG+};B7 zyF|W)P!^Hvlr~Ha-EPO$^4cUyzVbGgo$e4Ux`geFl`B-mNx)u{NNDdeeE5G%10mt$ z+NU0?uTA(;t5~@A7ZKvObepcYAxw zQpV3i7qkI~uTNQ>)34lf;Pr&o&dOk(fl9JJvK2g60 z5d$_qH$|ti2YhxPd``?ezZ^TQUQkfrVu5Z;3ksbiU~JD&Wd_O7+~0#_Dil5hXrCl%i$mt)}%W3oSkdTmoS)@VH= zKVCtZyf0;?rB~}kFawnj;&0ihwxCmNCF)oC+ ztWc*%*)K9xF#t7RQ19(o!5wF8+TVxJrte9_m?6}6LkB1Lcacs!!ro`sk7lFGh0l_5 zit_UrcMGO_qkbH=eaX4~T4%F7`HzSMbkB`R}jm;R@Ojt7v zpb@lN{53e3D9;cb+XLTm5sHTLrYTUpcm4oLmFOACn3Irl`#5gW$$1BqWQJj2^xg6{Ba7P zajdCX@jm+?kg|7_K%Ye(Q&pq=PFj6RWaTPUzT}Trb#u`sj6~n$dvTS1Lu?l78y+FA z?IS~3DG2*^{I?7WYLZ#N(@jpQ>8Lv0^+u#ek<3JZPbfKSE@KnytlVEjHK?lNvuVWeB*eCpjwEtn{Vu} z>R>k^Cjq)_j4TrsM#2!&Wsw470gKfX@!S3jyhc$BK0LGEo9mmc zDm3F1F-mtZg&sL29v<-UTEb>Z%KZCpPR>qFKOt$)J|%+HK2Fb-7|wRr=~Oh+E?qS@ z8atj%LeiQWPr_|OKF)#{lgbBjYAYzq3S(xfUOJ1i3V90zUM4)R;z=ke8SYIUB7ji4Jt+xM;xh z=Je;dN_D_094;IX2e$-2mGO)J@7p665E`dNM5{-S)?^wAFtYS3)O^giJIi zWzZtefy%9(nVdT4HWX%gJPQ8-^Kx!3EWdiu<-E`4m;BdUetvpI@a0 zb*{LZ%}Nr<;P;x;G`1aLu?two-q|1@IUd5{9uK0##qp!6*e|S=VzyxW_Zj4e<_i}c z8Ap1sz&M#;fwN#`TTl;|>~yugEE@Mp*I&7JS{3_Rc?vfIw!#dInBoYg%`Ek5(l7lYl1Dt-d5e^@p%#mY&5YSo~A{0Zj z4TF_;R-zG0bL;B$I*sTJ6AOmi4PQOwo$v5#zsSuWc-YL4Zdu!;2y*LE89@w&iIM+} z-k|UiBl^^$=fxYwHYrTY6R3h~fh2dmbNxoyuLX|gfUFimHO<$dwu&-V6G+mUWBc@> zi|;=IM;st=0#<#gx?ybSzzC-a-$+-k(dW{zxiHEL3ZZT=v?1>8j(HxulNnj)9RB7s z;Zlul9=q=c3Z2_XRBCY8;Uz*7*sEiDQvSn{2@XZ$Ei>^rteM}u& ziu2eXQ`19^J$zm;$t99mVxPe|1LBzQ;*jhY(GdO{Ygj6}cvb|8QQCr767e;nDYop) zp%==#d3<4z@xRwB^R?dDx6l#zqnI+&#>qpo)(o&-RCe)9R7HX3pj6cVSu!phsz$V= zVc>N(d6?4~%Qa*te}>q`#GxvEk4}HNda8YGtwUQ0oT`5V3lB_$!e`5fJ{y<%KoSRq zz{D~}qgar8vAdLq7)G72)0n6#(FFF6^|0P#DV6JjjNoQGDRz~o5L2`a>TR+=P}{QU zL8xPr_DA(9o~mhbuiIOsHoQKxCpWcj9_nJ$^RA=WWa1y$lU!>GAiL`e+n#J zOyW&8-LM&$aX7G;M7BINj@x9Bs6T$>U`mQ%YR?3M0ujBCIBi zkV%G)o*qifn^jQj|6e+ zLa0$elTF)!Y0<|glDGji`llJiNQSpF%gmA89^lz~<%I2R3q}FIF5d%l##;PJILg0e zsiC3ibZnd0mNvqA?eoPrm`ro0bNGjgv-KurrlH_o6&>V!o)?Eds4u!*^m@nD`fpXU z()+^NcgPMt`H7PVibz4?(Xu8kQ+S0E;=`!@{sWAvIaB3xAA@~-Ui>o)GCG**IWg+h zT>z0?O>S^-z?=>VqET0gHlz&IvylgIlHAJpcQvLG)k>+rQuA&Hf-Y)ODc!EswMgd| zunu=JF9JYN5`}lad)4f~0@ieOCECP8Bh*nx+=8yN%Ep^!N5k7+N@)R8-Ezgt?~IR& z3nNYJtZuJm_G<|aQG4v8i5bLv@9oVIz5cF0hzu&%Z@>jPtp6DZDAgV&1{;&^X*$-v z;uuqG@-!eHcWouUy3xZ0qo<3|i5ydrgBFDeM45}$5%59FSQLwy=Ib?xcwvw$TMy)U z=NY_!9b?<2=5ysSvBN5!Z}Y1wUOZMay_#WE)O=@>$qDj>){&7Q5J11!+1htlKBTDx zgUdwUe`AwVR6N+(Y4SME@3L_D>YqJ|FRda?^s3=3E>ZTeRce?9A7Ci~dQEvkwg`Yd zy;x7djY>-5nHuo2k;hgc-;A z-s}`t*xLG?GLiVP64uPY^(lM9U~G`S-xCwG3Gz`B|a)uD)gJ&8FyYjtdO zb#*^|`eZ3L0L}4P7kyN;SuK$|5REhM6{{AqvaT=DEwS|5b+R^ zg9u`HJY||P0D+<(j-Ypgm=CjC`<~c%z7z+$1XQAbO5!6JxsT}%xJQ2d(rPioiG>r% z#O$5}MAiFu@3hp^T~<%WfFBzrXaj(ys*17&&)b6!A3hBGIfGo>+^AlAaCz_i-Wd9Z zhl4Y)apJr+Iz2P3IL9QJU~xNBvGM%5l#~>4kk$tk8hrG2u-}WV)5FCkLeSA+oERgz zB<;7ACTqZ3Na0IY&HB5o26P!T?p3y&6%lys| zJfVVtUax{5)#FxmCmyXSDJcMjH(>HJ!ip3-$M<3`S7Wbac)#%dbIiU0wc;XGYT{aI zou!-ey~kt!$II=@c0oqb>#~=(RiTwN+UP;m^%WT#eek$yeAn}Hki@!?v6Q4u!3kXE z)$_>j-x@=q)T5z(^r8PqZpQ%OX?SH@x}4k#&! zWaJnK>U5TV|2E=D;dyavZmD{7ypI*1+2mDlHTYum4r^jlEAJnick;uNpn~f>d2seH>Uic=Mf0iu%-5UD9)d~C{R5h;sva)*xPj$7i$8=_AoVPhL?cPatI2!YM>rQD zvS83P3S}ZMxVTb{__vEOZ047N%}lG>t(gQ0cD2xJ%4z)4h=-3jBEgX{QZ{7!(E*FoX?eFf0e>1=DC|RVX%7a@oa6)M#*H}z4*32CQkGrb+7xC zW~GnkI&wdJsonUkld7t|e}@`BF6Enb9i>m77K;|_2qUF6IrOtNge8i+WiCW#BEB*~ zo3$To2mJ2cXgKO;YqW9izGNk2`^|oKA9x-KYizXlgHu@5k&oBvRJ8An3K!%TB+G9c zG_17y^kHK60Pk|RSogj}os}rEu53yWoW?M`773FmUXtF@b3y*Lf?LaXOFdxOVd zgdyzWU_rC|(*X4Nd~epCgWc4N)$BB1CQ7494+jL`!NH*s%9)+kH})rSb}@_D!I2K? zU*6h{$vqg>SW~GFf2&gX^XldL71u_(%zBaovJCOPkMq&u37p;}*GwHiT~W7mI4sDPic}QR|`{8=jb2nSR(^H*a9oC>~yryTfMCme>rq_C`@wylg;8OJDiCIsF18GDiAN!8#N&z z@a!VDO+;EYH0yiuYl2RZPAN+85m5M0Em0rs-+_=L35tPu(xQuYy1FEa_IO{vo2*sn zZ*6JjT*1mf6$j|ZFOq5MUTRgt)|PQ3e*;y=%JddtgNbz>&>0X;(I;n92?PCv8&Z!1?s(C47al*d(Jk#5gyGY zCoGCPJLrw;o<>bp7)zZXhuAr+JR)h*5y&t1*3MXS1IgY7~+45p>1IeSdV zm^93}X*mwF#y)<3JxYVovKa;f z0|S%(mqRstuL|_~aKoanO-*8e(8TGE~dx zzS(urUn#H6@wxLCKeV;?o7N*mI&Q%(@#7?s#6+S*bm!!|q9(S*L z=ZA-4KKUKCUH&8)gC9#uO7fA9{5oS~hyDU%*=9seGH}MK++ASpK`=BJ5AEHV(0qhY!Kd)j1DM#rJHo z&0_R^F_4VKD;e!_g*RSUxpohBm1A0@R#unZ@}RF z;KkL%i_6R6;$l2(+(>B!gEpn&uVJA8k5}Ep2>ea*Os&p=X0b9Ma*fq=X&*oj&Cc?E z<_k7dDT|4VfMC&7rnb^vPoEJ2k_6-zGoBucW~+)zZ(|NYFM0zKv9ZfhtCDB z&`BnYt!kC#xe8H%*Stc}>pYe4{2vnrFUZKA)!3t=14SWjY$+SxT*1fdF^6PUZk7_0 z$)cH=nVLRUFqlziVL|`qc7-X373_16mc0MLMY1E$(wiY=Qk=!K?&AEM@{QM&AA(m~ z_DGY(1YMgRH6`W9qUK?W+Me({ev=4PPkrTsY*Ko8$($&^8#Wy}of10aEKZ)}ZQAWy z7Lmz-ey{dSg2w2*51XU3AbbgvUk-|4>nSF-AESEam^i#!6)4z5dYJp8=F(=JztcLM zs{U<*6gv2FhQBE@QhK|TnMca6Z_5B`B@tN1Ii#AY<Vxwb^_Djk5iHQ$HGSiNZnpOIm(7rt8!PZP=DMf~N zsvO@MYU>A0cmh={>U4~ajayn;#Kd}~Z@wj`05ye`M#tcKjCZnMO9~343e{u9io1Yd zHCyrd{la=t@h3Fpl0KseTpY`=&@-<#X}XCW7X^M7A#Z+0PQ&%Ly|XUl?hf}i=Xj!5 zxxuj;vuC{}YH@yt-jA0Zk5M$}6z3aNiJuR1tCrZ%63jh3JXFdQKiB?fD?r*f=J`F@ zV!z0hrg(mS4y5|9Z@roOHU$IHNGgq1A8l;*nQkx6)jy_cDvrwctepV|Gz)_SKN_oa#p${D+PxL7s9a6Vha!7};h7sJBjQos%G ziHgOqz}j9(H^MTCk1I9Y-(qG1$H}Vhj}6x8_kKUpd!EhXw6n)9*VMll^ZwK14tYW3uXu;b76YhR{=XEkG{U?wcXmeLeH=8@WsVXZmH3>BeN6~yga;6KlB&x zA^5#b&RYVu^DaZ|4d@}=^lxK5u7eI(_{p(JW8-3D0nts7We^R)l$D|;$!&>jf!w=y z2Eso3%D;d8dS@gVB%)N-P&H;EFtl;9Q>H!f;lq9|P^v92H|TH+o*&ua9Wx<|1metk@Z~~4tOa#v<>5lmd$`5lmCgH_&3XNg zTt*U7qfm-kHp^Ac!7FDqx91+~MR79qEDI?-T`cMKi6)>|KBs(pi{BeVA^3Z8^73FI?8GDE{d;;kI*aro=G|Wj^z8ek_><0` zj&IJ7kB{jR&}BZ`7otO3%6Le&9CAPstOS2dtS$Wr>2kA{)c5_b6t*uQ>a~*pgibn#uq>*8AX`Vr&HTu~$MYS~rUM@ut$f15e}qzsZ{vJ&K$P7T!+;%W^+?!I zAZq#ohz=u2tHCY{$fG5vY}eFY=Wr6C%gV|Ipn^Iz^?hFwVL~N;GNMH08?U_j zdlrd;B%%J}Ch%joR!C)0+vx!>5rUw3NUGG4EI)R2I=XQaNK)ntnuC;-FdK1gRa^m~ z__<5=q2(0}&o4eTzS?#{phHIfO;N{PAE%_06yU9Q2= zTXE#rO0B^8$BW~ILh(b7BQddfd3iq&wK>B3YdG71YO&KVc(CtR5n^D&Ac9z`m~+k< z`TZw%O5GnIO}msLrXK+&-$*FHn_LoEa^z<6bj*IC-qt4T((;{rc6?UydFTD5E=y@! z+mdbQ)s;sy^ma?vevZTdi|&c1Hd^bm@mw$Fa%LyhV=@5z!jJIqPD~wQtS5qGC`c6o zQheO$cwA`zvJbn_@~imh=5{etp_`^%_!pm?;OFC{avJ~ZA|25(GevFztfEs}St@`o z=}qc_e=sx*+_ltZ+g_!qu5d)DahNLi1pNS1R=~J=ITM5MG8wTZbq+qq`Rtev2OW_V zo0-s*VpXfzld4Sc&<5Y+au!L9IAi12Sh7TqC3d^Q1a9Xlx`Mfq;-BwqX6mNc0XOI$ z<(}x&H$TiAFZi&n8V`&;^<)R|$^<9=vB^PUCH`RX+x~^w8TUar&Y^k6MG#)8$jf5L z?02T+U}r#i@Cy}TRfNyR{}?Dm22g<@7oT$})#g8pq(9?FFBN^5t=*Iz$E(oXSp`!G9}3yQo!$LJi>; zuK9$;WM)m(=hQ{YNjiWG&@<07HuxqTHm7M`tJ%*?Qv&SPKbCC`t@A%t`NO9n>nmE^KgdUKVL!E1ULtXyPAT)M8nTfiBTM50U&9*C+mL%=ZZ}^cf=}%j z&21#SW$(GN+TJ2{ak}Vset^J4aLj8mHrBYgPfPIxL&_S^sv?mz?Zck9dvvS+_o*O& zX@8$Q`W0uoMLVu8J($%1|KI>PAy0RgvLPl!sWGVZH@SLaa`wnZ#-VUt!@^2Xo8qHVWy=pzVKxSfoe!jnm z@xq}{Ra;Bz-G`*M4A+}8%jsCkmDVjPka)4In)7kK3UkuCLaRC<5%!o7fUgcjLep88 z?`O3SqsVZx2Z~6%iNEc3kwCE&e_L|;xzvl{pBoKg!ugh+@z{xxuqPiM+sF3yiP~;) z{r;AFszsPiv+6Jty%0ry_q(+@UX{5w=Wl~7G<9&$L!`~Z{o5qjvtr2YR4-%c&>IIG zsw0&ksR0gmfveR&I7)5ByRc*b8x@3N`l5rHo%)#7cGD+&sRisoJXoan(2*1Ak75-<8QwRqg?&|8{8~l;>t*cOPQ42cjAj%X=2RE^Ac5z6|O)d z2$oOpSn}=tSo)e;hb8em8z>>N*ozc&^qkG{4_N{qUYmkuvaLJaRDP?3>+kLV85%SOEELE z`}mqGQc|HPB7Hd4h`WtX>lu+%%H%UU<0YRD{6z}_J}rp<0tV4f z3mF5vj&aUa zW&?W76s0KB>=w}f>Y@U!m>Z>0G-J(77>jUH<8A5gdBHGqGMjre}yyz?m`iq ziJC`V z$8ZDf73=BJj{w&ZAkO^fgh37y1<~x10eXjXem?fNqrBpG)Ri3(cBC zZ+K*&x7~l6Wf?4cRVd`c@ zhH+7>|7Iw>Z=PNh&ezKR@k@DXrq+6l&)Mq@0w^>FVC+lcgcZm6Xu>00A^Qv<0ODa_ zVCdQ5fxk~9rfYYHcyqR zu@D|*oZhA7Y%uf(IFxs63p60GL!9_2;O`!^1g$lUAmGBzwtr=M;Va2@w!o@HTF{ZS z&vbL&M8g&$assDvl4-vIU3)3>^XJMoUW9co-`CN$Jx(${=7!7)k@{kG0Ionex#4hXpdmu6o#$gd6?vk1qe>RJTdw!_eD-)l) zm4?#B0q_B5xxX8F0LxjYA{ZZ=TaTCYmB6g=5Ta4=o1T`a5vM!ta~*7DdHKZfK3CUs zHfjorjGUaYc4s6|X<6CA{Ct2gGg0JpnZ_DCk?P3T>A?QUx z0_Z%VGy)=mS2b_JUunwqlqW@@W&@Iv$Uc4&eh;0UAn{Zx0+2WvF(V#MMOm5gBfx24 zgK78}qzhH~KegM`#V7+ThK(1FhK3F+aa37Z*@lzn+~x=x9_8--zKgD;tZaXCowc=< z6*6Mi47)BarO?#O@{>x<>wNd-<|cIL8wBzb;Bg-SM!}=wSsh{OP@QUlrv0zeDL2$U z&%tgYYrWQKh1gHOb6=0S1UD&^C5Wpa_jsy-=M-PTxmbx-&)XWk)fYTgTuIy``-c0? z-js+U2u0Z|Il3rU|ImIof^a4~SW$#O&f*2ve1f!L3Ly6)qIG);D8CfIB0)bnJ>76> zlFZwtCape|PcCg}xRi9Q+xsvuFz`B7kY6BC<5lhsAUi9V(E9vYk>HDnvbu?hg~j&6 z{cV_wkoe^1r6mRQiV6+YG&cU1g%xd|gKx?!_UBCroYEQ8}+L z=UV=@FvQ(Z^!Vyrog-yo8MdN^*YTGwuf~P2q4|#Q=3yW(hyTVZa0w0JIm#F(jsJZr+5VM&{=m8)2jN{4RVCv~r-KqP-d%0_Hgab$L@m zLt2>+Iu*nqnq>J9NyEz+3Z67J5%!9rBH0QJW9H_q`^y7!b!H;;AbcCmp0d4l4}%xz zpzkvVJJiXh{iO3Y$$6H}E8DPKT6!bx?@@GsWBkpKJZ<#Sr9XouXc@Lhu7G_eqzFdz z5oO0$v8wx6(z~}e5r|;G7b$w38W}5KJ12=1f-X^gJ@ChbhxVDZb)SuUFY8xY(om~9 zzPPs`A$dZSMuc~ap3kznv3AQlCg)!gV)p=V4`A}Q=U#~nO`<@b1frWmpbn@lkw6Rt3OZ8@`Q?#OQH82Iee*i?t8tOUcG$q^(+eq^D1 zEH^-@$V-o{o-VP&+j-+5cQ2l2vxyFWzApX5LL-5XM@)pFW>;xh)R`%p`C5z?ua6FQuPMo`7VAG>T_fLutYl1$^!GE2%?%{P91Atrdy;e+;_~7}miEvR`6vH6v|x_;aqj=WT52h?xL!B+Wo=ZS75H zXbrW%+rA&W!)cKxPW77KCwDZKoubs~BtYAL|B8C;lY(+PI-XD^(2t-XIFz*;49Zny z0XiEJ=--;~kU8t|5_(e+Qhrudm7F|#P!=;?U-$K#{VVL&bOyXNiIkys#&Jdm~Nl7mokMah8LJo)GQ=fei%LzLVxq zf@{tm%`c4S(b6%YWxfe^bvqf$;WjeSL>Ut!L1Yk z4k)lJ!c0DmIF;&K{P<8lCN(iqiheN|ChKJ-kdBHWoPrh}ZYnE5<8I=Uc6LKvf0}S}+T-b0~{W&#c8M%wutRi+| ztXRX>Im*EC4Kg^5uXlClax+EvcB;chOdc>@rU^7G;9_Gl6o1#Ug_gqMLxY366y$?L zLuu((TgL}$+NQ~D%r0|0mS-65D7<#zU$C!RX28OX2 zA3?k`a!8%lc0Y-p=YL+Zu#dY+3f5q)H;NH~Pg`cOXPOgz_yN#TddEw-8*S z2^3X-IosePgKP7sgBI{=u@2Grv3ibwj%4WS=$4h1a=lPik^_q9fc{{=cz}gP0o_N9 zu!ZXksBpeZ1pE8}C`>syIe-VJr>7r=>C4NzxVQlR>BGcCUS6IJ12N&y%jBYJ+9Ltk zs@coM#)SCzafj=l(sXjl%J$U*W+*5q6%`fwzUR~Gm3`-g>95m!!H^K}Q&~pOG zSINItbNRV3tgT&tho}Ax8u&tv4J&+qHM>7pm;3YQ$l7Y( zbhH>&PNL^?973XxeWtDFvlepaV?X;oxJLVHznP=0(&F>zxgaa*Gte36XQvV5Cj0*TU{U$79yCm6r!l)G+GrgtSKt7#7W~`9=#f6;BFh zP{7>(Ky1aB7{~t~V%KZPdjrjIr0Y8;PRa7;Bi?@zKrU8%B-U16B=e+MKbB1cX5E*= z2zu?XDqglw*Rna^Ykawm4}fa_fd31sqj~-pRQo@owT>!Hv>dK(jiw7GYiU#!g-CX- zNd+*lmtnDlNZ#%ES-V7+lt<4_M!C6|*=*Do{MI(1NmW)F0D1=M$8W@hpN-GVT+HuP4Vmz~lH(En>3B2&rFM3r3l0u` z^x0!~YhMEZbe5RsiHWIcAyA3gJ15-xO#4e3{jRaun2U6I0rD15GZFP=N7d@A_}w1J zQTq4$qoJmP+-gYc*uki>vfusPX6%oeg<9LQRhjX#aBFaOF#o~lXwcjY z>%D%4X4Wc~hq>@A5i58u5JB&92v0y$F)is5g-amn{JS?lRuKdqi1GrCV%$w^mPtl= zbpa>M=zH*q3lmqo-ZBRlmTN4z-(z%j<~BZJXHz| zKwcO#O!Ks4CZqMp7f=@;K`FPPp5uxFlI~aOJ;{idKuhT=3f^K5p*@;hK0SSHpQLak z!bQ@3MrKEZBJZSVLc+7v*OX%!^NMRh#55NEjp5$@zU&P6f~JGEJw1ZG89zmi&iJOr z$yDX;O(zkaxdMSuT3$~t@CfE$YYB;MkN-jF)?XOVk8bfX9#vUE@Y z^6^3Xcd|O^QVZxo5}yCsuf_2|Vj?&|xAU@r!@BJf+rmk9JNT!FNbH+d_qflgqn$T; zqn?d`qPacc}~w z4c&UY?Z}$%S5p>EOH13ibXGiV9yUA8c)Z(+*SH;*!keC&`m?#Ip|0_FspZnWkwbH1ix&F`YWA=_U6i*ORFdSL<27qqQ(lfOd!e zlK^hH%{x%I`r9~cDrv9@IsBS`QeeC_u0>iLU?VUbyd7*B7O5~G={;J4Q6f$VWuoif zkH#}!&u|wlKU=zm%M!B${7F1-tWD00NE2qy`-hh zgFtv`Y1ek^toJK(fk?KTi6xWP<@?dY!^fx9&f~SH=<*usOUBUY>6Z+QppVFy0klJ3Xm8;Y6 z`X&|}Ky$Jo_j8yultq3cf=Qq)Vm`fMK7QGMSLf=c7S@7UZ8q_(>oL`K1F;tG-^Kp*vkNPpi#~ ze{>7N^t3a|%FqJ(ukSWcP)bUGQhWiRS0axwQ&_4IOMHE?-WTQU?8F>d3U-#AoO`2D z`2BlpE4W0xbi#s_^;OJ84K}B`kd$>tUZHX(0k=$FNJV9lDM|L$tPI8 z>QYc+9RCJW=1FcCr%zV7_x+vq*~JTWx?EYsK$Ussyz!DfLS~vLR8CL72)SM4F(a+y z)|J3u5{zwwE0o~R<~RCxfdr>l%j5Rr5i0v}H8t@L}Mw-C%BjGHg&vY5^GYWY`d~X*ZE*QMfTHH%Wi>P#mNtZd7q@+5gAZ=)A9a!z5EjXmL>i~(A(H0T45RIs;Den8@2vQ~FZ z?rRM7zMp>kx#2IavIaT7ACbH(Fl(}l0WG#?c-Sz{LiuUnXXZ8kAI{zas>*c>8(kn<5hN4@0SOfW z>FzG2mF|{qkS-NaT0lTRQb6gF?v@gy8xiU5hC5NuKKq>W-+$b37Yv7cORjjo`Q#i` z3YCzRZLzJ&CIM!kH#e8tjh%x7zjZTvWhIx0gPDbeK~Y5|p6e2JYUxK~kUzJ2 zdTMI;r~2< zfUsD)$p`EmPt&Kn8jw)~XPf+S5Z+omPIqr>o1vG}(LJcQt%C70qkP0uQQIn4^{%P4 zxj}^*Dr7Ygh~>eY2ubp;uC84S3|uj>)}~w6{L-YN)6>&2F)^c}qrr7lc>mMe_J#)G zfRxYbbUYlARJe{)Wp^Uldi#3K%*^`D*!)u5CcUOS#Ua*A28Xc%k!tb2xI|8UuK;yN z5YuOjQCIHI<<|Yf{Onf)nbrRTaf?qyQbX7#{akwc@Ux&`#=u*7|ACUwiMklRNuf@fCie zF0X(75ss>N`*IGy@4S>@0bC*dln+YOksQyLkp9wWJUKbBiENhzEq-rj=TVyc$04Qj zZZdTFvADlX+-FZe52gMA+1R1_U8+N`8|9eTJM<7Y5YGjZ9#btXFC!tEo15h`cHj(H zc?jmH>-$v6k3Y$0SncdMJbQK%7uV-OB&FQ@HzQs;lZv*Qs_}sLmn8|x z=H|;%4Ao6sG&12o!JX!)90jVH2`d~k$j|FcSPO9qqh{wB7^psFWf2uV>(KmuRzTvZ zVf0@Bd-nNnfQ@rUlGRe>%<6Vks%O&U0^Y0WHyzh9ka5|9TbB((xrW?C{(-aY0B2(f zd^cqsKW-q=S1S$q@}-V2PCAZVPbCj6g!H4Zu=JP{Dwex=dg|j%vv0?4n&_koMKAc- zY3S+KHd{%JEw43JR8&yYP}kj-FLHmjdQfdc7hvR9M(}Ny2vUuYsc_3QD)lJp2RCZm z*1Ou<3EjpDa?a{JCB}HTHK+DY4hM-+*@+Pe28|#yYoYK7UNN;8>>uiieO!d#bpQFf z&4R+B7gZ?S)YKH7lvatmaO{G`og%{j37JDhWa)TA&38v9CsIDgYlxnysoL!jp8l&> zUs@eYNU!jxC}u#O7-U*uL*}9pivm^eRi&ELRG;+4(?m~hJfZdSAF_^91sxu(^!6;p zXQ!O@bRvGEr0aSy=H0|Uk0k#CXt$w0+at6y=dq27+eqkVoAa6>F7OYzGde_hv>_sh zovu~+rStwf@y1nNW^Dg2&88lFlLMMk|{l&I_Gp$GwtN5_$$U`iGY;WKVVV z_IWvJ|9F>SbLYk0w9+Y;R|B8#{t9TTdOp*erd7U3Rq7ihJYUJnq-0y3)32%S@Js2m zPbl%#W(s7Al}3J!Xw3Itp!r&W`a(5Fc`p?&@UK?J_uZwxQ20&U-jWuqkYJV0Cldw3 zu9tm(qj1b?si~>?{{g}uj4k}U_Iixfl_?Ncnq5iVUReO=H3Wvqs89Cm~ z9R)c$wL)?e8m>zPYC6NZ=f;po0itT{Gwtpd}bmdK(_sf&xX`{ zrpmfN4Ww>3XR?V&NfYDaJqk+p4x)8HsI?mX`~bkXqs!U_6z?z;{@(lp#uxE%M@ap5eFqCio!?*D9OU;Mq!=dQ~=mi*T46#&3MZS+DIU{7Fqq7tr$sKPztFh3xjFVqHL82y$ktu1?hsCgpX`Q!4nfeq%C$5A5+NRC}7K14ny2>tz{hT}s??F%zGhkn;JO zEV!DDzLg{^1C8<=2@XT5f_EIOO&j;_h1Z@ycL`cnUMmq{HrijAeMnh1k-T-D(0`&b zUUSvR`ssDPc0029vote zgY#k^nGzDKAZZn2b3D;OVZtBC{{D)H%{y6 zcX0STQiEHj5_bdGpgAKD{-WX0aWJ_OUB8~ZxMXH(x>@Y= zHv)foraz+eeRc0}d)~O1{=yf*f`pVPV8+P#EZSm%wuk_QXKwOZ{o=!?^{jOy8)kBC zNs0>T*^YHF9v2_{eAS36EX3?j`W6MnXB~Uv&^6i$FQ_ZlX^B{IHwll^t6x+gBgmdM z%6!nE>ZJ55fhInN{`2~-28^0kqr-z@p9`ZB+*IEbsD0{6Z9U`tlLKY3_Zn|^{!~Tz z%UgQUDn*iBKa2D`x(nJS{CC-_Jl+c*Tf=3(GmgQ&7X@nA9Ae_GMm;CGOSFrgt zb;>wM)|4Vc^{(w+a&V}>zikaZiKIvEDybP}WNZ2n5E}Jj+8t0unLHZ3Iu@TPt&&?)&eB#+c1hhv}wna5DOFyBe zo(nz16iO*E%3@F5kFUV1;isF+MV|UP+eaFraOVzc;`-;TcmV|zp#yhna>H!O`Fa6^ z=(X=QSYP*>+_)XvgP)_fMlQGVZ#!pyyxcj<$99n6srKG|p+QdjMdLG>Lrs<#S0^n- zjQopovfD%`OjvLn36^kZoEw~!4(O#PGnmTT6^hd4g1{LwJUo2z0@>jtdthL|($W%# z?K>48TDdzoojF5_&H-^6q9Gw#H~$k;w(P1_R&&zPaZo~%zP(+N;$tv~#=v_=NTv}6 zr>7M5nlL((^6Q%{vV7k0k7#_w@XRbL*4DR)H>uUG4F(l1*z^SmDcSM z(0CtdQOc-)D4kEz=TGCfBHWzpBW!eTGKhpEbKU0#+7lxIYy4urOLL_49|Sr3ErRB? z@uL%G`y;U8zn7zs=5~}G&wg;7-qnbYTSk+2<7V0Y-pK=`%@t0u>X0lo?Q;-$p+!o_ z(^FK`Cti^{641}y-d>60W#GOOFd?U?1kED3^z0uUO;E{D{bruVaZr+z17h0W-HnWl zY;JCKqrP*GrO3I_4?eG^rY1+Dl!BZbfdb0u13J3Z^>qQawfA7NEAZf<$3Ae-GVt27 z%7`Sa9=zABP;`w&4KeR~GfjLd0l$Dt3O7se@ZDkyqaZfOyUtWCt;nfA$V%uJMb|l# z_TI(b%wBpeWtPVq?L=l=>8E|zGtClu|G-g;f1Kev(YlQzBf+|>mXO&?Q_wL_njyX1 z`GAbxt0pJTcN-w(;a1OMU*T^8jJg7kGvs&o_O#1v>UZD#e2aU#F*Y{#k;b^(qerj5 znPk`4JU^uj{Pg7us7$BF`*0s1Au1{=W8^G-rJfe{zkLfA=^W(_Wik%Rk1#=JW>Rx7 zVEx9EYipgIo#^=Vib_gIh~L^SDCVontGiu%x0^#Q3#yC}C`E}WbUFwVA_7u(hFm+V z?|G+7eDHNCpBQ>_xi^Fb&?%UG!RMCNzo15BoT}v>WqiwxS9J^^% z$>z~|u7jU`$!!#p(#+7-VOqvMCXMS#(-Vih%{1E^H?wg4u!Ay_AIZXjc%-nfb(XlnYvZzh_E5blSSE~Y<{ZfQh`rTv_G!$>nLHb{G#4Q z0f_<<{zrq~Wl8VNF1lLAhK!lwE^Ji`pS0z|$vHkMQs3M-1hqi+zO)^EhT@Xz@k%C^ zpI%y8+8Or{YhH&iHdTOZiqW*YC(MN8^ZE>flx%@+wGQ0GHcna=vO~*1w#JtWw`KW1 zZ)am2y4UztR*N!z@@I^Xx*~|Zff;)5i--V*f!!y={txWaf@r-hB_>@=xT z>!})ehqRzbAB@7iQVhYw77-zj!eN$b``X?MLCjzZHZ~

|4R&2t+_i$JTZ1s^Ps`URY;ac!UtqfArBT8s%| zHx>J|bq`E_V-n}ebu8QR zeij%FsBm4=&^npo#`4}@?*2=82^T~1AM z{bF)wN@iweN=kzXGbmr&6YUySdQ;yZ_Y+FO^Q$6j%B7VU(8$Y@C&JN1%#Sk_&5Dm8 zBmqCU1v?8v7wV1w(u>PK)iC7lc-Z=OeEu&OYe-Xfrtlv?)^zuW0|_(d?S)<*N3Nh? zCFu6~w{~oQHT_#YZ(QQx9i0CKvCOKZ_I>VACs$jb;Mb#07)xs)Je}{ zOLmooGVTU3Lqo&vGH(5c?>}(s3%D!hm2#J0x0P}KP;8LtF0RObqoJYPT4C|>7vCJ- zZb%itouTH=f8LWULMi)xV@}fm-;HJU&ZD&tmUzYSP2)-bR(sHpZ#UZ922|UrQ4>slwZW`d z|JGcIGNk0v7aIn$h=_=;T)rG05fNiRhxg3g9av2d5s|~izFg~7BxIR4>GYO$-9P+N zu7{(b)Jy9vEEoB`vX2yV+*CtrG~rMD@|A*xK`D*P9lSuAG0`0yc^ zOrY3}kX{E*ihPlP00RRfbwK?O?9EUpsTi4SFVETz!qS#D$kz=YrbS5>7B)${L2&ys z1;ys5`fAl_O%er;8KZ7A{j^SE5ykz8#l_Cv52*ShkAt+T#WUWD%3{{0w`3UR3b%$!cdpgd#Jy0TL4PKbpWA;MafBBa z`P=Qy&pz`H85+C*?J{xfv-9&To6=yU3AtTB+U!#o*bGzFN|ud_%OMz|YQXCqwDy#J zeWEm!{5AMD%gV}loc{KD!(F#Wdm%)BC)1ewFD2S?lk{)o`|vu7?f1Ip&_*|(_0D&7 zoBUj8cT5yB4=E0?s5KQ2r-Z4@cZh0AwjDt&%L z&gp>tCiUsFJ86$!@2?ku&AY^6dSXGy+5y~wv69zDq(@k)U3uV{;Jzd1Z9LiaQJIO+Afj?#P4&g@kN(svWr3h~6b29>75{ zei44)+$Cjy|FJ%kHE-|KX@Ebfz7o}AkeAi@=B7tk zyE;YwK$eZ|1iwB7531NfTgrWzST958;c`dT{C8cl&r2ifWUg-GvI-&i6KR-|MYump zU82S7*SC%R_T8e7)hzIFiI+pf3Cgr~*Nd{N=>;Uiv{^6tGyNXR_z+V5er$h;SVZx# znNOzoVNV+~8KE~)#RREw^lQD35>SXWava^vcVqpnU^bBh>iTf`W#6Og+_TS$=5)Dh zmLe6*bTB#|U=ef1Id# zv2v?O)+G7m6xLtf)T=+Wh!byh@&kSmF5C_mTHE8_vPm7UYXE$ihV|9Gp zzE=n@2(|0pd^Gc_E3t|Cqeje^F~8QupVr-c*SkHAmF+$IWvri;eS`N_W?8O zmoDDOZdKCneWL9yP^*u+92ziVI{kn?Dd5ttuT`)?wsD7@W>_B2;F?YM@kQyA?AIy+ zEa;bJ0zcw*Qv&>H6YZ-OO|C21T|%mg)B&X^2gPa&%Wj9C_H2;`_%B3U)chWUA`3G- zbPhs!>;+Yp@Bo~*$>p2bL>Knp?HoPdz-wI|7xv)y&cep5&#-)N(l?0sbY3S<;~Uan zBYu+oNd70KnfYjOq|8}vu8KNS13j~X;Y`kSld7D z5Ky4@w-WeN-#~v09j(VUCk^KEoSd%Jg1;df7hE=Cgb{VN?k++?W@bw;#40I8ihqa^ zoltup6%`jD@2;!;YX2G2H%T%iRF|&%=8+os};0{2+3=LN?&DxOKNsv%910TjxDUdQ8R`BMq|6!%&i(+}z90gz zT74X7(3o=r}ME1u=* zHNVwkZu^dc5s;dwBPII=n&iV;GL*J#G7xIkKldp0jGFFTu-p8l(h$~WbRpw7;J zXo(ZC|1Vl1XQDt3zAX)#YtF-jk0mTrU(snKgKPL0QXDd?xr2P)%d|$<+_!j9ACz8L`1~)T%7f!*ZFC}Hd)OkrtpxWVbPnOo~9JX%>2JQtl!nwZ;3yvX6ewbSFY#wKKEa zRtHp|7V0ce=Zx8Fq>v$m*zIT7!-jizueo|O&USEdaZ!BOjF97BTv$-8bhOlaay?{b z0H59+I-R`JfzzT3d!+4o_!LnGz6>P9dd)F?lF(fEeU0{Z@*8c%=0iX2Tr4efzL^+) zKutUu=@!-B%ux6Ua5Nz%ruKnYgdu7P;d`MDK3LOp2nO9p$X<%C9m4%XE!@Y*} zn&n&oZk1Rp(X=BKPtm{5J2^VW2DS8y1*&VPXPTR_PdJW2Ia8)3md`E``_I`qPsi_d z@YR&FRc)7Bf^qmo-%Ly$DeKGMkied2(EReXy@7G1=+YHJLPE_c+|_yL{4DuwV?AE7 zI<~#Fh5ADG_VIv9J--4Lly~)9`hpzLML48eMe~B}HBM{jC6tk3`}tw1wBTq-a#jQY z+*=_!RwgE9W`(4Y(Q!0UkKjHZNr;f9xP-{osMWq3_L-Ff$|@9CA?9Gocq@ut3rGZR zMo!fy+Wd@+bx}$3)aVyoANtQ2GDM*p7ogJ9J9idMP1)Gk0E5nK?yZa%tUqeMiH0JC z8u*mjs?huq0$crMnwMe}fq@B;p@;DrF_Yu9LarR~K!+!($>nmfx2@PBe7>6Wah(W= z^q77{wkJCL|G^1-QkL3wh6WAb`~#b{w>T|S>g(T0>q{w3PjiMW`@KSXK|YLq^JX!m zyp@;3AEiO~r)GcBt+JwG=W?S#C?C|?9Ku=4pi{#Om~&P|8wo+HQ}ce2wQ676)AOuP zj<0!U6Yh+)nq#Z5M}`He@K|D_j3~<6l_v`&KR!8lD3Fq^{DNFTUY`8M)x>UlNEy;WO9P;I;3hQ{@bO0Lz_eYm;5>fk@6L ztpcEo=T&VSF{J!W0&0qxEpu8EqMn1GW>GEKeKYNDs-jdmU-Z4@wR z+1k~RvMhYr0mLIsWV=_@R(r^p_4!&O*8a{;KEJdKG@3W#L-j@<9s?#87Br`?*424` zGr5L=QS5~g@3ZmtEiTCYiWv<`)N{fXCpe)3+Un}AzqQC9SIM-BZ+(3e14@q{W2Y-7@Y)JfM~I1wFF90*yuyO*6ctc84DT42%<*y1ni8$; zde!Eu7n|=C;FR<86Cs^ z(01X?|Lqw#vFL95|3JlXMOqxI<>xSdMZ!Qp6WDgJ^|;577H3Ezzens%fdrSinDE|S zQvSVRFu!|%Ssa1jJuFJpf>aMsFjo*KE5-sE1^Qwzvy4e;Kwlo>yMX6zmfz74&sy0G z&(vW^&K_%ME6?eGiq}zJVia6?en!aAt5+QqtLWrpf1-Lo2aPokr#$E5ZEc$|E|wrP z;mG}*pBLEeS9lmod~vx_^Mp_~a1hXnatewO$E4%f#a|(XQoX%5mV4jfbE!^dR!7^{ zkkG)u*RNmu`*XQ$ktZzzsm74kVcD!&l9KX+qxLXZ_3mLJM5Y0BuZ>j>t(Nm@#WzNO zjj~*)Ok6qriG!%q04dTnv`gekKQwUHz!&%R^%Xd5le4q8@Af(^R=^lJfxQ~8kc|Vc zAr#v)GBa-tYFHy(bZ8LYy5+pI_z}bPv`^sN1zxX7i!!AX6cKG_2k^>JQYzrD@@NeK zmwsts%8!hXiQ3>;-0l|eS{vso*?^G?ygY?Nk?-C)Z~c^LTv%ucCITnB*TJwsHkL}R zW=ulD8JtA(aK~(~6wQB)KW<$25@;({QC3t`R)h^A?wexAzNyBB*_!2BO>Ge?1>FMY z?oXb4tK~naOB#}s#}#F4cJ2xrW`NvV2b0_*n+cuJDtimO?MrCmGjlpvYef6t#gL}J z`M`t_d}FAtt`4tx7TSd$g+PXF1)xQ=p_r(D?90(Cm#2<%Ay)VIM-VU)WH`WIdKm zEfH!oXbl+{8ftB6ULCGEeI;-*O$uG>1r;Ms-a>CyRxEslZQ;<^8}3la&8DYSY;CO^ zXafQU?h_VXA2O&Wgmxp3zsMxyd00`feNqU523`?~wA9czCnJxB1_uWAE$_m`8G$OV zrgr%K5>``7QyI-Gu9cI+Z74e|w>ouA7W5|j0&N1u^FO5>wX7VjW4264s^>@-iDJL3 z@97<{GHkoyO_n=vr|@DdFIlSOYWJ|V7-Z)VFWz;|6?}8CdK6)BcG1r?bu#3YmY!O* z(SS}IACC~q)vm4pdy3Pf%g3DJ-Od=6L}E?^whwW@Y2Nt}!yeC<`R$km>5#DpA(Y)7 zr_Z#twd=QL!C}Ztvc9}M5B?}&`p|#?Zo}pK+s?d!%{Pc|KAUa7xkuFwWk93ETEwLI zlwp!FcXY=D^0aDe9xyT8yZ4e<(f8fuH+5UDIe3{%T}+OLuyjK<*#9zLQF$iZWp@cH zuYa6=tn7ITmBMYBd4w)|4ETioW;(YgX2nQ_u$P_3Bb)W>E%X26OUO-&)org42@DG1 z6k?;u^!?l0LQcujs!nTGTmkE4YLb&rc1+ExC=?VH-FvoCd;3>k8nF`(^a$Cfn1aKl z%fLh@k&V|~_~)(K%6~XqtVnO^3e5g(aan#w=aMb;UHwWdN$=t?>6a!SpZhT}xz8@K zS{A6bc=lT+f@1Il_vz)pRT;m&+wkIq$XCP-91V4KTr^)nD#LD$X*stUTkhz4>uqyK zKC7>b9Npb{6T1@^h`CA8Lx}k*p!jnon>I|<9gajA zFG++$o#&n;_qbZ#q*crbN?V<0wlwuqKr_Y9cbas*1#Xro2*j>Tywn5T?}F#lpk+ajaoxX92P;!rj_j9*@}``ZbEw+i zb1*$Hg|;d~L(vc4M15X86nn_B{6y>_I9HkFqc9)>$ibm<9k1m~L4bR-udi`_n~0FG z>4t=>ICMC$NQD2%*2HtO?Gb%{wZ`%zA#t+~{c8*bE}b_qo@82}N zJJ`G(G>YUcdbd2I^-;JewkQ?uXbd>nAD5p18oi9j)~Zyd!qu*^dm2s

iaEhQ)S zI12)??v?a%5sxkU9)geqhz0^^Qp(DBI0SQRnpPptXpI{vLwLWn$4v|kQAV^i z-B^UOl9&TAq8OdWHv1D!iF|I3Q#Gex0Ud*-M+l*_4LH4KyS-}MXWR2UPqQG;G5)vf}L8VBtz8IH9w6wHjt*kUHXzzU+xmd9UzOk<% zA$hWiHI_3iZfoVnD^zar#0;6)*(>w42Vzh3rpaYFoz}*`*9lt5$~udR%G=oib0YpB zBPVBfK`@kk18e>J6>Qauj^GP&tQP&X!IDImT_|+Z!o_zmBs8@9>(?4p4Y!jH1{xX~ z4$IM|z0Yu$g$z&nt1;f$&}YIyw(h$G@;S6U?p3dQgnp~;oV##Ck&paxLpjaog2IzR z1|>}svD4R25hOck;i!R!s`PyMCDx?pat@!{9`9xoj@My_j!7)Zz_O0Z;cUp> z9`t!!j64ipXJCP}GPgEn_|c!CKoMcOzc#T5#%|EGF^_@*=R0VqB_$<2H%`|DwkFmG z`}wA1$!NjO~EFS8r%JeTYgu48{WNphe5;+D)DUrTTtPiE%j$yv?mP<8w@#KLeCM=FLQ$w3KTe? z4r|##j_;H;+Bi3`+xSRi=FKB=gP~@6CL5Ogyu0IH?#Rufq1`>weMmjGzk>*|v1okF%1?8XV{i#Ed~V{9E>(VWOB2 zVxyR`GV5(&W255>VR!sxP=&ndTGiuxg$vqd^0n|$*(XOF9~ZT_tb=WCcygTm-B+Sw zBR`8zg}+MYJDFfcF!*A`R&p)fUUB06D<(5fc_m}! zL5!9<=&z^a%)N#Hv)-@UoG|V8s@ZugS3x@T81Ig4_WI@gDp1v2hw3MySf)+cG` z%TD)g>Asi0Tswvs__a0SLOD2xrmeq7FqbCYEA1wYavTR}F@9u>GN*v-SP(u+(|hEg zE<5y#vJsAiVwwMon(%JowP|>7f;rQE2$eGVgDsq5w$;&giv4ODFQnYK-^F6r$KPSF zbr3x>HnO-<+(L}y?XLDB*`sHirvV3DaT!m;?nmtTiLbj$c!tKqg`muo{;(G9c3r*D z;8EZT37uw&Mi*AOQaHtiUg>mCD}BF?GQIuacH&G{T_d!j>C)m~*dei@L#H4v^6h)t z>w(KlOT%73C(O;`CoQN>|F&$EL~!mG;G#7{>#Xf<;V*bEwl9D+5ONSggB&m` zVQ^zX$lebqxO#s(AcdNSCg#0G8p~{x_f#lWid%> z`ZIFOT5jIh_9ns6?`;m-EW!rL`vhvbtmQI@-x8i61O1H*Q{5R>d@V*dN4Rq?}b8?5`KcVJ-XU(Wl(pwdG~F zq(TOGTM@c9#5esg;K+iPvqq_v%%evL1k8+YqoSaZ7KR7q48^tK8e*q$s}Yr+eE0?2V2-hiy*Uz{~gFCB& z4_fRMtVkgIh*Eh4`+NiD{~M|odsWb@Fot(&eUQwL2f6AFPrzH}V)E&tZzF$^o1IE2 z6H=Byk!xc8n>k;N#=mhxpOp~GL`Hrw=i#sX)~0H`OiWM_Jp&Um>NzV9|I}^d1h2jU z>y4F_6$G7tBe&8f37G-gHMDX70ftYn`6)9K8L_jsXEjzLLm$1ryZ6HMUsfywVugPg zv2f~x6!iF=IPpa%emVp&QwTeNE)Edfi2N<0o0zZS--A(hRvzcN6h`1RsTE_u}+2Vg@el z!T*s<1Dc1_HuMIXFLmUSxSX7mii(hs&^vjTt){CP(s%>}8GVbG#Qg7Jff-m!poNr! zlM@wTVrl8Im=;}jQL@a;93CH^GhoI>C}U=pDM-dh7~EM#5B8 zT##rVFAPopB+&|x%x3sZO1cgIH+mx?9Q4oVO~SrrN2!D{&NEajESPv(80Q`DVqp;X zG8v?6!s<)NE^&rI(B7lE+Nf1KkCzoL{r$OWg>OKbeXd*QI5l0Kgf*`=#uz7U4Aezg zr!bg--%S;$DXbgW>KtN{@>?*+sUi83y}szQIC5RfPa-x3@rZ_6xMKAo_XVDwoS!}XfETDgM7vXL0b?_b@eYsz*q|pkDWYg(~q<2uJ zxtDvca*d1#IIl2;y-{m>f&Q<~kLzPov%(Y+Fgg^Cbl!L2B>@*B9bl}xs6W(^dGU9q zc}<8eUAhD%jV|ZcZVdEg#gu!FcJ%7}YJeEQ!VyuJp}Eu}O&&tpPm3!SBiu?VXjG&I zz9Y`Hk~rVHub-YwgypN+5J9)1FPUmMxkT1LK;;)oT3QsLK8NY0f#BfamzOR*Xlql> zip*E-H+%MP*HLH#;yFbQSg*Yko9{neyDNbl=k~3J-`u%OVTu6ByTSI~RTc+iw0Y%M zac|y*9t)kkF;u#b34=^T$ZozcP@u60^f}VEIM)LSH&e_A)`R?Wl5IxdHL^j3$;?D+ z%!fleA%R3W&kp0cPOT@OL^#Z;=~UFPDB%~N_8~ZylJE}*srfcR{3|EJZ4%g|KZUNq zd1vG3+qZ;-1dt;A{4N)Wh9@LU+8i?f6HNzaZBiJS$J(lR26Q2NhqwL1D%5&d@QFG7 z_$hPpH_mps%;lM5623ZgU^f*e->P&CmVe;CVsX|SxHgbI5-R?}G-(KH`$^h6 zgNtq0@LY{^BQ!EL#>dCEj|nU;Zmm6GzH>(pSlbmdRY+~R?XR%~F=&7~NME3OGX3hD zJ?34A3Ut7L4+wM-&?F~Ht&?gKuLkJU@|!Qp|H;4bc^t92IqU2H03{G+`mE7W^{DGo zFxOORe^*kxk&1dJB?-kOzp!3a$zWl{o6b($^KBWZq%iHk`$n0V#8DBC z_K*wb(M207+|rBCs;nonCA?EUNmJa*Q9fN*kYOp>#&{?$UPk_PUg|znGfp;=3(eo4 zirt&jv?q2@y;Yh1QfuUHVd0mmr(k}3a^tF5;Lg06EhhWLU{8T1=F-_%Y)ZI^eS_oS zccI8@cZ+-F+!Px4Xp)z%{<1=NqYN!v8SBz-_+;@!&0?w43QyEqu+6Z}HbW_k`?+S- zBkb329t8T1SYLrW7Tg+GY;H$Ornx`&KAdHZgF)!(3)Jk`zphj1-nx0?1~j*}&Wi@E zn%+X)^^J4=$<^9`>#Cd1huy@omBbfO;QHp3XOeYquFY*Mbo98_4TYz<7}qdUP?^(> zY>fyTtOhj^eoyXRuiNyGTh)QiIn;g=Lt}>DyQl=6UqF}7umdf9r%_S8E_4eZ7vqrn z2ndDS&|15`^J3CSu=qqF%=TwX+f|2agG~xfytC*iD?bHp{h2MnL?|XICRRQR+XAy< z*>K`WMNmjcF5?hiCb+&2R*~0kB%6LtdKVYuV-i6y40+|cq&JGV2^}C^WhNQ@au7Ov ztHt6LGPn_c-KOs+#lqCe^mL#PCln=3a*2T`hU+F?W>Ha|DE#UOl+^n{eAikWX#eyN zUsMoZhQD`mp$I;`D*Fs2llZZeH#7iylL*dbq;u}&zcPzLZ}QLV8FwCJ-wj##TIzR` zo|$!@K>b+6{>Rn$Z~c6@%4Min*O>#Q!~Qp=VLNwz#d(5uP2N<~`A0R9CdV&mgWSL) z$Yvh!=0B`9{b02b`UzH>7^Z6qV6^!az7J1>@V(Qo@cp|=G(m5b=f-{1aOIpiQ*PyU4)Pe39wTesGozjepR60TtPMiLjIy zXa*|o5?)c5tu!>^#xB9Gp#SSk-9>J!^PiR)`Twxgj8W@pH}s5Jo2=qfzQWR3>Q`o) zLGyPJ486)^I@&aP>+fm$>^WKQ=TB!dD&kArroDvKd{-h-sXPStX!RYkH;-GDI!~s3x)9dbv?>R$uv!RYq*&kGs1p|#f_ex$MFBYnGC;I&lyY~(*2*HMY9V8E$)5z z9kiJIJ7|Hhv$KvBBdfmEqEruWu5n2y0x-a6V`PAjDnCntOx-EP)m!roW}y9F43&r zo_$=g;B`U0Kh$b^$}1{@{XflK$E$iNO2hMPrw<{d+O2yFdhA?1#_5PStWHMcyr?4| zwaXJDJ;#kci6;gBk_x0`g|?mAwp6|f0IQ|MT&z-c{OI|Ch)ynTwG8jqCiXqW=tYCCH3>?q$Kaw z8KrnG`-2TVgdiw%TeBVD4(@%dqob(!Ha`Az@nd4Uuy6nxI`}BIqv-CT{s3#3>Vt)` zsY6*Or~TghQmPocQ|D(K8Y{pOqX!1q+|H}9nFhA{FVzr4lr9E98EtWg^NohWm3v14+G>RKnpOshpfaVKV zcuvjioc9k&T*$}w8u~K6?dznCdUnv0?9a`U9@o)+qkpM%(sN8#m5nyRPzMjj1h%T2M{`2D4$)-H-Zifqg}qKW#nU zNl%L0myNFDb@<~@A`oCE__1}v9|iGPT|Hph8J<`63K6FtpdX|_b$oF0>)v~4frEHV z#j_h3pD*GuH3|O{kJ;MVs^8>)FPTA0OPfj+uxbQ-pUIDhCno4Fsu%h|0f6d-(AKdc zb9syQ+=B;NkI?lt)Zi4DPU=o%tek=Bvylq;UMdI2a zqc~UbQND*kx`5Bj+sBXo-Mj#?n9<{_H*Y>2t%$_)f3qo@$;K5U{itbF-KDDw6``!L zb2D=Q88_{%BrLd|aj{arf|=m_!@sw;H)|+YqUi^SOP~T&ZMG0uVNG~I-}l<& z#Bje4V&h;YG;3f94$}L4eAVF1p5B^;2N<%~AiD$p&S+=^o3}IXBs}EcU{{{7>qtD` ziUvEGhJs>h;j>u`@fj$wCL|?2Z$M{Qne4(H9~emExJBZ&QUG?sc@SP9e+O%-p@MBF zSBr;{QJ0ZWL_}EMzyLoj9V7x|1YTNtZ0B6h8|?K7ui33xFrI)&ck!G9mRXA3x&8gy zx2fMu-X&llz~|-q{rMHu`T@Fe?dUP7e4Xs)_5}!U7=~ek6s{P2kK>~w*+hQ2s7Idr zV^j3%pFbrh>j4c6`1*XkwAS)dp4Ue5s|#A&(J3uGorKqc>cInv48^U(lTV+hU~dMV z|NHf!P_OKeP_NlYmGMCV>0~Mz8gp}NfrG>O7^x_Wv5Lr5yrN=&x11r-(a}y04#R<# zW@af|&4s0x8yP`cE+~mwlp1+~%{!6zd1nW_0pd^kp&i$AJ!f$sA&!5od65Rj8JSB=HXmk4ig$bU>^4JU^=6{_Q4vVsLAGBRMvqWBO-FeLIHh7>8E zzk@;Hk{7|CF1x6q0S79Tj}?5W^V{mXFU10GwyC~O+CD9M9&gHtxp!OCwW8btJ+B&r zDK=<-g5hN25((wo;0eB42;Kh^dj5~%3P8^@pDC72ZQmq>71+(T<50>KB6ou~0uIET zJS=`)B@D`@mKLZR!ECdzQdTvCjKa30`MFSDiq6^xk;6 zy~P9qVbRnb$WVi7amxFhXVQ5l_!X(=0ccHKAhHdN6s=bu(~pJO{q=&3uey5AD71Q4qV6$2{fcA4F^-Q#V?`mPxsb``ep(tN$3F2^+p$K|$E8Vqsc$A1zm_cvj0{)3fXc_oUuLxn^*x zFWc@w^(*j%`l$C`E{}01`OCt0$~l8|v>MIL#h!6PSEi!!7AxUdxqW)h7p zyr2D06E`%3PbyH-BmPn5x}JV>MFoIPNAO;v`Tl^CEn1hut=R!z-=za*WlX(vOx05t z1=+DTVPOSNJEAUdtybaB(%MtHW9?})l2GH)>nH^k$9IRP9kd=~WMuC(ehQt5eiIS# z;W1UNEqzITJ|jOrcp2(T}!9eq9Dn zgQ-bwZf-D(fiVD1)Wo>Bv55&SZT`K0AGdAA`Ya08Yqn*xa)#vuj)OGjoUJsZ9PH__ zcsX{z*oW-;y?XFF*}TJd>IRcNN z<>P`QAvYx;ow+9g5yBrND2Tvj;s2&c2J{O)4J;KF7J@xxYs(HS8P6}vsLM=%Z;jB8CGO*TbqHGMBPYRjo#-KnKfMmP($id|BT!rZ8#xT@DRea*0uba?Ow=s=llCLvksm;CwoT;YVLM z9@f4t_l;3ZN;c_7NpydC9T3C|>1U$du=I&Z203eMokW3J_rt9*cFO?xC&R!B`>w{( zwZLNI=`$}!;|L$_^rWOLE(M@sg$9HeHB7_8F%gj~mSL(EsO|uKgp+m8p;-emV+Zb@A|fwUe^r&5 zSy4z7CM6{m7w=}})x;L?4+cGt&q8o+o6lqo<8pk_*6 zolRqT9l#jie%Ky)ap*$0VtRTS;k{SF9+0l4K}>3$NI-o9&ANORuRydgxr~#EDJ;?v zff!L!gDa@dm`@B`ykG;Wg{7rdh0k-4XIL^?zq(3COiZlypqYU^I`a8n7~}{R zd%n#_x-NfuV)C1ASsp+83*jFU!X8(xwef(5r?RZ6EAEK9Z?b>FD450~GuoLuMwXB0W7o69yu4T}ImgSg%-oTU#E?>sGILOD=&c2Fkx& z;{U_mTZdJ-uKU6mC<+J&0s_(kA`*&pNrRFi(p}QsAqWCe0uqt}0xI3zEm9)gB_J(b zV|@3}IoFyyJbI=e~dUF9f>ClVW4bVMBrm_-6z@bF`%8 zRDlRH_^xLO2wI~xZ9(ShV?AMn~{y25O znKQrdxH`PG*@T1X*Qii~KyeL#>0gnv28TXZ>G>Df9;c^u$7@z%)O{$TDNuL;E7lzb zK|%8I9EBIMYRbm06efyZ$Wy(*`JuX_Cl4N+iQir-(BC^?d|F<^ToyCEr?<9{zx6~S zl-zKm+lk!sAeVPT)PlKMF@^YfL*#KjNl?1t0Ru>W)Tu%6l;Q$$x z44)Ub?{J*_rk&i8+h5_u5v+!eh{yhbmG>MPpo7%yPrdhHe0~WGm^uvJ513Ny#-3kw zZ^CPfVzm*_W(*Jhv72OnHuJ9lX15Oz6WTvhZ$}62RZB?GR;N%69efHTcmRev_tIXc z@_nDlU=MM|2;BDlJ^nAFQkAvlm2Yu;yy_qGCfXD30OSS*l6Y{rSVUyc@XnO$N)8+y z9>@E)=YJMlW8%KKdFW1mS4gGAG)k1w!qj|+0BH%es<^qvxQv~wt!5~3&}X}fVK&6% z@p{#9ZWRh$fW)L>aT>h>=eII?_8-uE=2_G^v>IiMqPeG71iLssgX3>-FrU+Uf>z$X z#9P*A%+|;UI6^p-_sE`fVAWR4jaWuMHjkdFRK-1W_GTpas*z;>ps%Ry7Ii~`KEg`q z{7mKLlSi{a@I8gAe0sE!=e5;Bqyg&{6@6Q%8574D``y8Cah~t=+hghUh|~|jWP>8TpZe{ihnYlf&w}VdWCurpH_O4qA8Z9XKU2zuz%XWJ7G0jYiMySyUe%xJ7o=p za$b~mOQ^T}(?`b_sbBO9DLun>J1e*8$u}Y&{!;IY^WIjTu|=g-e?0pcyqy9K)>bpw z(FVeY>`xE4&5D|dvHv75P*R>nt7=6jve%*f?P5dD_HO}FO{KqK8iZ}+d7eRG%thK% zuXyB4cOObCe2K#or_qQfKwa0fdPn*Dm!{v%oS*N!dcb)7ESemXe2zMqg9Nz{?%_{b zRU@n>g7j3qT7J84(M$s`cWNIGH4bIAj6Xp;TYZ_Yaq%317+tH|XU zq^n%Re4V02w4B|-{eeeZ2FA+tvbg>++P2*J@C>nCin_asLsF50z?k?DD|g+pc5asd zf6`M#tnhpQZ*B;OECWi^-ueWl5h;qme$*F+*v<0_SqnE8 zad$s~e8J;;#@*FBUmu>~*?26+ANBU_$37Dnx;pKxs;)~wmjUFVDpX4J+mTw=!L6#P z)7*lO%OOMB$v;oeqVeslS!HIt=K~m4hlBMT_67Jp8hjidc_0iW#k5O4i;G_0GATku3Ta;sv6FB$4 z$9f)Fk6oc}z+xvxxiVuKfZYdIGW=nQ z>AZGX4EbDUsNlgbgBoS+@@ZE|Tid&#sPww`7IL$HCXjR?BE7yKF zpkUOV{Oklt6`XRQ41yvIRIn882PqAu{w_vFMp9CN!YL+}&km=xk#~u)RxADqv0EH` zz(mNRSy>F@iL|sAos0AHJ3D)zqp)DSC8b{N4At2l7BKVzgeW20Ufb(*7mjCDRIcC1 z$Ozi=(T6dsnNVB6&B(QRxHy!jX;*Us;tL|Q%|BIu%geu1fUA==yll5Fp@!!<{mZKh z2FAt!-pt=oH)1EgLdFFJ-|Gqksg4X8kU8*r9NB}|k<_YYcesG%=;#=Q7~QyW0|GhD zp^ZBsPgAI!)Y;6Pj;3;M>)dcO&hRH#B5+}s7t&z z;(ZP%e7WTL$(=gvVD+lI*hE8JYBiPgPI-TCkDSZ;9e}m7o>H{3`zzhN8z_undYUq< zt;ksYm$Wmy-+}vJbGv7RvA~kdjFrM&r*|cBZ01fGs0s5qI|np#v0Fv-{lS=QGc0bw z?ia}=f$PjKlFRY%xI8PswDw~MLIKFZDoiLEtDEfIw^(TTmJ>XIN~2%L zHMF(m6qLX+u0cno(*7BUv+JjI#9R*Rohf9~l?Vw56$x4#i-7zj&#=0_9^GL8+t^>l zgMw)qDMGlAx%H8^V{nY}W$+&q7l-F-exQVHy?}itpXAQhZx|wW$?V+YrgVWR1OP)A9EuY+0+om2!RMxYWX%^<1e z&|RS4A-c1r6t8EYGlrg?9tQ^{WOLt+pV9!y91Bo@Z$6AZ`SG-lTB@+7=CoV0%C@iw zjLKDOa*SU`C)F;OjxoHl{b7{D%=D(iV3ABL0Dix@rN!fLA?Ax*t+WH65}*iDt#Zr} zrRIj^2sR@Q1jI|;1biErOT&ut$311&`r+aO{u%gk)kVdVoE zHkjwKRB!r7r%TI71qKH*F*1>nlXH-S0F-G|Q~`8|oSM_=#bjfU<65jJJHFY56KEY) zA8CACGj-qSeKWxNrabm1{uV(V*Wz_!#%cn^hLn$v6-x6pVdX^%F~5M|09P4U`g>ts zCp2Rh4cCW$42&P|!ZZ<-!1vkN-w*$AZyJ0$216A1Bx3^uEm}`ucmyILi86uUfB@+3 zJi)d-K0Y3$v$(~!HB^UiFp&(f<1J>@-*4v6W`dIP>hj zHg=T3Ap9fqXOaszd+Ct*6GSuuh}{y+x`J?ue7fR9xlP*d%pZ`N&UZvAeq_IUS245y z29G@oY5tY@vqlP;KcMw3hBEt{Yq@?&NakL%8o z83(#w3-F6sJhy^CssP^2#qZx2=jY2aM$T{{peDb;Wvv58GPK=r=fgbn0$TOnk1Rkw zKbhUPwX+4|CMk7^snBY}PH=wDZxJggEmf1c%fW0ZqYw_P#I-d*6W~_A@|Y1KVBjyc zhMk0Kz4D@>qE=RUBj!f4hqiWQ<|96H<-#ewE4!#wv8u&|*gk|ClQuOXLZWV;KNkt@ z`0cz}*VUyyP^F_x2ms08vS@D+A3cMHIEtw|*-u97K%4<^<-ccSz&HgR4Husj#vPmA z`sGM>jxxOv+bD^sMs|A1u74Q+h2Ow5px`iSso|(Tm<z^z?Xr9|1-g|1pYT9bsnNgN_Cgow8NW z^0~b+I4eNOzaS@g$_~~8g2-(_PV-?MV4Q6{1#NSoVP`G8I0|jkD>%sWI=xN$PE|t# zWslfK;gwL63+EXmDpu~MNHT!j!yUfEo@7BJ3cn`*P^_DMqjoz~^Ki_0%mSu;AhpAx zACC&RX>qiRSy*7xO}ut3QiBQBNHYN&`HFeihosgoce`1B&NTVU7~MN2m%@E#2*Y$HJF1qhIb{~!|KjK|1?w2*oD&?^l z7#w_fBX8XNeyR^Ab0F?(Uv-T6Ql~wHpsv(~ionaV7E@Dmb0Ut)d!HvS&39Ez)4)bmIJQD>1n6o!jJbO$ z5W$+&^S+oG4?ho|SZ=$EuF*2{v7-F5TT^;??#LQ9g)_5UTgJ(nr0QgPUw@EeH5EJf z=P-3T#!PEZ4L!%H^Gxe9YNC+?2g9 zva-Ujl_#6o-`eWv=m5It&kH{V^uwD5HppE0l1kT|tYZ2*W_wLv^KfVOgAYXXl=--% zTTR~SqSvgycD40P()D`+hy(qi4kb0UqlKs1QtuVh6|dvr6_oJ?Pj4S;<#7txc_ zCq`!>SF11v5Wo}potU|A-2!C%{+*nZf&w_vX=p^KqflsxGY+j(X=&*NY8)@eXhW;n z05?xC=9uNmO-VUJrKG9~RB_@DpH{bB{a*TQVVspHkj*U01rJOP5Q>R?RMJ|#o(`jJ zcOzxxV(})@C9?JLilUMdhYqOyI4!}L@5UAX;!Mf)
CnzygLs?EGjeU`!;-1?B% z{fVyU(VpWCZkH{MF7;b<#Sd)nCQ@P~)ft_PCO->nqanB~6+y=Oh1!|Vs>gA3C7nT9 zNI~G`mZQJ)0sh3|w!(`qOw+rr+@YPHzhr;V|=r(r`jWwbpryR$-{Zg}fe8iQJ%@m2hjwfOigM^Bmi zap7X@>{kaf&g}7tR^MN}WPNd*Trx7;y*A7cta<@O^x7**?uxr__W8}N3oodE%MkTc z>k_T-kvgkatrFuY;^+btp^!z_Tq1+$*~ra}j*f=iD{pAUu}WW0&zkBTGZoddukWE~ z{?r+Bp9K2>E||gV68liH_w_x28Qk@IFWcrogPW$_>(zJAuzq8y8mNy7v>o(7ZDl88 zS|2URDla$q{>4pEQ4v71nOk7{3x?r_BXC-zwS~^K-4&paCC6EU1M>0RyWhK#De%Z^ z%Xs_R3z@g3Ips zL^#gXlapRI2`;0fEQze@GYe;D{GXmABkhiMmT?45K-z7KsCGF`#`(VbqjKHe0r{jM zGpmR60veiU`L+?d0M{Yy+n5+*R^rXFc_>6OGU&?2LN-Ef2nq-YB&}YGFEs9o<8@+A zI?_#09_#CyvF2jJ#@25Q4wTP2UDz~ru(q}~bGk{$><#&$5O{E4^z`XdFb@`@{E(du zDb_rRxs@3&bz>o?AhN@Lb+~|+P=A8$)VT7em+9w)1Zh8vMo!sIt7prn7UalBrQ2?4=`r74(y(upGiy<(W>_KJauS zOWr;?xbx+3jLnE3nDm8!jXk#3Ixr4g&lis4ToT6g-7&>wWxR>qw~dOF=xey^4PEB1Fl}CW8BW9zfo3yqv2hJeT;kcCl{CY{+>R z^~_dtmuzf=z0u0xL`&whngl^7@S!ka9t7i2jFcF7w_Th_=b1)7pvCM1f90U~+rYrH zl2K6Z&b5WYp=ova0_Vz>nupu8kel1_IJcdmQ(*g&QA09=YowG0a-k;D`g# zIrN`C9)%BvfdYr3CQR=>_s)P*lt${R;2KnIa1&<~((?~4?p8W)>18&BCnYgQNdpIg zgp@Q0x6&F>-C-ad$9~J^D=ir7hKK7~S=Bku2v6A!pL#(@KVgaN)JE9uw1>~Xr`0gu z@7bpku9@;bupby2xWVgaJ~>{|N#XvQv;EhtCbVq?0qzM2WG8meK56B#u&@Z)FUd+l)_Nfk^&(#tAD`abG`fcvRnt&Xl9%5Ey4*}P7{KygfB7-@6A9_H67yxn z1RXBB?~fdw3_QQNiIFnb76zJJ0Vq!}Qcz>O{!B<-{AG*9lX#^J=D0vNql>&WahjXe zRjI7x|5|8T5g?x0f+azEMZdyH%UleK92WGmOv&4y$T{?QrUz)161Y%nR3a9H7l=` zI`_Qlap$8QDDRzsjuO{pM3$Y;v_EE?d=uKTSkO6w$U0-YO5=L;|ckX$_AMD`G%=q%)QTouAN z1^5o-1GW#%$P&fT>gov$BTb9cC{!zcf(jKksKs5t3lI@#3orS;1fx!sT6Y&n<)B>N z1zl$jP|3%Xk(ADlALHkqa|8Yg3>bZdXHHzUZH;;+%dE8DDY>@>U%y*X{ow;fGT}IQ zVN!=Pzy=7$JqOq1;@F?raJqBTLwP%tm78nX7J37)pn15rT==veKX$r%Ed|`Fz!iF3 z&Z;&rY-Pe}3KE1#H_3qa#&0oNyg%hNwYUu@HWb0yC$mK01=IX|L&Uuj5g_;U>5%r6 z#}GJ1OtF6a=rPHPhQ|7-M3t{ZZEefh(yQnEgZ2g6*2R?{tJS*td(U)^g%ZD~ zU3WzBoG?-Z5nEvZ-DL*13SGlXx}x*-y_8Ypv!%vt=(G7Y>(8MqJ~o0jiwX;qJdc*4 zKHh=3kdV_kP4FK}A^y&kQR(X&u(^p3@B7$(;v4w|-#SceHZC{Jqq=}|wOZw-rnYKv zEqcq_wxn1WyoFd3=HdiAT-Dv&-n}7G2-k`FusGdUBPy^2*FQdsD9l^APHz3M7v}4+h1uQy*<)6V{cpA;)EP zANjFN49;aYXo<#*O(1t0pFH*+>9kx^olpuGS*+}(5O>ug$8236RQ z8+oY0Ve(u+=oJ-y5;Qcl@&)t3fUbD-b=;R!JUlAl7z*D6CEO)|5Hn{IXI(|rdKP}k zQnPN?hw4iuDJ#|%{s-Z6@K5WOHG@}L--h!RGE%@pw0b#loe~_3BGli}#9b$|*AobR zRewFANiiN6tw-V-ok2Yewze27xgQ{5Q1Ql}o8_@*mpc%>uQZjgglrmNvrU-86(r8!PVaGFmG0yj!`tTt;`<;Le*YU#Qf z@qOdUYIYM5UNoV!Th*$kL;E+bPi)sURioZWeS3F2`iqyy_pxL&7*RCjVJw8c-LJyR zS+KX9Ut#6&u|=@py_~$~jr-WU74NVkrO2T)EvC_Eg*xVND3EkObvj%=0Hcw@D5O=; zM@TrOj_7KmB|Po4qgl~ej8@M^n^aNsXFasLo;OH0~6$LX=D%WH#8$DA~#?C@G`MjgmayahXff5~l1RDI9 z&if}#HgP9`x24Qg+D48_3rVD@w%DB_^rSi1PSh|%^`B{*DU_^82O;Ug=!3#c1ne7Z za+A`oJLob};pKPFFt7)VR&p`0~_S4NeXxJOb$;rsbNJvZzR6q&#BYIMOS@Ny)1=F%l{_ zw3FtoTupuFpb}LAvFlhD{Df1K7~wU}jX?zl!m=cu0VEXw&IDplo<6;Y^#T_h8{jjF zr{PUCgL4BxLj%CsmR4307yuWYUx)E2sJe`bu3t1D5ChsENk}V{sgS(dO+cwgY%EcHmMdDfiut}v_LXLv;La5s z#lsrLbn!c51m$We`>&=vyakG1o4|97E#^6hKgwf*-G# z7n+VD5a7XK>~tpTIXArzQ`xbaIK~eQ}j0F@rGSbX(=d&;@L~pWOU~mu9+0jh>mu z_B8eG#S^Dhj~Lrd$Xua1)AC9*TfFI$l7Sd{-&db;%dKvf;Q);NRIzN8M?2>~)()%C z8Ie8X*fKLc(f6_Crj|<0!NxnErHHq-=KUNCK0X>vZN`kn8@~O|Qvtrh>58ff3f{PJ zk&!ep9pEalrHR5-H!cQ>rb0^sBp+z-GBPvah9l#)1%FIGfB&(yJ;3|u=;?{zq=do} zSP^(2LtD-%oBUC8@ue==6@RTfD>Jhq6(;zusZ)9abo2_$^=lqA{(&Ke z_t|DG)U6JRn;IK;T=_@x*tNi#Qm_eVHZVRyByYyrm%)-6HRlLx>vyXHm~|+82kEXnF_yPGV3r)vhDmYH=y`2wnuYSphaJNbgk5%xleU)C#u{=pYiRhy z!DDySQ&XECxxbcmj68ERy9M`u?~tbb6@JV8z>QivM+fXv(u-*2Rl6sHbLNDVBj$rb zb|ME*qV=pf&Yiz1gTjo#W3k2`{hV~?=U1j0TRMR5Ikg-*lS&^gDANq&2D53f?Ip0Qf~ zDYu}1K~jclNbNOCwVZ9rSRbtyaC5c#i(`kiVfqRqmdcMWHX7k?40{u|me}8ld@ZTQ zgfxGF2?#o!7Co0{H)lLXRidoagY(6P1)WlU#{)<#Kh=EdZm#Vu!#Nh<2_fVR2asdGE%i<=Ug zlP0kE%*-&|k>}A)xOV3YV9$4r^?SP7;N@|WBGe=EH<&=>A#v;24aOORsGh~N3vX5H zS9Wd5Vxr!vg01A}*Si87gNxgJT6yAW9i5$70DLeq{y=70zp)PPnHU!MQL`h>b$Tya zkC<<=txU~BS9hpnhKszdfVKm2rH1YWUPA1ZzYd){7;=KQn_kJ1 zD`S>v9>K7#>cbwurzsiD9#b>Vq>pb|+y==!)&=Ca0<9DXNI34_ub8lOw}O}ycXxN4 zZ@1ilYj3wW`;nscSqV(UuJ||U6+rDwM@@Y;rs*R5QB=KK!3SvsAC+d~L8Afp>sjx2 zg@uaL;Y;>B8XCz-NtSot0?_2xrkBl2&w6?RF%%NjInpcNJa<@Ja1y=#wP0lo-;#62 zRzW=N;B+6|KMa=~r(4s-_WNrh7OL>x@Sm0X1I$~BY z4Mu?%-3i=}CmxGZd{W$H_Lji7A?OLR{GAkka=@X6P){2qN)1-KeI^hFVJXtTlf>@^ z>+J}yKHMgmn$f$yY5_-ktazA&0O8Gu% zGwb9RCfSb!>yrzQB`UXn9>^%C7R*#?p>{h~+ijyQP)a0|_^3m|ss^($afTW3{iFj| z5&xz>HyW`^26vws8!uWjUxlrjsm;N`F_nUsgO>KSLE|+gd)vE_9IDk8w{J%s`YN_nJhgOI>O{7!UTE?QZN{Np0|{)=JHMXVL4D&($qB(bNmixw_l-BfjKZvzUZTIZuvy zq@!e7q$7M0_1EWppSpamItzA`_fCH^aTgz#xv}^7`|B@U#ui2W-^dcK%8Y5Qx9-6| zFsiKLC|oaHt5ORaNJ&V2Leg4ty~`r~ep(eBrIc1Yr`elm$M0OC{j64~lGR%MYy(;n zC#LkY+v7)}c)p+U!rF*1g^gPzm}JHHlBh*)%LVUPRT!V8T7OGPkP>h*-#fxH79ONq zH|N!$Rsf{#?IZXR*!XHP4{hSKJ^M+YOIx6O!+2%vvnhjLbN{&1 zt6L+@n?j_~dnJTVs&egF@Y?4(W@+MZG_93XxLPxrAKj!qJIFnbe%~_v zj0K@e>~j+Y3Jy!nj^eXt%fJgP0kYxrG|Umb$fru4^=E(r4ER}mV2cDv)K&(>-+>W3 zT46`O-nr%o*xSZPQD$MGt#3Cod^@bKwv8rsqFzGrm#&x&6@#B2ItS$L`U&fKBBVSQb0bB^^)8Xm^L&sZkeb8EWAK$>g5&+R#n zVm|Lxn2;w=V~XFw@f`}eGM_BIi%h!z$*teC1Q2O5K~MK*s@KEX)ayJwJUs=3+s!$& zb3Z}#N-gyhhJjp@fZ1YQzFY~!C2(g&p4K6qz;?XG={~?gDe|zJPR&Kb=c=K}&u7zL zb##N>8L&LH9H@2kKIMYzM(chA?X$A6dqssPAwy$hqXJhST&McAhI67dhzSjAA%!;$ zIuK_goqp8|q01Nrq)sS@5jcS%A?XaCDl6lbwLrzd0Np$&INkBVrxSkpZ>nkU=fSBT zpoC8AQvt`hxHxGAC6GVDr3M{1I7xs0{25mmoKW7q69S2qdFdF}lXfjFqVT-MVY6j& zR2i+#uY7Tqt94 zt!fXyqr9AN+&KMwI}ZpTa4#$-I!!*nu$Zi_0Nfw$J?}k$fVd=fQIoU=A9Pf!HUAhq zxd2n~e0zbf^|K`=A(N!6tbp6zDlpZF%8M#04#CYgN|F{VN0BaoWdR|he%XHVyA(QH zIdR20oc?&sB9pk*bK(zUnAd7F8KbCuq34l`fQbw^i(eWCGhe!J$uCCtLKEg~xrG^c zdDR)C;BEuY&wQpV<*YE5Y~p7*=oR7Wj~*`xjuG}hsGa<8cMnOIV5`DGh}{rd-r;dPr0q+frziW zY?!%lc~L2t`tQ0Bk9y+kzu{aw5@R?;VxG^IZ*OhRr=l`2yxBJ%3{^W(a*=6?TOYMD z6P*c3*-i*WkBMIUZ6i;I`?FgrZ?O~z`bJ+JxO33@=8uIBjR8;QQ36*%PR<&b1B;DtATH%4dEOYHi6UHwgP%fvZ-CT{=3hvgJcx|Mkrcfqzb0Q z5XISUPzX| zoE-R2eB3lNbD$IAV&izEYlr`BKVyrbR*Cde0-^f8F59G~pl0UhIM!*AxLWQk7xstT zH>0tx4tm+@5#+J2#@MAg@W^QiAYZ**w5=jny){~psI5I!SHsC0FDprzWj4P3J)9(D zvzLmQYgp4dZ)t`9kvUUVAfDA2&otr~24P&SngAq1_kkzO5s#f&_|nM26gH{GB=7c) zXV1zL)-Oo#40n<5Iyhi8O}C>Cvx76shw~&eE=54_TXd|lIUAU<43lIaTTM3koPp^J zOwhR)p%_Z$upmKHLqW%9J&i_As#a(aD4uqSw)hD6Rp76X*0(xVhFZb>*i`S8v+V5b zxIynp8+S6wa1O)LCtQpyoS1xi(_`w^C*G|T?luNh`RUq*bv3XF z*c;<<#5=cL5$(@4?2Pplt!=TL5POe7RS+9EbV)YmFO$u>S}jx|ejw;HX` zIcVf_H7dT2j*<`(euwE>+@J|NY}J6lz%-$Dgm@*|232f(F6uKOSK3**Bxlug%wL~W zHsjP90X5@Q>aJ@~qu-;iv|4ghH|i=cUqTnPyQgbGV&ta`Nbv_4w=5&}5XZnD90up# zmxr{Fpd&`DAhO@d2Kq)IJ_v>4Q++#Dl-HOM_!6umEPS@Du;$@8D2<*|_^vx(xgklt zS%bh3cwyl>7NR0eG9nlx?xv=fX-Ppt2~T!T;i;n|O3-GLq(5G6^Iz5EZ<+olYVy`X zrlU(b3%1JX!hA_`q@5HD)0E?ujSH#4;#+i#w&ylpGf4f_)^E zwZSawojpB}lt|KjuNq`-(M+0Zk00C3FDdxS|C$~NjGo>zy8HHx^a3Px+dN6uU`DBE z9IBS~4Io#bhun(~vg@m>PvFjP;(Y?&YyFL6&a6dgktA?JdSTg=GPGi6YwK!t>-T)* z(W5V@`N-u@H8mc9t9fVD12V!GE(ug5?{j&u2>>^bPkw33*>Qb*Ts^pJ=UjebJ+niEuygmbOA1$qSVJy6HiQs=TgxuEAs9rCzt=B@}_XyjF{ z+uyWN2z6Zry%qBplCuwRZ7ULVV}Wx+%F#MR%0m`9NqXxQ95--6HdRQsgJ(B@_C(?HBTV&Q0ZXEvb;l~eCvJg~n;N%p0?Vx=<*O|OP>b`1p$=GSX z!xf2SG|aSTB-(wBoo)vc^xZ-wUHUCHZI z@!P|*P}KX5VY8vi`pp;doS4DPFRxpR8KeH6ll)-CKOkzk8%~-QeG? zWd{YH226qD28h6B7r+xQ8ia6%d^pnjOgQ2%y1!HW2pO91KbUPvgFEr$B{2n{eFL}# zw-0>tgh7S@lTaGR#>Ihu4;WRxOIOa4dH680!vMsZ<>kW*w%ADP1Uj!->8qrJHl?*W z*)J6(`MMI{$41Ct;S}rs1JJ(kC!h^Rw#3PmtWmEqR2D;&iK}lS_y%vNJiC~8@6Oc* zz7MF&D>+~^VbAf985Ob(WkP;_e!Vzg^70H{_<>+@!Oa=Q^(H1Cz_R+zmRcXJ6thM- z>%YOVa|Zuw9Q$1Oc+QNw`C@v;G`oxr=7TMf!qzOJ%#aS5s}hVDXO=B@SfbYK`uGGI zh=SF~m-epA`;QU9d)%CBnxjghKpcMeIrj>t1TTO#$QT8Eg!kItmfMkB|Le!?~TxVFTV=p*Ak3Q5Gk>ZwxAJNDGFes$M_I3e08R8oU#bAyL2r?;Z^+ zk7MO~BU4-k4uwE4t{TvQH`c(kjOq;tB_a&qi-(?LIv0A&Kv6x8c$<0JnB$=2e~(jN z15V$pKg_U=3=IpIf|{7#KBBuC2xIsK+X~rCAZr(kr_F31{spG?=u8ox))`A6kEbzF zeAEApU%uUjJZodR-mrmFCuxR^5L z;YrJ4w%VFjal7E=klkwS*4%X&ESHuqdAWNMuO&ZjZdiZM2k_RJgu$9zM5c zsxx18+>B)IGlu`)*QoY+?A7Es{VqR$9zOlT7c&OHRbS}Z$`G>Jmg;hKV!HidGO{@lCqmt#K&Mq9_B9Y&N=cGYez_X-);V$7o_oridexEVCVEu# z&y^MQDX)!gZC@_glf%U(DMKJ_+&3Ep^W$2%=aeeZYeSjk$f%3972_+rdx!0CQ}U7L z_3(<)UIh^V!tHtOG#LV8k`$o{g7zX- z5!9}G?v?%js9n87u$q8|-f(%iMvJA*MpYF+;wRvJ4fKxvLw9x=Gxj*>hM?XDpALlA zWr<5|2jd}};rB=%@V#t6S8IPOx0$-4%OE5Gcv%7zR)gDOy;R#rzyl4<~B9MxJff=k_E^WVu)2O;LO%@awzzJ`8EEr76 znVXi@G-Iv86beWl9hj&oB&58B*uCFV*5NCC;2u?@fb_gylBK}vR-n{veEJz`t1{$N|{SAuj^T2Q~ zfA?-0Xvd8%c|aw-i}hGjlOGC25YnS5_|Og)WnD1{(73CR%^2Z%*g(Xt0UE;RGJQ38h*Hdmz zt9zlK-o(Um#uYWT0ZGkgOJNsO1}*pHhPcT*&N2mOEDL@9F}SxmC)$sQa&)Cyy&?Jn z-KC}(+iaLSlz3j+=fN=wMyxdK0euh zHJ&Z$o16UJp4Nse^%-*f{cqfs)s`}T_kUC`G4^s%3v%0+c011N4V!Du-$llar1{aR zR*ct8b~Ci7I8)hp*H2+y(&a>bB0bA3{U=!i(&=Ae)y%{Ll5^pOH)Srm1xN+WIdBey z(iOquR!U)LHVOoW)%dSc@7}!&G;bbWUhv>yBJ9Mi0r->c{{1M!k%)*0p@%uj@qipf zb}$h>w6Q7ZTp&TraLFK=qi8v@=gd9SRk71wJIC=#vE7U(CO2vNT059(Iuu2S#3q*` zr08>B@=0rKWE1#<06?0<>iiH00DYMGu5()|;UAEE9|Vs_dUjk-k%$c>B&?sIDz=#u z_w5C;AP0$@ZhkF?#V9IT%|sb;P)5!2EXK>(*x34j0W>%W?!L(8K>Vd{&{u+b7aY2x zp`k(?ZvbOmGro(kd>@DmA4s2sap}_a^df-br*m831kf7nLqB$p022Za`QX|Rzt`LpV=fv#`bUx=>pPi08!I{=L}wJ(S- z*(}GoUg`qh5WFpP})h}>It6l8O%vLgF;?Iup^72A1pNnrV1%zXu zV-muSKRp4tIBK+4JwUY$VqO#brQIViyLxdE2QlPzN+fIo5IFd=!GX!tiIQ$y4Wz|o zqV{DckcDjTz{UDqY$AFNp;(f=Hz=;0^73*vmGa1S*FkJH6@GwJ6 zrvuLf=of)8o!m23YN7s4!cVwiZp)HB%33 z0Rt9bFW~zb0Bi+pJs^j})}sY_OHeHWjtHF+9FX9K0JYonv<`bJFmu6)iqG&f0hv)( zESm^jBaD2H5UA-VE{Y` z$0`pXLPA6hylXIbYYYc9A$Sdxxo&vt68ocYqOs9Y*cdf55+rGDt0oXO{64XKF1F(0 ztT=^QInX7 z2lgXivk1C0N!kb^E$1^svR#S1Wu>Ki-~uR)VKJPq3(E~CHzh+_O}cF;Vb9M_mNW&O z5;aZc70XRMeSFbKU6g85s{Bfkze5t6%{TZbRY?#d)@ovMcW+YGN1?8jQOa@pzE;cW?&@0L zvV@2dk*B@;#>acr5Ib2@7CR|0FBt8(__kz7FZ>~k*F@EWA;T3vA}7qIdGh!d`tD2J z)aJh=Eg71>BrWHBRUrT-4?lrZ2EX(;xGueEl?17;;5*Kkpg}* z0>I0JJ@Oe8nl?6D!1%ruE(u$lsAw~|p_~$#|3pbdfNL;RbX;Us+sC7D@H6Pllx^C3 z;H!dk4#igmat~K%(byW04{9X7Lap%}4+Dy6K({sIc-sD~uTQ?vU^I?xFU$6`iz%G- zu3y|n=eUbCw|#gvhI()2w=Ux|j1o;PEPw}44fKp>YJb&bFu56PJq_KeAhn`4=J0XG zdV!gLz!A^Flo^8+Y!kfriWw zW?wL@1ULkgD}8AaHu^7htwXmtjw@3R4xC{N>Fs^K{tiYM*NBK#c3m_3A^&b3PB<8O z@E)Q{ypw~)uG#HFSU(qDc&#%;&;sK_Hs;pTU%QsKmJ`qW1zV65-pO$V0GxX zT!FANmQ5dMZ`R!IfPPR$-HhyjT!kp z*>#1}NNh%fh83g)96U#(Z;H}%PY$*Ku}3Vhc)4s&KZrJj^U~DR6oy3iT=`*rl&@@p zJ0rP0>6IF-p;6#T&zpX3`4Fw4z0;HsWGPR4bcLlaI)=bGSEC6f7=qqi9KgGef@bzlGt?ooo{iz)s zwqBJrpu2-?g72m7`LhiG8EZSwpl9~KO-@Ea7UO3FFdrDV+}B1P4Z$r0stFL=Izh!o z)&?9?lySratiRiN)e%rr5?p?ZT25}KyF(XwC(j(i2@JUu*u|(D?p_byb}OmX^z$!s zTW?}wAg0bMHz%hBo5O07DY65KLVHLyWH(jLQS*bwh!zB@aEdqpci&n0%znYO|GW<+j6j=MVexXB2j_9xbCK0>DiXjg301Ja zUHCho!l*i;nBXeug*|+jh?M%`4&chE2Ko|LeoaOZ!I9VenwUTXhI9xx*JoM+-w&i* z;}EFnUU>0<5*O-)oyiaV%c#03q={2STTsY3IvZdHg^)$m3i{T>#44@w_DkJYS(e&Z zn=j3m8rN@M5RV-i^CEPHUqBmIi$ymZKJCS`BL!X%ObS62X0hUt0+3I_?J7L*`_6GAC+8T zOyqtKyr6Vgahprtb%DV~*Y`6NACwqj7rgaDty=NP6F(XX|@YS6Mc$-}M%9`(F9DoQ?mlsAOjKq6SQ z+Fd|LiID3NIKn!+A>~;7&|c3=kDmNkZ|!T0s3C{11VFfda{S zrKOJG>H#3PkW;iMwadv?8;A`+kpc50B$!PDH@dzCx{vGu)6=mj#NaU%Dn3_IvVAF$ zAFdjVKW@f!qMv=>>goz(*guHZpjm0*)!c6@#bbptECP>te{@tK~SJ9_>OB7U+!rPk;1RhWCH7KRVv9{=Z3Vfv{LU;U%!| zYw|(I1XjzY<_!+c9W0kuum)v3;|nMPC-KR#EqZA{V5K5}zXjGF-c_JMqP-e8 zV%{1asj<%D`MJA0S~|U?qy)MJrcM|i!2t!q&cHZo^f)F3?@#?7!`Qq$64B~OBQ4ONX15<=M-2_-9>TA;NK z3<^RWoFMT4{|aDm8+9j)eEm9W&jVbti{p!ci$H5qd%CCYD-T{%uddyGd1&x|x~c!i zyD6LkIf3M`xn<};0cW^QWl%I?&1hxaA0f?ULPb~Fz&U?fsb`s&V*GWmo zzJ2rEeo}t^mt6eqYj6a01D64=eOAly^3YJ=&+9@Kaqx9~JUs7y;)aF>ZzbShhMefCZD+JFLqyj`++(hU~AVytIGo!<+7s z_uS+4uXer^=!-ZR+yq5!ihO|f6u_Q<{Qw_vhSdzp6&R6BZS?k0DU_-Z zMw_ZoWIDh|CJ{la`t<2Ow5X&Zt@TBh>toaj2?%y318{VAl1QPMQSd1eK83^DVK3h8 z=l8~JDu|$q>Ae1O`Nxk+U48eOl%|=bn($PHw~q!#Mz|A#&M(+;_l($@vissJz^}7iomV4 zFPMK?TZ0Ft`8$OC|bXObxz*}^+@w#AxasFhL&~~RqUWH$r;_>Qq&L6S^vwC z6>G|iWE6epM~UIfiGQagNH%!&pZPoYUPY=o`B?B#`oh|SaccMpz)YGB3Z1Zg- zAo47stJHugzU2AscPO|MS!f=u!Tj{265vq~ir^grGv*ZeST~5=!`j<~*)yc@1hx36 zm%`1vXx0vgert0xafo<)S9DAaCyad?_JAG;UhQz~U&|V(1hcBxSYls!D2jmwZ)dj+ z{iYJ*6L@OMh^qVl+7Io{7PD1I%}SzEQBH(7B89t|9SpOQ5dahbB zbG>7LPM3k;a=PNk_&C{cuHr|?rRy_UMZSi_idU(hva;T;S0)4E8q;aWPO>G#L`@BJ zAMh%ys;Rj_1%MxJ`BS<`IH=Lj-hGt{uw00^J`6HI1}vrkAQUz>HguZcNdX<42!?9T zAn+Z5aZxm4IroLq%|%nRHc3D4n&$si-Is?`y{_+9>@1ljNg}0@kYp%RhDu3;Lda62 zM5fTnm@<^g3Z+DB8YqM$GLLC6m1LenW)hb9cYoN!IlFz%+2{K`*Y!QWbL~IQ)wZ+N z`*}a_`#kq^4++e*_SIbp+t>>?i;5Kr^3SlY39_u)DdNy3YEp3~B6@GSJyS&I!JgUu z{D8w0jB%vac$5JMbFxCh7%9B2&cGj;H^K-xS#Tab9WuBRtFNzK_o|LOi7=Z@xA(wR z28FS1ASRreky{{s5EXrfA4Fe_T3-=DDoLc6o;>;&rXgSTS+_1=sBQmV#kV6MBII$O zyRz>eHKtFbK6M;FJ+bfl!3I(Hi%S+Az2SA_$s3 zAMZ&l9e=^h!jb_=1<+Kq;^@x-KblA!rBG7pEgRF$TS2&h-Wj;F^+VW^H((6W;AF(} zy4f0oC|FxxE8twZbRFl_I?H4?1M14P`$I$g{A>V(gzzkX9RG^3%c-2gMSjY1MDtqg zL_CjGa@aGillyoE1ot(DRLI?X6vDZ8A!E0q?B-M*UPJ2VPPq$XlftHRg@CnhGI-?_N3OgTLMQJdaMCYK!9$;}zknBC!h z2+=$h_8Pq8WIx6qS>_6E>GA0GA|h7tEx~cUKUj+8R4zDLYp#(Vo;@oMDb3s}{MlIH|C+WVbUi!$sI!8im{_UNKprywIEbMqZIz9@nr@oDS<8TZT?P7*!Z zdAuNz<;^V!2m0NrXv#dnRyO*OAmoKN+D22k1ut5sG2Xc*!CTOuB_`wSF+iC-zKq%t zG;2|$s_f=Ed&DS)SOglzmA-wOqX-KZOyq$h{{zV;PQHIB**x~VK8v1T&<-O2$iKb$ z9Z7wHW9#Zu$ zA;&h)VeZ=wwjaIOeP01JucuSi4(8V4q84Rk4*}PvzmaN^U4(FUWK+5K!v*(=V9M56 z6BDoFa3iv;;vrjMX&tQ|uHUmZ0028{$EqE#3`(B&SdXXHCIEcCz&)wrZWd5b`|>#_ z<7T@@id2+L)542f@}l`$3GkXYn0V;V(PTUk`T-ZCi+^gw7$4`W>uCp8&S9^f> zqET*9YdBvAdDh-z-fH?l-{~Aw#^(@i3 z=^0)1zflMMM%}qvma|_%R8nRuZy3jUE$^vYvO-%*j@I@_Ze@s6EqAz+mLM;*#di_+ z#JZppRWJL`$ZLDs`e>W+RGLaA-~N2P<)L=Hux;Whj#O9wkAC?!R~}ED3&{P&v;&_{ zal_g|lJeBGP2UPGl^q@AxTeKHjeJw~Zt?0V{k|ZjD^HYK{~oXYO<6hPxcvc_vAvkf z75W8kVnQBB1%k`Ss{-ngSZ(3kdoU~Ww%Vyim`Geo;eG*4TVl0b6W#4S$S7Q^cI=uC zMuY>J>wO3h`fJy^dwN8Kg<&C}@?F8EB_Hy(z>|CT>{+l0Bz`@;dm3x;uW*-(-Z7_n zw@QvJ*72{3_e~r=dvfb7uQOQQKpKRPQ^p3D<6wMn7>H-pyJsQ~>1cGtwxUdEC$<5_lL+HYh!d`TI6aBProj96 z>z!tPk$_%2CZ-#2gz@>MP|sa-n;3e`@&BPAU}jgkwAcV_14}he3gs$%W=RXT*e7l zH9iii7k+BTrzcu4faZ2Jn@3AX?fh%yW=m3px<5Rij%#RK|8V+6wFPFe=%ZIyu)z=r zEJb{d1ijd6+?#x^Ufn@@^zh-A_I4?0X;{0!)B-@_!d0*I^Tw)ReUf8#FPr;Pi?_)Q z9Wl-20l`APhuIf=tp7^hSlbbC!TFLHPm)|&?}|6Qqib&Ofhivhz3w~lHhClI*aA~e zVeJ@Dh3Tdc!~zB3x_W zSQv6V$5YJVUC!)-13AQZBB;Iy%5w6u!M#O;FPYWDO7;Q*lNf zHnNytOIcKbwj^Uj{`wN0I7kB9rK%c(TYi-v^9S4PRxr`9b2o~J6uo^5Gva)LS>?sD z93=~2j5HVlz?ug9jJj#9pPIaT{u=lG*#zU?8^QUx2=-H$#Z7;9ab5!QLA zqjbN9MqQG*4hW%odYJDHAvZ4r9&;f2Fz{30aG(CZ`Dz@4Jyp7~y)4duR1}{!? zs2v<09=1&0E-G4LBEb^4HOEEJ12tl=gB#^WF-G-h5q_nGu{t~(NQUo}bgNh&{NG7x z2Qh${7N>TDA@MXO#vtAZbFpnv9^l8-GYMNvi%>if!mG1qVK8z5QsT85cg1ygoog>r*U*45 zL&oZGSo$SaPfjA`4*q+fB1-cxZ@VwLeS7Vhq-W%;hEBVdQ7;2gDCUqTi0{#ee~@>C z(43&VcWL{i)1wLY<;!Xd{bb7(J&c%>MwXAkk2-0zdU<>t5Ls}Y&(^0V;S1m?;qX@9 zvj@FkUbw%m21J>vu(i!DO>&~t6I}NaLmW>75yzib%4YU2~K(GQY4bb)Y zsAuCsmxduyIJEhA?bqLammSw(ROsop6yyA@;EaJ%nJru~<08{!Q{6j~o2z_MI=X^a z2A%>(7})rU@5!NTl|G2sZJp76m=>M5t5z;KJx_mD20{}9q4Q5iLyM=Upv?PAgQ4kz1dM-JBYG&Jk)zOhgnK zpvh$cetw%#03dZYx&k=?Y+%7<0cnQ*3Fn?n5vql3w;X*vJz&vowqtHHqi%5quMj2* z0fovcKO(XNZqOJtVJE$P^M;luFnvH@e*!EHSfxLF`gG4KzaqRX+qOC4RmLxd{|hhEn8Gj)}gqYoX?dPt#s``alfL^Qu z_7teSMCuVZpP?Y=VS9ohZ6a|YqX7{^Z`;0QXE(xN2K{!E8s7nR>B~6-*kx=Ty@Rk+ z0lgMhvgWwjDKOjowscYjXBh@;@cLr7p1`hLDnDw_gh&aKtEv*Jk{3%MWgg%LZ}#v; zED$~myj$%dMF+d;a*jAaA$XHb!oi$(i>b!RlWmBDL~z@NfVVteLE^`AagW08D+A6+ zVmD>JxWEb9TV@NIB*7QQ^A^4`a2;$ZK*dEC;$*2>s!Yn-iARF!$&{xZ&I~v|3FG(L zUg82YrpmwW7#bA2qbq_pI8^qWDda9lhdLDb$-B^n&c9g+1okqt438x#0D5 z42_`?Ls+3#EptajkBjnn78~6`v;Njp>*m~4`)+n+6lkn+>EKNbSCR`AOR+yBBnVRo z>0$bLw5x&HFt16{t4F0(Q?m{04I)f7VVhMAZ_O_bX>^<5y-{HxQRO7YrDt~911=o8a&STqoE`hd9#@TFdzg6=( zi&j+~{Z#M_6jyTN&1MN%q06fvoZq!!VM(C0%Q@|>=T1!aFUlPX2xcSm)C;r3?drBe z6Be#@NcTvTdE0}`9Se0cs;fb*WLHn^C>|;b?PcS-eEHaWfo}!*6w{LLU)mbm%PHWO z3LtNmj)C>Nrbdf%DJ(gskOqeGaUVP_lhj`WTn~*Ix!vj+SkU1eoksJP`1@#-m7lf?|wt$rGO^*r3 zo`gao9DrK5xauyzNbDi}##n?Gz%mOT_%h39K6%Ga&DKrqgkiC>LKaHK$G$+?o<4fi z1`@ps?j|Vh`1t$ccp(DsKqWg^WlSB`tZM*^6k8B>rX{$zY-)@hMx&|h+wj6+eKK69 z#<2noSr4)0k@51@Xh-R$H?ZB6cNzilO;7Kc%O*x7qDm9_TevY%@(p-R98EY;fSat} zA&m-SHLd*@A9unT8g<}m517P^IcI7|FS)9U>KWGowpNgK^i;8x>*5tFj$xtj@gcQw z!NbBqHCd=Q%^|}ED)yY-8-=6F_ceaIF_+@;!s5QeEAI+%fG=!WvoozHuqebg#l>BZ z8xq_|yt!>dV_=)Tz&An1Vk_^aJ|OTGYNhB3NAL)tMQ@emE1sNy!Wp(F_|{`Myyj2v z@@2Wdn>SCGqOQ8*@PNDkJ{y@l1NnkHZbw{U*Qr!+4W#W)FgOdaQVdl3Pl1Do&_3KW zVb0&)?vtNCuW#FGt%EP-Ccy#of13o4eJ{WipWxblf?e-_oNK#~>90(Jp-`K6$^Nf| zYsmrccrWEOXTS1ZuKjWvl@Mw{l*Pmm3@g~ntrxLWC<2Xe7ULxna2{~~aJuNto143Z zut=q-=xo-I&CR1VBblzp)0}7IX3%I(fw~QR8X*g9vjlFx@A`i44Wx9 z$(_+OA~)p43%e`x-$8Bc+lm{>TGj!u?9U%J-;5meE!j&oOVZD*4S1YigCvqP=Z(xJamWdC$ z1a2(b^198o`9l{2mLF-u?J|9r*%YeKA?FzJ#K+BV93t#%p^8AYC}mj}b?6E7%L8GJ z^Qq?9i?`=b#FyHzxKu3Q4>^Xe2UnyUa!pm1$k8|mJBgJfBx2Gy0r`3;3#$9mc^#$S zK|h`UKhqlO)83?S);Wr}cwp?nk1vqgj%cZnK815Lw zZ&r!0cqB1SMdLZ4&r7Za3UGzND@3zne7Kusp^7%dfqRITe|i*E5eif0JU$ugd4(D@ zd7?3&KMDe>dcfOT{{FVZT}Qn0+|746D>S0E$N7|(?PMG;eB8Sf+Dlh&*uP&|>Bz8i zDy#|u8oa|xF=21r2+^FI0R#hhR>Z}NUzW$F2o=Mv;qFSdD*wlw*~MW|o!>FV8L)<+rl3>YqCQI6C@Tt4#O-!bnOYXyC{TJmVdjnyjq; z1D*&gjXZzXbM@5*7YteOGwXMpORMKngQ@cTiCjSr);HE2jT-+&e$GEJ+yQwy5b$)ASRH~awytayKwgJKwq zHWLNZ^Ha)?!9~XvM3@8q3vxMKHNhrgM)Padu0@mAVD=EYTybWCsl1-c2X*d3^6M-Z zZX-Ip0|GGw0E`JF*RJ5;utEJdGU5=c?F0n|2od!-%7D-Cwsc!vxbR9ZR&(DzK>o~@ z*nYU%A`WjvSTQj_M2wAcbOr062M#NUXVOsD3DyAl$uW^YdxcN2N5~&$Txs?EyDf)D z36qZGWT1(a9Yf7yD{-h1nJg%ri4!nuW~v|6x_8Rwe$>Lw3IgQZmtc?^#BIrUZ{nmO zcY5K@Buj4o4f-k8f|+B`474Jz0c}J7OvdzhH(JB9R#r2ZRMphh;-gHUFm5&fJWyz@ zrB!}$zY*vr%HabDq}n^f6TP2IhoR$kw`n;|RoJ4!wiXf3#K{UGAlw)7;wo_n>INq1 ziiqA}qDr)Ws6ec+QicQIJ`6?6u5zRQL~7~ecx4tV6H~x$p_K0v!B&~e;^T0+H=EPZ zxxiW$31nt~e83i=C><3UDRqZey-9*Dn6F^mKi0hew6Yr^GL0WUqMp_|a%30~p2E!r z*sm1L?og_a&-2<|1Za_c`DFb=LY$fI7_veeRQ5`%9{glnKTwQ@K>>k_ga>`*^u>s%JBE&LJME^yvF9IXhWH8HAO)83^8EW( z-uRAd1=0`KfgA>WU0&3+xNG!vbyKj-p|KOud$C?jjJlEp`~h2r&aKuk%T+$J3=wXf zux?i)Wqf-l49imGaDqh&Y!W4MUPaO_Du40&_gd?@aHq@nbtA|LBzkB#X%DChdu9WL z;i49dwvxcsMDGL$jlmv&tF$yx^rkGb#$(5Y)78_He{v!v4hVQi@X;*SDWW6?h++xu zW__x?+jx1Hj4#k~B8<^{{jHs(`|!3#a2E!~?kT5eg+$gEw*54Ev2SC}BJZIEp(7_C zsgliKXana}gnmKZ0lAgbc{Y6}#F@ED}N(Cfv@j6=|!oRmb^ijl$z#cQ4y3-K+j z`E9VQw3;uGz)5j7KQ_}_&jm59ptvHR+QB?^dRNrLglcJBm~*a8NMS31A_0tC*liA; z8Q|x@%5IkL5aCDyP9UTkfOtT@v6bPYBvaVXWaKw4M*r}@!-GCZ<8j3q8(%5$Y(S|g zXEIBj5T^pVPX;h8g)p|I_ERlOvnVZB*NqF@GK_N@sXhfoN*H-yexoEE>a7kPTxOCxcbV_voZ*Y&^1 z0urG6l?CK1-o1^lBa79B3$q5?G_9K-G)sm`oP3Atd}4B{S6<47$&pR>XFp?c9tc`` zHFUR4wtnoi!=MsXL(1_@ObWV~#)K?(Eo{;WEbR!xeQrYboWCzj;n;ev5!qva+wSZZ z!lTSFLMqAD;Ns*FKPr`J*T>|~cQhllz))&Xqa^cFPs#K@LuW7`ujn%V9b%MPM$3h$ z{%82SJUu&y9m7hcL(WUggHU837Frc4xfaEZ$GLgd#AvjW%_5hQi)2h&=#P&sp<7+l z-K#if@MF2D=+f36*RQ1H;0)Ifr!%YbrK!1;^@U?PR6-DuO>+NH9dG=jI-ch}L22sS zcC(n^yv*bIXIl6l`ALi`CWTs+FKfkZ((ktHiVu_UHhC>;9(@0Cd(*AcHGQ_yFYyNl zbEc0Rll3|~@$9p){rbq$x4mr6?mVY&R=i&GOhv{jX<^Rr?sXehsPlQBu2GcUP!voI z1+VkHF`WuHVS9b$r>v?~WD(N+B?1XDOMbD*|CrSl%IHB0W2)Kp_avmV>h%YA_{SLZ zk*!YMIuX*ickkXkouKQ`TVoHXVEIb{!qSvO86W^}Y9dh8 z+hEV!c@OvxDrs)#m1rF>>~co{E2bI~{aMt?97L3S#=ivT9K3P-nt1$RxLfQf z3a?r7Q`U2LZ>aIwQ$iEd;o zs%1c9GHcg9#)ts;5E%s~dj=crBXf$45^VtHAZXnn2fH5j^f%RuZXgk?ZAHQEZ<5mnqbOC3MUetafT66A?sFR!ghJM!Yvs9ExM08P7q=vL^$qzQ@kPI z;Cox6p{Ya+sdWU#pno#hs_GrVFbHF@0HI}ETz0r@0Ry+Uo8H$VQXu`<-kRLL47 zTONZ&I0_0h()j-7;}oK`#o!A1sv{I!Re(hX8IJZkI!mXOvrzp5Gl9XZDV`k0 zV=APrQc^X&)8br$9k=@f$@Bf4`hfi>jR+k&zVHg|U+)?~P(pClmEL2G9kHfMo7xGB ziOoRnwXpkGWce~?FU&OwojJpNh-5!^+M&EdItKxBehv42yg$rRoCmWZ^x1r@g-E^8 z)z=sH+ueZT162Q_``dM*D}a4Q3b;*fR{rX@i+@>*xzbEMp!RO`PWhwl9UY*dLw~>B zns=gX)-dy(C9pp$cy;_Z2{Je-?Yu=u$OLW|DTAf1BZ)md7Jj=`M|f5F(Q`u8`RlkW zJ@ou1qYidOcN73%+y6S4%fv8($Lj*}jAmuXZ)YY;XG}~?&?+(XyJ5LOk8xio-n4g1 z)6Nhxv`EL%)DK#c!ZFqbGsg;NK#Q^M!Z#C~Hvld%9a(xhC?rJ0Z#Ni<&dzdq;_w(F zX4`Pc2M=TlzB6o9cYvE&C(G~x#6(}ZRLqdCrF;tWtM1RSDun;DY<$9xvy;oc%~tie zq?16X$$mYb%GB87#yYsWZEg}Ql`qTuFymCwUBQ62xP%dFp-VQ5t2B`q+ckE-1cf9{ ze)@aJ@d5d@2|H9a1n4^ZIw@IMn8S?0*Vh{{CI=1UM)5IZ#9VPC-?ao^A~n?!fD?8Z z#szw@pN+(sx%(G@lyvNOXSwNB!gpXw^-J(cSYUW~IQsXqv%?!OO=NAqT+vv|#Kx9f z+Y43CK21#w#IYbteP3v-Er(}%g%cZdo(vzf@Qt58!{QxQAJ)}3|2jMM*u=YrUFFiJ zwo*+uMCSYN$Q|FWdA;wpSLK7h@jcd7*!YX@anN7zJuc!0aRyp{IyO0_gP#Ulk-z=* zL5^NaX(a~H%$wJH6^F{Fp>;(U>9FlSBC2^#tdxM52Bbd%m?y$&QMa0az#*N<=r5#OoLnlUe5dEWrg9E>;tq|;K zz?_JVCUTqu*6wqpHG4yGYZQ)rZo$_FeEoX*_57H%5)$?n#JY>x%;yd8)iw&c^aT?z!3z{ z^U~KJB$1VU7ipDAdSb^?QOi}O8L>isEnHNSY!!xtMMj}}k5QgseB{W+v=E{0+)I`; zM@7Lsx8{YJ%;L{gRg|Xe`5p%V^#E6fL(afo?|2iHJ!(K^kGcKAM4OIDig~Fcw<8 zY#C;|UOZk*jEo1L9F|dN9i%{ja(L+kG%vWT8}}WK zH3GW^fK`A@-cylz&FB?!F#R29WWStq;J^W+#Omqk(X3HLZ%p!4LaL@omvqx935BiI zcpreU(~)aWq`4rB8v&&@uMFJ%Y|wYb&J6?#haa8;bK<~yA$8kq{LvMqPSJhk0G{a= zgq1F>dLw?lv}1YCYB0!&h??6txGOM8MpAHC>8^4(aQ+ribK)p>J;x6N0dkx<#{ymd zFYA3icL(LuyG?{DZ*#R@f~Z@la5@X1=<*1nUV}g%Q%muI2A-Zh zRXIc+Jsa?F44BZa9GR@PNYZmrXvEa7ysj?&Y^{_5OMOEF$|8ad8CFW18Z8{_uM_si zqLYc0-AhRHbnWiw!5Hc`DI=&IuTWQaTpZJ=6^{<4tjF=PKaGk4EQEm>QRQtl1dz6( zeHl+rie(31!(p;!!-C`$1-!%%dXpFUkx$x>Jco|w$)C742@YVP0zeT&WE!~{Gu z7BMn{w($;Ii(MHP9`382o=nntkbH0}(qIlmb{oVG|D@i=B-u(Qo;K~@9Y)wM@zYp%O|-0h`s$VxEh3d#c%gO1@q-3 zIlRe*HW&z!!ZB3#_MSLZy?gg=5-Y0$jHA=~Go#Bp9ZyA`zK6k8M#qK}+cuwB?JCaN z<;c$@NStrx!HcRDZAvfhRBy`DxdHRjr(L62GcGF?qGq9QXS^wZf+-XlD4v_Rk*V+j zqjOK?W>84(G$g!1Lg%2SH5Er*Jhdao;hGzHw|{=cUTJf%+)ys$RW1a!kCoiaye0o< zeK$IO;tjosqIK4eW}D-cOoZ8!EgCZs_23Gc2+j?OAdC2jsQIPMsEio4C#29%<1?OJ z`DmPoEMvT`3+Ded2(}S?`!(_~;Pa`h6k{9Xr=x^ZFSbi6Pgo790Idp~;4j^4yN2d`S2&_Kro?&|jrU8W6_^-E}YX>=( z@Tc#~7&T0+PDl5KdssEP0@>uP0CU&xU`L5S;xcE8q*o9W+_5$QXwO!(Oc34@P(T zM1X_!d*;-8*2uWD714)Bf==Zqa^0QFoAP5<(HS42#2o6(iYSv#@7nA;MD~`k$ncIR z+msVNX8o*!H(o4Od-;+jAfG*H{=`#1ow%Y5h#6i!2#hRgeCP~7!2!_abuK-xv3CHq z`WyuNJS`(&zypnQ$g+}aGEK7S)s@kinVIM1aCkh{2x=Lmcx|~YIQ9xlj>>h_SMtc6oXlJ9p(CR$gUv-%xVV-*}jk+IJ9O_4LVVkyi$g zUVbbuhaJa))4i_JV0G_9(8Ei##Pu$mjO11!-qNMUBlm*wXUfs-lLn?Tan3xp~ z&da@EZQ}Uocz0V^JKFw3gc|ssq;N3k>gwu%j-o7!I{XyWB_*Nr<}!b95$!54Gqw%L z_gkc_l)Aik+B4*sBz6D5PY(haQZwkrnn#FFI9c^~$x6>(y^@VThLFox?FS#K{wArk z-d^!9_sS;f=SEkc{y10|#zKE?3?itKHGwZp)Vp zTq9ig>qMf9Q^QE_ScATXmB2TFy=g;5HRcI#{zW3uqrE{M`SWu{{z(|omDc<)H-(VH z&?%A=O>gLm^i9x{E}g7)b@BN%3)Cu%3J|xnb#OdgMRgT&{kpRlzA}|GQ=2kA0Di5h zAz++YpL@&I?4}&__%bP>cD{wJ9E&`K)LD7(N6o!sc9CZHBG$VrA@pg%@@0RBCIUpS zr>_sfY5yEB=0b^np=qGr6$Mk zo99wJ?R{9N*0cFOSA_0o>E*@6Ep?mZu#YH8Tv@r@W!pM#rcu|+Jg3_|2IOWzE*#8+ISOrLh*I z2)+`vOGw^1S>62IKH2#8JB1+;n)ib8wbzo*o?OFu;_DtbdmPy~rNcGur6d2^X1aUU zSWuiKNBcaR-*f6z*p1avPu9MF4uN}WN%L1E%uDWLIHaBY(pzBNy^**VSMrLy z>0V>0n^$6!zfy;jxn}M}C&e5eRoRtN-!pt|CqlaBOx;Ve93_dvZzoA)#nk25q|Pws zUNu?^9RPRDGsw Hbm{*AWQY`( diff --git a/docs/reference/settings.mdx b/docs/reference/settings.mdx index a60f61bf..42ce8d58 100644 --- a/docs/reference/settings.mdx +++ b/docs/reference/settings.mdx @@ -139,7 +139,7 @@ A live preview card shows a sample fleet-status tile so you can see a color choi | Control | What it does | |---------|--------------| | **Density** | **Comfortable** (default spacing, roomier rows and tiles for review and orientation) or **Compact** (tighter rows and tiles, fits more stacks, tasks, and audit entries on screen at once). Affects the dashboard stack table, the resource gauge strip, the Settings Hub sidebar, the Schedules and Audit Log tables, and every other data table in Sencho. Typography, color, and layout structure stay the same; only vertical padding compresses. | -| **Log chip color** | **Unified** uses the accent color for every service's log chip. **Per service** assigns each service a stable label color for faster visual scanning across a busy log stream. | +| **Log chip color** | Applies on multi-service or multi-container stacks (chips are hidden for a single service with a single container). **Unified** uses the accent color for every service's log chip. **Per service** assigns each service a stable label color for faster visual scanning across a busy log stream. | ### Navigation diff --git a/frontend/src/components/EditorLayout/EditorView.tsx b/frontend/src/components/EditorLayout/EditorView.tsx index 4488f336..3f4d8504 100644 --- a/frontend/src/components/EditorLayout/EditorView.tsx +++ b/frontend/src/components/EditorLayout/EditorView.tsx @@ -181,8 +181,8 @@ export interface EditorViewProps { action: 'start' | 'stop' | 'restart', serviceName: string, ) => Promise; - // Declared-service facts for the multi-service header split (§12). Empty - // on single-service stacks and older remotes (capability-gated fetch), so + // Declared-service facts for the multi-service header layout. Empty on + // single-service stacks and older remotes (capability-gated fetch), so // ContainersHealth falls back to the flat single-service layout. Optional // so callers/tests that never deal in services can omit them. effectiveServices?: EffectiveServiceSpec[]; @@ -388,9 +388,8 @@ export function EditorView(props: EditorViewProps) { }); }; - // Declared-service headers (§12) need the same expandable, scroll-wrapped - // layout as a multi-container stack even when only one container of a - // multi-service stack is currently running. + // Multi-service stacks need the same expandable, scroll-wrapped layout as a + // multi-container stack even when only one container is currently running. const isMultiContainerLayout = safeContainers.length > 1 || effectiveServices.length > 1; // Below md, render the segmented full-screen mobile detail instead of the @@ -401,6 +400,17 @@ export function EditorView(props: EditorViewProps) { return ; } + const stackLogsSection = ( + + ); + return (
@@ -518,23 +528,9 @@ export function EditorView(props: EditorViewProps) { Hidden when containers are expanded to fill the column. */} {!containersExpanded && (isMultiContainerLayout ? (
- + {stackLogsSection}
- ) : ( - - ))} + ) : stackLogsSection)}
)} diff --git a/frontend/src/components/EditorLayout/MobileStackDetail.test.tsx b/frontend/src/components/EditorLayout/MobileStackDetail.test.tsx index 1a94442f..073fae25 100644 --- a/frontend/src/components/EditorLayout/MobileStackDetail.test.tsx +++ b/frontend/src/components/EditorLayout/MobileStackDetail.test.tsx @@ -1,16 +1,22 @@ -import { describe, it, expect, vi } from 'vitest'; +import { describe, it, expect, vi, afterEach } from 'vitest'; import { render, screen, fireEvent } from '@testing-library/react'; import { useState, type ReactNode } from 'react'; import { MobileStackDetail } from './MobileStackDetail'; import type { EditorViewProps } from './EditorView'; +import type { ContainerInfo } from './EditorView'; +import type { EffectiveServiceSpec } from '@/types/effectiveServices'; // The detail's heavy children stream logs, parse compose, and render container // stats; stub them with markers so this test focuses on segment behavior and the -// mobile editing flow. +// mobile editing flow. Capture showServiceChips for wiring assertions. +let lastShowServiceChips: boolean | undefined; vi.mock('./editor-view-blocks', () => ({ StackIdentityHeader: () =>
identity-header
, ContainersHealth: () =>
health-pane
, - StackLogsSection: () =>
logs-pane
, + StackLogsSection: ({ showServiceChips }: { showServiceChips: boolean }) => { + lastShowServiceChips = showServiceChips; + return
logs-pane
; + }, })); // Prop-aware so the edit affordance (canEdit + onEditCompose) is exercised, not // just the read-only marker. @@ -298,3 +304,65 @@ describe('MobileStackDetail mobile editing', () => { expect(setContent).not.toHaveBeenCalled(); }); }); + +function containerStub(id: string, name: string): ContainerInfo { + return { + Id: id, + Names: [name], + State: 'running', + Status: 'Up 1 minute', + }; +} + +function serviceStub(name: string): EffectiveServiceSpec { + return { + name, + declaredImage: `${name}:latest`, + hasBuild: false, + expectedReplicas: 1, + dependsOn: [], + hasHealthcheck: false, + }; +} + +describe('MobileStackDetail showServiceChips wiring', () => { + afterEach(() => { + lastShowServiceChips = undefined; + }); + + it('passes false for one container and one declared service', () => { + render( + , + ); + expect(lastShowServiceChips).toBe(false); + }); + + it('passes true for two containers', () => { + render( + , + ); + expect(lastShowServiceChips).toBe(true); + }); + + it('passes true for one container and two declared services', () => { + render( + , + ); + expect(lastShowServiceChips).toBe(true); + }); +}); diff --git a/frontend/src/components/EditorLayout/MobileStackDetail.tsx b/frontend/src/components/EditorLayout/MobileStackDetail.tsx index fd57963b..dce74aa1 100644 --- a/frontend/src/components/EditorLayout/MobileStackDetail.tsx +++ b/frontend/src/components/EditorLayout/MobileStackDetail.tsx @@ -62,7 +62,7 @@ export function MobileStackDetail(props: EditorViewProps) { openLogViewer, openBashModal, serviceAction, - effectiveServices, + effectiveServices = [], serviceUpdateStatuses, serviceUpdateInProgress, onRequestServiceUpdate, @@ -88,6 +88,7 @@ export function MobileStackDetail(props: EditorViewProps) { const [segment, setSegment] = useState('logs'); const safeContainers = containers || []; + const isMultiContainerLayout = safeContainers.length > 1 || effectiveServices.length > 1; const isRunning = safeContainers.some(c => c.State === 'running'); const canEditStack = can('stack:edit', 'stack', stackName); @@ -241,7 +242,12 @@ export function MobileStackDetail(props: EditorViewProps) {
)} {segment === 'logs' && ( - + )} {segment === 'compose' && (
diff --git a/frontend/src/components/EditorLayout/__tests__/EditorView.test.tsx b/frontend/src/components/EditorLayout/__tests__/EditorView.test.tsx index 90a95a30..0317d52f 100644 --- a/frontend/src/components/EditorLayout/__tests__/EditorView.test.tsx +++ b/frontend/src/components/EditorLayout/__tests__/EditorView.test.tsx @@ -17,11 +17,15 @@ vi.mock('@/lib/monacoLoader', () => ({ }, })); -// Stub heavy children; this test only asserts the Monaco language prop. +// Capture StackLogsSection props for showServiceChips wiring tests. +let lastShowServiceChips: boolean | undefined; vi.mock('../editor-view-blocks', () => ({ StackIdentityHeader: () =>
identity-header
, ContainersHealth: () =>
health-pane
, - StackLogsSection: () =>
logs-pane
, + StackLogsSection: ({ showServiceChips }: { showServiceChips: boolean }) => { + lastShowServiceChips = showServiceChips; + return
logs-pane
; + }, })); vi.mock('../../StackAnatomyPanel', () => ({ default: () =>
anatomy-pane
, @@ -98,6 +102,7 @@ describe('EditorView Monaco language prop', () => { lastLanguage = undefined; lastValue = undefined; lastReadOnly = undefined; + lastShowServiceChips = undefined; }); it('passes language="ini" when the env tab is active', () => { @@ -149,6 +154,7 @@ describe('EditorView single edit gate', () => { lastLanguage = undefined; lastValue = undefined; lastReadOnly = undefined; + lastShowServiceChips = undefined; }); it('shows Save & Deploy immediately without an Edit button when compose editor is open', () => { @@ -191,3 +197,65 @@ describe('EditorView single edit gate', () => { expect(closeComposeEditor).toHaveBeenCalledTimes(1); }); }); + +function containerStub(id: string, name: string): EditorViewProps['containers'][number] { + return { + Id: id, + Names: [name], + State: 'running', + Status: 'Up 1 minute', + }; +} + +function serviceStub(name: string) { + return { + name, + declaredImage: `${name}:latest`, + hasBuild: false, + expectedReplicas: 1, + dependsOn: [] as string[], + hasHealthcheck: false, + }; +} + +describe('EditorView showServiceChips wiring', () => { + afterEach(() => { + lastShowServiceChips = undefined; + }); + + it('passes false for one container and one declared service', () => { + render( + , + ); + expect(lastShowServiceChips).toBe(false); + }); + + it('passes true for two containers', () => { + render( + , + ); + expect(lastShowServiceChips).toBe(true); + }); + + it('passes true for one container and two declared services', () => { + render( + , + ); + expect(lastShowServiceChips).toBe(true); + }); +}); diff --git a/frontend/src/components/EditorLayout/__tests__/StackLogsSection.test.tsx b/frontend/src/components/EditorLayout/__tests__/StackLogsSection.test.tsx new file mode 100644 index 00000000..44b3dda4 --- /dev/null +++ b/frontend/src/components/EditorLayout/__tests__/StackLogsSection.test.tsx @@ -0,0 +1,77 @@ +/** + * StackLogsSection forwards showServiceChips to StructuredLogViewer and leaves + * the raw-terminal contract unchanged. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, fireEvent } from '@testing-library/react'; +import type { ReactNode } from 'react'; +import { StackLogsSection } from '../editor-view-blocks'; + +let lastViewerShowServiceChips: boolean | undefined; + +vi.mock('../../StructuredLogViewer', () => ({ + default: ({ showServiceChips }: { showServiceChips?: boolean }) => { + lastViewerShowServiceChips = showServiceChips; + return
; + }, +})); + +vi.mock('../../Terminal', () => ({ + default: ({ stackName }: { stackName: string }) => ( +
{stackName}
+ ), +})); + +vi.mock('../../ErrorBoundary', () => ({ + default: ({ children }: { children: ReactNode }) => <>{children}, +})); + +describe('StackLogsSection showServiceChips forwarding', () => { + beforeEach(() => { + lastViewerShowServiceChips = undefined; + }); + + it('forwards true to StructuredLogViewer in structured mode', () => { + render( + , + ); + expect(screen.getByTestId('structured-log-viewer')).toBeInTheDocument(); + expect(lastViewerShowServiceChips).toBe(true); + }); + + it('forwards false to StructuredLogViewer in structured mode', () => { + render( + , + ); + expect(screen.getByTestId('structured-log-viewer')).toBeInTheDocument(); + expect(lastViewerShowServiceChips).toBe(false); + }); + + it('renders TerminalComponent in raw mode without requiring chip props', () => { + const setLogsMode = vi.fn(); + render( + , + ); + expect(screen.getByTestId('raw-terminal')).toHaveTextContent('web'); + expect(screen.queryByTestId('structured-log-viewer')).toBeNull(); + expect(lastViewerShowServiceChips).toBeUndefined(); + + fireEvent.click(screen.getByRole('button', { name: /Structured/i })); + expect(setLogsMode).toHaveBeenCalledWith('structured'); + }); +}); diff --git a/frontend/src/components/EditorLayout/editor-view-blocks.tsx b/frontend/src/components/EditorLayout/editor-view-blocks.tsx index c70bb0b3..496ab00f 100644 --- a/frontend/src/components/EditorLayout/editor-view-blocks.tsx +++ b/frontend/src/components/EditorLayout/editor-view-blocks.tsx @@ -346,8 +346,8 @@ export function ContainersHealth({ containersLoadError = null, onRetryContainersLoad, }: ContainersHealthProps) { - // Multi-service only (§12): a single-service stack keeps the existing flat - // layout untouched, including its per-container Start/Stop/Restart kebab. + // Multi-service only: a single-service stack keeps the existing flat layout + // untouched, including its per-container Start/Stop/Restart kebab. const isMultiService = effectiveServices.length > 1; const [copiedUrlId, setCopiedUrlId] = useState(null); const copiedUrlTimerRef = useRef(null); @@ -445,9 +445,9 @@ export function ContainersHealth({ ) : null; // One container card. `hideServiceMenu` drops the per-container - // Start/Stop/Restart kebab on multi-service stacks, where the declared- - // service header above owns that action instead (§12 point 4: child cards - // keep only logs, shell, ports, metrics). + // Start/Stop/Restart kebab on multi-service stacks; the declared-service + // header above owns lifecycle actions. Child cards keep logs, shell, ports, + // and metrics only. const renderContainerCard = (container: ContainerInfo, hideServiceMenu: boolean) => { let mainPort: number | undefined; let mainPortPrivate: number | undefined; @@ -807,6 +807,8 @@ export interface StackLogsSectionProps { stackName: string; logsMode: 'structured' | 'raw'; setLogsMode: (mode: 'structured' | 'raw') => void; + /** True when the stack has more than one service or container; gates log chips. */ + showServiceChips: boolean; /** When set, the structured viewer shows an expand control that collapses * the Command Center to give the logs more vertical room. */ logsExpanded?: boolean; @@ -814,7 +816,7 @@ export interface StackLogsSectionProps { } // Logs pane: structured / raw-terminal toggle + the live viewer. -export function StackLogsSection({ stackName, logsMode, setLogsMode, logsExpanded, onToggleLogsExpand }: StackLogsSectionProps) { +export function StackLogsSection({ stackName, logsMode, setLogsMode, showServiceChips, logsExpanded, onToggleLogsExpand }: StackLogsSectionProps) { return (
@@ -844,7 +846,7 @@ export function StackLogsSection({ stackName, logsMode, setLogsMode, logsExpande
{logsMode === 'structured' ? ( - + ) : (
diff --git a/frontend/src/components/StructuredLogViewer.tsx b/frontend/src/components/StructuredLogViewer.tsx index e9086de2..150f16d1 100644 --- a/frontend/src/components/StructuredLogViewer.tsx +++ b/frontend/src/components/StructuredLogViewer.tsx @@ -3,11 +3,17 @@ import { Button } from './ui/button'; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from './ui/tooltip'; import { Download, RefreshCw, Maximize2, Minimize2 } from 'lucide-react'; import { cn } from '@/lib/utils'; -import { useLogChipColorMode } from '@/hooks/use-log-chip-color-mode'; +import { useLogChipColorMode, type LogChipColorMode } from '@/hooks/use-log-chip-color-mode'; import { hashLabel } from '@/lib/label-colors'; interface StructuredLogViewerProps { stackName: string; + /** + * When true, render a service chip on prefixed log rows. + * Pass true only for multi-service or multi-container stacks; defaults to + * false so single-service streams stay uncluttered. + */ + showServiceChips?: boolean; /** When set, renders an expand/collapse control next to the download button. */ expanded?: boolean; onToggleExpand?: () => void; @@ -20,7 +26,7 @@ interface LogRow { ts: string | null; level: LogLevel; message: string; - /** Normalized service name extracted from the log prefix, or null for synthetic / old-format rows. */ + /** Display name from the log prefix (normalized Compose service name), or null for synthetic / old-format rows. */ containerName: string | null; /** True when this row was synthesized by the client (e.g. reconnect sentinel). */ synthetic?: boolean; @@ -69,7 +75,36 @@ function formatTs(iso: string | null): string { return `${hh}:${mm}:${ss}`; } -export default function StructuredLogViewer({ stackName, expanded, onToggleExpand }: StructuredLogViewerProps) { +function stackDisplayName(stackName: string): string { + return stackName.replace(/\.(yml|yaml)$/, ''); +} + +function LogServiceChip({ name, colorMode }: { name: string; colorMode: LogChipColorMode }) { + const perService = colorMode === 'per-service'; + const labelKey = hashLabel(name); + return ( + + {name} + + ); +} + +export default function StructuredLogViewer({ stackName, showServiceChips = false, expanded, onToggleExpand }: StructuredLogViewerProps) { const [rows, setRows] = useState([]); const [filter, setFilter] = useState('all'); const [following, setFollowing] = useState(true); @@ -91,7 +126,7 @@ export default function StructuredLogViewer({ stackName, expanded, onToggleExpan setFollowing(true); followingRef.current = true; - const cleanStackName = stackName.replace(/\.(yml|yaml)$/, ''); + const cleanStackName = stackDisplayName(stackName); const wsProtocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; const activeNodeId = localStorage.getItem('sencho-active-node') || ''; const wsUrl = `${wsProtocol}//${window.location.host}/api/stacks/${cleanStackName}/logs${activeNodeId ? `?nodeId=${activeNodeId}` : ''}`; @@ -238,12 +273,12 @@ export default function StructuredLogViewer({ stackName, expanded, onToggleExpan const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; - a.download = `${stackName.replace(/\.(yml|yaml)$/, '')}-logs.txt`; + a.download = `${stackDisplayName(stackName)}-logs.txt`; a.click(); URL.revokeObjectURL(url); }; - const label = `logs · ${stackName.replace(/\.(yml|yaml)$/, '')}`; + const label = `logs · ${stackDisplayName(stackName)}`; return (
@@ -346,25 +381,8 @@ export default function StructuredLogViewer({ stackName, expanded, onToggleExpan {row.level} - {row.containerName && ( - - {row.containerName} - + {showServiceChips && row.containerName && ( + )} {row.message} diff --git a/frontend/src/components/__tests__/StructuredLogViewer.test.tsx b/frontend/src/components/__tests__/StructuredLogViewer.test.tsx index e335903e..716c52ad 100644 --- a/frontend/src/components/__tests__/StructuredLogViewer.test.tsx +++ b/frontend/src/components/__tests__/StructuredLogViewer.test.tsx @@ -1,7 +1,7 @@ /** * Unit tests for StructuredLogViewer's log-row lifecycle (stack switching, - * row clearing, auto-follow reset, level filter), container name chip - * rendering, and chip color mode (unified / per-service). + * row clearing, auto-follow reset, level filter), service chip rendering, + * and chip color mode (unified / per-service). */ import { render, screen, cleanup, fireEvent, act } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; @@ -182,51 +182,78 @@ describe('StructuredLogViewer', () => { expect(MockWS.instances[1].url).not.toContain('.yaml'); }); - // ── Container name chip ──────────────────────────────────────────── + // ── Service chip ─────────────────────────────────────────────────── - it('renders a container name chip when the WebSocket message includes a prefix', async () => { - const { container } = render(); + it('renders a service chip when showServiceChips is true and the line has a prefix', async () => { + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: 'redis | 2025-01-01T12:00:00Z connected\n' }); }); - expect(container.textContent).toContain('redis'); - expect(container.textContent).toContain('connected'); + const chip = screen.getByTitle('redis'); + expect(chip).toHaveTextContent('redis'); + expect(screen.getByText('connected')).toBeInTheDocument(); }); - it('does not render a container name chip for old-format lines with no prefix', async () => { - const { container } = render(); + it('hides the chip by default but keeps the message and download attribution', async () => { + let capturedBlob: Blob | null = null; + vi.stubGlobal('URL', { + ...URL, + createObjectURL: vi.fn((blob: Blob) => { + capturedBlob = blob; + return 'blob:fake'; + }), + revokeObjectURL: vi.fn(), + }); + + render(); + await act(async () => { + MockWS.instances[0].onopen?.(); + MockWS.instances[0].onmessage?.({ data: 'redis | 2025-01-01T12:00:00Z connected\n' }); + }); + + expect(screen.queryByTitle('redis')).toBeNull(); + expect(screen.getByText('connected')).toBeInTheDocument(); + + await userEvent.click(screen.getByLabelText('Download logs')); + expect(capturedBlob).not.toBeNull(); + const text = await capturedBlob!.text(); + expect(text).toContain('[redis]'); + expect(text).toContain('connected'); + }); + + it('does not render a service chip for old-format lines with no prefix', async () => { + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: '2025-01-01T12:00:00Z plain message\n' }); }); - expect(container.textContent).toContain('plain message'); - expect(container.querySelector('.select-none')).toBeNull(); + expect(screen.getByText('plain message')).toBeInTheDocument(); + expect(screen.queryByTitle('plain message')).toBeNull(); }); it('renders a dotted container name correctly', async () => { - const { container } = render(); + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: 'api.v1 | 2025-01-01T12:00:00Z ready\n' }); }); - expect(container.textContent).toContain('api.v1'); - expect(container.textContent).toContain('ready'); + expect(screen.getByTitle('api.v1')).toHaveTextContent('api.v1'); + expect(screen.getByText('ready')).toBeInTheDocument(); }); it('handles pipe in message body without false prefix extraction', async () => { - const { container } = render(); + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: 'redis | 2025-01-01T12:00:00Z value | other\n' }); }); - expect(container.textContent).toContain('redis'); - expect(container.textContent).toContain('value'); - expect(container.textContent).toContain('other'); + expect(screen.getByTitle('redis')).toHaveTextContent('redis'); + expect(screen.getByText(/value \| other/)).toBeInTheDocument(); }); // ── Download ──────────────────────────────────────────────────────── @@ -242,7 +269,7 @@ describe('StructuredLogViewer', () => { revokeObjectURL: vi.fn(), }); - render(); + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: 'redis | 2025-01-01T12:00:00Z connected\n' }); @@ -287,14 +314,13 @@ describe('StructuredLogViewer', () => { // ── Chip color mode ────────────────────────────────────────────────── it('in unified mode (default), chip has brand classes and no inline style', async () => { - const { container } = render(); + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: 'redis | 2025-01-01T12:00:00Z connected\n' }); }); - const chip = container.querySelector('.select-none') as HTMLElement; - expect(chip).not.toBeNull(); + const chip = screen.getByTitle('redis'); expect(chip.className).toContain('text-brand/80'); expect(chip.className).toContain('bg-brand/10'); expect(chip.getAttribute('style')).toBeNull(); @@ -302,27 +328,26 @@ describe('StructuredLogViewer', () => { it('in per-service mode, chip has inline label-token style', async () => { localStorage.setItem(LOG_CHIP_COLOR_KEY, 'per-service'); - const { container } = render(); + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: 'redis | 2025-01-01T12:00:00Z connected\n' }); }); - const chip = container.querySelector('.select-none') as HTMLElement; - expect(chip).not.toBeNull(); + const chip = screen.getByTitle('redis'); const style = chip.getAttribute('style') ?? ''; expect(style).toContain('--label-'); expect(style).toContain('-bg'); }); it('updates chip style when setting changes from unified to per-service', async () => { - const { container } = render(); + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: 'redis | 2025-01-01T12:00:00Z connected\n' }); }); - const chip = container.querySelector('.select-none') as HTMLElement; + const chip = screen.getByTitle('redis'); expect(chip.getAttribute('style')).toBeNull(); localStorage.setItem(LOG_CHIP_COLOR_KEY, 'per-service'); diff --git a/frontend/src/components/settings/AppearanceSection.tsx b/frontend/src/components/settings/AppearanceSection.tsx index 05099453..684671f7 100644 --- a/frontend/src/components/settings/AppearanceSection.tsx +++ b/frontend/src/components/settings/AppearanceSection.tsx @@ -447,7 +447,7 @@ export function AppearanceSection({ { expect(screen.getByText('Constrained graphics')).toBeTruthy(); }); + it('states that log chip color applies on multi-service or multi-container stacks', () => { + render(); + expect( + screen.getByText(/Applies to service chips on multi-service or multi-container stacks/i), + ).toBeTruthy(); + }); + it('readability locks the header + chart controls and disables the glow slider', () => { const { container } = render(); // Baseline: nothing reduced, so no slider is disabled. From ec0f59a85e3a54f29670d2424e4e04d21b07d6c4 Mon Sep 17 00:00:00 2001 From: Anso Date: Thu, 23 Jul 2026 23:14:25 -0400 Subject: [PATCH 09/13] fix: dedupe healthcheck alerts and share crash rate limits (#1690) * fix: dedupe healthcheck alerts and share crash rate limits Health flaps no longer spam the bell: emit only on transition into unhealthy, keep a prune-surviving 60m dedup stamp that advances only when history persists, and share the fixed-window rate cap with crash alerts using typed roll-up copy. Docs now match fixed-window and non-persisted overflow behavior. * fix: harden health alert rate refund and shutdown cleanup Bind rate-token refunds to the issuing fixed window, clear deferred-retry markers on recovery/destroy/shutdown, and refuse deferred health dispatches after the service stops so the 20/min cap and no-duplicate guarantees hold under async races. --- .../__tests__/docker-event-service.test.ts | 589 +++++++++++++++++- backend/src/services/DockerEventService.ts | 273 ++++++-- docs/features/alerts-notifications.mdx | 18 +- 3 files changed, 823 insertions(+), 57 deletions(-) diff --git a/backend/src/__tests__/docker-event-service.test.ts b/backend/src/__tests__/docker-event-service.test.ts index e06a97a8..6e9f7c53 100644 --- a/backend/src/__tests__/docker-event-service.test.ts +++ b/backend/src/__tests__/docker-event-service.test.ts @@ -97,6 +97,8 @@ let service: DockerEventService; beforeEach(() => { vi.clearAllMocks(); vi.useFakeTimers(); + mockDispatchAlert.mockReset(); + mockDispatchAlert.mockResolvedValue({ persisted: true }); mockGetGlobalSettings.mockReturnValue({ global_crash: '1' }); mockIsOwnContainer.mockReset(); mockIsOwnContainer.mockReturnValue(false); @@ -119,6 +121,11 @@ afterEach(() => { vi.useRealTimers(); }); +async function flushMicrotasks(): Promise { + await Promise.resolve(); + await Promise.resolve(); +} + // ── Classification via event stream ──────────────────────────────────── describe('DockerEventService - die classification', () => { @@ -367,6 +374,7 @@ describe('DockerEventService - die classification', () => { }, }, }); + await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledWith( 'error', @@ -442,9 +450,586 @@ describe('DockerEventService - rate limiting', () => { // After the rate window, a summary warning fires. await vi.advanceTimersByTimeAsync(61_000); const summaryCalls = mockDispatchAlert.mock.calls.filter(c => - typeof c[2] === 'string' && c[2].includes('additional containers crashed')); + typeof c[2] === 'string' && c[2].includes('rate-limited')); expect(summaryCalls).toHaveLength(1); - expect(summaryCalls[0][2]).toContain('2 additional'); + expect(summaryCalls[0][2]).toBe( + '2 additional container crash alerts were rate-limited in the last minute.', + ); + }); +}); + +// ── Health alert transition / dedup / rate sharing ───────────────────── + +describe('DockerEventService - health alert gates', () => { + it('emits once for duplicate unhealthy while already unhealthy and keeps unhealthySince', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { + ID: 'h1', + Attributes: { name: 'api', 'com.docker.compose.project': 'web' }, + }, + }); + await flushMicrotasks(); + const first = service.getContainerState('h1'); + expect(first?.healthStatus).toBe('unhealthy'); + expect(first?.unhealthySince).toBeTypeOf('number'); + const since = first!.unhealthySince!; + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h1', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + expect(service.getContainerState('h1')?.unhealthySince).toBe(since); + expect(service.getContainerState('h1')?.healthStatus).toBe('unhealthy'); + }); + + it('suppresses a second flap while the first health dispatch is still pending', async () => { + let resolveFirst!: (value: { persisted: boolean }) => void; + mockDispatchAlert.mockImplementationOnce( + () => new Promise<{ persisted: boolean }>((resolve) => { resolveFirst = resolve; }), + ); + + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h2', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'h2', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h2', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + resolveFirst({ persisted: true }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + }); + + it('retries a deferred flap when the in-flight dispatch fails to persist', async () => { + let resolveFirst!: (value: { persisted: boolean }) => void; + mockDispatchAlert + .mockImplementationOnce( + () => new Promise<{ persisted: boolean }>((resolve) => { resolveFirst = resolve; }), + ) + .mockResolvedValue({ persisted: true }); + + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h2b', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'h2b', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h2b', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + resolveFirst({ persisted: false }); + await flushMicrotasks(); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + }); + + it('does not advance health dedup when persisted is false, allowing a later transition', async () => { + mockDispatchAlert + .mockResolvedValueOnce({ persisted: false }) + .mockResolvedValue({ persisted: true }); + + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h3', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'h3', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h3', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + }); + + it('keeps health dedup across the 10-minute prune window and re-emits after 60 minutes', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h4', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + // Recover so lastActivity ages without fresh unhealthy events; prune + // may drop containerState after 10m but health dedup must survive. + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'h4', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + await vi.advanceTimersByTimeAsync(11 * 60_000); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h4', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(50 * 60_000); + + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'h4', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h4', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + }); + + it('does not let crash and health dedup suppress one another', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: 'h5', Attributes: { exitCode: '1', name: 'api' } }, + }); + await vi.advanceTimersByTimeAsync(600); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + stream.push({ + Type: 'container', + Action: 'start', + Actor: { ID: 'h5', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h5', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + expect(mockDispatchAlert.mock.calls[1][2]).toContain('Healthcheck failed'); + }); + + it('does not consume rate tokens for deduped duplicate health events', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h6', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + for (let i = 0; i < 5; i++) { + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'h6', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h6', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + } + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + for (let i = 0; i < 19; i++) { + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: `crash-${i}`, Attributes: { exitCode: '1', name: `n-${i}` } }, + }); + } + await vi.advanceTimersByTimeAsync(600); + const crashCalls = mockDispatchAlert.mock.calls.filter(c => + typeof c[2] === 'string' && c[2].includes('Crash')); + expect(crashCalls).toHaveLength(19); + }); + + it('emits an accurate health-only rate-limit roll-up', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + for (let i = 0; i < 22; i++) { + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: `hh-${i}`, Attributes: { name: `svc-${i}` } }, + }); + await flushMicrotasks(); + } + + const healthCalls = mockDispatchAlert.mock.calls.filter(c => + typeof c[2] === 'string' && c[2].includes('Healthcheck failed')); + expect(healthCalls).toHaveLength(20); + + await vi.advanceTimersByTimeAsync(61_000); + const summaryCalls = mockDispatchAlert.mock.calls.filter(c => + typeof c[2] === 'string' && c[2].includes('rate-limited')); + expect(summaryCalls).toHaveLength(1); + expect(summaryCalls[0][2]).toBe( + '2 additional container health alerts were rate-limited in the last minute.', + ); + }); + + it('emits an accurate mixed crash and health rate-limit roll-up', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + for (let i = 0; i < 10; i++) { + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: `mc-${i}`, Attributes: { exitCode: '1', name: `c-${i}` } }, + }); + } + await vi.advanceTimersByTimeAsync(600); + + for (let i = 0; i < 10; i++) { + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: `mh-${i}`, Attributes: { name: `h-${i}` } }, + }); + await flushMicrotasks(); + } + + for (let i = 10; i < 12; i++) { + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: `mc-${i}`, Attributes: { exitCode: '1', name: `c-${i}` } }, + }); + } + await vi.advanceTimersByTimeAsync(600); + + for (let i = 10; i < 13; i++) { + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: `mh-${i}`, Attributes: { name: `h-${i}` } }, + }); + await flushMicrotasks(); + } + + await vi.advanceTimersByTimeAsync(61_000); + const summaryCalls = mockDispatchAlert.mock.calls.filter(c => + typeof c[2] === 'string' && c[2].includes('rate-limited')); + expect(summaryCalls).toHaveLength(1); + expect(summaryCalls[0][2]).toBe( + '5 additional container alerts were rate-limited in the last minute (2 crash, 3 health).', + ); + }); + + it('keeps Auto-Heal health state when global_crash is off', async () => { + mockGetGlobalSettings.mockReturnValue({ global_crash: '0' }); + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h7', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + expect(mockDispatchAlert).not.toHaveBeenCalled(); + expect(service.getContainerState('h7')).toMatchObject({ + healthStatus: 'unhealthy', + }); + expect(service.getContainerState('h7')?.unhealthySince).toBeTypeOf('number'); + }); + + it('does not refund a rate token into the next fixed window', async () => { + let resolvePersist!: (value: { persisted: boolean }) => void; + mockDispatchAlert.mockImplementation( + () => + new Promise<{ persisted: boolean }>((resolve) => { + resolvePersist = resolve; + }), + ); + + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'cross-window', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + // Cross the fixed window, then consume a token in the new window so + // refund would inflate W2's count if it ignored the issuing window. + await vi.advanceTimersByTimeAsync(60_000); + mockDispatchAlert.mockResolvedValue({ persisted: true }); + stream.push({ + Type: 'container', + Action: 'die', + Actor: { + ID: 'cw-roll', + Attributes: { exitCode: '1', name: 'roll' }, + }, + }); + await vi.advanceTimersByTimeAsync(600); + expect( + mockDispatchAlert.mock.calls.filter( + (c) => typeof c[2] === 'string' && c[2].includes('Crash Detected'), + ), + ).toHaveLength(1); + + // Fail the original health persist: old code refunds into W2 (count 1→0). + resolvePersist({ persisted: false }); + await flushMicrotasks(); + + // Fill the rest of W2 to the 20-alert cap (19 more crashes). + for (let i = 0; i < 19; i++) { + stream.push({ + Type: 'container', + Action: 'die', + Actor: { + ID: `cw-${i}`, + Attributes: { exitCode: '1', name: `svc-${i}` }, + }, + }); + } + await vi.advanceTimersByTimeAsync(600); + + // Cap must still hold: a cross-window refund would have allowed a 21st. + stream.push({ + Type: 'container', + Action: 'die', + Actor: { + ID: 'cw-overflow', + Attributes: { exitCode: '1', name: 'overflow' }, + }, + }); + await vi.advanceTimersByTimeAsync(600); + + const crashCalls = mockDispatchAlert.mock.calls.filter( + (c) => typeof c[2] === 'string' && c[2].includes('Crash Detected'), + ); + expect(crashCalls).toHaveLength(20); + }); + + it('clears pending health retry when the container recovers before persist fails', async () => { + let resolvePersist!: (value: { persisted: boolean }) => void; + mockDispatchAlert.mockImplementation( + () => + new Promise<{ persisted: boolean }>((resolve) => { + resolvePersist = resolve; + }), + ); + + service = new DockerEventService(1, 'local'); + await service.start(); + + // First unhealthy starts an in-flight dispatch. + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + // Flap while in flight: recover then re-fail so a pending-retry marker is set. + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + // Recover again before the first persist settles; marker must be dropped. + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + resolvePersist({ persisted: false }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + // Later unrelated persist failure must not spuriously retry from a stale marker. + mockDispatchAlert.mockResolvedValueOnce({ persisted: false }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + }); + + it('does not dispatch deferred health retry after shutdown', async () => { + let resolvePersist!: (value: { persisted: boolean }) => void; + mockDispatchAlert.mockImplementation( + () => + new Promise<{ persisted: boolean }>((resolve) => { + resolvePersist = resolve; + }), + ); + + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'shutdown-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + service.shutdown(); + resolvePersist({ persisted: false }); + await flushMicrotasks(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + }); + + it('clears health alert bookkeeping on destroy', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + stream.push({ + Type: 'container', + Action: 'destroy', + Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + }); + + it('does not retry a deferred health alert after destroy', async () => { + let resolvePersist!: (value: { persisted: boolean }) => void; + mockDispatchAlert.mockImplementation( + () => + new Promise<{ persisted: boolean }>((resolve) => { + resolvePersist = resolve; + }), + ); + + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + // Set a pending-retry marker while the first dispatch is in flight. + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + stream.push({ + Type: 'container', + Action: 'destroy', + Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + resolvePersist({ persisted: false }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); }); }); diff --git a/backend/src/services/DockerEventService.ts b/backend/src/services/DockerEventService.ts index 638b00ca..7ea5033b 100644 --- a/backend/src/services/DockerEventService.ts +++ b/backend/src/services/DockerEventService.ts @@ -45,13 +45,21 @@ export interface ContainerHealthSnapshot { /** Grace window after a `die` before classifying, to absorb out-of-order kill events. */ const DIE_GRACE_WINDOW_MS = 500; -/** Max crash alerts emitted per node within RATE_WINDOW_MS. Overflow is batched. */ +/** Max crash/health alerts emitted per node within RATE_WINDOW_MS. Overflow is batched. */ const RATE_LIMIT_MAX = 20; const RATE_WINDOW_MS = 60_000; /** Dedup window for repeat crash alerts of the same container. */ const CRASH_DEDUP_MS = 60 * 60_000; +/** + * Dedup window for repeat health alerts of the same container. Same duration as + * crash dedup, but stored outside containerState so the 10-minute prune cannot + * shrink the advertised 60-minute window. Unlike crash dedup, recovery (start / + * healthy) does not clear this stamp. + */ +const HEALTH_DEDUP_MS = CRASH_DEDUP_MS; + /** Interval for pruning stale container state from memory. */ const PRUNE_INTERVAL_MS = 60_000; const STATE_STALE_AFTER_MS = 10 * 60_000; @@ -109,6 +117,11 @@ interface DockerEventPayload { type LifecycleStatus = 'disconnected' | 'connecting' | 'connected' | 'stopped'; +/** Rate-limit token bound to the fixed window that issued it. */ +interface RateToken { + readonly windowStart: number; +} + export class DockerEventService { private readonly nodeId: number; private readonly nodeName: string; @@ -127,12 +140,35 @@ export class DockerEventService { /** Pending die timers keyed by container ID (for the 500ms grace window). */ private pendingDieTimers: Map = new Map(); - /** Rate-limiter bookkeeping. */ - private rateWindowStart = 0; - private rateCount = 0; - private suppressedCount = 0; + /** + * Shared fixed-window rate limiter for crash and health alerts (per local + * node / this service instance). Dedup and transition checks run before + * consuming a token so suppressed duplicates do not count toward the cap. + * Tokens carry the issuing window start so a late refund cannot inflate a + * newer window's budget. + */ + private containerAlertRateWindowStart = 0; + private containerAlertRateCount = 0; + private suppressedCrashAlertCount = 0; + private suppressedHealthAlertCount = 0; private summaryTimer: NodeJS.Timeout | null = null; + /** + * Process-local health-alert dedup keyed by container id. Kept outside + * containerState so pruneStaleState cannot drop an active 60m window. + * Entries expire after HEALTH_DEDUP_MS via pruneStaleState; the map clears + * on process restart. Not cleared on healthy/start recovery. + */ + private healthAlertDedupAt = new Map(); + /** In-flight health dispatches; prevents concurrent duplicate flaps. */ + private healthAlertInFlight = new Set(); + /** + * Container ids that entered unhealthy again while a health dispatch was + * in flight. Retried once after the in-flight call finishes if history + * did not persist (so a failed write cannot permanently silence the alert). + */ + private healthAlertPendingRetry = new Set(); + /** Parse-error tracking for flooded bad payloads. */ private parseErrorWindowStart = 0; private parseErrorCount = 0; @@ -166,6 +202,8 @@ export class DockerEventService { /** Close the stream, cancel timers, and clear state. */ public shutdown(): void { + // Flush overflow roll-up while still live so pending summaries are not dropped. + this.flushRateLimitSummaryNow(); this.status = 'stopped'; this.clearReconnectTimer(); if (this.pruneTimer) { @@ -180,6 +218,17 @@ export class DockerEventService { this.pendingDieTimers.clear(); this.detachStream(); this.containerState.clear(); + this.healthAlertDedupAt.clear(); + this.healthAlertInFlight.clear(); + this.healthAlertPendingRetry.clear(); + this.suppressedCrashAlertCount = 0; + this.suppressedHealthAlertCount = 0; + this.containerAlertRateCount = 0; + } + + /** Status helper so await-crossing stopped checks are not narrowed away. */ + private isStopped(): boolean { + return this.status === 'stopped'; } // ======================================================================== @@ -460,27 +509,104 @@ export class DockerEventService { state.lastActivityAt = Date.now(); if (action.includes('unhealthy')) { - if (state.healthStatus !== 'unhealthy') { + const enteringUnhealthy = state.healthStatus !== 'unhealthy'; + if (enteringUnhealthy) { state.unhealthySince = Date.now(); } + // Auto-Heal reads healthStatus / unhealthySince; update before any alert gate. state.healthStatus = 'unhealthy'; + if (!enteringUnhealthy) return; if (!this.isCrashAlertsEnabled()) return; + void this.dispatchHealthAlert(id, state); + } else { + // Recovery clears Auto-Heal timing but must not erase health-alert dedup, + // or a healthy↔unhealthy flap would re-alert immediately. + state.unhealthySince = undefined; + state.healthStatus = action.includes('starting') ? 'starting' : 'healthy'; + // Drop deferred-retry markers so a later unrelated persist failure cannot + // spuriously retry from a flap that already recovered. + this.healthAlertPendingRetry.delete(id); + } + } + + /** + * Dispatch a healthcheck failure alert with transition already confirmed. + * Dedup and rate checks run before the token is consumed. Advance dedup + * only when dispatchAlert returns `{ persisted: true }` (history row written). + */ + private async dispatchHealthAlert(id: string, state: InternalContainerState): Promise { + if (this.isStopped()) return; + + const now = Date.now(); + if (this.isWithinAlertDedupWindow(this.healthAlertDedupAt.get(id), now, HEALTH_DEDUP_MS)) { + return; + } + if (this.healthAlertInFlight.has(id)) { + this.healthAlertPendingRetry.add(id); + return; + } + const token = this.consumeRateToken(); + if (!token) { + this.suppressedHealthAlertCount += 1; + this.scheduleSummary(); + return; + } + + this.healthAlertInFlight.add(id); + let persisted = false; + try { + if (this.isStopped()) { + this.refundRateToken(token); + this.healthAlertPendingRetry.delete(id); + return; + } const name = state.name ?? id.slice(0, 12); - void this.emitError( + const result = await this.emitError( 'monitor_alert', `Healthcheck failed: ${name} is unhealthy.`, state.stackName, state.name, state.isSelf === true, ); - } else { - state.unhealthySince = undefined; - if (action.includes('starting')) { - state.healthStatus = 'starting'; + persisted = result.persisted; + if (persisted) { + this.healthAlertDedupAt.set(id, Date.now()); + this.healthAlertPendingRetry.delete(id); } else { - state.healthStatus = 'healthy'; + // Refund only into the window that issued the token so a late + // persist failure cannot inflate a newer window's budget. + this.refundRateToken(token); + console.error( + `[DockerEvents:${this.nodeName}] Health alert history not persisted for ${id}; dedup not advanced`, + ); } + } finally { + this.healthAlertInFlight.delete(id); } + + if (this.isStopped()) { + this.healthAlertPendingRetry.delete(id); + return; + } + + // A concurrent healthy→unhealthy flap was deferred while we were in + // flight. If the first write failed, retry so the alert is not lost + // while the container stays unhealthy with no further transitions. + if (this.shouldRetryHealthAlert(id, state, persisted)) { + this.healthAlertPendingRetry.delete(id); + await this.dispatchHealthAlert(id, state); + } + } + + private shouldRetryHealthAlert( + id: string, + state: InternalContainerState, + persisted: boolean, + ): boolean { + return !this.isStopped() + && !persisted + && this.healthAlertPendingRetry.has(id) + && state.healthStatus === 'unhealthy'; } private onStart(id: string): void { @@ -495,6 +621,7 @@ export class DockerEventService { state.healthStatus = 'starting'; state.lastStartAt = Date.now(); state.lastActivityAt = Date.now(); + this.healthAlertPendingRetry.delete(id); } private onDestroy(id: string): void { @@ -504,6 +631,9 @@ export class DockerEventService { clearTimeout(pending); this.pendingDieTimers.delete(id); } + this.healthAlertPendingRetry.delete(id); + this.healthAlertInFlight.delete(id); + this.healthAlertDedupAt.delete(id); } private async classifyDie(id: string, event: DockerEventPayload, dieAt: number): Promise { @@ -545,7 +675,7 @@ export class DockerEventService { // Dedup early: crashloops repeatedly reach this point with exit 137, // and the OOM fallback below issues a Docker inspect. Skipping the // inspect on deduped crashes avoids hammering the daemon. - if (state.lastCrashAlertAt && now - state.lastCrashAlertAt < CRASH_DEDUP_MS) { + if (this.isWithinAlertDedupWindow(state.lastCrashAlertAt, now, CRASH_DEDUP_MS)) { return; } @@ -584,6 +714,7 @@ export class DockerEventService { state: InternalContainerState | null, info: { name: string; exitCode: number; stackName?: string; isSelf?: boolean }, ): Promise { + if (this.isStopped()) return; // Respect the existing global crash-alerts toggle so users who have // disabled these notifications in Settings remain opted out. if (!this.isCrashAlertsEnabled()) return; @@ -592,8 +723,9 @@ export class DockerEventService { ? `Container OOM Kill: ${info.name} was killed by the OOM killer (out of memory).` : `Container Crash Detected: ${info.name} exited unexpectedly (Code: ${info.exitCode}).`; - if (!this.consumeRateToken()) { - this.suppressedCount += 1; + const token = this.consumeRateToken(); + if (!token) { + this.suppressedCrashAlertCount += 1; this.scheduleSummary(); return; } @@ -625,32 +757,70 @@ export class DockerEventService { return value; } - /** Return true if an alert can be emitted now. Side effect: increments counters. */ - private consumeRateToken(): boolean { + /** + * Consume one shared crash/health rate token for the current fixed window. + * Returns null when the window is exhausted. Rolling into a new window first + * flushes any pending overflow summary so counters cannot span windows. + */ + private consumeRateToken(): RateToken | null { const now = Date.now(); - if (now - this.rateWindowStart >= RATE_WINDOW_MS) { - this.rateWindowStart = now; - this.rateCount = 0; + if (now - this.containerAlertRateWindowStart >= RATE_WINDOW_MS) { + this.flushRateLimitSummaryNow(); + this.containerAlertRateWindowStart = now; + this.containerAlertRateCount = 0; } - if (this.rateCount >= RATE_LIMIT_MAX) return false; - this.rateCount += 1; - return true; + if (this.containerAlertRateCount >= RATE_LIMIT_MAX) return null; + this.containerAlertRateCount += 1; + return { windowStart: this.containerAlertRateWindowStart }; + } + + /** Refund a token only when the issuing window is still active. */ + private refundRateToken(token: RateToken): void { + if (token.windowStart !== this.containerAlertRateWindowStart) return; + if (this.containerAlertRateCount > 0) this.containerAlertRateCount -= 1; + } + + private isWithinAlertDedupWindow( + lastAlertAt: number | undefined, + now: number, + windowMs: number, + ): boolean { + return lastAlertAt !== undefined && now - lastAlertAt < windowMs; + } + + private buildRateLimitSummaryMessage(crash: number, health: number): string { + const total = crash + health; + if (crash > 0 && health > 0) { + return `${total} additional container alerts were rate-limited in the last minute (${crash} crash, ${health} health).`; + } + if (health > 0) { + return `${health} additional container health alerts were rate-limited in the last minute.`; + } + return `${crash} additional container crash alerts were rate-limited in the last minute.`; + } + + /** Emit and clear any pending overflow roll-up immediately (e.g. on window roll). */ + private flushRateLimitSummaryNow(): void { + if (this.summaryTimer) { + clearTimeout(this.summaryTimer); + this.summaryTimer = null; + } + const crash = this.suppressedCrashAlertCount; + const health = this.suppressedHealthAlertCount; + this.suppressedCrashAlertCount = 0; + this.suppressedHealthAlertCount = 0; + if (crash + health <= 0) return; + if (this.status === 'stopped') return; + void this.emitWarning('monitor_alert', this.buildRateLimitSummaryMessage(crash, health)); } private scheduleSummary(): void { if (this.summaryTimer) return; - const remaining = RATE_WINDOW_MS - (Date.now() - this.rateWindowStart); + const remaining = RATE_WINDOW_MS - (Date.now() - this.containerAlertRateWindowStart); const delay = Math.max(1_000, remaining); this.summaryTimer = setTimeout(() => { - const count = this.suppressedCount; this.summaryTimer = null; - this.suppressedCount = 0; - if (count > 0) { - void this.emitWarning( - 'monitor_alert', - `${count} additional containers crashed in the last minute.`, - ); - } + this.flushRateLimitSummaryNow(); }, delay); } @@ -707,15 +877,24 @@ export class DockerEventService { } private pruneStaleState(): void { - if (this.containerState.size === 0) return; - const cutoff = Date.now() - STATE_STALE_AFTER_MS; - for (const [id, state] of this.containerState) { - // Keep state for a container with an unresolved crash so Auto-Heal can - // still act on it past the default stale window (policy thresholds run - // up to 24h). It is cleared on `start` and removed on `destroy`. - if (state.crashedAt !== undefined) continue; - if (state.lastActivityAt < cutoff) { - this.containerState.delete(id); + if (this.containerState.size > 0) { + const cutoff = Date.now() - STATE_STALE_AFTER_MS; + for (const [id, state] of this.containerState) { + // Keep state for a container with an unresolved crash so Auto-Heal can + // still act on it past the default stale window (policy thresholds run + // up to 24h). It is cleared on `start` and removed on `destroy`. + if (state.crashedAt !== undefined) continue; + if (state.lastActivityAt < cutoff) { + this.containerState.delete(id); + } + } + } + // Health dedup lives outside containerState so the 60m window survives + // ordinary 10m prune. Drop only entries whose window has fully expired. + if (this.healthAlertDedupAt.size > 0) { + const dedupCutoff = Date.now() - HEALTH_DEDUP_MS; + for (const [id, at] of this.healthAlertDedupAt) { + if (at < dedupCutoff) this.healthAlertDedupAt.delete(id); } } } @@ -734,16 +913,16 @@ export class DockerEventService { : { stackName, containerName, actor: 'system:docker-events' }; } - private async emitError(category: NotificationCategory, message: string, stackName?: string, containerName?: string, systemOnly = false): Promise { - await this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly)); + private async emitError(category: NotificationCategory, message: string, stackName?: string, containerName?: string, systemOnly = false): Promise<{ persisted: boolean }> { + return this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly)); } - private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise { - await this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName)); + private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<{ persisted: boolean }> { + return this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName)); } - private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise { - await this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName)); + private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<{ persisted: boolean }> { + return this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName)); } // ======================================================================== diff --git a/docs/features/alerts-notifications.mdx b/docs/features/alerts-notifications.mdx index ee67c6b3..bc949794 100644 --- a/docs/features/alerts-notifications.mdx +++ b/docs/features/alerts-notifications.mdx @@ -308,10 +308,11 @@ Real-time on the Docker event stream: - **Crash** at `error`/`monitor_alert`: `Container Crash Detected: exited unexpectedly (Code: ).` - **OOM kill** at `error`/`monitor_alert`: `Container OOM Kill: was killed by the OOM killer (out of memory).` When a `die` arrives with exit code 137 but no preceding `oom` event, Sencho inspects the container and reclassifies as OOM if the `OOMKilled` flag is set. -- **Healthcheck failure** at `error`/`monitor_alert`: `Healthcheck failed: is unhealthy.` +- **Healthcheck failure** at `error`/`monitor_alert`: `Healthcheck failed: is unhealthy.` Emitted only on the transition into Docker `unhealthy` (not on every repeated unhealthy event). - **Mass exit** kicks in when a daemon-disconnect gap is followed by 20% or more of containers exiting on reconnect, in which case a single `info`/`system` summary `Docker daemon interruption detected: N containers exited during connection gap.` is emitted instead of N crash alerts. -- **Rate limit** caps crash dispatches at 20 per 60-second window per node, then a single `warning`/`monitor_alert` `N additional containers crashed in the last minute.` -- **Dedup** is 60 minutes per container after a non-rate-suppressed dispatch. +- **Rate limit** caps combined crash and health dispatches at 20 per fixed 60-second window per node. Overflow does not write individual history rows; at the end of the window Sencho emits a single `warning`/`monitor_alert` roll-up such as `N additional container crash alerts were rate-limited in the last minute.`, `N additional container health alerts were rate-limited in the last minute.`, or `N additional container alerts were rate-limited in the last minute (X crash, Y health).` +- **Crash dedup** is 60 minutes per container after a non-rate-suppressed crash dispatch. Cleared when that container starts again. +- **Health dedup** is 60 minutes per container after a health alert that was persisted to notification history. It survives healthy/starting recovery and the 10-minute prune of in-memory container tracking state (dedup lives in a separate map), and clears only when the window expires or the Sencho process restarts. ### Daemon connectivity @@ -406,7 +407,7 @@ Three retention controls live under **Settings · Operations · Data Retention** A hard 100-row per-node cap inside `notification_history` always applies on top of the user-tunable retention; new inserts evict the oldest rows beyond 100 even if your retention window is longer. -A separate rate limit applies to crash and health alerts only: 20 emits per 60-second window per node, with a single `warning`/`monitor_alert` roll-up `N additional containers crashed in the last minute.` issued at the end of the window. +A separate rate limit applies to crash and health alerts only: 20 emits per fixed 60-second window per node (not a rolling window). When the cap is hit, individual overflow alerts are not written to history or the bell; at the end of the window Sencho emits a single `warning`/`monitor_alert` roll-up that names crash-only, health-only, or mixed suppression. ## Crash detection toggle @@ -432,8 +433,9 @@ The **Host Alerts** panel carries the **Host thresholds** rows (CPU limit, RAM l | Notification fanout to channels | One attempt by default; optional 0-3 extra in-process attempts with a fixed 1s delay; 10-second timeout per attempt; no durable queue | | Bell live updates | Pushed live over the notifications WebSocket per node | | Bell safety-net reconcile | 60 seconds | -| Crash dedup window per container | 60 minutes | -| Crash rate-limit window | 20 alerts per 60 seconds per node, then a single roll-up | +| Crash dedup window per container | 60 minutes (cleared on container start) | +| Health dedup window per container | 60 minutes after a persisted health alert; survives recovery; process-local | +| Crash/health rate-limit window | 20 alerts per fixed 60 seconds per node, then a single typed roll-up (overflow individuals are not persisted) | Switching the active node tears down per-stack rule editors and reloads channel state, but does not affect the bell, which keeps every node's WebSocket open in parallel. @@ -455,8 +457,8 @@ Switching the active node tears down per-stack rule editors and reloads channel Check **Settings · Monitoring · Container Alerts · Container crash & health alerts**. The toggle is the master switch for the Docker event service. The panel cache reads the database every 500 ms; if the database read errors, Sencho defaults the toggle to off so the failure mode is silent rather than spammy. Repair the toggle, save, and the next Docker event reaches the dispatcher. - - Sencho rate-limits crash and health dispatches to 20 emits per rolling 60-second window per node, then emits a single `warning`/`monitor_alert` roll-up `N additional containers crashed in the last minute.` once the window closes. Every alert is still persisted to `notification_history` and visible in the bell up to the 100-row per-node cap; only the channel fanout is throttled. + + Sencho rate-limits crash and health dispatches together to 20 emits per fixed 60-second window per node, then emits a single `warning`/`monitor_alert` roll-up such as `N additional container crash alerts were rate-limited in the last minute.`, a health-only variant, or a mixed `(X crash, Y health)` summary once the window closes. Rate-limited overflow individuals are not written to `notification_history` and do not appear in the bell; only the roll-up is persisted for that overflow. Separately, health flaps on the same container are deduped for 60 minutes after a persisted unhealthy alert, even if Docker recovered and failed again inside that window. Sencho deliberately hides rows whose category is one of `deploy_success`, `stack_started`, `stack_stopped`, `stack_restarted`, or `image_update_applied` AND whose `actor_username` is set to a real user. The reasoning: those are confirmations of the action you just clicked and are already shown as a toast. The rows are still persisted to `notification_history` and still dispatched to global channels and matching routes; only the bell render hides them. From 524cc56d2ff43d1d38b737c0a8751509e72d29bc Mon Sep 17 00:00:00 2001 From: Anso Date: Thu, 23 Jul 2026 23:25:07 -0400 Subject: [PATCH 10/13] feat: surface stack Monitor from header and service cards (#1693) Make Alerts and Auto-heal reachable from the stack More menu and container cards, with optional Compose service prefill in add forms. --- docs/features/alerts-notifications.mdx | 6 +- docs/features/auto-heal-policies.mdx | 10 ++-- docs/features/stack-management.mdx | 5 +- frontend/src/components/EditorLayout.tsx | 4 ++ .../components/EditorLayout/EditorView.tsx | 7 +++ .../EditorLayout/MobileStackDetail.tsx | 4 ++ .../components/EditorLayout/ShellOverlays.tsx | 1 + .../__tests__/ContainersHealth.test.tsx | 39 ++++++++++++ .../__tests__/StackIdentityHeader.test.tsx | 11 ++++ .../EditorLayout/editor-view-blocks.tsx | 38 +++++++++++- .../hooks/useOverlayState.test.ts | 13 ++++ .../EditorLayout/hooks/useOverlayState.ts | 16 +++-- .../src/components/StackAlertSheet.test.tsx | 60 +++++++++++++++++++ frontend/src/components/StackAlertSheet.tsx | 59 ++++++++++++++---- 14 files changed, 247 insertions(+), 26 deletions(-) diff --git a/docs/features/alerts-notifications.mdx b/docs/features/alerts-notifications.mdx index bc949794..97043bb4 100644 --- a/docs/features/alerts-notifications.mdx +++ b/docs/features/alerts-notifications.mdx @@ -167,7 +167,11 @@ Each stack carries its own set of threshold rules that fire when a metric stays Rules live on the node where the stack runs and are evaluated locally on a 30-second tick. -Open the rules editor by right-clicking a stack in the sidebar and choosing **Alerts**, or by pressing `A` while focused on a stack. +Open the rules editor from any of these entry points: + +- Stack header **More actions** → **Monitor** (Alerts tab) +- Container card **Monitor** (prefers that Compose service in add forms) +- Sidebar context menu **Alerts**, or press `A` while focused on a stack The Stack › PLEX › MONITOR sheet with the Alerts tab active, a green notification-channel banner under the NOTIFICATION CHANNELS heading, an ACTIVE RULES section listing 'CPU Usage (%) > 80' with the secondary line 'Trigger after 5m • Cooldown 60m', and an ADD NEW RULE form with Metric, Operator, Threshold, Duration, Cooldown fields and an Add Rule submit button. diff --git a/docs/features/auto-heal-policies.mdx b/docs/features/auto-heal-policies.mdx index 1b978208..0c8c77bd 100644 --- a/docs/features/auto-heal-policies.mdx +++ b/docs/features/auto-heal-policies.mdx @@ -28,10 +28,12 @@ Policies live next to your stack-level alert rules in the stack's **Monitor** sh ## Workflow -1. In the sidebar, right-click the stack you want to protect (or focus the stack and press **H**). -2. Click **Auto-Heal** in the **inspect** group. The stack's **Monitor** sheet opens directly on the **Auto-heal** tab. -3. In **Add new policy**, pick a **Service** (or leave the combobox on **All services** to cover every container in the stack) and tune the four thresholds. -4. Click **Add Policy**. The new policy appears in **Active policies** above the form, with its enable toggle already on. +1. Open the stack's **Monitor** sheet any of these ways: + - Stack header **More actions** → **Monitor**, then switch to the **Auto-heal** tab + - Container card **Monitor** (prefers that Compose service in **Add new policy**), then switch to **Auto-heal** + - Sidebar: right-click the stack (or focus it and press **H**) → **Auto-Heal** in the **inspect** group (opens directly on the **Auto-heal** tab) +2. In **Add new policy**, pick a **Service** (or leave the combobox on **All services** to cover every container in the stack) and tune the four thresholds. +3. Click **Add Policy**. The new policy appears in **Active policies** above the form, with its enable toggle already on. The Stack › plex › Monitor sheet on the Auto-heal tab, showing one policy scoped to the plex service in the Active policies section with its ON toggle, and the Add new policy form below with Service, Unhealthy for, Cooldown, Max restarts / hr, and Auto-disable after fields, plus the Add Policy button diff --git a/docs/features/stack-management.mdx b/docs/features/stack-management.mdx index be87bb04..08cdf81d 100644 --- a/docs/features/stack-management.mdx +++ b/docs/features/stack-management.mdx @@ -368,6 +368,7 @@ The stack header groups actions by frequency of use. The most common action is t | Secondary | **Update** | `docker compose pull` + `up -d` (or build-aware rebuild when services declare `build:`) | Pulls registry images and recreates containers. When one or more services use `build:`, Update rebuilds those images from source (`compose build --pull`), pulls any remaining registry images, then recreates containers. | | Overflow | **Rollback** | Restores backup | Reverts compose and env files to the pre-deploy snapshot and redeploys. Only shown when a backup exists. | | Overflow | **Scan config** | Trivy config scan | Scans the compose file for misconfigurations (admin role). | +| Overflow | **Monitor** | Opens Monitor sheet | Opens the stack **Monitor** sheet on the Alerts tab (alert rules and Auto-heal). Same sheet as sidebar **Alerts** / **Auto-Heal**. | | Overflow | **Mute** | Creates a mute rule | Quick presets to mute notifications, deploy-success noise, or monitor alerts for this stack, plus a link to manage its mute rules in full. See [Alerts and Notifications](/features/alerts-notifications). | | Overflow | **Delete** | `down --volumes` + removes files | Stops and removes containers and volumes, then deletes the stack directory. | @@ -377,6 +378,7 @@ The stack header groups actions by frequency of use. The most common action is t |-----------|--------|---------|--------------| | Primary | **Start** | `docker compose up -d` | Starts the stack. | | Secondary | **Update** | `docker compose pull` + `up -d` (or build-aware rebuild when services declare `build:`) | Pulls registry images and recreates containers. When one or more services use `build:`, Update rebuilds those images from source (`compose build --pull`), pulls any remaining registry images, then recreates containers. | +| Overflow | **Monitor** | Opens Monitor sheet | Opens the stack **Monitor** sheet on the Alerts tab. | | Overflow | **Delete** | Removes files | Deletes the stack directory. | Use the sidebar context menu or `⌘↓` / `Ctrl+↓` for **Take down** on a stopped stack. @@ -414,8 +416,9 @@ Press `B` again or toggle the bulk mode button to leave bulk mode. ## Controlling a single service -Inside the stack detail view, each container card has a **Service actions** kebab on the right side. Use it to act on that service without touching the rest of the stack. +Inside the stack detail view, each container card has quick actions (View logs, Monitor, Open bash when available) and a **Service actions** kebab on the right side. Use them to act on that service without touching the rest of the stack. +- **Monitor**: opens the stack **Monitor** sheet with that Compose service preferred in the add forms. Shown only when the container reports a Compose service name. - **Restart service**: stops and starts all containers for that service. - **Stop service**: stops all containers for that service. Only shown when the service is running. - **Start service**: starts all containers for that service. Only shown when the service is stopped or exited. diff --git a/frontend/src/components/EditorLayout.tsx b/frontend/src/components/EditorLayout.tsx index be11b41e..be3611eb 100644 --- a/frontend/src/components/EditorLayout.tsx +++ b/frontend/src/components/EditorLayout.tsx @@ -660,6 +660,10 @@ export default function EditorLayout() { changeEnvFile={stackActions.changeEnvFile} openLogViewer={stackActions.openLogViewer} openBashModal={stackActions.openBashModal} + onOpenMonitor={stackName ? () => overlayState.openAlertSheet(stackName) : undefined} + onOpenServiceMonitor={stackName + ? (serviceName) => overlayState.openAlertSheet(stackName, { serviceName }) + : undefined} serviceAction={stackActions.serviceAction} effectiveServices={effectiveServices} serviceUpdateStatuses={serviceUpdateStatuses} diff --git a/frontend/src/components/EditorLayout/EditorView.tsx b/frontend/src/components/EditorLayout/EditorView.tsx index 3f4d8504..d2ad501b 100644 --- a/frontend/src/components/EditorLayout/EditorView.tsx +++ b/frontend/src/components/EditorLayout/EditorView.tsx @@ -177,6 +177,8 @@ export interface EditorViewProps { // Container / service actions openLogViewer: (containerId: string, containerName: string) => void; openBashModal: (containerId: string, containerName: string) => void; + onOpenMonitor?: () => void; + onOpenServiceMonitor?: (serviceName: string) => void; serviceAction: ( action: 'start' | 'stop' | 'restart', serviceName: string, @@ -278,6 +280,8 @@ export function EditorView(props: EditorViewProps) { changeEnvFile, openLogViewer, openBashModal, + onOpenMonitor, + onOpenServiceMonitor, serviceAction, effectiveServices = [], serviceUpdateStatuses = [], @@ -447,6 +451,7 @@ export function EditorView(props: EditorViewProps) { showTakeDown={showTakeDown} isSelfStack={isSelfStack} stackMuteActions={stackMuteActions} + onOpenMonitor={onOpenMonitor} />
{recoveryResult && loadingAction == null && ( @@ -484,6 +489,7 @@ export function EditorView(props: EditorViewProps) { activeNode={activeNode} openLogViewer={openLogViewer} openBashModal={openBashModal} + onOpenServiceMonitor={onOpenServiceMonitor} serviceAction={serviceAction} effectiveServices={effectiveServices} serviceUpdateStatuses={serviceUpdateStatuses} @@ -508,6 +514,7 @@ export function EditorView(props: EditorViewProps) { activeNode={activeNode} openLogViewer={openLogViewer} openBashModal={openBashModal} + onOpenServiceMonitor={onOpenServiceMonitor} serviceAction={serviceAction} effectiveServices={effectiveServices} serviceUpdateStatuses={serviceUpdateStatuses} diff --git a/frontend/src/components/EditorLayout/MobileStackDetail.tsx b/frontend/src/components/EditorLayout/MobileStackDetail.tsx index dce74aa1..c2d409dd 100644 --- a/frontend/src/components/EditorLayout/MobileStackDetail.tsx +++ b/frontend/src/components/EditorLayout/MobileStackDetail.tsx @@ -61,6 +61,8 @@ export function MobileStackDetail(props: EditorViewProps) { changeEnvFile, openLogViewer, openBashModal, + onOpenMonitor, + onOpenServiceMonitor, serviceAction, effectiveServices = [], serviceUpdateStatuses, @@ -159,6 +161,7 @@ export function MobileStackDetail(props: EditorViewProps) { showTakeDown={showTakeDown} isSelfStack={isSelfStack} stackMuteActions={stackMuteActions} + onOpenMonitor={onOpenMonitor} />
@@ -229,6 +232,7 @@ export function MobileStackDetail(props: EditorViewProps) { activeNode={activeNode} openLogViewer={openLogViewer} openBashModal={openBashModal} + onOpenServiceMonitor={onOpenServiceMonitor} serviceAction={serviceAction} effectiveServices={effectiveServices} serviceUpdateStatuses={serviceUpdateStatuses} diff --git a/frontend/src/components/EditorLayout/ShellOverlays.tsx b/frontend/src/components/EditorLayout/ShellOverlays.tsx index b7dfb8de..e0e8cc1c 100644 --- a/frontend/src/components/EditorLayout/ShellOverlays.tsx +++ b/frontend/src/components/EditorLayout/ShellOverlays.tsx @@ -117,6 +117,7 @@ export function ShellOverlays({ onOpenChange={(open) => { if (!open) closeStackMonitor(); }} stackName={stackMonitor?.stackName ?? ''} initialTab={stackMonitor?.tab ?? 'alerts'} + initialService={stackMonitor?.serviceName} /> {/* Pre-update readiness check */} diff --git a/frontend/src/components/EditorLayout/__tests__/ContainersHealth.test.tsx b/frontend/src/components/EditorLayout/__tests__/ContainersHealth.test.tsx index 4d2dd253..083e9283 100644 --- a/frontend/src/components/EditorLayout/__tests__/ContainersHealth.test.tsx +++ b/frontend/src/components/EditorLayout/__tests__/ContainersHealth.test.tsx @@ -509,4 +509,43 @@ describe('containers load states', () => { await user.click(screen.getByRole('button', { name: /retry/i })); expect(onRetry).toHaveBeenCalledTimes(1); }); + + it('shows Monitor when Service is set and calls onOpenServiceMonitor', async () => { + const onOpenServiceMonitor = vi.fn(); + const user = userEvent.setup(); + const c = { ...container([{ PrivatePort: 80, PublicPort: 8080 }]), Service: 'web' } as ContainerInfo; + render( + , + ); + + await user.click(screen.getByRole('button', { name: 'Monitor web' })); + expect(onOpenServiceMonitor).toHaveBeenCalledWith('web'); + }); + + it('hides Monitor when the container has no Service label', () => { + render( + , + ); + expect(screen.queryByRole('button', { name: /Monitor / })).toBeNull(); + }); }); diff --git a/frontend/src/components/EditorLayout/__tests__/StackIdentityHeader.test.tsx b/frontend/src/components/EditorLayout/__tests__/StackIdentityHeader.test.tsx index f61b63f7..43e98509 100644 --- a/frontend/src/components/EditorLayout/__tests__/StackIdentityHeader.test.tsx +++ b/frontend/src/components/EditorLayout/__tests__/StackIdentityHeader.test.tsx @@ -103,4 +103,15 @@ describe('StackIdentityHeader', () => { expect(screen.queryByRole('menuitem', { name: /Take down/i })).toBeNull(); }); + + it('shows Monitor in More actions and calls onOpenMonitor', async () => { + const user = userEvent.setup(); + const onOpenMonitor = vi.fn(); + renderHeader({ onOpenMonitor, backupInfo: { exists: false, timestamp: null } }); + + await user.click(screen.getByRole('button', { name: 'More actions' })); + await user.click(screen.getByRole('menuitem', { name: 'Monitor' })); + + expect(onOpenMonitor).toHaveBeenCalledTimes(1); + }); }); diff --git a/frontend/src/components/EditorLayout/editor-view-blocks.tsx b/frontend/src/components/EditorLayout/editor-view-blocks.tsx index 496ab00f..09735b60 100644 --- a/frontend/src/components/EditorLayout/editor-view-blocks.tsx +++ b/frontend/src/components/EditorLayout/editor-view-blocks.tsx @@ -22,7 +22,8 @@ import { Maximize2, Minimize2, AlertCircle, - RefreshCw + RefreshCw, + HeartPulse, } from 'lucide-react'; import { useCallback, useEffect, useRef, useState } from 'react'; import { Button } from '../ui/button'; @@ -144,6 +145,8 @@ export interface StackIdentityHeaderProps { /** True when this stack is the running Sencho instance on the active node. */ isSelfStack?: boolean; stackMuteActions?: ReturnType; + /** Opens the stack Monitor sheet on the Alerts tab. */ + onOpenMonitor?: () => void; } // Breadcrumb + serif title + state pill + action bar. The action buttons grow @@ -170,6 +173,7 @@ export function StackIdentityHeader({ showTakeDown, isSelfStack = false, stackMuteActions, + onOpenMonitor, }: StackIdentityHeaderProps) { const selfProtected = isSelfStack; return ( @@ -206,7 +210,7 @@ export function StackIdentityHeader({ const canScan = trivy.available && isAdmin; const canMute = stackMuteActions?.canMute ?? false; const hasOverflowExtras = canRollback || canScan; - const hasOverflow = hasOverflowExtras || canDelete || canMute; + const hasOverflow = hasOverflowExtras || canDelete || canMute || onOpenMonitor; if (!canDeploy && !hasOverflow) return null; return (
@@ -278,8 +282,14 @@ export function StackIdentityHeader({ {stackMisconfigScanning ? 'Scanning...' : 'Scan config'} )} + {onOpenMonitor && ( + + + Monitor + + )} {stackMuteActions && } - {(canRollback || canScan || stackMuteActions?.canMute) && canDelete && } + {(canRollback || canScan || onOpenMonitor || stackMuteActions?.canMute) && canDelete && } {canDelete && ( void; openBashModal: (containerId: string, containerName: string) => void; + /** Opens Monitor (Alerts tab); preselects the Compose service in add forms when listed. */ + onOpenServiceMonitor?: (serviceName: string) => void; serviceAction: (action: 'start' | 'stop' | 'restart', serviceName: string) => Promise; // Declared Compose services from the effective model. Multi-service // headers (owning Update/Rebuild + badge + Start/Stop/Restart) render only @@ -335,6 +347,7 @@ export function ContainersHealth({ activeNode, openLogViewer, openBashModal, + onOpenServiceMonitor, serviceAction, effectiveServices = [], serviceUpdateStatuses = [], @@ -474,6 +487,7 @@ export function ContainersHealth({ : ''; const containerName = container?.Names?.[0]?.replace(/^\//, '') || container?.Id?.slice(0, 12) || 'container'; + const composeService = container.Service; const isActive = container.State === 'running' || container.State === 'paused'; const health = container.healthStatus; const uptime = isActive ? extractUptime(container.Status) : null; @@ -565,6 +579,24 @@ export function ContainersHealth({ View logs + {onOpenServiceMonitor && composeService && ( + + + + + + Monitor {composeService} + + + )} {isAdmin && ( diff --git a/frontend/src/components/EditorLayout/hooks/useOverlayState.test.ts b/frontend/src/components/EditorLayout/hooks/useOverlayState.test.ts index 8438a881..2da64452 100644 --- a/frontend/src/components/EditorLayout/hooks/useOverlayState.test.ts +++ b/frontend/src/components/EditorLayout/hooks/useOverlayState.test.ts @@ -74,6 +74,19 @@ describe('useOverlayState', () => { expect(result.current.stackMonitor).toEqual({ stackName: 'web-stack', tab: 'alerts' }); }); + it.each([ + ['openAlertSheet', 'alerts' as const], + ['openAutoHeal', 'auto-heal' as const], + ])('%s can carry a serviceName for form preselect', (openFn, tab) => { + const { result } = renderHook(() => useOverlayState()); + act(() => result.current[openFn as 'openAlertSheet' | 'openAutoHeal']('web-stack', { serviceName: 'api' })); + expect(result.current.stackMonitor).toEqual({ + stackName: 'web-stack', + tab, + serviceName: 'api', + }); + }); + it('openAutoHeal opens stack monitor on the auto-heal tab', () => { const { result } = renderHook(() => useOverlayState()); act(() => result.current.openAutoHeal('web-stack')); diff --git a/frontend/src/components/EditorLayout/hooks/useOverlayState.ts b/frontend/src/components/EditorLayout/hooks/useOverlayState.ts index 8ddfabd5..86aeea97 100644 --- a/frontend/src/components/EditorLayout/hooks/useOverlayState.ts +++ b/frontend/src/components/EditorLayout/hooks/useOverlayState.ts @@ -25,6 +25,12 @@ type PolicyBlock = { }; type Container = { id: string; name: string }; +type StackMonitorState = { + stackName: string; + tab: 'alerts' | 'auto-heal'; + serviceName?: string; +}; + // Kept here (not in useStackActions) so overlay state can hold load options // without a circular type import between the two hooks. export type LoadFileOptions = { @@ -109,12 +115,12 @@ export function useOverlayState() { return () => window.removeEventListener(SENCHO_OPEN_LOGS_EVENT, handler); }, [openLogViewer]); // openLogViewer is stable (useCallback with empty deps) - const [stackMonitor, setStackMonitor] = useState<{ stackName: string; tab: 'alerts' | 'auto-heal' } | null>(null); - const openAlertSheet = useCallback((stackName: string) => { - setStackMonitor({ stackName, tab: 'alerts' }); + const [stackMonitor, setStackMonitor] = useState(null); + const openAlertSheet = useCallback((stackName: string, opts?: { serviceName?: string }) => { + setStackMonitor({ stackName, tab: 'alerts', serviceName: opts?.serviceName }); }, []); - const openAutoHeal = useCallback((stackName: string) => { - setStackMonitor({ stackName, tab: 'auto-heal' }); + const openAutoHeal = useCallback((stackName: string, opts?: { serviceName?: string }) => { + setStackMonitor({ stackName, tab: 'auto-heal', serviceName: opts?.serviceName }); }, []); const closeStackMonitor = useCallback(() => setStackMonitor(null), []); diff --git a/frontend/src/components/StackAlertSheet.test.tsx b/frontend/src/components/StackAlertSheet.test.tsx index 46f413cf..a598be3d 100644 --- a/frontend/src/components/StackAlertSheet.test.tsx +++ b/frontend/src/components/StackAlertSheet.test.tsx @@ -245,4 +245,64 @@ describe('StackAlertSheet Alerts tab', () => { // New node's list has only worker, so the api-targeted rule is missing. await waitFor(() => expect(screen.getByText(/Not in compose/i)).toBeInTheDocument()); }); + + it('prefills the Service combobox from initialService when listed', async () => { + mockHappyPath(['api', 'database']); + render( + {}} + stackName="my-stack" + initialService="api" + />, + ); + + await waitFor(() => { + expect(screen.getByText('Add Rule')).toBeInTheDocument(); + expect(screen.getAllByRole('combobox')[0]).toHaveTextContent('api'); + }); + }); + + it('ignores Alerts initialService when service-scoped capability is absent', async () => { + nodeState.activeNodeMeta = { version: '0.90.0', capabilities: [] }; + mockHappyPath(['api', 'database']); + render( + {}} + stackName="my-stack" + initialService="api" + />, + ); + + await waitFor(() => expect(screen.getByText('Add Rule')).toBeInTheDocument()); + expect(screen.queryByText('All services')).toBeNull(); + expect( + mockedFetch.mock.calls.some(([url]) => String(url).includes('/services')), + ).toBe(false); + }); + + it('prefills Auto-heal Service from initialService when listed', async () => { + mockedFetch.mockImplementation(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes('/auto-heal/policies')) return jsonRes([]); + if (url.includes('/services')) return jsonRes(['api', 'database']); + return jsonRes(null, false); + }); + + render( + {}} + stackName="my-stack" + initialTab="auto-heal" + initialService="api" + />, + ); + + await waitFor(() => { + expect(screen.getByText('Add Policy')).toBeInTheDocument(); + expect(screen.getAllByRole('combobox')[0]).toHaveTextContent('api'); + }); + }); }); diff --git a/frontend/src/components/StackAlertSheet.tsx b/frontend/src/components/StackAlertSheet.tsx index fbe70db0..bd1766d2 100644 --- a/frontend/src/components/StackAlertSheet.tsx +++ b/frontend/src/components/StackAlertSheet.tsx @@ -78,6 +78,8 @@ interface StackAlertSheetProps { onOpenChange: (open: boolean) => void; stackName: string; initialTab?: MonitorTab; + /** Prefill Add-form service comboboxes when opening from a service card. */ + initialService?: string; } interface AgentStatus { @@ -124,6 +126,10 @@ function actionColorClass(action: AutoHealHistoryEntry['action']): string { return 'text-muted-foreground'; } +function preselectListedService(initialService: string | undefined, names: string[]): string { + return initialService && names.includes(initialService) ? initialService : ''; +} + function actionLabel(action: AutoHealHistoryEntry['action']): string { switch (action) { case 'restarted': return 'Restarted'; @@ -136,7 +142,13 @@ function actionLabel(action: AutoHealHistoryEntry['action']): string { } } -export function StackAlertSheet({ open, onOpenChange, stackName, initialTab = 'alerts' }: StackAlertSheetProps) { +export function StackAlertSheet({ + open, + onOpenChange, + stackName, + initialTab = 'alerts', + initialService, +}: StackAlertSheetProps) { const [activeTab, setActiveTab] = useState(initialTab); useEffect(() => { @@ -147,6 +159,7 @@ export function StackAlertSheet({ open, onOpenChange, stackName, initialTab = 'a { id: 'alerts', label: 'Alerts' }, { id: 'auto-heal', label: 'Auto-heal' }, ]; + const prefillService = open ? initialService : undefined; return ( } - {activeTab === 'auto-heal' && } + {activeTab === 'alerts' && ( + + )} + {activeTab === 'auto-heal' && ( + + )} ); } -function AlertsTab({ stackName }: { stackName: string }) { +function AlertsTab({ stackName, initialService }: { stackName: string; initialService?: string }) { const { isAdmin } = useAuth(); const { activeNode, activeNodeMeta } = useNodes(); const isRemote = activeNode?.type === 'remote'; @@ -211,7 +228,9 @@ function AlertsTab({ stackName }: { stackName: string }) { const res = await apiFetch(`/stacks/${encodeURIComponent(stackName)}/services`); if (!res.ok) throw new Error(`services ${res.status}`); const names = await res.json() as string[]; - if (!cancelled) setServicesState({ status: 'success', options: names }); + if (cancelled) return; + setServicesState({ status: 'success', options: names }); + setService(preselectListedService(initialService, names)); } catch (e) { console.error('[StackAlertSheet] Failed to fetch stack services', e); if (!cancelled) setServicesState({ status: 'error' }); @@ -219,7 +238,7 @@ function AlertsTab({ stackName }: { stackName: string }) { })(); return () => { cancelled = true; }; - }, [stackName, activeNode?.id, canScopeService]); + }, [stackName, activeNode?.id, canScopeService, initialService]); const fetchAlerts = async () => { try { @@ -567,7 +586,15 @@ function AlertsTab({ stackName }: { stackName: string }) { ); } -function AutoHealTab({ stackName, open }: { stackName: string; open: boolean }) { +function AutoHealTab({ + stackName, + open, + initialService, +}: { + stackName: string; + open: boolean; + initialService?: string; +}) { const { isAdmin } = useAuth(); const [policies, setPolicies] = useState([]); const [loading, setLoading] = useState(false); @@ -584,17 +611,25 @@ function AutoHealTab({ stackName, open }: { stackName: string; open: boolean }) useEffect(() => { if (!open || !stackName) return; setLoading(true); + setService(''); apiFetch(`/auto-heal/policies?stackName=${encodeURIComponent(stackName)}`) .then(res => res.json() as Promise) .then(data => setPolicies(data)) .catch(() => toast.error('Failed to load auto-heal policies.')) .finally(() => setLoading(false)); - apiFetch(`/stacks/${encodeURIComponent(stackName)}/services`) - .then(res => res.json() as Promise) - .then(names => setServiceOptions(names.map(n => ({ value: n, label: n })))) - .catch(() => { /* services list is optional, silently skip */ }); - }, [open, stackName]); + void (async () => { + try { + const res = await apiFetch(`/stacks/${encodeURIComponent(stackName)}/services`); + if (!res.ok) throw new Error(`services ${res.status}`); + const names = await res.json() as string[]; + setServiceOptions(names.map(n => ({ value: n, label: n }))); + setService(preselectListedService(initialService, names)); + } catch (e) { + console.error('[StackAlertSheet] Failed to fetch stack services', e); + } + })(); + }, [open, stackName, initialService]); const handleToggle = async (id: number, enabled: boolean) => { setSaving(true); From 79914fe7503923edebe64e9f006b4fbe5b7d189f Mon Sep 17 00:00:00 2001 From: Anso Date: Fri, 24 Jul 2026 09:41:21 -0400 Subject: [PATCH 11/13] fix: recognize clean one-shot completions in health gate and drift (#1691) * fix: recognize clean one-shot completions in health gate and drift Treat exit 0 with restart policy no/absent as successful completion so init and migration jobs no longer fail post-update observation or show as service-missing, while long-running restart policies still fail closed. * fix: ignore residual health on clean one-shots and honor deploy.restart_policy Completed exit-0 jobs with no-restart intent no longer fail the health gate on leftover starting/unhealthy state, and Drift treats deploy.restart_policy with Compose precedence so any/on-failure services are not mistaken for one-shots. * fix: require explicit Compose restart no for one-shot recognition Docker inspect reports restart no for both intentional jobs and bare services that omit restart, so Health Gate and Drift now require declared restart:""no"" (or deploy.restart_policy condition none) and load Compose intent once per gate. --- .../compose-network-inspector.test.ts | 2 +- .../dependency-graph-builder.test.ts | 2 +- .../src/__tests__/docker-controller.test.ts | 18 ++ backend/src/__tests__/drift-detection.test.ts | 219 +++++++++++++++++- backend/src/__tests__/drift-ledger.test.ts | 4 +- .../src/__tests__/health-gate-prepare.test.ts | 208 ++++++++++++++++- .../src/__tests__/health-gate-service.test.ts | 211 ++++++++++++++++- .../src/__tests__/networking-operator.test.ts | 4 +- .../src/__tests__/networking-summary.test.ts | 2 +- .../src/__tests__/one-shot-completion.test.ts | 92 ++++++++ backend/src/__tests__/preflight-rules.test.ts | 6 +- .../sanitize-network-inspect.test.ts | 2 +- backend/src/helpers/composeDependencyParse.ts | 3 + .../src/helpers/effectiveToDeclaredCompose.ts | 2 + backend/src/services/DockerController.ts | 7 + backend/src/services/DriftDetectionService.ts | 66 ++++-- backend/src/services/HealthGateService.ts | 127 ++++++++-- .../src/services/preflight/effectiveModel.ts | 2 +- backend/src/services/preflight/rules.ts | 2 +- backend/src/utils/oneShotCompletion.ts | 77 ++++++ docs/features/health-gated-updates.mdx | 16 +- docs/features/stack-drift.mdx | 12 +- 22 files changed, 1015 insertions(+), 69 deletions(-) create mode 100644 backend/src/__tests__/one-shot-completion.test.ts create mode 100644 backend/src/utils/oneShotCompletion.ts diff --git a/backend/src/__tests__/compose-network-inspector.test.ts b/backend/src/__tests__/compose-network-inspector.test.ts index d8d0d062..035d1599 100644 --- a/backend/src/__tests__/compose-network-inspector.test.ts +++ b/backend/src/__tests__/compose-network-inspector.test.ts @@ -24,7 +24,7 @@ function effSvc(over: Partial = {}): EffService { function container(over: Partial = {}): DependencyContainer { return { id: 'c1', name: 'web1', service: 'web', composeProject: 'myapp', stack: 'myapp', - state: 'running', image: 'nginx:1.27', networks: [], volumes: [], ports: [], ...over, + state: 'running', exitCode: null, image: 'nginx:1.27', networks: [], volumes: [], ports: [], ...over, }; } diff --git a/backend/src/__tests__/dependency-graph-builder.test.ts b/backend/src/__tests__/dependency-graph-builder.test.ts index 7ec5982e..3bc0f3a7 100644 --- a/backend/src/__tests__/dependency-graph-builder.test.ts +++ b/backend/src/__tests__/dependency-graph-builder.test.ts @@ -32,7 +32,7 @@ function snap(partial: Partial): DependencySnapshot { function container(p: Partial & { id: string }): DependencyContainer { return { name: p.id, service: null, composeProject: null, stack: null, - state: 'running', image: 'img:latest', networks: [], volumes: [], ports: [], ...p, + state: 'running', exitCode: null, image: 'img:latest', networks: [], volumes: [], ports: [], ...p, }; } diff --git a/backend/src/__tests__/docker-controller.test.ts b/backend/src/__tests__/docker-controller.test.ts index 30cc4e99..e55e7082 100644 --- a/backend/src/__tests__/docker-controller.test.ts +++ b/backend/src/__tests__/docker-controller.test.ts @@ -1316,6 +1316,7 @@ describe('DockerController - getDependencySnapshot', () => { expect(c.networks).toEqual([{ name: 'web_frontend', id: 'net1', ip: '172.18.0.2' }]); expect(c.volumes).toEqual(['web_data']); // bind mount dropped expect(c.ports).toEqual([{ ip: '0.0.0.0', publishedPort: 8080, privatePort: 80, protocol: 'tcp' }]); // unpublished 9090 dropped + expect(c.exitCode).toBeNull(); expect(snap.networks.find((n) => n.name === 'bridge')?.isSystem).toBe(true); const frontend = snap.networks.find((n) => n.name === 'web_frontend'); @@ -1338,6 +1339,23 @@ describe('DockerController - getDependencySnapshot', () => { expect(snap.containers[0].stack).toBeNull(); expect(snap.containers[0].composeProject).toBeNull(); }); + + it('parses exitCode from list Status for exited containers', async () => { + mockDocker.listContainers.mockResolvedValue([ + { Id: 'a', Names: ['/job-0'], Image: 'busybox', State: 'exited', Status: 'Exited (0) 5 minutes ago', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] }, + { Id: 'b', Names: ['/crash-0'], Image: 'busybox', State: 'exited', Status: 'Exited (137) 1 minute ago', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] }, + { Id: 'c', Names: ['/up-0'], Image: 'busybox', State: 'running', Status: 'Up 3 hours', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] }, + { Id: 'd', Names: ['/odd-0'], Image: 'busybox', State: 'exited', Status: 'Exited', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] }, + ]); + mockDocker.listNetworks.mockResolvedValue([]); + mockDocker.listVolumes.mockResolvedValue({ Volumes: [] }); + + const snap = await DockerController.getInstance(1).getDependencySnapshot([]); + expect(snap.containers.find(c => c.id === 'a')?.exitCode).toBe(0); + expect(snap.containers.find(c => c.id === 'b')?.exitCode).toBe(137); + expect(snap.containers.find(c => c.id === 'c')?.exitCode).toBeNull(); + expect(snap.containers.find(c => c.id === 'd')?.exitCode).toBeNull(); + }); }); // --- getBulkStackStatuses uptime (runningSince from StartedAt) ----------------- diff --git a/backend/src/__tests__/drift-detection.test.ts b/backend/src/__tests__/drift-detection.test.ts index e927c303..efb4694a 100644 --- a/backend/src/__tests__/drift-detection.test.ts +++ b/backend/src/__tests__/drift-detection.test.ts @@ -36,7 +36,7 @@ function declared(services: DeclaredService[], parseError?: string): DeclaredCom function container(p: Partial & { id: string }): DependencyContainer { return { name: p.id, service: null, composeProject: null, stack: 'app', - state: 'running', image: 'img:latest', networks: [], volumes: [], ports: [], ...p, + state: 'running', exitCode: null, image: 'img:latest', networks: [], volumes: [], ports: [], ...p, }; } @@ -108,6 +108,147 @@ describe('assembleStackDrift - status', () => { expect(report.findings).toEqual([]); }); + it('does not emit service-missing for a clean one-shot beside a running service', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'migrate', image: 'migrate:1', restart: 'no' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }), + container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: 0 }), + ], + }); + expect(findingKinds(report)).not.toContain('service-missing'); + expect(report.status).toBe('in-sync'); + expect(report.hasContainers).toBe(true); + }); + + it('still emits service-missing when a one-shot exits non-zero', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'migrate', image: 'migrate:1', restart: 'no' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }), + container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: 1 }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + expect(report.findings.find(f => f.kind === 'service-missing')?.service).toBe('migrate'); + }); + + it('still treats exited unless-stopped as missing even with exit 0', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([service({ name: 'web', restart: 'unless-stopped' })]), + containers: [container({ id: 'c1', service: 'web', state: 'exited', exitCode: 0 })], + }); + expect(report.status).toBe('missing-runtime'); + expect(report.hasContainers).toBe(false); + }); + + it('fails closed when exitCode is null even with restart no', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'migrate', image: 'migrate:1', restart: 'no' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }), + container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: null }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + expect(report.findings.find(f => f.kind === 'service-missing')?.service).toBe('migrate'); + }); + + it('does not treat absent declared restart as a clean one-shot', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'daemon-default', image: 'daemon:1' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1' }), + container({ id: 'c2', service: 'daemon-default', image: 'daemon:1', state: 'exited', exitCode: 0 }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + expect(report.findings.find((f) => f.kind === 'service-missing')?.service).toBe('daemon-default'); + }); + + it('all-one-shot stack with dedicated network is not missing-runtime but keeps network-missing and hasContainers false', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: { + services: [service({ name: 'migrate', restart: 'no', networks: ['jobs'] })], + networks: { jobs: { external: false } }, + volumes: {}, + projectName: 'app', + }, + containers: [ + container({ + id: 'c1', service: 'migrate', state: 'exited', exitCode: 0, + networks: [{ name: 'app_jobs', id: 'j', ip: '' }], + }), + ], + networks: [depNet('app_jobs')], + }); + expect(report.status).not.toBe('missing-runtime'); + expect(findingKinds(report)).not.toContain('service-missing'); + expect(report.hasContainers).toBe(false); + expect(findingKinds(report)).toContain('network-missing'); + expect(report.status).toBe('drifted'); + }); + + it('all-one-shot stack without network findings is in-sync with hasContainers false', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([service({ name: 'migrate', restart: 'no' })]), + containers: [container({ id: 'c1', service: 'migrate', state: 'exited', exitCode: 0 })], + }); + expect(report.status).toBe('in-sync'); + expect(report.hasContainers).toBe(false); + expect(report.findings).toEqual([]); + }); + + it('emits service-missing when normalized restart is always (deploy any)', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'worker', image: 'worker:1', restart: 'always' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1' }), + container({ id: 'c2', service: 'worker', image: 'worker:1', state: 'exited', exitCode: 0 }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + expect(report.findings.find((f) => f.kind === 'service-missing')?.service).toBe('worker'); + }); + + it('emits service-missing when normalized restart is on-failure', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'worker', image: 'worker:1', restart: 'on-failure' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1' }), + container({ id: 'c2', service: 'worker', image: 'worker:1', state: 'exited', exitCode: 0 }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + }); + it('counts a restarting container as deployed', () => { const report = assembleStackDrift({ stack: 'app', @@ -456,6 +597,82 @@ describe('declaredFromEffectiveModel', () => { expect(converted.services[0].ports).toEqual([{ hostIp: '127.0.0.1', publishedPort: 8080, protocol: 'tcp' }]); }); + it('preserves restart policy including no, unless-stopped, and absent', () => { + const withNo = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ name: 'migrate', restart: 'no' })], + networks: {}, + volumes: {}, + }); + expect(withNo.services[0].restart).toBe('no'); + + const withUnless = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ name: 'web', restart: 'unless-stopped' })], + networks: {}, + volumes: {}, + }); + expect(withUnless.services[0].restart).toBe('unless-stopped'); + + const absent = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ name: 'job', restart: undefined })], + networks: {}, + volumes: {}, + }); + expect(absent.services[0].restart).toBeNull(); + }); + + it('normalizes deploy.restart_policy conditions with Compose precedence', () => { + const none = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ + name: 'migrate', + restart: 'unless-stopped', + deploy: { restart_policy: { condition: 'none' } }, + })], + networks: {}, + volumes: {}, + }); + expect(none.services[0].restart).toBe('no'); + + const any = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ + name: 'worker', + restart: 'no', + deploy: { restart_policy: { condition: 'any' } }, + })], + networks: {}, + volumes: {}, + }); + expect(any.services[0].restart).toBe('always'); + + const onFailure = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ + name: 'worker', + restart: undefined, + deploy: { restart_policy: { condition: 'on-failure' } }, + })], + networks: {}, + volumes: {}, + }); + expect(onFailure.services[0].restart).toBe('on-failure'); + + const defaultAny = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ + name: 'worker', + restart: 'no', + deploy: { restart_policy: {} }, + })], + networks: {}, + volumes: {}, + }); + expect(defaultAny.services[0].restart).toBe('always'); + }); + it('normalizes networks so drift matches the rendered model', () => { const model: EffectiveModel = { projectName: 'myapp', diff --git a/backend/src/__tests__/drift-ledger.test.ts b/backend/src/__tests__/drift-ledger.test.ts index f81fbcd2..1bb4b7f6 100644 --- a/backend/src/__tests__/drift-ledger.test.ts +++ b/backend/src/__tests__/drift-ledger.test.ts @@ -323,7 +323,7 @@ describe('DriftLedgerService.reconcileStack', () => { // A running container on a different image than compose declares => image-mismatch. const driftedContainer = (stack: string) => ({ id: `${stack}-c1`, name: `${stack}-web-1`, service: 'web', composeProject: stack, stack, - state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [], + state: 'running', exitCode: null, image: 'nginx:1.26', networks: [], volumes: [], ports: [], }); beforeEach(() => { @@ -377,7 +377,7 @@ describe('drift route (GET read-only, POST recheck persists)', () => { getDependencySnapshot: vi.fn().mockResolvedValue({ containers: [{ id: 'c1', name: `${STACK}-web-1`, service: 'web', composeProject: STACK, stack: STACK, - state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [], + state: 'running', exitCode: null, image: 'nginx:1.26', networks: [], volumes: [], ports: [], }], networks: [], volumes: [], }), diff --git a/backend/src/__tests__/health-gate-prepare.test.ts b/backend/src/__tests__/health-gate-prepare.test.ts index 258be3e2..3a4a3cd4 100644 --- a/backend/src/__tests__/health-gate-prepare.test.ts +++ b/backend/src/__tests__/health-gate-prepare.test.ts @@ -29,6 +29,7 @@ const { state } = vi.hoisted(() => ({ settings: {} as Record, listContainers: vi.fn(), inspect: vi.fn(), + renderConfig: vi.fn(), }, })); @@ -68,6 +69,15 @@ vi.mock('../services/DockerController', () => ({ }, })); +vi.mock('../services/ComposeService', () => ({ + getComposeCommandTimeoutMs: () => 30_000, + ComposeService: { + getInstance: () => ({ + renderConfig: state.renderConfig, + }), + }, +})); + // AutoHeal suppression is exercised elsewhere; here we only need the calls to // not throw when the gate finalizes a service run. vi.mock('../services/AutoHealService', () => ({ @@ -85,6 +95,8 @@ type Fixture = { restartCount?: number; startedAt?: string; imageId?: string; + exitCode?: number | null; + restartPolicy?: string | null; }; function setContainers(fixtures: Fixture[]): void { @@ -100,16 +112,36 @@ function setContainers(fixtures: Fixture[]): void { return Promise.resolve({ State: { Status: f.state ?? 'running', + ExitCode: f.exitCode === undefined ? (f.state === 'exited' ? 1 : 0) : f.exitCode, Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined, StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z', }, RestartCount: f.restartCount ?? 0, Image: f.imageId ?? 'sha256:app', + HostConfig: { RestartPolicy: { Name: f.restartPolicy ?? 'unless-stopped' } }, Config: { Labels: { 'com.docker.compose.service': f.service } }, }); }); } +function setDeclaredRestarts(services: Record): void { + const rendered = { + name: 'web', + services: Object.fromEntries( + Object.entries(services).map(([name, restart]) => [ + name, + restart === undefined ? { image: `${name}:1` } : { image: `${name}:1`, restart }, + ]), + ), + }; + state.renderConfig.mockResolvedValue({ + rendered: JSON.stringify(rendered), + stderr: '', + code: 0, + timedOut: false, + }); +} + const svc = () => HealthGateService.getInstance(); async function ticks(n: number): Promise { @@ -137,6 +169,8 @@ beforeEach(() => { state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' }; state.listContainers.mockReset(); state.inspect.mockReset(); + state.renderConfig.mockReset(); + setDeclaredRestarts({ app: 'unless-stopped', db: 'unless-stopped' }); svc().start(); }); @@ -216,7 +250,7 @@ describe('primary vs collateral attribution', () => { await ticks(1); setContainers([ { id: 'p1', name: 'web-app-1', service: 'app' }, - { id: 's1', name: 'web-db-1', service: 'db', state: 'exited' }, + { id: 's1', name: 'web-db-1', service: 'db', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' }, ]); await ticks(1); const report = svc().getReport(0, 'web', runId!); @@ -224,6 +258,178 @@ describe('primary vs collateral attribution', () => { expect(report.failureSource).toBe('collateral'); }); + it('passes when a collateral one-shot exits 0 with restart no', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { id: 's1', name: 'web-migrate-1', service: 'migrate', restartPolicy: 'no' }, + ]); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { id: 's1', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('passes a collateral one-shot with residual unhealthy health', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { id: 's1', name: 'web-migrate-1', service: 'migrate', restartPolicy: 'no' }, + ]); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { + id: 's1', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'unhealthy', + }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('fails when a collateral daemon with omitted restart exits 0', async () => { + setDeclaredRestarts({ app: 'unless-stopped', 'daemon-default': undefined }); + const token = await prepareService([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { id: 's1', name: 'web-daemon-1', service: 'daemon-default', restartPolicy: 'no' }, + ]); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { + id: 's1', name: 'web-daemon-1', service: 'daemon-default', + state: 'exited', exitCode: 0, restartPolicy: 'no', + }, + ]); + await ticks(1); + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.failureSource).toBe('collateral'); + expect(report.reason).toContain('exited during observation'); + }); + + it('passes when the primary service is a completed one-shot', async () => { + setDeclaredRestarts({ job: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, restartPolicy: 'no', imageId: 'sha256:app' }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('passes a primary one-shot with residual unhealthy health', async () => { + setDeclaredRestarts({ job: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { + id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'unhealthy', imageId: 'sha256:app', + }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('passes a primary one-shot with residual starting health', async () => { + setDeclaredRestarts({ job: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { + id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'starting', imageId: 'sha256:app', + }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('fails when a primary one-shot exits with null exit code', async () => { + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: null, restartPolicy: 'no', imageId: 'sha256:app' }, + ]); + await ticks(1); + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.reason).toContain('exited during observation'); + expect(report.failureSource).toBe('primary'); + }); + + it('fails when a primary one-shot exits 0 under unless-stopped', async () => { + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'unless-stopped' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped', imageId: 'sha256:app' }, + ]); + await ticks(1); + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.reason).toContain('exited during observation'); + expect(report.failureSource).toBe('primary'); + }); + + it('fails when a primary one-shot exits non-zero', async () => { + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 1, restartPolicy: 'no', imageId: 'sha256:app' }, + ]); + await ticks(1); + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.reason).toContain('exited during observation'); + expect(report.failureSource).toBe('primary'); + }); + it('fails with failureSource collateral when a healthy sibling vanishes before the first poll', async () => { // Sibling is healthy at prepare, then gone before arming. Seeding expected // from the prepare baseline must still track it so the gate fails. diff --git a/backend/src/__tests__/health-gate-service.test.ts b/backend/src/__tests__/health-gate-service.test.ts index 161bc570..0ba4458c 100644 --- a/backend/src/__tests__/health-gate-service.test.ts +++ b/backend/src/__tests__/health-gate-service.test.ts @@ -29,6 +29,7 @@ const { state } = vi.hoisted(() => ({ settings: {} as Record, listContainers: vi.fn(), inspect: vi.fn(), + renderConfig: vi.fn(), }, })); @@ -80,6 +81,15 @@ vi.mock('../services/DockerController', () => ({ }, })); +vi.mock('../services/ComposeService', () => ({ + getComposeCommandTimeoutMs: () => 30_000, + ComposeService: { + getInstance: () => ({ + renderConfig: state.renderConfig, + }), + }, +})); + import { HealthGateService } from '../services/HealthGateService'; type ContainerFixture = { @@ -89,25 +99,57 @@ type ContainerFixture = { health?: string | null; restartCount?: number; startedAt?: string; + exitCode?: number | null; + restartPolicy?: string | null; + service?: string; + imageId?: string; }; /** Configure the docker mocks from a simple fixture list. */ function setContainers(fixtures: ContainerFixture[]): void { - state.listContainers.mockResolvedValue(fixtures.map(f => ({ Id: f.id, Names: [`/${f.name}`], State: f.state ?? 'running' }))); + state.listContainers.mockResolvedValue(fixtures.map(f => ({ + Id: f.id, + Names: [`/${f.name}`], + State: f.state ?? 'running', + Labels: f.service ? { 'com.docker.compose.service': f.service } : {}, + }))); state.inspect.mockImplementation((id: string) => { const f = fixtures.find(c => c.id === id); if (!f) return Promise.reject(Object.assign(new Error('no such container'), { statusCode: 404 })); return Promise.resolve({ State: { Status: f.state ?? 'running', + ExitCode: f.exitCode === undefined ? (f.state === 'exited' ? 1 : 0) : f.exitCode, Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined, StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z', }, RestartCount: f.restartCount ?? 0, + Image: f.imageId ?? 'sha256:img', + HostConfig: { RestartPolicy: { Name: f.restartPolicy ?? '' } }, + Config: { Labels: f.service ? { 'com.docker.compose.service': f.service } : {} }, }); }); } +/** Declared Compose restart map used for one-shot recognition (not inspect). */ +function setDeclaredRestarts(services: Record): void { + const rendered = { + name: 'web', + services: Object.fromEntries( + Object.entries(services).map(([name, restart]) => [ + name, + restart === undefined ? { image: `${name}:1` } : { image: `${name}:1`, restart }, + ]), + ), + }; + state.renderConfig.mockResolvedValue({ + rendered: JSON.stringify(rendered), + stderr: '', + code: 0, + timedOut: false, + }); +} + const svc = () => HealthGateService.getInstance(); const latest = (stack = 'web') => svc().getReport(0, stack); @@ -125,7 +167,9 @@ beforeEach(() => { state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' }; state.listContainers.mockReset(); state.inspect.mockReset(); - setContainers([{ id: 'aaa', name: 'web-app-1' }]); + state.renderConfig.mockReset(); + setDeclaredRestarts({ app: 'unless-stopped' }); + setContainers([{ id: 'aaa', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }]); svc().start(); }); @@ -149,7 +193,7 @@ describe('HealthGateService verdicts', () => { it('fails fast when a container exits', async () => { svc().beginStack(0, 'web', 'update', 'tester'); await ticks(1); // baseline - setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited' }]); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' }]); await ticks(1); const report = latest(); expect(report.status).toBe('failed'); @@ -157,6 +201,167 @@ describe('HealthGateService verdicts', () => { expect(state.activity.some(a => a.category === 'health_gate_failed')).toBe(true); }); + it('passes when a clean one-shot exits 0 with explicit declared restart no', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' }, + ]); + await ticks(1); + expect(latest().status).toBe('observing'); + await ticks(6); + expect(latest().status).toBe('passed'); + }); + + it('fails when a daemon with omitted Compose restart exits 0 (inspect also reports no)', async () => { + // QA P0: Docker HostConfig.RestartPolicy.Name is "no" for both omit and + // explicit restart:"no". Declared intent must decide, not inspect. + setDeclaredRestarts({ 'daemon-default': undefined }); + setContainers([ + { + id: 'daemon', name: 'web-daemon-default-1', service: 'daemon-default', + state: 'running', restartPolicy: 'no', + }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { + id: 'daemon', name: 'web-daemon-default-1', service: 'daemon-default', + state: 'exited', exitCode: 0, restartPolicy: 'no', + }, + ]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + + it('passes a clean one-shot even when residual health is unhealthy', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { + id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'unhealthy', + }, + ]); + await ticks(1); + expect(latest().status).toBe('observing'); + await ticks(6); + expect(latest().status).toBe('passed'); + }); + + it('passes a clean one-shot even when residual health is still starting', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { + id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'starting', + }, + ]); + await ticks(1); + expect(latest().status).toBe('observing'); + await ticks(6); + expect(latest().status).toBe('passed'); + }); + + it('still fails unhealthy on a long-running container', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { + id: 'app', name: 'web-app-1', service: 'app', state: 'running', + restartPolicy: 'unless-stopped', health: 'unhealthy', + }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' }, + ]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('unhealthy'); + }); + + it('still ends unknown when a long-running healthcheck is starting at window end', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { + id: 'app', name: 'web-app-1', service: 'app', state: 'running', + restartPolicy: 'unless-stopped', health: 'starting', + }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' }, + ]); + await ticks(1); + expect(latest().status).toBe('observing'); + await ticks(6); + expect(latest().status).toBe('unknown'); + expect(latest().reason).toContain('still starting'); + }); + + it('fails when exit 0 has unless-stopped restart policy', async () => { + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped' }]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + + it('fails when exit 0 has always restart policy', async () => { + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'always' }]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + + it('fails closed when exit code is null on an exited container', async () => { + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: null, restartPolicy: 'no' }]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + + it('fails when a one-shot exits non-zero', async () => { + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'no' }]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + it('fails fast when a healthcheck reports unhealthy', async () => { svc().beginStack(0, 'web', 'update', 'tester'); await ticks(1); diff --git a/backend/src/__tests__/networking-operator.test.ts b/backend/src/__tests__/networking-operator.test.ts index 466fcb95..7d2a6f6d 100644 --- a/backend/src/__tests__/networking-operator.test.ts +++ b/backend/src/__tests__/networking-operator.test.ts @@ -142,7 +142,7 @@ describe('networking operator routes', () => { it('blocks admin delete when network is attached', async () => { vi.spyOn(DockerController, 'getInstance').mockReturnValue({ getDependencySnapshot: vi.fn().mockResolvedValue({ - containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'nginx', networks: [{ name: 'orphan_net', id: NET_ID, ip: '' }], volumes: [], ports: [] }], + containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'orphan_net', id: NET_ID, ip: '' }], volumes: [], ports: [] }], networks: [{ id: NET_ID, name: 'orphan_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null }], volumes: [], }), @@ -220,7 +220,7 @@ describe('evaluateNetworkDeleteGuard', () => { it('blocks a network that still has an attached container', () => { const snapshot = { volumes: [], - containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'img', networks: [{ name: 'app_net', id: 'n1', ip: '' }], volumes: [], ports: [] }], + containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'img', networks: [{ name: 'app_net', id: 'n1', ip: '' }], volumes: [], ports: [] }], networks: [{ id: 'n1', name: 'app_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK }], }; expect(evaluateNetworkDeleteGuard('n1', snapshot, []).code).toBe('attached'); diff --git a/backend/src/__tests__/networking-summary.test.ts b/backend/src/__tests__/networking-summary.test.ts index c1ca54dc..caa3d011 100644 --- a/backend/src/__tests__/networking-summary.test.ts +++ b/backend/src/__tests__/networking-summary.test.ts @@ -60,7 +60,7 @@ describe('networking summary', () => { it('flags a stack with an undeclared runtime network as network drift', async () => { vi.spyOn(DockerController, 'getInstance').mockReturnValue({ getDependencySnapshot: vi.fn().mockResolvedValue({ - containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'nginx', networks: [{ name: `${STACK}_default`, id: 'd', ip: '' }, { name: `${STACK}_rogue`, id: 'r', ip: '' }], volumes: [], ports: [] }], + containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: `${STACK}_default`, id: 'd', ip: '' }, { name: `${STACK}_rogue`, id: 'r', ip: '' }], volumes: [], ports: [] }], networks: [ { id: 'd', name: `${STACK}_default`, driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK }, { id: 'r', name: `${STACK}_rogue`, driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK }, diff --git a/backend/src/__tests__/one-shot-completion.test.ts b/backend/src/__tests__/one-shot-completion.test.ts new file mode 100644 index 00000000..0faddd26 --- /dev/null +++ b/backend/src/__tests__/one-shot-completion.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from 'vitest'; +import { + isCleanOneShotCompletion, + isNoRestartPolicy, + normalizeComposeRestartIntent, +} from '../utils/oneShotCompletion'; + +describe('isNoRestartPolicy', () => { + it('accepts only explicit no', () => { + expect(isNoRestartPolicy('no')).toBe(true); + }); + + it('rejects absent, empty, and restarting policies', () => { + expect(isNoRestartPolicy(undefined)).toBe(false); + expect(isNoRestartPolicy(null)).toBe(false); + expect(isNoRestartPolicy('')).toBe(false); + expect(isNoRestartPolicy('unless-stopped')).toBe(false); + expect(isNoRestartPolicy('always')).toBe(false); + expect(isNoRestartPolicy('on-failure')).toBe(false); + }); +}); + +describe('isCleanOneShotCompletion', () => { + const clean = { + state: 'exited', + exitCode: 0 as number | null, + restartPolicy: 'no' as string | null | undefined, + }; + + it('returns true only for exited + exit 0 + explicit restart no', () => { + expect(isCleanOneShotCompletion(clean)).toBe(true); + }); + + it('returns false for absent or empty declared restart', () => { + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: undefined })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: null })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: '' })).toBe(false); + }); + + it('returns false for non-exited states', () => { + expect(isCleanOneShotCompletion({ ...clean, state: 'running' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, state: 'restarting' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, state: 'created' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, state: 'dead' })).toBe(false); + }); + + it('returns false for non-zero and null exit codes (fail closed)', () => { + expect(isCleanOneShotCompletion({ ...clean, exitCode: 1 })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, exitCode: 137 })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, exitCode: null })).toBe(false); + }); + + it('returns false for restarting policies even with exit 0', () => { + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'unless-stopped' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'always' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'on-failure' })).toBe(false); + }); +}); + +describe('normalizeComposeRestartIntent', () => { + it('falls back to service restart when deploy.restart_policy is unset', () => { + expect(normalizeComposeRestartIntent('no')).toBe('no'); + expect(normalizeComposeRestartIntent('unless-stopped')).toBe('unless-stopped'); + expect(normalizeComposeRestartIntent(undefined)).toBeNull(); + expect(normalizeComposeRestartIntent(null, {})).toBeNull(); + expect(normalizeComposeRestartIntent('always', { replicas: 2 })).toBe('always'); + }); + + it('maps deploy.restart_policy.condition with Compose defaults and precedence', () => { + expect(normalizeComposeRestartIntent('unless-stopped', { + restart_policy: { condition: 'none' }, + })).toBe('no'); + expect(normalizeComposeRestartIntent(null, { + restart_policy: { condition: 'any' }, + })).toBe('always'); + expect(normalizeComposeRestartIntent('no', { + restart_policy: { condition: 'on-failure' }, + })).toBe('on-failure'); + expect(normalizeComposeRestartIntent('no', { + restart_policy: {}, + })).toBe('always'); + }); + + it('fails closed on malformed restart_policy shapes', () => { + expect(normalizeComposeRestartIntent('no', { restart_policy: null })).toBe('always'); + expect(normalizeComposeRestartIntent('no', { restart_policy: 'none' })).toBe('always'); + expect(normalizeComposeRestartIntent('no', { restart_policy: [] })).toBe('always'); + expect(normalizeComposeRestartIntent('no', { + restart_policy: { condition: 'weird' }, + })).toBe('always'); + }); +}); diff --git a/backend/src/__tests__/preflight-rules.test.ts b/backend/src/__tests__/preflight-rules.test.ts index aaab42cd..b6ffb191 100644 --- a/backend/src/__tests__/preflight-rules.test.ts +++ b/backend/src/__tests__/preflight-rules.test.ts @@ -212,7 +212,11 @@ describe('hygiene rules', () => { }); it('flags a missing restart policy and healthcheck', () => { const bare = model([svc({ restart: undefined, hasHealthcheck: false })]); - expect(ids(runRules(ctx({ model: bare })), 'no-restart-policy')).toHaveLength(1); + const restartFindings = ids(runRules(ctx({ model: bare })), 'no-restart-policy'); + expect(restartFindings).toHaveLength(1); + expect(restartFindings[0].remediation).toMatch(/one-shot|init jobs/i); + expect(restartFindings[0].remediation).toMatch(/restart: "no"/); + expect(restartFindings[0].remediation).toMatch(/unless-stopped/); expect(ids(runRules(ctx({ model: bare })), 'no-healthcheck')).toHaveLength(1); const withDeployRestart = model([svc({ restart: undefined, deploy: { restart_policy: { condition: 'any' } }})]); expect(ids(runRules(ctx({ model: withDeployRestart })), 'no-restart-policy')).toHaveLength(0); diff --git a/backend/src/__tests__/sanitize-network-inspect.test.ts b/backend/src/__tests__/sanitize-network-inspect.test.ts index 316f5934..7358e52f 100644 --- a/backend/src/__tests__/sanitize-network-inspect.test.ts +++ b/backend/src/__tests__/sanitize-network-inspect.test.ts @@ -13,7 +13,7 @@ describe('sanitizeNetworkInspect connected containers', () => { networks: [{ id: 'net1', name: 'app_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: 'app', stack: 'app' }], volumes: [], containers: [{ - id: 'c1', name: 'app-web-1', service: 'web', composeProject: 'app', stack: 'app', state: 'running', image: 'nginx', + id: 'c1', name: 'app-web-1', service: 'web', composeProject: 'app', stack: 'app', state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'app_net', id: 'net1', ip: '172.20.0.5/16' }], volumes: [], ports: [], }], }; diff --git a/backend/src/helpers/composeDependencyParse.ts b/backend/src/helpers/composeDependencyParse.ts index f8059fd0..e73f94ab 100644 --- a/backend/src/helpers/composeDependencyParse.ts +++ b/backend/src/helpers/composeDependencyParse.ts @@ -26,6 +26,8 @@ export interface DeclaredService { image?: string; /** Service `network_mode:` as declared (e.g. `host`), or undefined. */ networkMode?: string; + /** Service `restart` from raw YAML parse. The effective-model path may set this via normalizeComposeRestartIntent (including deploy.restart_policy). Null when unset. */ + restart?: string | null; } /** A top-level networks:/volumes: entry. */ @@ -200,6 +202,7 @@ export function parseComposeDependencies(content: string): DeclaredCompose { ports, image: asString(svc.image), networkMode: asString(svc.network_mode), + restart: asString(svc.restart) ?? null, }); } diff --git a/backend/src/helpers/effectiveToDeclaredCompose.ts b/backend/src/helpers/effectiveToDeclaredCompose.ts index 64325c07..bf73690d 100644 --- a/backend/src/helpers/effectiveToDeclaredCompose.ts +++ b/backend/src/helpers/effectiveToDeclaredCompose.ts @@ -1,5 +1,6 @@ import type { EffectiveModel, EffResource } from '../services/preflight/effectiveModel'; import type { DeclaredCompose, DeclaredPort, DeclaredResource, DeclaredService } from './composeDependencyParse'; +import { normalizeComposeRestartIntent } from '../utils/oneShotCompletion'; /** Map one rendered network/volume entry to the declared shape drift expects. */ function toDeclaredResource(key: string, res: EffResource, projectName: string): DeclaredResource { @@ -41,6 +42,7 @@ export function declaredFromEffectiveModel(model: EffectiveModel): DeclaredCompo ), image: s.image, networkMode: s.networkMode, + restart: normalizeComposeRestartIntent(s.restart, s.deploy), })); return { diff --git a/backend/src/services/DockerController.ts b/backend/src/services/DockerController.ts index 8ee42120..7c46a91a 100644 --- a/backend/src/services/DockerController.ts +++ b/backend/src/services/DockerController.ts @@ -210,6 +210,12 @@ export interface DependencyContainer { /** Resolved Sencho stack, or null when the container is not Sencho-managed. */ stack: string | null; state: string; + /** + * Exit code from list Status when a parenthesized code is present + * (e.g. "Exited (0) …", "Restarting (1) …"); null when none (e.g. "Up …", bare "Exited"). + * Required so Drift can fail closed on unknown codes. + */ + exitCode: number | null; image: string; networks: { name: string; id: string; ip: string }[]; /** Named-volume sources mounted by the container (bind mounts excluded). */ @@ -1833,6 +1839,7 @@ class DockerController { composeProject: c.Labels?.['com.docker.compose.project'] ?? null, stack: DockerController.resolveContainerStack(c.Labels, projectToStack, knownSet, absDirToStack, resolvedBase), state: c.State ?? 'unknown', + exitCode: parseExitCode(typeof c.Status === 'string' ? c.Status : undefined), image: c.Image ?? '', networks, volumes, diff --git a/backend/src/services/DriftDetectionService.ts b/backend/src/services/DriftDetectionService.ts index 92fdab29..3151b4d2 100644 --- a/backend/src/services/DriftDetectionService.ts +++ b/backend/src/services/DriftDetectionService.ts @@ -8,6 +8,7 @@ import { parseEffectiveModel } from './preflight/effectiveModel'; import { compareStackNetworks, fromDeclaredCompose } from './network/normalize'; import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog'; import { getErrorMessage } from '../utils/errors'; +import { isCleanOneShotCompletion } from '../utils/oneShotCompletion'; const MAX_RENDER_ERROR = 600; @@ -178,16 +179,17 @@ function networkDriftFindings( } /** - * Pure diff step (no Docker / FS access) so it is directly unit-testable. Only - * running containers are compared, since a stopped container publishes no ports - * and is not "deployed": a declared service with no running container is - * service-missing, a running container with no matching service is - * service-undeclared, and image / port differences are checked only for services - * present on both sides so a missing/undeclared service is not double-reported. + * Pure diff step (no Docker / FS access) so it is directly unit-testable. + * Running/restarting containers drive image/port comparison. Clean one-shot + * completions (exit 0 + explicit declared restart "no", including normalized + * `deploy.restart_policy.condition: none`) satisfy service presence without + * counting as hasContainers. Omitting restart does not qualify. Network + * comparison still uses only running/restarting attachments. */ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftReport { const { stack, declared, containers, parseError } = input; const networks = input.networks ?? []; + // Public contract: at least one running/restarting container (not "satisfied"). const hasContainers = containers.some((c) => RUNNING_STATES.has(c.state)); // A parse failure means the declared model is untrustworthy: report drift @@ -196,30 +198,44 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe return { stack, status: 'drifted', hasComposeFile: false, hasContainers, findings: [], parseError }; } - const runtimeByService = new Map(); - for (const c of containers) { - if (!RUNNING_STATES.has(c.state)) continue; - const name = c.service ?? c.name; - const agg = runtimeByService.get(name) ?? { images: new Set(), ports: new Set() }; - if (c.image) agg.images.add(normalizeImageRef(c.image)); - for (const p of c.ports) agg.ports.add(portKey(p.publishedPort, p.protocol)); - runtimeByService.set(name, agg); - } - - // Nothing running: the stack is defined on disk but not deployed. One status - // conveys this; per-service findings would just be noise. - if (!hasContainers) { - return { stack, status: 'missing-runtime', hasComposeFile: true, hasContainers: false, findings: [] }; - } - const declaredByName = new Map(); for (const svc of declared.services) declaredByName.set(svc.name, svc); + const runtimeByService = new Map(); + const oneShotSatisfied = new Set(); + for (const c of containers) { + const name = c.service ?? c.name; + if (RUNNING_STATES.has(c.state)) { + const agg = runtimeByService.get(name) ?? { images: new Set(), ports: new Set() }; + if (c.image) agg.images.add(normalizeImageRef(c.image)); + for (const p of c.ports) agg.ports.add(portKey(p.publishedPort, p.protocol)); + runtimeByService.set(name, agg); + continue; + } + const declaredRestart = declaredByName.get(name)?.restart; + if (isCleanOneShotCompletion({ + state: c.state, + exitCode: c.exitCode, + restartPolicy: declaredRestart, + })) { + oneShotSatisfied.add(name); + } + } + + const servicePresent = (serviceName: string): boolean => + runtimeByService.has(serviceName) || oneShotSatisfied.has(serviceName); + + // Nothing running and no declared service satisfied by a clean one-shot: the + // stack is defined on disk but not deployed. One status conveys this. + if (!hasContainers && !declared.services.some((svc) => servicePresent(svc.name))) { + return { stack, status: 'missing-runtime', hasComposeFile: true, hasContainers: false, findings: [] }; + } + const findings: StackDriftFinding[] = []; - // Declared service with no running container. + // Declared service with no running container and no clean one-shot completion. for (const svc of declared.services) { - if (!runtimeByService.has(svc.name)) { + if (!servicePresent(svc.name)) { findings.push({ kind: 'service-missing', service: svc.name, @@ -239,7 +255,7 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe } } - // Image / port divergence for services present on both sides. + // Image / port divergence for services present on both sides (running only). for (const [name, svc] of declaredByName) { const runtime = runtimeByService.get(name); if (!runtime) continue; diff --git a/backend/src/services/HealthGateService.ts b/backend/src/services/HealthGateService.ts index 0e51f1ce..a5c413df 100644 --- a/backend/src/services/HealthGateService.ts +++ b/backend/src/services/HealthGateService.ts @@ -5,8 +5,11 @@ import { AutoHealService } from './AutoHealService'; import { sanitizeForLog } from '../utils/safeLog'; import { getErrorMessage } from '../utils/errors'; import { withTimeout } from '../utils/withTimeout'; +import { isCleanOneShotCompletion } from '../utils/oneShotCompletion'; +import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompose'; +import { parseEffectiveModel } from './preflight/effectiveModel'; import type { HealthGateContainer, HealthGateReport } from './updateGuard/types'; -import { getComposeCommandTimeoutMs } from './ComposeService'; +import { ComposeService, getComposeCommandTimeoutMs } from './ComposeService'; const POLL_INTERVAL_MS = 5_000; // A prepared-but-never-begun token (prepare called, mutation then failed before @@ -47,6 +50,10 @@ interface ObservedContainer { service: string | null; /** Image id the container is running (service gates check convergence on it). */ imageId: string; + /** Inspect State.ExitCode; null when unavailable. */ + exitCode: number | null; + /** HostConfig.RestartPolicy.Name (report/debug only; not used for one-shot intent). */ + restartPolicy: string | null; } interface ActiveGate { @@ -89,6 +96,13 @@ interface ActiveGate { collateralBaselineByName: Map; /** Role of each expected container (service gates), for failure attribution. */ roleByName: Map; + /** + * Declared Compose restart intent by service name (normalized). Loaded once + * per gate; null until the first load attempt completes. Used for one-shot + * recognition instead of Docker inspect (which cannot distinguish omit vs + * explicit `restart: "no"`). + */ + declaredRestartByService: Map | null; } /** A pre-mutation baseline container captured at prepare time. */ @@ -101,6 +115,8 @@ interface PreparedBaseline { restartCount: number; startedAt: string | null; imageId: string; + exitCode: number | null; + restartPolicy: string | null; } function isRegressionEligibleSibling(baseline: PreparedBaseline): boolean { @@ -118,9 +134,35 @@ function observedFromPreparedBaseline(baseline: PreparedBaseline): ObservedConta health: baseline.health, service: baseline.service, imageId: baseline.imageId, + exitCode: baseline.exitCode, + restartPolicy: baseline.restartPolicy, }; } +/** Running, or a clean one-shot exit (exit 0 + explicit declared restart "no"). */ +function isObservedContainerSatisfied( + gate: ActiveGate, + current: ObservedContainer | undefined, +): boolean { + if (!current) return false; + if (current.state === 'running') return true; + return isDeclaredCleanOneShot(gate, current); +} + +/** + * One-shot recognition from declared Compose intent only. Unlabeled containers + * and services missing from the effective model fail closed. + */ +function isDeclaredCleanOneShot(gate: ActiveGate, current: ObservedContainer): boolean { + const map = gate.declaredRestartByService; + if (!map || !current.service || !map.has(current.service)) return false; + return isCleanOneShotCompletion({ + state: current.state, + exitCode: current.exitCode, + restartPolicy: map.get(current.service), + }); +} + /** An in-memory prepare snapshot awaiting attachExpectedImage + beginPrepared. */ interface PreparedGate { token: string; @@ -329,6 +371,7 @@ export class HealthGateService { collateralEligibleNames: new Set(), collateralBaselineByName: new Map(), roleByName: new Map(), + declaredRestartByService: null, }; this.active.set(key, gate); this.scheduleNextPoll(gate); @@ -488,6 +531,7 @@ export class HealthGateService { prep.collateralBaseline.filter(b => prep.collateralEligibleNames.has(b.name)), ), roleByName: new Map(), + declaredRestartByService: null, }; this.active.set(key, gate); this.scheduleNextPoll(gate); @@ -587,6 +631,9 @@ export class HealthGateService { if (gate.finalized || this.active.get(key) !== gate) return; gate.consecutivePollErrors = 0; + await this.ensureDeclaredRestartMap(gate); + if (gate.finalized || this.active.get(key) !== gate) return; + const elapsedMs = Date.now() - gate.startedAt; if (gate.expected === null) { @@ -629,12 +676,16 @@ export class HealthGateService { } gate.missingLastPoll.delete(name); - if (current.state === 'exited' && baseline.restarts === current.restarts) { - // An exit with no restart attempt is terminal for the window. + const cleanOneShot = isDeclaredCleanOneShot(gate, current); + // An exit with no restart attempt is terminal for the window, unless + // this is an expected one-shot (exit 0 + explicit declared restart "no"). + if (current.state === 'exited' && baseline.restarts === current.restarts && !cleanOneShot) { this.finalize(gate, 'failed', `container ${name} exited during observation`, summary); return; } - if (current.health === 'unhealthy') { + // Residual Docker health on a completed one-shot is not a gate failure; + // long-running containers still fail on unhealthy. + if (!cleanOneShot && current.health === 'unhealthy') { this.finalize(gate, 'failed', `container ${name} reported unhealthy`, summary); return; } @@ -657,14 +708,19 @@ export class HealthGateService { if (elapsedMs < gate.windowSeconds * 1000) return; - // Window complete: pass requires everything running and healthy wherever a - // healthcheck exists. A health state still 'starting' is not a pass. - const stillStarting = observed.filter(c => c.health === 'starting'); + // Window complete: pass requires everything running (or a clean one-shot + // completion) and healthy wherever a healthcheck exists. A health state + // still 'starting' is not a pass (except residual health on a clean one-shot). + const stillStarting = observed.filter( + c => c.health === 'starting' && !isDeclaredCleanOneShot(gate, c), + ); if (stillStarting.length > 0) { this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary); return; } - const notRunning = [...gate.expected.keys()].filter(name => byName.get(name)?.state !== 'running'); + const notRunning = [...gate.expected.keys()].filter( + name => !isObservedContainerSatisfied(gate, byName.get(name)), + ); if (notRunning.length > 0) { this.finalize(gate, 'failed', `not running at the end of the window: ${notRunning.join(', ')}`, summary); return; @@ -700,6 +756,9 @@ export class HealthGateService { if (gate.finalized || this.active.get(key) !== gate) return; gate.consecutivePollErrors = 0; + await this.ensureDeclaredRestartMap(gate); + if (gate.finalized || this.active.get(key) !== gate) return; + const elapsedMs = Date.now() - gate.startedAt; const serviceName = gate.serviceName ?? ''; @@ -781,11 +840,12 @@ export class HealthGateService { } gate.missingLastPoll.delete(name); - if (current.state === 'exited' && baseline.restarts === current.restarts) { + const cleanOneShot = isDeclaredCleanOneShot(gate, current); + if (current.state === 'exited' && baseline.restarts === current.restarts && !cleanOneShot) { this.finalize(gate, 'failed', `${noun} ${name} exited during observation`, summary, role); return; } - if (current.health === 'unhealthy') { + if (!cleanOneShot && current.health === 'unhealthy') { this.finalize(gate, 'failed', `${noun} ${name} reported unhealthy`, summary, role); return; } @@ -808,24 +868,28 @@ export class HealthGateService { if (elapsedMs < gate.windowSeconds * 1000) return; const stillStarting = observed.filter( - c => c.health === 'starting' && (c.service === serviceName || gate.collateralEligibleNames.has(c.name)), + c => c.health === 'starting' + && !isDeclaredCleanOneShot(gate, c) + && (c.service === serviceName || gate.collateralEligibleNames.has(c.name)), ); if (stillStarting.length > 0) { this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary); return; } - const runningPrimary = observed.filter(c => c.service === serviceName && c.state === 'running'); - if (runningPrimary.length !== gate.expectedReplicas) { + const satisfiedPrimary = observed.filter( + c => c.service === serviceName && isObservedContainerSatisfied(gate, c), + ); + if (satisfiedPrimary.length !== gate.expectedReplicas) { this.finalize( gate, 'failed', - `service ${serviceName} has ${runningPrimary.length} running replica(s), expected ${gate.expectedReplicas}`, + `service ${serviceName} has ${satisfiedPrimary.length} satisfied replica(s), expected ${gate.expectedReplicas}`, summary, 'primary', ); return; } const collateralNotRunning = [...gate.expected.keys()] .filter(name => gate.roleByName.get(name) === 'collateral') - .filter(name => byName.get(name)?.state !== 'running'); + .filter(name => !isObservedContainerSatisfied(gate, byName.get(name))); if (collateralNotRunning.length > 0) { this.finalize(gate, 'failed', `sibling(s) not running at the end of the window: ${collateralNotRunning.join(', ')}`, summary, 'collateral'); return; @@ -855,6 +919,35 @@ export class HealthGateService { return this.listStackContainers(gate.nodeId, gate.stackName); } + /** + * Load declared Compose restart intent once per gate. Fail closed to an empty + * map on render/parse errors so inspect "no" cannot false-qualify one-shots. + */ + private async ensureDeclaredRestartMap(gate: ActiveGate): Promise { + if (gate.declaredRestartByService !== null) return; + gate.declaredRestartByService = new Map(); + try { + const result = await ComposeService.getInstance(gate.nodeId).renderConfig(gate.stackName); + if (result.rendered === null) { + console.warn( + '[HealthGate] declared restart map unavailable for %s (compose render failed)', + sanitizeForLog(gate.stackName), + ); + return; + } + const model = parseEffectiveModel(JSON.parse(result.rendered), gate.stackName); + const declared = declaredFromEffectiveModel(model); + gate.declaredRestartByService = new Map( + declared.services.map((s) => [s.name, s.restart ?? null]), + ); + } catch (error) { + console.warn( + '[HealthGate] declared restart map load failed for %s:', + sanitizeForLog(gate.stackName), getErrorMessage(error, 'unknown'), + ); + } + } + /** List and inspect a stack's containers into the gate's observation shape. */ private async listStackContainers(nodeId: number, stackName: string): Promise { const docker = DockerController.getInstance(nodeId).getDocker(); @@ -877,6 +970,8 @@ export class HealthGateService { health: inspect.State?.Health?.Status ?? null, service: labels['com.docker.compose.service'] ?? null, imageId: inspect.Image ?? '', + exitCode: typeof inspect.State?.ExitCode === 'number' ? inspect.State.ExitCode : null, + restartPolicy: inspect.HostConfig?.RestartPolicy?.Name || null, }; } catch (e: unknown) { // Removed between list and inspect; the missing-container logic will @@ -903,6 +998,8 @@ export class HealthGateService { restartCount: c.restartCount, startedAt: c.startedAt, imageId: c.imageId, + exitCode: c.exitCode, + restartPolicy: c.restartPolicy, })); } diff --git a/backend/src/services/preflight/effectiveModel.ts b/backend/src/services/preflight/effectiveModel.ts index 44d6f41e..e5bf0efd 100644 --- a/backend/src/services/preflight/effectiveModel.ts +++ b/backend/src/services/preflight/effectiveModel.ts @@ -56,7 +56,7 @@ export interface EffService { networkMode?: string; restart?: string; hasHealthcheck: boolean; - /** Raw deploy block (read for key presence only, never values; undefined = none). */ + /** Raw deploy block (preflight uses key presence; Drift also reads restart_policy.condition). Undefined = none. */ deploy?: Record; containerName?: string; user?: string; diff --git a/backend/src/services/preflight/rules.ts b/backend/src/services/preflight/rules.ts index e58ca8e3..2e39aef1 100644 --- a/backend/src/services/preflight/rules.ts +++ b/backend/src/services/preflight/rules.ts @@ -363,7 +363,7 @@ const noRestartPolicy: PreflightRule = { message: `Service "${s.name}" has no restart policy, so it will not come back after a crash or host reboot.`, sourcePath: s.name, service: s.name, - remediation: 'Add restart: unless-stopped.', + remediation: 'For long-running services, add restart: unless-stopped. For one-shot or init jobs that should finish and stay stopped, restart: "no" is appropriate.', })); }, }; diff --git a/backend/src/utils/oneShotCompletion.ts b/backend/src/utils/oneShotCompletion.ts new file mode 100644 index 00000000..4c5206bb --- /dev/null +++ b/backend/src/utils/oneShotCompletion.ts @@ -0,0 +1,77 @@ +/** + * Helpers for clean one-shot completion and Compose restart-intent normalization. + * Used by Health Gate and Drift service-presence only; does not change bulk + * status, Auto-Heal, or atomic-deploy helpers. + */ + +/** + * True only for an explicit Compose `restart: "no"` (after normalization). + * Absent / null / empty do not qualify: Docker reports HostConfig restart + * "no" for both intentional one-shots and bare long-running services that + * omit `restart:`, so consumers must pass declared Compose intent, not inspect. + */ +export function isNoRestartPolicy(policy: string | null | undefined): boolean { + return policy === 'no'; +} + +/** + * Normalize Compose restart intent for Drift / declared-model consumers. + * + * Compose precedence: when `deploy.restart_policy` is set, its `condition` + * wins; otherwise the service-level `restart` field is used. Conditions map to + * Docker-like policy names so {@link isNoRestartPolicy} stays the single gate: + * `none` → `no`, `any` → `always`, `on-failure` → `on-failure`. Missing + * condition defaults to `any` (Compose default). Unknown shapes fail closed. + * Absent service restart (no deploy policy) stays null and is not one-shot eligible. + */ +export function normalizeComposeRestartIntent( + serviceRestart: string | null | undefined, + deploy?: Record | null, +): string | null { + if (deploy && Object.prototype.hasOwnProperty.call(deploy, 'restart_policy')) { + const policy = deploy['restart_policy']; + if (policy === null || typeof policy !== 'object' || Array.isArray(policy)) { + return 'always'; + } + const condition = (policy as Record).condition; + // Missing, empty, or non-string → Compose default `any` → always. + if (typeof condition !== 'string' || condition === '') { + return 'always'; + } + switch (condition.toLowerCase()) { + case 'none': + return 'no'; + case 'on-failure': + return 'on-failure'; + case 'any': + return 'always'; + default: + return 'always'; + } + } + if (serviceRestart == null || serviceRestart === '') return null; + return serviceRestart; +} + +export interface OneShotCompletionInput { + state: string; + /** Exact Docker exit code; null means unknown (fail closed). */ + exitCode: number | null; + /** + * Declared Compose restart intent after normalization (`"no"` for explicit + * one-shots / `deploy.restart_policy.condition: none`). Do not pass Docker + * inspect HostConfig values here. + */ + restartPolicy: string | null | undefined; +} + +/** + * True only for an exited container with exit code exactly 0 and explicit + * declared restart `"no"`. Null/absent restart, null exit codes, and restarting + * policies never qualify. + */ +export function isCleanOneShotCompletion(input: OneShotCompletionInput): boolean { + return input.state === 'exited' + && input.exitCode === 0 + && isNoRestartPolicy(input.restartPolicy); +} diff --git a/docs/features/health-gated-updates.mdx b/docs/features/health-gated-updates.mdx index 8abafdae..be9c27fa 100644 --- a/docs/features/health-gated-updates.mdx +++ b/docs/features/health-gated-updates.mdx @@ -49,11 +49,13 @@ The 3-second crash probe from [atomic deployments](/features/atomic-deployments) During the window, the gate checks every container that came up with the stack: -- all expected containers stay **running**, -- Docker healthchecks report **healthy** wherever a service defines one, -- no container **exits**, **disappears**, or enters a **restart loop**. +- all expected containers stay **running** (or finish cleanly as a one-shot; see below), +- Docker healthchecks report **healthy** on long-running containers wherever a service defines one (residual health on a completed one-shot is ignored), +- no container **crashes** (non-zero exit), **disappears**, or enters a **restart loop**. -A clear failure ends the observation immediately. Passing requires the full window, and a healthcheck still in its start period when the window ends records the verdict as unknown rather than claiming success. +A one-shot or init service that finishes with exit code `0` and explicitly declares `restart: "no"` (or `deploy.restart_policy.condition: none`) counts as a successful completion, not a gate failure. Omitting `restart:` does not qualify: Compose defaults to no-restart, and Docker reports the same inspect value for both intentional jobs and bare long-running services. Residual Docker health state on a completed one-shot (`starting` or `unhealthy` from an inherited healthcheck) does not fail or leave the gate unknown. A clean exit under `unless-stopped`, `always`, or `on-failure`, or with no declared restart, still fails the gate. An exit whose code cannot be read also fails the gate. + +A clear failure ends the observation immediately. Passing requires the full window, and a healthcheck still in its start period on a long-running container when the window ends records the verdict as unknown rather than claiming success. After a **service-scoped** update or restore, the gate still watches the whole project, but it attributes failures: a problem on the updated service is a primary failure; a previously healthy sibling that regresses during the window is a collateral failure. Siblings that were already unhealthy before the update do not fail the gate on their own. @@ -78,7 +80,7 @@ The gate also runs for updates you did not click: scheduled image updates, webho Open **Settings > Infrastructure > Stacks > Deploy Guardrails** on the node you want to configure: - **Observe health after updates** turns the gate on or off for that node. On by default; turning it off only stops the observation and its timeline events, never the update itself. -- **Observation window** sets how long containers are watched, from 15 to 600 seconds. The default is 90 seconds; raise it for stacks that take a while to settle, since a healthcheck still starting at the end of the window records an unknown verdict instead of a pass. +- **Observation window** sets how long containers are watched, from 15 to 600 seconds. The default is 90 seconds; raise it for stacks that take a while to settle, since a long-running healthcheck still starting at the end of the window records an unknown verdict instead of a pass. Stacks settings page showing the Deploy Guardrails subsection, with the Observe health after updates toggle set to On and the Observation window field set to 90 seconds @@ -110,10 +112,10 @@ When a deploy or update fails, Sencho classifies the failure from the compose ou Unknown means one of the verdict-affecting signals could not be verified, most often because Docker or the node did not answer in time. The update path is never blocked by an unknown verdict; check the node's connection if it persists, and proceed when you are confident in the stack's state. - The gate fails on the first clear problem it observes: a container exit, an unhealthy healthcheck, a restart loop, or a container disappearing mid-window. Check the named container's logs from the stack page. If the container legitimately restarts during startup (a migration step, for example), raise the observation window under **Settings > Infrastructure > Stacks > Deploy Guardrails** so the gate watches past the settle period. + The gate fails on the first clear problem it observes: a non-zero exit, an unexpected clean exit of a service that is not an explicit one-shot, an unhealthy healthcheck on a long-running container, a restart loop, or a container disappearing mid-window. A one-shot that exits `0` with explicit `restart: "no"` (or `deploy.restart_policy.condition: none`) does not fail the gate, including when residual Docker health on that completed container is still starting or unhealthy. Check the named container's logs from the stack page. If the container legitimately restarts during startup (a migration step that should stay up, for example), raise the observation window under **Settings > Infrastructure > Stacks > Deploy Guardrails** so the gate watches past the settle period. - Unknown means the observation could not finish: Sencho restarted mid-window, a newer update superseded the observation, Docker became unreachable, a healthcheck was still starting when the window ended, or no containers appeared to observe. The stack timeline records the reason. Check the containers directly; an unknown verdict makes no claim either way. + Unknown means the observation could not finish: Sencho restarted mid-window, a newer update superseded the observation, Docker became unreachable, a long-running healthcheck was still starting when the window ended, or no containers appeared to observe. Residual starting health on a completed one-shot does not produce unknown. The stack timeline records the reason. Check the containers directly; an unknown verdict makes no claim either way. The modal holds its auto-close while the gate observes so the verdict is not lost. You can close it at any time; the observation continues server-side and the verdict lands on the stack timeline. If the gate result repeatedly cannot be retrieved, the modal gives up with an unknown verdict instead of waiting forever. diff --git a/docs/features/stack-drift.mdx b/docs/features/stack-drift.mdx index ae7c5897..2297f106 100644 --- a/docs/features/stack-drift.mdx +++ b/docs/features/stack-drift.mdx @@ -28,9 +28,9 @@ The status badge at the top of the tab summarizes the comparison between the eff | Status | Meaning | |--------|---------| -| **In sync** | Running containers match the effective model: same services, images, and published ports. | -| **Drifted** | Something running differs from the model. Specific findings are listed below the badge. | -| **Not running** | The stack has no running containers. The file is present but nothing is up. | +| **In sync** | Declared services match runtime: same running services, images, and published ports, or a declared one-shot finished cleanly (`Exited (0)` with explicit `restart: "no"` or `deploy.restart_policy.condition: none`). Omitting `restart:` does not count as a one-shot. | +| **Not running** | The stack has no running containers, and no declared service is satisfied by a finished one-shot. The file is present but nothing is up. | +| **Drifted** | Something differs from the model. Specific findings are listed below the badge. | | **Unreachable** | Docker could not be reached on the active node, so drift cannot be assessed. | ## Since your last deploy @@ -62,10 +62,10 @@ When a stack is drifted, each reason is listed in the **Findings** section again |---------|---------------| | **Image** | A running container uses a different image than the Compose file declares. Expected and actual values are shown side by side. | | **Ports** | The published ports of a service differ from what the Compose file declares. | -| **Service missing** | The Compose file declares a service, but no running container matches it. | +| **Service missing** | The Compose file declares a service, but no running container matches it and no clean one-shot completion satisfies it. | | **Undeclared service** | A container is running for the stack but no matching service exists in the Compose file. | | **Network undeclared** | A running service is attached to a network that is not declared in the Compose file. | -| **Network missing** | A network declared in the Compose file is not used by any running service or is absent from the runtime. | +| **Network missing** | A network declared in the Compose file is not used by any running service or is absent from the runtime. Network usage still counts only running or restarting containers, so a finished one-shot does not keep a dedicated Compose network marked as in use. | ### Image comparison @@ -157,7 +157,7 @@ Viewing the tab requires read access to the stack; any role that can see a stack - That is expected. Drift compares the file on disk against what is actually running, so a stack with no running containers reports **Not running** even when you stopped it intentionally. Deploy it to return it to **In sync**. + That is expected for long-running services. Drift compares the file on disk against what is actually running, so a stack with no running containers and no finished one-shot jobs reports **Not running** even when you stopped it intentionally. Deploy it to return it to **In sync**. A stack made only of one-shot jobs that already exited `0` with explicit `restart: "no"` (or `deploy.restart_policy.condition: none`) can show **In sync** (or **Drifted** for other findings such as unused networks) with no containers currently running. Click **re-check** after the deploy finishes. During a rolling update, replicas can briefly run different images, and the report captures that moment. Once every container is on the declared image, the finding clears. From e33eda3c38423b047f1c870ea7a452cf11b1741f Mon Sep 17 00:00:00 2001 From: Anso Date: Fri, 24 Jul 2026 15:21:22 -0400 Subject: [PATCH 12/13] fix: pin postcss override to clear backend audit vulnerability (#1701) vitest pulls in vite, which pulls in postcss@8.5.15, flagged high severity for a source-map path traversal advisory. Pin postcss via npm overrides (same pattern already used for brace-expansion) so the fix is scoped to the vulnerable transitive dependency instead of letting a broad npm audit fix churn unrelated packages. --- backend/package-lock.json | 14 +++++++------- backend/package.json | 3 ++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index 37e09d66..756d7fa9 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -5081,9 +5081,9 @@ "optional": true }, "node_modules/nanoid": { - "version": "3.3.12", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", - "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", "dev": true, "funding": [ { @@ -5428,9 +5428,9 @@ } }, "node_modules/postcss": { - "version": "8.5.15", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", - "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "version": "8.5.23", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz", + "integrity": "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==", "dev": true, "funding": [ { @@ -5448,7 +5448,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.16", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, diff --git a/backend/package.json b/backend/package.json index c6bd4607..467de07d 100644 --- a/backend/package.json +++ b/backend/package.json @@ -30,7 +30,8 @@ "node": ">=26.0.0" }, "overrides": { - "brace-expansion": "^5.0.7" + "brace-expansion": "^5.0.7", + "postcss": "^8.5.23" }, "devDependencies": { "@eslint/js": "^10.0.1", From 17a8dc8a94b18b396492735de8cd89bc2cf45c97 Mon Sep 17 00:00:00 2001 From: Anso Date: Fri, 24 Jul 2026 15:57:18 -0400 Subject: [PATCH 13/13] fix(blueprints): fail closed on marker ownership for apply and withdraw (#1694) * fix(blueprints): fail closed on marker ownership for apply and withdraw Require a matching .blueprint.json under the stack lock, persist required_blueprint_id on deletion intents, remove the legacy remote apply fallback, and protect the marker in the file explorer. * fix(blueprints): add CodeQL path barriers on ownership probes Use the canonical resolve-and-startsWith sanitizer inline at the marker and stack-directory fs sinks so js/path-injection clears. * fix(blueprints): block delete on failed withdraw and defer marker write Refuse Blueprint DELETE when pre-delete withdraw does not complete, and write .blueprint.json only after a successful deploy so failed applies cannot orphan stacks or claim an unapplied revision. * test(blueprints): align lock-order assert with deferred marker write Update the per-stack lock ordering expectations to compose, cleanup, deploy, then marker after the partial-apply fix. * fix(deps): bump postcss past GHSA-r28c-9q8g-f849 for npm audit Raise the Vitest/Vite transitive postcss to 8.5.23 so Backend CI audit --audit-level=high passes. --- .../blueprints-compose-apply.test.ts | 80 ++++ .../__tests__/blueprints-edge-cases.test.ts | 82 +++- .../blueprints-preview-apply.test.ts | 36 ++ .../blueprints-remote-deploy.test.ts | 97 ++-- .../blueprints-route-validation.test.ts | 68 +++ backend/src/__tests__/blueprints.test.ts | 30 +- ...ase-stack-cleanup-intent-migration.test.ts | 64 +++ .../deployed-stack-deletion-service.test.ts | 39 ++ .../__tests__/filesystem-stack-paths.test.ts | 4 +- .../src/__tests__/stack-files-routes.test.ts | 41 ++ backend/src/helpers/blueprintMarker.ts | 28 ++ backend/src/routes/blueprints.ts | 87 +++- backend/src/routes/stacks.ts | 39 ++ backend/src/services/BlueprintService.ts | 425 ++++++++++-------- backend/src/services/DatabaseService.ts | 10 +- .../services/DeployedStackDeletionService.ts | 199 ++++++-- backend/src/services/FileSystemService.ts | 13 +- .../services/blueprintPreviewProjection.ts | 27 +- docs/features/blueprint-model.mdx | 9 +- 19 files changed, 1092 insertions(+), 286 deletions(-) create mode 100644 backend/src/__tests__/database-stack-cleanup-intent-migration.test.ts create mode 100644 backend/src/helpers/blueprintMarker.ts diff --git a/backend/src/__tests__/blueprints-compose-apply.test.ts b/backend/src/__tests__/blueprints-compose-apply.test.ts index 7e8d6b21..8baf72f6 100644 --- a/backend/src/__tests__/blueprints-compose-apply.test.ts +++ b/backend/src/__tests__/blueprints-compose-apply.test.ts @@ -89,6 +89,7 @@ describe('Blueprint compose apply (real filesystem)', () => { const stackDir = path.join(process.env.COMPOSE_DIR!, stackName); expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe(composeContent); await expectMissing(path.join(stackDir, 'docker-compose.yml')); + expect(await fsPromises.readFile(path.join(stackDir, '.blueprint.json'), 'utf-8')).toBe(markerContent); const resolved = await FileSystemService.getInstance(nodeId).getComposeFilename(stackName); expect(resolved).toBe('compose.yaml'); @@ -104,6 +105,10 @@ describe('Blueprint compose apply (real filesystem)', () => { await fsPromises.writeFile(path.join(stackDir, 'compose.yml'), 'services:\n a:\n image: a\n'); await fsPromises.writeFile(path.join(stackDir, 'docker-compose.yaml'), 'services:\n b:\n image: b\n'); await fsPromises.writeFile(path.join(stackDir, 'docker-compose.yml'), 'services:\n c:\n image: c\n'); + await fsPromises.writeFile( + path.join(stackDir, '.blueprint.json'), + JSON.stringify({ blueprintId: 2, revision: 2, lastApplied: 1 }, null, 2), + ); const composeContent = 'services:\n app:\n image: redis:7\n'; const markerContent = JSON.stringify({ blueprintId: 2, revision: 3, lastApplied: Date.now() }, null, 2); @@ -131,8 +136,83 @@ describe('Blueprint compose apply (real filesystem)', () => { await expectMissing(path.join(stackDir, 'compose.yml')); await expectMissing(path.join(stackDir, 'docker-compose.yaml')); await expectMissing(path.join(stackDir, 'docker-compose.yml')); + expect(await fsPromises.readFile(path.join(stackDir, '.blueprint.json'), 'utf-8')).toBe(markerContent); expect(deploySpy).toHaveBeenCalledTimes(1); }); + + it('does not write a new marker when deploy fails; rolls back a newly created stack', async () => { + const nodeId = seedLocalNode(); + const stackName = `bp-partial-${counter}`; + const composeContent = 'services:\n web:\n image: traefik:v3\n'; + const markerContent = JSON.stringify({ blueprintId: 7, revision: 1, lastApplied: Date.now() }, null, 2); + const stackDir = path.join(process.env.COMPOSE_DIR!, stackName); + + vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue(new Error('docker unavailable')); + + await expect( + BlueprintService.getInstance().applyLocalUnderLock( + nodeId, + stackName, + composeContent, + markerContent, + '/api/blueprints/test/apply', + ), + ).rejects.toThrow(/docker unavailable/); + + await expectMissing(path.join(stackDir, '.blueprint.json')); + await expectMissing(stackDir); + }); + + it('keeps the prior marker when a re-apply deploy fails', async () => { + const nodeId = seedLocalNode(); + const stackName = `bp-reapply-fail-${counter}`; + const stackDir = path.join(process.env.COMPOSE_DIR!, stackName); + const priorMarker = JSON.stringify({ blueprintId: 8, revision: 2, lastApplied: 1 }, null, 2); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n old:\n image: nginx\n'); + await fsPromises.writeFile(path.join(stackDir, '.blueprint.json'), priorMarker); + + vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue(new Error('deploy blew up')); + + await expect( + BlueprintService.getInstance().applyLocalUnderLock( + nodeId, + stackName, + 'services:\n new:\n image: redis:7\n', + JSON.stringify({ blueprintId: 8, revision: 3, lastApplied: Date.now() }, null, 2), + '/api/blueprints/test/apply', + ), + ).rejects.toThrow(/deploy blew up/); + + expect(await fsPromises.readFile(path.join(stackDir, '.blueprint.json'), 'utf-8')).toBe(priorMarker); + expect(await fsPromises.access(stackDir).then(() => true, () => false)).toBe(true); + }); + + it('refuses to overwrite an existing unmanaged stack directory inside the lock', async () => { + const { BlueprintNameConflictError } = await import('../services/BlueprintService'); + const nodeId = seedLocalNode(); + const stackName = `bp-hijack-${counter}`; + const stackDir = path.join(process.env.COMPOSE_DIR!, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + const original = 'services:\n mine:\n image: nginx:alpine\n'; + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), original); + + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined); + + await expect( + BlueprintService.getInstance().applyLocalUnderLock( + nodeId, + stackName, + 'services:\n bp:\n image: redis:7\n', + JSON.stringify({ blueprintId: 99, revision: 1, lastApplied: Date.now() }, null, 2), + '/api/blueprints/test/apply', + ), + ).rejects.toBeInstanceOf(BlueprintNameConflictError); + + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe(original); + await expectMissing(path.join(stackDir, '.blueprint.json')); + expect(deploySpy).not.toHaveBeenCalled(); + }); }); describe('FileSystemService.removeAlternateRootComposeFiles', () => { diff --git a/backend/src/__tests__/blueprints-edge-cases.test.ts b/backend/src/__tests__/blueprints-edge-cases.test.ts index f7beba38..026f2c22 100644 --- a/backend/src/__tests__/blueprints-edge-cases.test.ts +++ b/backend/src/__tests__/blueprints-edge-cases.test.ts @@ -111,9 +111,9 @@ describe('Blueprint route edge cases', () => { describe('Blueprint delete guard', () => { // Rows with no stack of ours on the node must not block delete, AND the route must not run the - // withdraw primitive for them: withdrawFromNode proceeds on a missing marker and would - // down/delete a same-name stack Sencho never owned. A name_conflict is exactly that unmanaged - // stack, so it is excluded even though it carries a last_deployed_at timestamp. + // withdraw primitive for them: there is nothing Sencho owns to remove. A name_conflict is an + // unmanaged same-name stack, so it is excluded even though it may carry a last_deployed_at + // timestamp. When withdraw does run, ownership is enforced under the delete lock. it.each([ { label: 'never-deployed pending review', status: 'pending_state_review' as const, last_deployed_at: null }, { label: 'first-deploy failure', status: 'failed' as const, last_deployed_at: null }, @@ -184,6 +184,34 @@ describe('Blueprint delete guard', () => { expect(DatabaseService.getInstance().getBlueprint(bp.id)).toBeUndefined(); }); + it('refuses to delete a stateless blueprint when pre-delete withdraw fails', async () => { + const node = seedNode(); + const bp = seedBlueprint([node.id]); + DatabaseService.getInstance().upsertDeployment({ + blueprint_id: bp.id, + node_id: node.id, + status: 'failed', + applied_revision: bp.revision, + last_deployed_at: Date.now(), + last_error: 'Remote node lacks withdraw-local', + }); + const withdrawSpy = vi.spyOn(BlueprintService.getInstance(), 'withdrawFromNode').mockResolvedValue({ + status: 'failed', + error: 'Remote node does not support atomic blueprint withdraw', + }); + + const res = await request(app) + .delete(`/api/blueprints/${bp.id}`) + .set('Cookie', adminCookie); + + expect(res.status).toBe(409); + expect(res.body.code).toBe('withdraw_failed_blocking_delete'); + expect(res.body.nodeId).toBe(node.id); + expect(withdrawSpy).toHaveBeenCalledTimes(1); + expect(DatabaseService.getInstance().getBlueprint(bp.id)).toBeDefined(); + expect(DatabaseService.getInstance().listDeployments(bp.id)).toHaveLength(1); + }); + it('refuses to delete when a pending review still has a deployed stack (revision drift)', async () => { const node = seedNode(); const bp = seedBlueprint([node.id], 'stateful'); @@ -253,38 +281,54 @@ describe('BlueprintService marker edge cases', () => { }); it('refuses to withdraw when the marker belongs to a different blueprint', async () => { + const fs = await import('fs'); + const path = await import('path'); + const composeDir = process.env.COMPOSE_DIR!; const localNode = DatabaseService.getInstance().getNodes()[0]; - const bp = seedBlueprint([localNode.id]); + DatabaseService.getInstance().getDb() + .prepare('UPDATE nodes SET compose_dir = ? WHERE id = ?') + .run(composeDir, localNode.id); + const refreshed = DatabaseService.getInstance().getNode(localNode.id)!; + const bp = seedBlueprint([refreshed.id]); const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!; - - vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue({ - blueprintId: bp.id + 999, - revision: 1, - lastApplied: 0, + DatabaseService.getInstance().upsertDeployment({ + blueprint_id: bp.id, + node_id: refreshed.id, + status: 'active', + applied_revision: bpObj.revision, + last_deployed_at: Date.now(), }); - const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, localNode); + const stackDir = path.join(composeDir, bpObj.name); + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n app:\n image: nginx\n'); + fs.writeFileSync( + path.join(stackDir, '.blueprint.json'), + JSON.stringify({ blueprintId: bp.id + 999, revision: 1, lastApplied: 0 }), + ); + + const { DeployedStackDeletionService } = await import('../services/DeployedStackDeletionService'); + const deleteSpy = vi.spyOn(DeployedStackDeletionService.getInstance(), 'deleteDeployedStack'); + + const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, refreshed); expect(result.status).toBe('name_conflict'); - // The deployment row must record the conflict, not silently disappear. - const dep = DatabaseService.getInstance().getDeployment(bp.id, localNode.id); + // deleteDeployedStack is invoked but returns name_conflict without mutating. + expect(deleteSpy).toHaveBeenCalled(); + const dep = DatabaseService.getInstance().getDeployment(bp.id, refreshed.id); expect(dep).toBeDefined(); expect(dep?.status).toBe('name_conflict'); + expect(fs.existsSync(path.join(stackDir, 'compose.yaml'))).toBe(true); }); }); describe('BlueprintService developer-mode diagnostics', () => { - // withdrawFromNode emits its "withdraw inputs" diagnostic line before reading the - // marker, so a cross-blueprint marker stub lets us assert the gate without Docker. + // withdrawFromNode emits its "withdraw inputs" diagnostic before the deletion + // service runs. An absent stack directory is enough to exercise logging without Docker. function arrangeWithdraw() { const localNode = DatabaseService.getInstance().getNodes()[0]; const bp = seedBlueprint([localNode.id]); const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!; - vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue({ - blueprintId: bp.id + 999, - revision: 1, - lastApplied: 0, - }); return { bpObj, localNode }; } diff --git a/backend/src/__tests__/blueprints-preview-apply.test.ts b/backend/src/__tests__/blueprints-preview-apply.test.ts index d466b367..6a084973 100644 --- a/backend/src/__tests__/blueprints-preview-apply.test.ts +++ b/backend/src/__tests__/blueprints-preview-apply.test.ts @@ -326,6 +326,7 @@ describe('POST /api/blueprints/:id/apply confirm binding', () => { const conflict = await BlueprintService.getInstance().hasNameConflict( created.body.name as string, DatabaseService.getInstance().getNode(node.id)!, + created.body.id as number, ); expect(conflict).toBe(true); @@ -342,6 +343,41 @@ describe('POST /api/blueprints/:id/apply confirm binding', () => { .toBe('services:\n app:\n image: nginx\n'); }); + it('hasNameConflict is false for a matching marker and true for a foreign marker', async () => { + const node = seedNode(); + counter += 1; + const created = await request(app) + .post('/api/blueprints') + .set('Cookie', adminCookie) + .send(validBlueprintBody(node.id)); + expect(created.status).toBe(201); + + const composeDir = process.env.COMPOSE_DIR!; + DatabaseService.getInstance().getDb() + .prepare('UPDATE nodes SET compose_dir = ? WHERE id = ?') + .run(composeDir, node.id); + const nodeObj = DatabaseService.getInstance().getNode(node.id)!; + const stackName = created.body.name as string; + const blueprintId = created.body.id as number; + const stackDir = path.join(composeDir, stackName); + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n app:\n image: nginx\n'); + fs.writeFileSync( + path.join(stackDir, '.blueprint.json'), + JSON.stringify({ blueprintId, revision: 1, lastApplied: 1 }), + ); + + const { BlueprintService } = await import('../services/BlueprintService'); + const svc = BlueprintService.getInstance(); + expect(await svc.hasNameConflict(stackName, nodeObj, blueprintId)).toBe(false); + + fs.writeFileSync( + path.join(stackDir, '.blueprint.json'), + JSON.stringify({ blueprintId: blueprintId + 99, revision: 1, lastApplied: 1 }), + ); + expect(await svc.hasNameConflict(stackName, nodeObj, blueprintId)).toBe(true); + }); + it('blocks rename while a non-withdrawn deployment exists', async () => { const node = seedNode(); counter += 1; diff --git a/backend/src/__tests__/blueprints-remote-deploy.test.ts b/backend/src/__tests__/blueprints-remote-deploy.test.ts index d9a145b7..9bf91e72 100644 --- a/backend/src/__tests__/blueprints-remote-deploy.test.ts +++ b/backend/src/__tests__/blueprints-remote-deploy.test.ts @@ -101,7 +101,7 @@ describe('BlueprintService remote deploy', () => { expect(dep?.applied_revision).toBe(bpObj.revision); }); - it('falls back to the legacy create/write/deploy flow when the remote lacks apply-local (404)', async () => { + it('fails closed when the remote lacks apply-local (404); no legacy mutations', async () => { const node = seedRemoteNode(); const bp = seedBlueprint([node.id]); const nodeObj = DatabaseService.getInstance().getNode(node.id)!; @@ -110,25 +110,17 @@ describe('BlueprintService remote deploy', () => { vi.spyOn(axios, 'get').mockResolvedValue({ status: 200, data: [] }); const putSpy = vi.spyOn(axios, 'put').mockResolvedValue({ status: 200, data: {} }); const postSpy = vi.spyOn(axios, 'post') - .mockResolvedValueOnce({ status: 404, data: {} }) // apply-local missing on older node - .mockResolvedValueOnce({ status: 201, data: {} }) // legacy create stack - .mockResolvedValueOnce({ status: 200, data: {} }); // legacy deploy + .mockResolvedValueOnce({ status: 404, data: {} }); const result = await BlueprintService.getInstance().deployToNode(bpObj, nodeObj); - expect(result.status).toBe('active'); + expect(result.status).toBe('failed'); + expect(result.error).toMatch(/apply-local|Upgrade/i); + expect(postSpy).toHaveBeenCalledTimes(1); expect(postSpy.mock.calls[0][0]).toMatch(/\/api\/blueprints\/apply-local$/); - expect(postSpy.mock.calls[1][0]).toMatch(/\/api\/stacks$/); - expect(putSpy).toHaveBeenCalledTimes(2); // compose + marker - const composePutUrl = String(putSpy.mock.calls[0][0]); - const markerPutUrl = String(putSpy.mock.calls[1][0]); - expect(composePutUrl).toMatch(/[?&]path=compose\.yaml(?:&|$)/); - expect(markerPutUrl).toMatch(/[?&]path=\.blueprint\.json(?:&|$)/); - expect(postSpy.mock.calls[2][0]).toMatch(/\/deploy$/); - const [composePutOrder, markerPutOrder] = putSpy.mock.invocationCallOrder; - const deployOrder = postSpy.mock.invocationCallOrder[2]; - expect(composePutOrder).toBeLessThan(markerPutOrder); - expect(markerPutOrder).toBeLessThan(deployOrder); + expect(putSpy).not.toHaveBeenCalled(); + const dep = DatabaseService.getInstance().getDeployment(bp.id, node.id); + expect(dep?.status).toBe('failed'); }); it('maps a remote apply lock-conflict (409) to status=failed', async () => { @@ -192,7 +184,7 @@ describe('BlueprintService remote deploy', () => { expect(dep?.status).toBe('name_conflict'); }); - it('withdraws a remote deployment by deleting the stack and removing the row', async () => { + it('withdraws a remote deployment via withdraw-local and removes the row', async () => { const node = seedRemoteNode(); const bp = seedBlueprint([node.id]); const nodeObj = DatabaseService.getInstance().getNode(node.id)!; @@ -204,17 +196,73 @@ describe('BlueprintService remote deploy', () => { applied_revision: bpObj.revision, }); - vi.spyOn(axios, 'get').mockResolvedValue({ status: 404, data: {} }); // readMarker → null → proceed - vi.spyOn(axios, 'post').mockResolvedValue({ status: 200, data: {} }); // remote down (best-effort) - const delSpy = vi.spyOn(axios, 'delete').mockResolvedValue({ status: 200, data: {} }); + const postSpy = vi.spyOn(axios, 'post').mockResolvedValue({ + status: 200, + data: { status: 'withdrawn' }, + }); + const delSpy = vi.spyOn(axios, 'delete'); const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj); expect(result.status).toBe('withdrawn'); - expect(delSpy.mock.calls[0][0]).toMatch(/\/api\/stacks\//); + expect(postSpy.mock.calls[0][0]).toMatch(/\/api\/blueprints\/withdraw-local$/); + expect(delSpy).not.toHaveBeenCalled(); expect(DatabaseService.getInstance().getDeployment(bp.id, node.id)).toBeUndefined(); }); + it('maps remote withdraw-local lock conflict to failed without fallback delete', async () => { + const node = seedRemoteNode(); + const bp = seedBlueprint([node.id]); + const nodeObj = DatabaseService.getInstance().getNode(node.id)!; + const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!; + DatabaseService.getInstance().upsertDeployment({ + blueprint_id: bp.id, + node_id: node.id, + status: 'active', + applied_revision: bpObj.revision, + }); + + vi.spyOn(axios, 'post').mockResolvedValue({ + status: 409, + data: { + error: `${bpObj.name} is busy: another operation (update) is already in progress`, + code: 'stack_op_in_progress', + }, + }); + const delSpy = vi.spyOn(axios, 'delete'); + + const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj); + + expect(result.status).toBe('failed'); + expect(result.error).toMatch(/already in progress/); + expect(delSpy).not.toHaveBeenCalled(); + expect(DatabaseService.getInstance().getDeployment(bp.id, node.id)?.status).toBe('failed'); + }); + + it('fails closed when the remote lacks withdraw-local (404); no legacy delete', async () => { + const node = seedRemoteNode(); + const bp = seedBlueprint([node.id]); + const nodeObj = DatabaseService.getInstance().getNode(node.id)!; + const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!; + DatabaseService.getInstance().upsertDeployment({ + blueprint_id: bp.id, + node_id: node.id, + status: 'active', + applied_revision: bpObj.revision, + }); + + const postSpy = vi.spyOn(axios, 'post').mockResolvedValue({ status: 404, data: { error: 'Not Found' } }); + const delSpy = vi.spyOn(axios, 'delete'); + + const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj); + + expect(result.status).toBe('failed'); + expect(result.error).toMatch(/withdraw-local|Upgrade/i); + expect(postSpy.mock.calls[0][0]).toMatch(/\/api\/blueprints\/withdraw-local$/); + expect(delSpy).not.toHaveBeenCalled(); + expect(DatabaseService.getInstance().getDeployment(bp.id, node.id)?.status).toBe('failed'); + }); + it('does not clear hub role assignments when withdrawing a remote deployment', async () => { const bcrypt = await import('bcrypt'); const db = DatabaseService.getInstance(); @@ -237,15 +285,14 @@ describe('BlueprintService remote deploy', () => { user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: bpObj.name, }); - vi.spyOn(axios, 'get').mockResolvedValue({ status: 404, data: {} }); - vi.spyOn(axios, 'post').mockResolvedValue({ status: 200, data: {} }); - const delSpy = vi.spyOn(axios, 'delete').mockResolvedValue({ status: 200, data: {} }); + vi.spyOn(axios, 'post').mockResolvedValue({ status: 200, data: { status: 'withdrawn' } }); + const delSpy = vi.spyOn(axios, 'delete'); const rbacSpy = vi.spyOn(db, 'deleteRoleAssignmentsByResource'); const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj); expect(result.status).toBe('withdrawn'); - expect(delSpy.mock.calls[0][0]).toMatch(/\/api\/stacks\//); + expect(delSpy).not.toHaveBeenCalled(); expect(rbacSpy).not.toHaveBeenCalled(); expect(db.getAllRoleAssignments(userId) .some((a) => a.resource_type === 'stack' && a.resource_id === bpObj.name)).toBe(true); diff --git a/backend/src/__tests__/blueprints-route-validation.test.ts b/backend/src/__tests__/blueprints-route-validation.test.ts index ce4de5dd..6848ff93 100644 --- a/backend/src/__tests__/blueprints-route-validation.test.ts +++ b/backend/src/__tests__/blueprints-route-validation.test.ts @@ -129,3 +129,71 @@ describe('POST /api/blueprints/apply-local (node-to-node atomic apply)', () => { expect(StackOpLockService.getInstance().get(1, 'apply-local-busy')?.action).toBe('update'); }); }); + +describe('POST /api/blueprints/withdraw-local', () => { + let viewerCookie: string; + + beforeAll(async () => { + const bcrypt = (await import('bcrypt')).default; + const passwordHash = await bcrypt.hash('bp-wd-viewer-pass', 1); + DatabaseService.getInstance().addUser({ + username: 'bp-wd-viewer', + password_hash: passwordHash, + role: 'viewer', + }); + const res = await request(app) + .post('/api/auth/login') + .send({ username: 'bp-wd-viewer', password: 'bp-wd-viewer-pass' }); + const cookies = res.headers['set-cookie'] as string | string[]; + viewerCookie = Array.isArray(cookies) ? cookies[0] : cookies; + }); + + it('rejects an invalid stack name', async () => { + const res = await request(app) + .post('/api/blueprints/withdraw-local') + .set('Cookie', adminCookie) + .send({ stackName: '../escape', blueprintId: 1 }); + expect(res.status).toBe(400); + expect(res.body.error).toContain('Invalid stack name'); + }); + + it('rejects a non-positive blueprintId', async () => { + const res = await request(app) + .post('/api/blueprints/withdraw-local') + .set('Cookie', adminCookie) + .send({ stackName: 'wd-local-stack', blueprintId: 0 }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/blueprintId/i); + }); + + it('returns 403 for a viewer without stack:delete', async () => { + const res = await request(app) + .post('/api/blueprints/withdraw-local') + .set('Cookie', viewerCookie) + .send({ stackName: 'wd-local-stack', blueprintId: 1 }); + expect(res.status).toBe(403); + }); + + it('returns 409 self_stack_protected for Sencho own stack', async () => { + const selfStackGuard = await import('../helpers/selfStackGuard'); + vi.spyOn(selfStackGuard, 'refuseIfSelfStack').mockImplementation(async (_req, res) => { + res.status(409).json({ error: 'self', code: 'self_stack_protected' }); + return true; + }); + const res = await request(app) + .post('/api/blueprints/withdraw-local') + .set('Cookie', adminCookie) + .send({ stackName: 'sencho-self', blueprintId: 1 }); + expect(res.status).toBe(409); + expect(res.body.code).toBe('self_stack_protected'); + }); + + it('returns already_absent when the stack directory is missing', async () => { + const res = await request(app) + .post('/api/blueprints/withdraw-local') + .set('Cookie', adminCookie) + .send({ stackName: `wd-absent-${Date.now()}`, blueprintId: 42 }); + expect(res.status).toBe(200); + expect(res.body.status).toBe('already_absent'); + }); +}); diff --git a/backend/src/__tests__/blueprints.test.ts b/backend/src/__tests__/blueprints.test.ts index 337298e4..261a0e22 100644 --- a/backend/src/__tests__/blueprints.test.ts +++ b/backend/src/__tests__/blueprints.test.ts @@ -399,7 +399,7 @@ describe('BlueprintReconciler developer-mode diagnostics', () => { }); describe('BlueprintService per-stack lock', () => { - it('deploy under a free lock writes compose then marker, then deploys', async () => { + it('deploy under a free lock writes compose, cleans siblings, deploys, then writes the marker', async () => { const nodeId = seedNode(); const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] }); const node = DatabaseService.getInstance().getNode(nodeId)!; @@ -419,7 +419,7 @@ describe('BlueprintService per-stack lock', () => { source: 'blueprint', actor: 'system:blueprint', }); - // Compose is written first, then the marker, both before sibling cleanup and deploy. + // Compose is written first; the marker is deferred until after sibling cleanup and deploy. expect(writeSpy).toHaveBeenCalledTimes(2); expect(writeSpy.mock.calls[0][1]).toBe('compose.yaml'); expect(writeSpy.mock.calls[0][2]).toBe(bp.compose_content); @@ -429,9 +429,9 @@ describe('BlueprintService per-stack lock', () => { const [composeOrder, markerOrder] = writeSpy.mock.invocationCallOrder; const [cleanupOrder] = cleanupSpy.mock.invocationCallOrder; const [deployOrder] = deploySpy.mock.invocationCallOrder; - expect(composeOrder).toBeLessThan(markerOrder); - expect(markerOrder).toBeLessThan(cleanupOrder); + expect(composeOrder).toBeLessThan(cleanupOrder); expect(cleanupOrder).toBeLessThan(deployOrder); + expect(deployOrder).toBeLessThan(markerOrder); }); it('deploy skips, writes no stack files, and records failed when the stack lock is held', async () => { @@ -476,7 +476,10 @@ describe('BlueprintService local withdraw clears stack-scoped role assignments', async function arrangeLocalWithdraw() { const db = DatabaseService.getInstance(); + const composeDir = process.env.COMPOSE_DIR!; const nodeId = seedNode(); + db.getDb().prepare('UPDATE nodes SET compose_dir = ? WHERE id = ?').run(composeDir, nodeId); + const bp = seedBlueprint({ name: `rbac-wd-${counter}`, classification: 'stateless', nodeIds: [nodeId] }); const node = db.getNode(nodeId)!; db.upsertDeployment({ @@ -499,7 +502,15 @@ describe('BlueprintService local withdraw clears stack-scoped role assignments', user_id: userId, role: 'deployer', resource_type: 'node', resource_id: String(otherNodeId), }); - vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue(null); + // Matching on-disk marker so ownership passes; FS/down are stubbed. + const fs = await import('fs'); + const path = await import('path'); + const stackDir = path.join(composeDir, bp.name); + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync( + path.join(stackDir, '.blueprint.json'), + JSON.stringify({ blueprintId: bp.id, revision: bp.revision, lastApplied: Date.now() }), + ); const { ComposeService } = await import('../services/ComposeService'); const { FileSystemService } = await import('../services/FileSystemService'); vi.spyOn(ComposeService.prototype, 'downStack').mockResolvedValue(undefined); @@ -522,15 +533,16 @@ describe('BlueprintService local withdraw clears stack-scoped role assignments', db.deleteUser(userId); }); - it('clears the target assignment when deleteStack treats the directory as already absent (ENOENT)', async () => { - // Shared deletion always calls deleteStack; FileSystemService maps ENOENT to success. + it('clears the target assignment when the stack directory is already absent', async () => { const { bp, node, userId, deleteStackSpy, db } = await arrangeLocalWithdraw(); - deleteStackSpy.mockResolvedValue(undefined); + const fs = await import('fs'); + const path = await import('path'); + fs.rmSync(path.join(process.env.COMPOSE_DIR!, bp.name), { recursive: true, force: true }); const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node); expect(outcome.status).toBe('withdrawn'); - expect(deleteStackSpy).toHaveBeenCalledWith(bp.name); + expect(deleteStackSpy).not.toHaveBeenCalled(); expect(hasAssignment(userId, 'stack', bp.name)).toBe(false); expect(db.getAllRoleAssignments(userId)).toHaveLength(2); db.deleteUser(userId); diff --git a/backend/src/__tests__/database-stack-cleanup-intent-migration.test.ts b/backend/src/__tests__/database-stack-cleanup-intent-migration.test.ts new file mode 100644 index 00000000..9723050b --- /dev/null +++ b/backend/src/__tests__/database-stack-cleanup-intent-migration.test.ts @@ -0,0 +1,64 @@ +/** + * Legacy installations upgrade stack_update_cleanup_pending with nullable + * required_blueprint_id via maybeAddCol. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +let tmpDir: string; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +describe('stack_update_cleanup_pending required_blueprint_id migration', () => { + it('upgrades a legacy table without the column and preserves null for old rows', () => { + const db = DatabaseService.getInstance(); + const raw = db.getDb(); + + // Simulate a pre-migration installation: drop the column if present, then + // re-add it the same way initSchema's maybeAddCol does. + try { + raw.exec('ALTER TABLE stack_update_cleanup_pending DROP COLUMN required_blueprint_id'); + } catch { + // Column may already be absent in a hand-built fixture. + } + + const now = Date.now(); + raw.prepare(` + INSERT INTO stack_update_cleanup_pending ( + id, node_id, stack_name, status, target_kind, rollback_tags_json, + override_paths_json, prune_volumes_requested, created_at, updated_at + ) VALUES (?, ?, ?, 'prepared', 'local_socket', '[]', '[]', 0, ?, ?) + `).run('legacy-1', 1, 'legacy-stack', now, now); + + try { + raw.exec('ALTER TABLE stack_update_cleanup_pending ADD COLUMN required_blueprint_id INTEGER'); + } catch { + // Idempotent if a parallel path re-added it. + } + + const legacy = db.getCleanupPending('legacy-1'); + expect(legacy).toBeDefined(); + expect(legacy?.required_blueprint_id ?? null).toBeNull(); + + db.insertCleanupPending({ + id: 'owned-1', + node_id: 2, + stack_name: 'bp-stack', + status: 'prepared', + target_kind: 'local_socket', + rollback_tags_json: '[]', + override_paths_json: '[]', + prune_volumes_requested: 0, + required_blueprint_id: 42, + created_at: now, + updated_at: now, + }); + expect(db.getCleanupPending('owned-1')?.required_blueprint_id).toBe(42); + }); +}); diff --git a/backend/src/__tests__/deployed-stack-deletion-service.test.ts b/backend/src/__tests__/deployed-stack-deletion-service.test.ts index 27052fd8..d8dfc0b4 100644 --- a/backend/src/__tests__/deployed-stack-deletion-service.test.ts +++ b/backend/src/__tests__/deployed-stack-deletion-service.test.ts @@ -89,6 +89,7 @@ describe('DeployedStackDeletionService ready transaction', () => { rollback_tags_json: '[]', override_paths_json: '[]', prune_volumes_requested: 0, + required_blueprint_id: null, created_at: now, updated_at: now, }; @@ -111,6 +112,7 @@ describe('DeployedStackDeletionService ready transaction', () => { rollback_tags_json: '[]', override_paths_json: '[]', prune_volumes_requested: 0, + required_blueprint_id: null, created_at: now, updated_at: now, }); @@ -129,6 +131,7 @@ describe('DeployedStackDeletionService ready transaction', () => { rollback_tags_json: '[]', override_paths_json: '[]', prune_volumes_requested: 0, + required_blueprint_id: null, created_at: now, updated_at: now, }); @@ -154,3 +157,39 @@ describe('overrideDeletionContainmentBase', () => { }); }); +describe('DeployedStackDeletionService blueprint ownership probe', () => { + it('returns failed (not name_conflict) when marker read fails with non-ENOENT I/O', async () => { + const { promises: fsPromises } = await import('fs'); + const { vi } = await import('vitest'); + const composeDir = process.env.COMPOSE_DIR!; + const stackName = `del-probe-${Date.now()}`; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n'); + await fsPromises.writeFile( + path.join(stackDir, '.blueprint.json'), + JSON.stringify({ blueprintId: 7, revision: 1, lastApplied: 0 }), + ); + + const accessErr = Object.assign(new Error('EACCES'), { code: 'EACCES' }); + const readSpy = vi.spyOn(fsPromises, 'readFile').mockRejectedValueOnce(accessErr); + + const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({ + nodeId: NODE, + stackName, + pruneVolumes: false, + actor: 'test', + requireBlueprintId: 7, + }); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.code).toBe('failed'); + expect(result.error).toMatch(/EACCES|Failed to read|permission/i); + } + expect(readSpy).toHaveBeenCalled(); + readSpy.mockRestore(); + await fsPromises.rm(stackDir, { recursive: true, force: true }); + }); +}); + diff --git a/backend/src/__tests__/filesystem-stack-paths.test.ts b/backend/src/__tests__/filesystem-stack-paths.test.ts index abf7d2e7..00b29665 100644 --- a/backend/src/__tests__/filesystem-stack-paths.test.ts +++ b/backend/src/__tests__/filesystem-stack-paths.test.ts @@ -95,9 +95,10 @@ describe('FileSystemService stack methods', () => { expect(fileNames).toEqual(['.env', 'compose.yaml']); }); - it('marks compose.yaml and .env as protected', async () => { + it('marks compose.yaml, .env, and .blueprint.json as protected', async () => { await fs.writeFile(path.join(stackDir, 'compose.yaml'), ''); await fs.writeFile(path.join(stackDir, '.env'), ''); + await fs.writeFile(path.join(stackDir, '.blueprint.json'), '{}'); await fs.writeFile(path.join(stackDir, 'custom.conf'), ''); const service = FileSystemService.getInstance(); @@ -106,6 +107,7 @@ describe('FileSystemService stack methods', () => { const byName = Object.fromEntries(entries.map(e => [e.name, e])); expect(byName['compose.yaml'].isProtected).toBe(true); expect(byName['.env'].isProtected).toBe(true); + expect(byName['.blueprint.json'].isProtected).toBe(true); expect(byName['custom.conf'].isProtected).toBe(false); }); diff --git a/backend/src/__tests__/stack-files-routes.test.ts b/backend/src/__tests__/stack-files-routes.test.ts index 37373309..2195bb10 100644 --- a/backend/src/__tests__/stack-files-routes.test.ts +++ b/backend/src/__tests__/stack-files-routes.test.ts @@ -1116,6 +1116,37 @@ describe('PUT /api/stacks/:stackName/files/content', () => { expect(content).toBe('community-write'); }); + it('blocks root trust file writes while a stack op lock is held', async () => { + const { StackOpLockService } = await import('../services/StackOpLockService'); + StackOpLockService.getInstance().tryAcquire(1, STACK, 'deploy', 'admin'); + try { + const composeRes = await request(app) + .put(`/api/stacks/${STACK}/files/content`) + .query({ path: 'compose.yaml' }) + .set('Cookie', adminCookie) + .send({ content: 'services:\n app:\n image: nginx\n' }); + expect(composeRes.status).toBe(409); + expect(composeRes.body.code).toBe('stack_op_in_progress'); + + const markerRes = await request(app) + .put(`/api/stacks/${STACK}/files/content`) + .query({ path: '.blueprint.json' }) + .set('Cookie', adminCookie) + .send({ content: '{"blueprintId":1,"revision":1,"lastApplied":0}' }); + expect(markerRes.status).toBe(409); + expect(markerRes.body.code).toBe('stack_op_in_progress'); + + const nestedRes = await request(app) + .put(`/api/stacks/${STACK}/files/content`) + .query({ path: 'config/app.conf' }) + .set('Cookie', adminCookie) + .send({ content: 'ok' }); + expect(nestedRes.status).toBe(204); + } finally { + StackOpLockService.getInstance().release(1, STACK); + } + }); + it('returns 400 when content is not a string', async () => { const res = await request(app) .put(`/api/stacks/${STACK}/files/content`) @@ -1931,6 +1962,16 @@ describe('protected stack files', () => { expect(res.body.code).toBe('PROTECTED_FILE'); }); + it('DELETE /files refuses .blueprint.json with 409 PROTECTED_FILE', async () => { + await fs.writeFile(path.join(stacksDir, STACK, '.blueprint.json'), '{"blueprintId":1,"revision":1}\n'); + const res = await request(app) + .delete(`/api/stacks/${STACK}/files`) + .query({ path: '.blueprint.json' }) + .set('Cookie', adminCookie); + expect(res.status).toBe(409); + expect(res.body.code).toBe('PROTECTED_FILE'); + }); + it('PATCH /files/rename refuses compose.yaml as source with 409 PROTECTED_FILE', async () => { const res = await request(app) .patch(`/api/stacks/${STACK}/files/rename`) diff --git a/backend/src/helpers/blueprintMarker.ts b/backend/src/helpers/blueprintMarker.ts new file mode 100644 index 00000000..d341f594 --- /dev/null +++ b/backend/src/helpers/blueprintMarker.ts @@ -0,0 +1,28 @@ +/** + * Dependency-neutral blueprint marker parse helpers. + * Used by BlueprintService and DeployedStackDeletionService without a service import cycle. + */ + +export const BLUEPRINT_MARKER_FILENAME = '.blueprint.json'; + +export interface BlueprintMarker { + blueprintId: number; + revision: number; + lastApplied: number; +} + +export function parseBlueprintMarker(content: string): BlueprintMarker | null { + try { + const parsed: unknown = JSON.parse(content); + if (!parsed || typeof parsed !== 'object') return null; + const obj = parsed as Record; + if (typeof obj.blueprintId !== 'number' || typeof obj.revision !== 'number') return null; + return { + blueprintId: obj.blueprintId, + revision: obj.revision, + lastApplied: typeof obj.lastApplied === 'number' ? obj.lastApplied : 0, + }; + } catch { + return null; + } +} diff --git a/backend/src/routes/blueprints.ts b/backend/src/routes/blueprints.ts index ecdef6db..84c113d8 100644 --- a/backend/src/routes/blueprints.ts +++ b/backend/src/routes/blueprints.ts @@ -7,7 +7,9 @@ import { type BlueprintSelector, type DriftMode, } from '../services/DatabaseService'; -import { BlueprintService } from '../services/BlueprintService'; +import { BlueprintService, BlueprintNameConflictError, BlueprintOwnershipProbeError } from '../services/BlueprintService'; +import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService'; +import { refuseIfSelfStack } from '../helpers/selfStackGuard'; import { BlueprintReconciler, messageForConfirmedOutcomes, @@ -317,12 +319,9 @@ blueprintsRouter.delete('/:id', async (req: Request, res: Response): Promise n.id === dep.node_id); if (!node) continue; try { - await BlueprintService.getInstance().withdrawFromNode(blueprint, node); + const outcome = await BlueprintService.getInstance().withdrawFromNode(blueprint, node); + if (outcome.status !== 'withdrawn') { + res.status(409).json({ + error: `Cannot delete blueprint: withdraw on node "${node.name}" ended as ${outcome.status}. Resolve that deployment, then retry.`, + code: 'withdraw_failed_blocking_delete', + nodeId: node.id, + withdrawStatus: outcome.status, + }); + return; + } } catch (err) { console.warn(`[Blueprints] Pre-delete withdraw failed for blueprint ${id} on node ${node.id}:`, err); + res.status(409).json({ + error: `Cannot delete blueprint: withdraw on node "${node.name}" failed. Resolve that deployment, then retry.`, + code: 'withdraw_failed_blocking_delete', + nodeId: node.id, + }); + return; } } DatabaseService.getInstance().deleteBlueprint(id); @@ -384,11 +398,68 @@ blueprintsRouter.post('/apply-local', async (req: Request, res: Response): Promi } res.json({ deployed: true }); } catch (error) { + if (error instanceof BlueprintNameConflictError) { + res.status(409).json({ error: error.message, code: 'name_conflict' }); + return; + } + if (error instanceof BlueprintOwnershipProbeError) { + console.error('[Blueprints] apply-local ownership probe failed:', sanitizeForLog(error.message)); + res.status(500).json({ error: error.message }); + return; + } console.error('[Blueprints] apply-local error:', sanitizeForLog(getErrorMessage(error, 'apply failed'))); res.status(500).json({ error: getErrorMessage(error, 'Blueprint apply failed') }); } }); +// Node-to-node atomic blueprint withdraw. Ownership is validated under the +// delete lock on this node. Requires stack:delete and refuses Sencho's own stack. +blueprintsRouter.post('/withdraw-local', async (req: Request, res: Response): Promise => { + const body = (req.body ?? {}) as { stackName?: unknown; blueprintId?: unknown }; + if (typeof body.stackName !== 'string' || !isValidStackName(body.stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (typeof body.blueprintId !== 'number' || !Number.isInteger(body.blueprintId) || body.blueprintId <= 0) { + res.status(400).json({ error: 'blueprintId must be a positive integer' }); + return; + } + if (!requirePermission(req, res, 'stack:delete', 'stack', body.stackName)) return; + if (await refuseIfSelfStack(req, res, body.stackName)) return; + + try { + const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({ + nodeId: req.nodeId, + stackName: body.stackName, + pruneVolumes: false, + actor: req.user?.username ?? 'system:blueprint', + requireBlueprintId: body.blueprintId, + }); + if (result.ok) { + res.json({ + status: result.status === 'already_absent' ? 'already_absent' : 'withdrawn', + }); + return; + } + if (result.code === 'name_conflict') { + res.status(409).json({ error: result.error, code: 'name_conflict' }); + return; + } + if (result.code === 'lock_conflict') { + res.status(409).json({ + error: result.error, + code: 'stack_op_in_progress', + inProgress: { action: result.existingAction }, + }); + return; + } + res.status(500).json({ error: result.error }); + } catch (error) { + console.error('[Blueprints] withdraw-local error:', sanitizeForLog(getErrorMessage(error, 'withdraw failed'))); + res.status(500).json({ error: getErrorMessage(error, 'Blueprint withdraw failed') }); + } +}); + blueprintsRouter.post('/:id/apply', async (req: Request, res: Response): Promise => { if (!requireAdmin(req, res)) return; const id = parseIntParam(req, res, 'id'); diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index 4fe318de..6df26f2a 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -135,6 +135,40 @@ function releaseStackOpLock(req: Request, stackName: string): void { StackOpLockService.getInstance().release(req.nodeId, stackName); } +/** Root compose + blueprint marker on the stack-source root must not change while a lifecycle op holds the stack lock. */ +const STACK_OP_LOCKED_ROOT_TRUST_FILES = new Set([ + 'compose.yaml', + 'compose.yml', + 'docker-compose.yaml', + 'docker-compose.yml', + '.blueprint.json', +]); + +function rejectIfStackOpBlocksRootTrustFileWrite( + req: Request, + res: Response, + stackName: string, + relPath: string, + root: StackFileRoot, +): boolean { + if (root.kind !== 'stack-source') return false; + if (relPath.includes('/')) return false; + const base = relPath.toLowerCase(); + if (!STACK_OP_LOCKED_ROOT_TRUST_FILES.has(base)) return false; + const existing = StackOpLockService.getInstance().get(req.nodeId, stackName); + if (!existing) return false; + res.status(409).json({ + error: `${stackName} is busy: another operation (${existing.action}) is already in progress`, + code: 'stack_op_in_progress', + inProgress: { + action: existing.action, + startedAt: existing.startedAt, + user: existing.user, + }, + }); + return true; +} + function stackFileEtag(mtimeMs: number): string { return `W/"${Math.floor(mtimeMs)}"`; } @@ -2884,6 +2918,10 @@ stacksRouter.post( return res.status(400).json({ error: 'Invalid filename' }); } const targetRelPath = relPath ? `${relPath}/${originalName}` : originalName; + if (rejectIfStackOpBlocksRootTrustFileWrite(req, res, stackName, targetRelPath, root)) { + await cleanupUploadTemp(req); + return; + } const overwrite = String(req.query.overwrite) === '1'; // The multer wrapper stashed the route-entry timestamp on the request so // the success path and the rejection paths share one window. Fall back to @@ -2987,6 +3025,7 @@ stacksRouter.put('/:stackName/files/content', async (req: Request, res: Response const expectedVersion = req.header('if-match') || undefined; const root = await resolveRootForOp(req, res, stackName, 'write'); if (!root) return; + if (rejectIfStackOpBlocksRootTrustFileWrite(req, res, stackName, relPath, root)) return; const startedAt = Date.now(); logFileDiag('write start', { stackName, relPath, nodeId: req.nodeId, bytes: Buffer.byteLength(content, 'utf-8'), hasIfMatch: expectedVersion !== undefined, rootKind: root.kind }); try { diff --git a/backend/src/services/BlueprintService.ts b/backend/src/services/BlueprintService.ts index e9a8aa69..adddbee8 100644 --- a/backend/src/services/BlueprintService.ts +++ b/backend/src/services/BlueprintService.ts @@ -19,8 +19,12 @@ import { assertPolicyGateAllows, buildSystemPolicyGateOptions, describePolicyBlo import { enforcePolicyForImageRefs } from './PolicyEnforcement'; import { BlueprintAnalyzer } from './BlueprintAnalyzer'; import { sanitizeForLog } from '../utils/safeLog'; - -const MARKER_FILENAME = '.blueprint.json'; +import { isPathWithinBase } from '../utils/validation'; +import { + BLUEPRINT_MARKER_FILENAME, + parseBlueprintMarker, + type BlueprintMarker, +} from '../helpers/blueprintMarker'; /** On-disk compose name for Blueprint applies. Must match createStack scaffold and Sencho discovery priority. */ const COMPOSE_FILENAME = 'compose.yaml'; const REMOTE_HTTP_TIMEOUT_MS = 30_000; @@ -41,10 +45,32 @@ function diagnosticLog(message: string, fields: Record { try { if (node.type === 'local') { - const baseDir = NodeRegistry.getInstance().getComposeDir(node.id); - const markerPath = path.resolve(baseDir, blueprintName, MARKER_FILENAME); - if (!markerPath.startsWith(path.resolve(baseDir))) return null; - const content = await fsPromises.readFile(markerPath, 'utf-8'); - return BlueprintService.parseMarker(content); + const markerRead = await this.readLocalMarkerFromDisk(node.id, blueprintName); + return markerRead.kind === 'present' ? markerRead.marker : null; } const target = NodeRegistry.getInstance().getProxyTarget(node.id); if (!target) return null; - const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/files/content?path=${encodeURIComponent(MARKER_FILENAME)}`; + const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/files/content?path=${encodeURIComponent(BLUEPRINT_MARKER_FILENAME)}`; const res = await axios.get(url, { headers: this.remoteHeaders(target.apiToken), timeout: REMOTE_HTTP_TIMEOUT_MS, @@ -146,7 +175,7 @@ export class BlueprintService { const body = res.data; const content = typeof body === 'string' ? body : (typeof body?.content === 'string' ? body.content : null); if (content == null) return null; - return BlueprintService.parseMarker(content); + return parseBlueprintMarker(content); } catch { return null; } @@ -154,47 +183,77 @@ export class BlueprintService { /** * Returns true when a stack directory by this name exists on the target - * node but does not carry our marker file. The reconciler must not - * deploy in that case: there is a real user-authored stack with the - * same name and we must not overwrite it. + * node and the on-disk marker is missing, malformed, or references a + * different blueprint ID. Throws BlueprintOwnershipProbeError when the + * directory or marker cannot be probed (non-ENOENT I/O or remote list failure). */ - async hasNameConflict(blueprintName: string, node: Node): Promise { - try { - if (node.type === 'local') { - const baseDir = NodeRegistry.getInstance().getComposeDir(node.id); - const stackDir = path.resolve(baseDir, blueprintName); - if (!stackDir.startsWith(path.resolve(baseDir))) return true; - try { - const stat = await fsPromises.stat(stackDir); - if (!stat.isDirectory()) return false; - } catch { - return false; // directory doesn't exist → no conflict - } - const markerPath = path.join(stackDir, MARKER_FILENAME); - try { - await fsPromises.stat(markerPath); - return false; // marker present → ours - } catch { - return true; // directory exists but no marker → conflict - } + async hasNameConflict(blueprintName: string, node: Node, blueprintId: number): Promise { + if (node.type === 'local') { + const baseDir = NodeRegistry.getInstance().getComposeDir(node.id); + const stackDir = path.resolve(baseDir, blueprintName); + if (!isPathWithinBase(stackDir, baseDir)) return true; + try { + const stat = await fsPromises.stat(stackDir); + if (!stat.isDirectory()) return false; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') return false; + throw BlueprintService.ownershipProbeError(blueprintName, BlueprintService.formatError(err)); } - const target = NodeRegistry.getInstance().getProxyTarget(node.id); - if (!target) return false; - const baseUrl = target.apiUrl.replace(/\/$/, ''); - const listUrl = `${baseUrl}/api/stacks`; - const listRes = await axios.get(listUrl, { + const markerRead = await this.readLocalMarkerFromDisk(node.id, blueprintName); + if (markerRead.kind === 'failed') { + throw BlueprintService.ownershipProbeError(blueprintName, markerRead.error); + } + return markerRead.kind === 'missing' || markerRead.marker.blueprintId !== blueprintId; + } + const target = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!target) { + throw new BlueprintOwnershipProbeError( + `Cannot verify stack ownership on remote node "${node.name}": no proxy target configured`, + ); + } + const baseUrl = target.apiUrl.replace(/\/$/, ''); + let listRes; + try { + listRes = await axios.get(`${baseUrl}/api/stacks`, { headers: this.remoteHeaders(target.apiToken), timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true, }); - if (listRes.status !== 200) return false; - const stacks = Array.isArray(listRes.data) ? listRes.data as Array<{ name?: string }> : []; - const exists = stacks.some(s => s?.name === blueprintName); - if (!exists) return false; - const marker = await this.readMarker(blueprintName, node); - return marker == null; - } catch { - return false; + } catch (err) { + throw new BlueprintOwnershipProbeError( + `Cannot verify stack ownership on remote node "${node.name}": ${BlueprintService.formatError(err)}`, + ); + } + if (listRes.status !== 200) { + throw new BlueprintOwnershipProbeError( + `Cannot verify stack ownership on remote node "${node.name}" (HTTP ${listRes.status})`, + ); + } + const stacks = Array.isArray(listRes.data) ? listRes.data as Array<{ name?: string }> : []; + const exists = stacks.some(s => s?.name === blueprintName); + if (!exists) return false; + const marker = await this.readMarker(blueprintName, node); + return marker == null || marker.blueprintId !== blueprintId; + } + + /** Read and parse a local on-disk marker without going through the remote HTTP path. */ + private async readLocalMarkerFromDisk(nodeId: number, stackName: string): Promise { + try { + // Canonical js/path-injection barrier inline with the read sink. + const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId)); + const safePath = path.resolve(baseResolved, stackName, BLUEPRINT_MARKER_FILENAME); + if (!safePath.startsWith(baseResolved + path.sep)) { + return { kind: 'failed', error: 'Invalid stack path for blueprint marker' }; + } + const content = await fsPromises.readFile(safePath, 'utf-8'); + const marker = parseBlueprintMarker(content); + if (!marker) return { kind: 'missing' }; + return { kind: 'present', marker }; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') return { kind: 'missing' }; + return { kind: 'failed', error: BlueprintService.formatError(err) }; } } @@ -222,7 +281,7 @@ export class BlueprintService { }); try { this.setStatus(blueprint.id, node.id, 'deploying'); - if (await this.hasNameConflict(blueprint.name, node)) { + if (await this.hasNameConflict(blueprint.name, node, blueprint.id)) { this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: `A stack named "${blueprint.name}" already exists on this node and is not managed by Sencho.`, }); @@ -249,6 +308,12 @@ export class BlueprintService { sanitizeForLog(blueprint.name), node.id, Date.now() - started); return { status: 'active' }; } catch (err) { + if (err instanceof BlueprintNameConflictError) { + this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: err.message }); + console.warn('[BlueprintService] deploy name conflict blueprint=%s node=%s durationMs=%s', + sanitizeForLog(blueprint.name), node.id, Date.now() - started); + return { status: 'name_conflict', error: 'name_conflict' }; + } const message = BlueprintService.formatError(err); this.setStatus(blueprint.id, node.id, 'failed', { last_error: message }); console.error('[BlueprintService] deploy failed blueprint=%s node=%s durationMs=%s error=%s', @@ -280,20 +345,15 @@ export class BlueprintService { }); try { this.setStatus(blueprint.id, node.id, 'withdrawing'); - // Refuse to withdraw a directory we do not own - const marker = await this.readMarker(blueprint.name, node); - if (marker && marker.blueprintId !== blueprint.id) { - this.setStatus(blueprint.id, node.id, 'name_conflict', { - last_error: `Marker on this node points to a different blueprint (id=${marker.blueprintId}); refusing to withdraw.`, - }); - return { status: 'name_conflict' }; - } + // Ownership is validated on the node that owns the stack, inside the delete lock. if (node.type === 'local') { diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'local' }); - await this.withdrawLocal(blueprint, node); + const localOutcome = await this.withdrawLocal(blueprint, node); + if (localOutcome.status !== 'withdrawn') return localOutcome; } else { diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'remote' }); - await this.withdrawRemote(blueprint, node); + const remoteOutcome = await this.withdrawRemote(blueprint, node); + if (remoteOutcome.status !== 'withdrawn') return remoteOutcome; } DatabaseService.getInstance().deleteDeployment(blueprint.id, node.id); console.info('[BlueprintService] withdraw complete blueprint=%s node=%s durationMs=%s', @@ -382,15 +442,22 @@ export class BlueprintService { // ---- local primitives ---- + /** Returns whether the stack directory exists. Throws on non-ENOENT I/O. */ private async stackDirExists(nodeId: number, blueprintName: string): Promise { const baseDir = NodeRegistry.getInstance().getComposeDir(nodeId); const stackDir = path.resolve(baseDir, blueprintName); - if (!stackDir.startsWith(path.resolve(baseDir))) return false; + if (!isPathWithinBase(stackDir, baseDir)) { + throw new BlueprintOwnershipProbeError(`Invalid stack path for "${blueprintName}"`); + } try { const stat = await fsPromises.stat(stackDir); return stat.isDirectory(); - } catch { - return false; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') return false; + throw new BlueprintOwnershipProbeError( + `Cannot access stack directory "${blueprintName}": ${BlueprintService.formatError(err)}`, + ); } } @@ -423,14 +490,14 @@ export class BlueprintService { } /** - * Create the stack if needed, write the compose and marker files, run the - * deploy policy gate, and deploy, all under the per-stack operation lock so - * none of it can race a manual deploy/update/rollback/backup on the same - * stack and node. Runs on the node that owns the stack: deployLocal calls it - * for the hub's own node, and the /api/blueprints/apply-local route calls it - * on a remote node receiving a blueprint apply from its hub (so the file - * writes hold the remote's lock, not just the deploy). On lock conflict - * nothing is written and { ran: false } is returned. + * Create the stack if needed, write the compose file, run the deploy policy + * gate and deploy, then write the marker, all under the per-stack operation + * lock. The marker is written only after a successful deploy so a failed + * apply cannot claim an applied revision that never ran. Runs on the node + * that owns the stack: deployLocal calls it for the hub's own node, and the + * /api/blueprints/apply-local route calls it on a remote node receiving a + * blueprint apply from its hub. On lock conflict nothing is written and + * { ran: false } is returned. */ async applyLocalUnderLock( nodeId: number, @@ -439,47 +506,83 @@ export class BlueprintService { markerContent: string, auditPath: string, ): Promise<{ ran: true } | { ran: false; existingAction: StackOpAction }> { + const expected = parseBlueprintMarker(markerContent); + if (!expected) { + throw new Error('Invalid blueprint marker'); + } const fs = FileSystemService.getInstance(nodeId); const lock = await StackOpLockService.getInstance().runExclusive( nodeId, stackName, 'deploy', 'system', async () => { - if (!(await this.stackDirExists(nodeId, stackName))) { + let createdStack = false; + if (await this.stackDirExists(nodeId, stackName)) { + const existing = await this.readLocalMarkerFromDisk(nodeId, stackName); + if (existing.kind === 'failed') { + throw new BlueprintOwnershipProbeError( + `Cannot verify ownership of stack "${stackName}": ${existing.error}`, + ); + } + if (existing.kind === 'missing' || existing.marker.blueprintId !== expected.blueprintId) { + throw new BlueprintNameConflictError( + `A stack named "${stackName}" already exists on this node and is not managed by this blueprint.`, + ); + } + } else { await fs.createStack(stackName); + createdStack = true; } await fs.writeStackFile(stackName, COMPOSE_FILENAME, composeContent); - await fs.writeStackFile(stackName, MARKER_FILENAME, markerContent); // Clear lower-priority compose siblings so discovery cannot shadow compose.yaml. - // Local + modern apply-local only; legacy remote has no sibling DELETE. await fs.removeAlternateRootComposeFiles(stackName); - await assertPolicyGateAllows( - stackName, - nodeId, - buildSystemPolicyGateOptions('blueprint', { auditPath }), - ); - await ComposeService.getInstance(nodeId).deployStack( - stackName, - undefined, - false, - { source: 'blueprint', actor: 'system:blueprint' }, - ); + try { + await assertPolicyGateAllows( + stackName, + nodeId, + buildSystemPolicyGateOptions('blueprint', { auditPath }), + ); + await ComposeService.getInstance(nodeId).deployStack( + stackName, + undefined, + false, + { source: 'blueprint', actor: 'system:blueprint' }, + ); + await fs.writeStackFile(stackName, BLUEPRINT_MARKER_FILENAME, markerContent); + } catch (err) { + if (createdStack) { + try { + await fs.deleteStack(stackName); + } catch (cleanupErr) { + console.warn( + '[BlueprintService] Failed to roll back newly created stack "%s" after apply error: %s', + sanitizeForLog(stackName), + sanitizeForLog(BlueprintService.formatError(cleanupErr)), + ); + } + } + throw err; + } }, ); return lock.ran ? { ran: true } : { ran: false, existingAction: lock.existing.action }; } - private async withdrawLocal(blueprint: Blueprint, node: Node): Promise { + private async withdrawLocal(blueprint: Blueprint, node: Node): Promise { const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({ nodeId: node.id, stackName: blueprint.name, pruneVolumes: false, actor: 'system:blueprint', + requireBlueprintId: blueprint.id, }); - if (!result.ok) { - if (result.code === 'lock_conflict') { - throw new Error(result.error); - } - throw new Error(result.error); + if (result.ok) { + return { status: 'withdrawn' }; } + if (result.code === 'name_conflict') { + this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: result.error }); + return { status: 'name_conflict' }; + } + this.setStatus(blueprint.id, node.id, 'failed', { last_error: result.error }); + return { status: 'failed', error: result.error }; } // ---- remote primitives ---- @@ -500,10 +603,7 @@ export class BlueprintService { const baseUrl = target.apiUrl.replace(/\/$/, ''); const headers = this.remoteHeaders(target.apiToken); - // Atomic apply: the remote runs create + write compose/marker + deploy - // under its own per-stack lock, so the file writes cannot race a manual - // operation on that node. Older nodes without this route answer 404; we - // fall back to the legacy multi-call flow there (not lock-atomic). + // Atomic apply: the remote validates ownership and writes under its stack lock. const res = await axios.post( `${baseUrl}/api/blueprints/apply-local`, { @@ -514,11 +614,17 @@ export class BlueprintService { { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, ); if (res.status === 404) { - console.warn(`[BlueprintService] remote node ${node.id} lacks /api/blueprints/apply-local; using legacy non-atomic apply`); - await this.deployRemoteLegacy(blueprint, node, marker); - return; + throw new BlueprintRemoteUpgradeRequiredError( + `Remote node "${node.name}" does not support atomic blueprint apply (/api/blueprints/apply-local). Upgrade that Sencho instance, then retry.`, + ); } if (res.status === 409) { + if (BlueprintService.extractApiCode(res.data) === 'name_conflict') { + throw new BlueprintNameConflictError( + BlueprintService.extractApiError(res.data) + || `A stack named "${blueprint.name}" already exists on this node and is not managed by this blueprint.`, + ); + } throw new Error(`blueprint apply skipped: ${BlueprintService.extractApiError(res.data) || 'another operation is already in progress'}`); } if (res.status >= 400) { @@ -526,105 +632,62 @@ export class BlueprintService { } } - /** - * Legacy remote apply for nodes that predate /api/blueprints/apply-local: - * create, write compose, write marker, deploy as separate calls. The remote - * deploy locks, but the preceding file writes do not, so this is not atomic - * against a concurrent manual operation on that node. Kept only as a - * compatibility fallback. - */ - private async deployRemoteLegacy(blueprint: Blueprint, node: Node, marker: BlueprintMarker): Promise { + private async withdrawRemote(blueprint: Blueprint, node: Node): Promise { const target = NodeRegistry.getInstance().getProxyTarget(node.id); if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`); const baseUrl = target.apiUrl.replace(/\/$/, ''); const headers = this.remoteHeaders(target.apiToken); - // 1. Ensure stack exists. POST returns 409 when already exists; we treat that as success. - const createRes = await axios.post(`${baseUrl}/api/stacks`, - { stackName: blueprint.name }, - { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, - ); - if (createRes.status >= 400 && createRes.status !== 409) { - throw new Error(`create stack: HTTP ${createRes.status} ${BlueprintService.extractApiError(createRes.data)}`); - } - - // 2. Write the compose file - await this.remotePutFile(baseUrl, headers, blueprint.name, COMPOSE_FILENAME, blueprint.compose_content); - - // 3. Write the marker (last so a partial failure leaves us in name_conflict-recoverable state) - await this.remotePutFile(baseUrl, headers, blueprint.name, MARKER_FILENAME, JSON.stringify(marker, null, 2)); - - // 4. Deploy - const deployRes = await axios.post( - `${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}/deploy`, - {}, - { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, - ); - if (deployRes.status >= 400) { - throw new Error(`deploy: HTTP ${deployRes.status} ${BlueprintService.extractApiError(deployRes.data)}`); - } - } - - private async withdrawRemote(blueprint: Blueprint, node: Node): Promise { - const target = NodeRegistry.getInstance().getProxyTarget(node.id); - if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`); - const baseUrl = target.apiUrl.replace(/\/$/, ''); - const headers = this.remoteHeaders(target.apiToken); - - // down (best-effort) + let res; try { - await axios.post( - `${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}/down`, - {}, + res = await axios.post( + `${baseUrl}/api/blueprints/withdraw-local`, + { stackName: blueprint.name, blueprintId: blueprint.id }, { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, ); } catch (err) { - console.warn(`[BlueprintService] remote down failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(err)}`); + const message = BlueprintService.formatError(err); + this.setStatus(blueprint.id, node.id, 'failed', { last_error: message }); + return { status: 'failed', error: message }; } - // delete the stack directory entirely - const delRes = await axios.delete( - `${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}`, - { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, - ); - if (delRes.status >= 400 && delRes.status !== 404) { - throw new Error(`remote delete: HTTP ${delRes.status} ${BlueprintService.extractApiError(delRes.data)}`); + if (res.status === 404) { + throw new BlueprintRemoteUpgradeRequiredError( + `Remote node "${node.name}" does not support atomic blueprint withdraw (/api/blueprints/withdraw-local). Upgrade that Sencho instance, then retry.`, + ); } - } - - private async remotePutFile( - baseUrl: string, - headers: Record, - stackName: string, - relPath: string, - content: string, - ): Promise { - const url = `${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/files/content?path=${encodeURIComponent(relPath)}`; - const res = await axios.put(url, - { content }, - { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, - ); - if (res.status >= 400) { - throw new Error(`PUT ${relPath}: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`); + if (res.status === 200) { + return { status: 'withdrawn' }; } + if (res.status === 409) { + const error = BlueprintService.extractApiError(res.data) || 'withdraw refused'; + if (BlueprintService.extractApiCode(res.data) === 'name_conflict') { + this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: error }); + return { status: 'name_conflict' }; + } + // stack_op_in_progress and any other 409: match local withdraw lock-conflict → failed + this.setStatus(blueprint.id, node.id, 'failed', { last_error: error }); + return { status: 'failed', error }; + } + const message = `blueprint withdraw: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`; + this.setStatus(blueprint.id, node.id, 'failed', { last_error: message }); + return { status: 'failed', error: message }; } static parseMarker(content: string): BlueprintMarker | null { - try { - const parsed = JSON.parse(content); - if (parsed && typeof parsed === 'object' - && typeof parsed.blueprintId === 'number' - && typeof parsed.revision === 'number') { - return { - blueprintId: parsed.blueprintId, - revision: parsed.revision, - lastApplied: typeof parsed.lastApplied === 'number' ? parsed.lastApplied : 0, - }; - } - } catch { - // fall through - } - return null; + return parseBlueprintMarker(content); + } + + private static ownershipProbeError(blueprintName: string, detail: string): BlueprintOwnershipProbeError { + return new BlueprintOwnershipProbeError( + `Cannot verify stack ownership for "${blueprintName}": ${detail}`, + ); + } + + static extractApiCode(body: unknown): string { + if (!body || typeof body !== 'object') return ''; + const code = (body as Record).code; + return typeof code === 'string' ? code : ''; } static formatError(err: unknown): string { diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 9f66c424..ace540a4 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -251,6 +251,8 @@ export interface StackUpdateCleanupPendingRow { rollback_tags_json: string; override_paths_json: string; prune_volumes_requested: number; + /** Blueprint ID that authorized this deletion; null for manual deletes. */ + required_blueprint_id: number | null; created_at: number; updated_at: number; } @@ -1828,6 +1830,7 @@ export class DatabaseService { `); maybeAddCol('stack_update_recovery_generations', 'artifacts_retired', 'INTEGER NOT NULL DEFAULT 0'); + maybeAddCol('stack_update_cleanup_pending', 'required_blueprint_id', 'INTEGER'); // Distributed API model columns maybeAddCol('nodes', 'api_url', "TEXT DEFAULT ''"); @@ -3933,12 +3936,12 @@ export class DatabaseService { this.db.prepare( `INSERT INTO stack_update_cleanup_pending ( id, node_id, stack_name, status, target_kind, rollback_tags_json, - override_paths_json, prune_volumes_requested, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + override_paths_json, prune_volumes_requested, required_blueprint_id, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( row.id, row.node_id, row.stack_name, row.status, row.target_kind, row.rollback_tags_json, row.override_paths_json, row.prune_volumes_requested, - row.created_at, row.updated_at, + row.required_blueprint_id, row.created_at, row.updated_at, ); } @@ -4409,6 +4412,7 @@ export class DatabaseService { rollback_tags_json: JSON.stringify(localCleanup.tags), override_paths_json: JSON.stringify(localCleanup.overridePaths), prune_volumes_requested: 0, + required_blueprint_id: null, created_at: now, updated_at: now, }); diff --git a/backend/src/services/DeployedStackDeletionService.ts b/backend/src/services/DeployedStackDeletionService.ts index dce2d133..b9ca7b92 100644 --- a/backend/src/services/DeployedStackDeletionService.ts +++ b/backend/src/services/DeployedStackDeletionService.ts @@ -23,6 +23,10 @@ import { StackOpLockService, stackOpSkipMessage } from './StackOpLockService'; import { getErrorMessage } from '../utils/errors'; import { sanitizeForLog } from '../utils/safeLog'; import { isPathWithinBase, isValidStackName } from '../utils/validation'; +import { + BLUEPRINT_MARKER_FILENAME, + parseBlueprintMarker, +} from '../helpers/blueprintMarker'; /** * Directory that may contain recovery override files for a tombstone sweep. @@ -46,13 +50,30 @@ export interface DeleteDeployedStackInput { stackName: string; pruneVolumes: boolean; actor: string; + /** When set, deletion requires an on-disk .blueprint.json matching this blueprint ID. */ + requireBlueprintId?: number; /** When true, skip acquiring a new lock (caller already holds delete via continuation). */ continuationIntentId?: string; } export type DeleteDeployedStackResult = - | { ok: true } - | { ok: false; code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed'; error: string; existingAction?: string }; + | { ok: true; status: 'deleted' | 'already_absent' } + | { + ok: false; + code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed' | 'name_conflict' | 'failed'; + error: string; + existingAction?: string; + }; + +type DirProbe = { kind: 'absent' } | { kind: 'present' } | { kind: 'error'; error: string }; +type MarkerProbe = + | { kind: 'match' } + | { kind: 'name_conflict'; error: string } + | { kind: 'failed'; error: string }; + +function blueprintMarkerMismatchError(stackName: string): string { + return `Stack "${stackName}" exists without a matching blueprint marker; refusing to withdraw.`; +} function collectArtifactsFromGenerations( generations: Array<{ override_path: string | null; services_json: string }>, @@ -100,6 +121,51 @@ function parseJsonStringArray(raw: string): string[] { } } + +async function probeStackDirectory(nodeId: number, stackName: string): Promise { + // Canonical js/path-injection barrier inline with the stat sink. + const baseResolved = path.resolve(FileSystemService.getInstance(nodeId).getBaseDir()); + const safePath = path.resolve(baseResolved, stackName); + if (!safePath.startsWith(baseResolved + path.sep)) { + return { kind: 'error', error: 'Invalid stack path' }; + } + try { + const stat = await fs.stat(safePath); + return stat.isDirectory() ? { kind: 'present' } : { kind: 'absent' }; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === 'ENOENT') return { kind: 'absent' }; + return { kind: 'error', error: getErrorMessage(error, 'Failed to access stack directory') }; + } +} + +async function probeBlueprintMarkerOwnership( + nodeId: number, + stackName: string, + requireBlueprintId: number, +): Promise { + // Canonical js/path-injection barrier inline with the read sink. + const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId)); + const safePath = path.resolve(baseResolved, stackName, BLUEPRINT_MARKER_FILENAME); + if (!safePath.startsWith(baseResolved + path.sep)) { + return { kind: 'failed', error: 'Invalid stack path for blueprint marker' }; + } + try { + const content = await fs.readFile(safePath, 'utf-8'); + const marker = parseBlueprintMarker(content); + if (!marker || marker.blueprintId !== requireBlueprintId) { + return { kind: 'name_conflict', error: blueprintMarkerMismatchError(stackName) }; + } + return { kind: 'match' }; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + return { kind: 'name_conflict', error: blueprintMarkerMismatchError(stackName) }; + } + return { kind: 'failed', error: getErrorMessage(error, 'Failed to read blueprint marker') }; + } +} + export class DeployedStackDeletionService { private static instance: DeployedStackDeletionService; @@ -164,6 +230,42 @@ export class DeployedStackDeletionService { const { nodeId, stackName, pruneVolumes } = input; const db = DatabaseService.getInstance(); + // Continuation loads ownership from the persisted intent; first call uses input. + let requiredBlueprintId: number | null = + typeof input.requireBlueprintId === 'number' ? input.requireBlueprintId : null; + + if (existingIntentId) { + const existing = db.getDeletionIntentById(existingIntentId); + if (!existing || existing.status !== 'prepared') { + return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' }; + } + if (existing.required_blueprint_id != null) { + requiredBlueprintId = existing.required_blueprint_id; + } + } + + let skipPhysical = false; + if (requiredBlueprintId != null) { + const dirProbe = await probeStackDirectory(nodeId, stackName); + if (dirProbe.kind === 'error') { + return { ok: false, code: 'failed', error: dirProbe.error }; + } + if (dirProbe.kind === 'absent') { + skipPhysical = true; + } else { + const ownership = await probeBlueprintMarkerOwnership(nodeId, stackName, requiredBlueprintId); + if (ownership.kind === 'failed') { + return { ok: false, code: 'failed', error: ownership.error }; + } + if (ownership.kind === 'name_conflict') { + if (existingIntentId) { + db.updateCleanupPendingStatus(existingIntentId, 'cancelled'); + } + return { ok: false, code: 'name_conflict', error: ownership.error }; + } + } + } + let intentId = existingIntentId; if (!intentId) { const { tags, overridePaths } = collectArtifacts(nodeId, stackName); @@ -177,6 +279,7 @@ export class DeployedStackDeletionService { rollback_tags_json: JSON.stringify(tags), override_paths_json: JSON.stringify(overridePaths), prune_volumes_requested: pruneVolumes ? 1 : 0, + required_blueprint_id: requiredBlueprintId, created_at: now, updated_at: now, }; @@ -197,38 +300,53 @@ export class DeployedStackDeletionService { return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' }; } - try { - await ComposeService.getInstance(nodeId).downStack(stackName); - } catch (downErr) { - console.warn( - '[DeployedStackDeletion] Compose down failed or no-op for %s:', - sanitizeForLog(stackName), - downErr, - ); - } - - if (intent.prune_volumes_requested === 1) { + if (!skipPhysical) { try { - await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]); - } catch (pruneErr) { + await ComposeService.getInstance(nodeId).downStack(stackName); + } catch (downErr) { console.warn( - '[DeployedStackDeletion] Volume prune failed for %s, continuing delete:', + '[DeployedStackDeletion] Compose down failed or no-op for %s:', sanitizeForLog(stackName), - pruneErr, + downErr, ); } + + if (intent.prune_volumes_requested === 1) { + try { + await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]); + } catch (pruneErr) { + console.warn( + '[DeployedStackDeletion] Volume prune failed for %s, continuing delete:', + sanitizeForLog(stackName), + pruneErr, + ); + } + } + + try { + await FileSystemService.getInstance(nodeId).deleteStack(stackName); + } catch (fsErr) { + db.updateCleanupPendingStatus(intentId, 'cancelled'); + return { + ok: false, + code: 'fs_failed', + error: getErrorMessage(fsErr, 'Failed to remove stack files'), + }; + } } - try { - await FileSystemService.getInstance(nodeId).deleteStack(stackName); - } catch (fsErr) { - db.updateCleanupPendingStatus(intentId, 'cancelled'); - return { - ok: false, - code: 'fs_failed', - error: getErrorMessage(fsErr, 'Failed to remove stack files'), - }; - } + const finalized = await this.finalizeLogicalDeletion(input, intentId); + if (!finalized.ok) return finalized; + return { ok: true, status: skipPhysical ? 'already_absent' : 'deleted' }; + } + + /** Ready transaction, secondary DB/RBAC cleanup, mesh opt-out, sweep, invalidate. */ + private async finalizeLogicalDeletion( + input: DeleteDeployedStackInput, + intentId: string, + ): Promise { + const { nodeId, stackName } = input; + const db = DatabaseService.getInstance(); if (!db.commitStackDeletionReadyTransaction(intentId, nodeId, stackName)) { return { @@ -282,7 +400,7 @@ export class DeployedStackDeletionService { stackName, ts: Date.now(), }); - return { ok: true }; + return { ok: true, status: 'deleted' }; } /** @@ -497,6 +615,31 @@ export class DeployedStackDeletionService { continue; } + if (intent.required_blueprint_id != null) { + const ownership = await probeBlueprintMarkerOwnership( + nodeId, + stackName, + intent.required_blueprint_id, + ); + if (ownership.kind === 'name_conflict') { + db.updateCleanupPendingStatus(intent.id, 'cancelled'); + console.warn( + '[DeployedStackDeletion] Startup cancelled blueprint deletion for %s: %s', + sanitizeForLog(stackName), + sanitizeForLog(ownership.error), + ); + continue; + } + if (ownership.kind === 'failed') { + console.warn( + '[DeployedStackDeletion] Startup ownership probe failed for %s (leaving prepared): %s', + sanitizeForLog(stackName), + sanitizeForLog(ownership.error), + ); + continue; + } + } + const result = await this.deleteDeployedStack({ nodeId, stackName, diff --git a/backend/src/services/FileSystemService.ts b/backend/src/services/FileSystemService.ts index f0e038b1..fecb9dd0 100644 --- a/backend/src/services/FileSystemService.ts +++ b/backend/src/services/FileSystemService.ts @@ -58,6 +58,13 @@ const PROTECTED_STACK_FILES = new Set([ '.env', ]); +// Explorer-only protection: includes the blueprint ownership marker without +// putting it in PROTECTED_STACK_FILES (backup/rollback orphan removal). +const EXPLORER_PROTECTED_STACK_FILES = new Set([ + ...PROTECTED_STACK_FILES, + '.blueprint.json', +]); + // Bookkeeping markers Sencho writes into the backup slot. They are never copied // back into the stack directory on restore: `.timestamp` records when the backup // was taken; `.checksums` is the integrity manifest verified before a restore. @@ -169,7 +176,7 @@ function isProtectedRelPath(relPath: string): boolean { if (normalized.includes('/')) return false; // Fold case so e.g. a request for COMPOSE.YAML cannot dodge the gate on a // case-insensitive filesystem where it resolves to the real compose.yaml. - return PROTECTED_STACK_FILES.has(fsCaseKey(normalized)); + return EXPLORER_PROTECTED_STACK_FILES.has(fsCaseKey(normalized)); } function protectedFileError(relPath: string): Error & { code: string } { @@ -1643,7 +1650,7 @@ export class FileSystemService { type, size, mtime, - isProtected: protectedEnabled && PROTECTED_STACK_FILES.has(dirent.name), + isProtected: protectedEnabled && EXPLORER_PROTECTED_STACK_FILES.has(dirent.name), }; }) ); @@ -2096,7 +2103,7 @@ export class FileSystemService { type, size: stat.isDirectory() ? 0 : stat.size, mtime: stat.mtimeMs, - isProtected: (scope?.protectedEnabled ?? true) && PROTECTED_STACK_FILES.has(name), + isProtected: (scope?.protectedEnabled ?? true) && EXPLORER_PROTECTED_STACK_FILES.has(name), }; } } diff --git a/backend/src/services/blueprintPreviewProjection.ts b/backend/src/services/blueprintPreviewProjection.ts index f32e2c71..d4991406 100644 --- a/backend/src/services/blueprintPreviewProjection.ts +++ b/backend/src/services/blueprintPreviewProjection.ts @@ -457,15 +457,30 @@ function asApprovedBlueprint(blueprint: Blueprint): Blueprint & BlueprintApprova } /** Upgrade create rows to blockers when an unmanaged same-name stack already exists. */ -async function applyCreateNameConflictBlockers(blueprintName: string, raw: RawAction[]): Promise { +function blockCreateForOwnership(row: RawAction, detail: string): void { + row.action = 'blocked_name_conflict'; + row.severity = 'blocker'; + row.detail = detail; +} + +async function applyCreateNameConflictBlockers( + blueprintName: string, + blueprintId: number, + raw: RawAction[], +): Promise { const { BlueprintService } = await import('./BlueprintService'); const svc = BlueprintService.getInstance(); for (const row of raw) { if (row.action !== 'create') continue; - if (!(await svc.hasNameConflict(blueprintName, row.node))) continue; - row.action = 'blocked_name_conflict'; - row.severity = 'blocker'; - row.detail = 'Unmanaged stack with this name already exists on this node'; + try { + if (!(await svc.hasNameConflict(blueprintName, row.node, blueprintId))) continue; + blockCreateForOwnership(row, 'Unmanaged stack with this name already exists on this node'); + } catch (err) { + blockCreateForOwnership( + row, + err instanceof Error ? err.message : 'Cannot verify stack ownership on this node', + ); + } } } @@ -477,7 +492,7 @@ export async function buildBlueprintPreview(blueprintId: number): Promise Blueprint detail sheet with a deployment row in Awaiting confirmation state @@ -309,7 +309,7 @@ A future Volume Migration feature will automate this with app-aware backup tooli - A directory by the blueprint's name already exists on that node and does not carry the `.blueprint.json` marker. The most likely cause is a manually created stack with the same name. Resolution: rename either the existing stack or the blueprint, then click **Apply now** on the detail sheet to retry. + A directory by the blueprint's name already exists on that node without a matching `.blueprint.json` marker (missing, malformed, or owned by another blueprint). The most likely cause is a manually created stack with the same name. Resolution: rename either the existing stack or the blueprint, then click **Apply now** on the detail sheet to retry. The reconciler will not auto-deploy a stateful blueprint to a node it has never run on. Click **Confirm deploy** on the row, then choose **Deploy fresh** in the dialog. Sencho will create empty named volumes and start the stack. @@ -323,6 +323,9 @@ A future Volume Migration feature will automate this with app-aware backup tooli The row moves to **Failed** with the remote error. Reconnect the node (see [Pilot Agent](/features/pilot-agent) and [Multi-node management](/features/multi-node)), verify whether the stack directory contains `compose.yaml` and `.blueprint.json`, then click **Apply now**. If the directory exists without a matching marker, Sencho treats it as a name conflict until you rename or remove the remote stack manually. + + Blueprint apply and withdraw on a remote node require that node's Sencho build to expose the atomic node-local endpoints (`/api/blueprints/apply-local` and `/api/blueprints/withdraw-local`). Older remotes refuse the mutation with a failed deployment row rather than writing files without ownership checks. Upgrade the remote instance to a matching Sencho release, then retry **Apply now** or withdraw. + The deployment row moves to **Failed** and records the Docker or registry error. Resolve the daemon, socket, registry credentials, rate limit, disk, or volume-permission issue on the affected node, then click **Apply now**. Watch that node's stack activity and security scan status after retry.