fix(fleet): harden fleet snapshots with DRY capture, audit fixes, and design compliance (#555)

Extract duplicated snapshot capture functions from index.ts and SchedulerService.ts into a shared module (snapshot-capture.ts). Fix audit log route patterns that used singular 'snapshot' instead of plural 'snapshots'. Apply design system to FleetSnapshots component (card styling, strokeWidth, font-mono, tabular-nums, ScrollArea). Add safePage pagination, loading toast for creation, and fix the restore dialog race condition with a controlled AlertDialog. Add diagnostic logging gated behind Developer Mode. Add tests for restore endpoint, admin role enforcement, and edge cases. Update docs with troubleshooting section and refresh screenshots.
2026-07-26 11:49:16 +00:00 · 2026-04-13 13:26:55 -04:00
parent 620e537eda
commit 809bf76c20
10 changed files with 426 additions and 205 deletions
@@ -358,3 +358,183 @@ describe('Fleet snapshot lifecycle', () => {
    expect(check.status).toBe(404);
  });
 });
+
+// ─── Snapshot Restore Endpoint ───
+
+describe('Fleet snapshot restore', () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  let snapshotId: number;
+
+  beforeAll(async () => {
+    const { LicenseService: LS } = await import('../services/LicenseService');
+    vi.spyOn(LS.getInstance(), 'getTier').mockReturnValue('paid');
+    const res = await request(app)
+      .post('/api/fleet/snapshots')
+      .set('Authorization', authHeader)
+      .send({ description: 'Restore test snapshot' });
+    snapshotId = res.body.id;
+    vi.restoreAllMocks();
+  });
+
+  it('POST /api/fleet/snapshots/:id/restore returns 401 without auth', async () => {
+    const res = await request(app)
+      .post(`/api/fleet/snapshots/${snapshotId}/restore`)
+      .send({ nodeId: 1, stackName: 'test' });
+    expect(res.status).toBe(401);
+  });
+
+  it('POST /api/fleet/snapshots/:id/restore returns 403 on free tier', async () => {
+    mockTier('community');
+    const res = await request(app)
+      .post(`/api/fleet/snapshots/${snapshotId}/restore`)
+      .set('Authorization', authHeader)
+      .send({ nodeId: 1, stackName: 'test' });
+    expect(res.status).toBe(403);
+    expect(res.body.code).toBe('PAID_REQUIRED');
+  });
+
+  it('returns 400 with missing nodeId/stackName', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .post(`/api/fleet/snapshots/${snapshotId}/restore`)
+      .set('Authorization', authHeader)
+      .send({});
+    expect(res.status).toBe(400);
+    expect(res.body.error).toMatch(/nodeId and stackName are required/i);
+  });
+
+  it('returns 400 with invalid stackName (path traversal)', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .post(`/api/fleet/snapshots/${snapshotId}/restore`)
+      .set('Authorization', authHeader)
+      .send({ nodeId: 1, stackName: '../etc/passwd' });
+    expect(res.status).toBe(400);
+    expect(res.body.error).toMatch(/invalid stack name/i);
+  });
+
+  it('returns 400 for NaN snapshot ID', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .post('/api/fleet/snapshots/abc/restore')
+      .set('Authorization', authHeader)
+      .send({ nodeId: 1, stackName: 'mystack' });
+    expect(res.status).toBe(400);
+    expect(res.body.error).toMatch(/invalid snapshot id/i);
+  });
+
+  it('returns 404 for non-existent snapshot', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .post('/api/fleet/snapshots/99999/restore')
+      .set('Authorization', authHeader)
+      .send({ nodeId: 1, stackName: 'mystack' });
+    expect(res.status).toBe(404);
+    expect(res.body.error).toMatch(/snapshot not found/i);
+  });
+
+  it('returns 404 when no files match the nodeId/stackName combo', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .post(`/api/fleet/snapshots/${snapshotId}/restore`)
+      .set('Authorization', authHeader)
+      .send({ nodeId: 999, stackName: 'nonexistent-stack' });
+    expect(res.status).toBe(404);
+    expect(res.body.error).toMatch(/no files found/i);
+  });
+});
+
+// ─── Snapshot Admin Role Enforcement ───
+
+describe('Fleet snapshot admin enforcement', () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  let viewerHeader: string;
+
+  beforeAll(async () => {
+    const { DatabaseService: DS } = await import('../services/DatabaseService');
+    const db = DS.getInstance();
+    const bcrypt = await import('bcrypt');
+    const viewerHash = await bcrypt.hash('snapshotviewer', 1);
+    try {
+      db.addUser({ username: 'snapshotviewer', password_hash: viewerHash, role: 'viewer' });
+    } catch {
+      // User may already exist
+    }
+    const viewerToken = jwt.sign({ username: 'snapshotviewer', role: 'viewer' }, TEST_JWT_SECRET, { expiresIn: '1m' });
+    viewerHeader = `Bearer ${viewerToken}`;
+  });
+
+  it('POST /api/fleet/snapshots returns 403 for viewer', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .post('/api/fleet/snapshots')
+      .set('Authorization', viewerHeader)
+      .send({ description: 'test' });
+    expect(res.status).toBe(403);
+    expect(res.body.code).toBe('ADMIN_REQUIRED');
+  });
+
+  it('DELETE /api/fleet/snapshots/1 returns 403 for viewer', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .delete('/api/fleet/snapshots/1')
+      .set('Authorization', viewerHeader);
+    expect(res.status).toBe(403);
+    expect(res.body.code).toBe('ADMIN_REQUIRED');
+  });
+
+  it('POST /api/fleet/snapshots/1/restore returns 403 for viewer', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .post('/api/fleet/snapshots/1/restore')
+      .set('Authorization', viewerHeader)
+      .send({ nodeId: 1, stackName: 'test' });
+    expect(res.status).toBe(403);
+    expect(res.body.code).toBe('ADMIN_REQUIRED');
+  });
+
+  it('GET /api/fleet/snapshots succeeds for viewer (read-only)', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .get('/api/fleet/snapshots')
+      .set('Authorization', viewerHeader);
+    expect(res.status).toBe(200);
+  });
+});
+
+// ─── Snapshot Edge Cases ───
+
+describe('Fleet snapshot edge cases', () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  it('snapshot with 0 stacks captured (empty COMPOSE_DIR)', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .post('/api/fleet/snapshots')
+      .set('Authorization', authHeader)
+      .send({ description: 'Empty snapshot' });
+    expect(res.status).toBe(201);
+    expect(res.body.stack_count).toBe(0);
+  });
+
+  it('DELETE on already-deleted snapshot returns 404', async () => {
+    mockTier('paid');
+    const createRes = await request(app)
+      .post('/api/fleet/snapshots')
+      .set('Authorization', authHeader)
+      .send({ description: 'To delete twice' });
+    const id = createRes.body.id;
+
+    await request(app)
+      .delete(`/api/fleet/snapshots/${id}`)
+      .set('Authorization', authHeader);
+
+    mockTier('paid');
+    const res = await request(app)
+      .delete(`/api/fleet/snapshots/${id}`)
+      .set('Authorization', authHeader);
+    expect(res.status).toBe(404);
+  });
+});
@@ -60,6 +60,7 @@ function invalidateNodeCaches(nodeId: number): void {

 import { isDebugEnabled } from './utils/debug';
 import { getErrorMessage } from './utils/errors';
+import { captureLocalNodeFiles, captureRemoteNodeFiles, SnapshotNodeData } from './utils/snapshot-capture';
 import { GlobalLogEntry, normalizeContainerName, parseLogTimestamp, detectLogLevel, demuxDockerLog } from './utils/log-parsing';
 import SelfUpdateService from './services/SelfUpdateService';
 import semver from 'semver';
@@ -905,9 +906,9 @@ const AUDIT_ROUTE_SUMMARIES: Record<string, string> = {
  'PUT /webhooks': 'Updated webhook',
  'DELETE /webhooks': 'Deleted webhook',
  'PUT /settings': 'Updated settings',
-  'POST /fleet/snapshot': 'Created fleet backup',
-  'DELETE /fleet/snapshot': 'Deleted fleet backup',
-  'POST /fleet/snapshot/restore': 'Restored fleet backup',
+  'POST /fleet/snapshots': 'Created fleet backup',
+  'DELETE /fleet/snapshots': 'Deleted fleet backup',
+  'POST /fleet/snapshots/*/restore': 'Restored fleet backup',
  'PUT /sso/config': 'Updated SSO configuration',
  'DELETE /sso/config': 'Deleted SSO configuration',
  'POST /api-tokens': 'Created API token',
@@ -2017,93 +2018,6 @@ async function fetchRemoteNodeOverview(node: Node): Promise<FleetNodeOverview> {

 // ─── Fleet Snapshots (Skipper+) ───

-interface SnapshotNodeData {
-  nodeId: number;
-  nodeName: string;
-  stacks: Array<{
-    stackName: string;
-    files: Array<{ filename: string; content: string }>;
-  }>;
-}
-
-async function captureLocalNodeFiles(node: Node): Promise<SnapshotNodeData> {
-  const fsService = FileSystemService.getInstance(node.id);
-  const stackNames = await fsService.getStacks();
-  const stacks: SnapshotNodeData['stacks'] = [];
-
-  for (const stackName of stackNames) {
-    const files: Array<{ filename: string; content: string }> = [];
-    try {
-      const composeContent = await fsService.getStackContent(stackName);
-      files.push({ filename: 'compose.yaml', content: composeContent });
-    } catch (e) {
-      console.warn(`[Fleet Snapshot] Could not read compose file for stack "${stackName}", skipping:`, (e as Error).message);
-      continue;
-    }
-    try {
-      const envContent = await fsService.getEnvContent(stackName);
-      files.push({ filename: '.env', content: envContent });
-    } catch {
-      // No .env file - that's fine
-    }
-    stacks.push({ stackName, files });
-  }
-
-  return { nodeId: node.id, nodeName: node.name, stacks };
-}
-
-async function captureRemoteNodeFiles(node: Node): Promise<SnapshotNodeData> {
-  if (!node.api_url || !node.api_token) {
-    throw new Error('Remote node not configured');
-  }
-
-  const baseUrl = node.api_url.replace(/\/$/, '');
-  const headers = { Authorization: `Bearer ${node.api_token}` };
-
-  const stacksRes = await fetch(`${baseUrl}/api/stacks`, {
-    headers,
-    signal: AbortSignal.timeout(15000),
-  });
-  if (!stacksRes.ok) throw new Error('Failed to fetch stacks from remote node');
-  const stackNames = await stacksRes.json() as string[];
-
-  const stacks: SnapshotNodeData['stacks'] = [];
-
-  for (const stackName of stackNames) {
-    const files: Array<{ filename: string; content: string }> = [];
-    try {
-      const composeRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, {
-        headers,
-        signal: AbortSignal.timeout(15000),
-      });
-      if (composeRes.ok) {
-        const content = await composeRes.text();
-        files.push({ filename: 'compose.yaml', content });
-      }
-    } catch (e) {
-      console.warn(`[Fleet Snapshot] Failed to fetch remote compose for stack "${stackName}":`, (e as Error).message);
-      continue;
-    }
-    try {
-      const envRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, {
-        headers,
-        signal: AbortSignal.timeout(15000),
-      });
-      if (envRes.ok) {
-        const content = await envRes.text();
-        files.push({ filename: '.env', content });
-      }
-    } catch {
-      // No .env - skip
-    }
-    if (files.length > 0) {
-      stacks.push({ stackName, files });
-    }
-  }
-
-  return { nodeId: node.id, nodeName: node.name, stacks };
-}
-
 // Create fleet snapshot
 app.post('/api/fleet/snapshots', authMiddleware, async (req: Request, res: Response): Promise<void> => {
  if (!requireAdmin(req, res)) return;
@@ -2119,6 +2033,7 @@ app.post('/api/fleet/snapshots', authMiddleware, async (req: Request, res: Respo
    const nodes = db.getNodes();
    const username = req.user?.username || 'admin';

+    const captureStart = Date.now();
    const results = await Promise.allSettled(
      nodes.map(async (node) => {
        if (node.type === 'remote') {
@@ -2175,6 +2090,12 @@ app.post('/api/fleet/snapshots', authMiddleware, async (req: Request, res: Respo
    }

    console.log('[Fleet] Snapshot created:', capturedNodes.length, 'nodes,', totalStacks, 'stacks');
+    if (isDebugEnabled()) {
+      console.debug(`[Fleet:debug] Snapshot ${snapshotId} capture completed in ${Date.now() - captureStart}ms, ${allFiles.length} file(s) stored`);
+      for (const skip of skippedNodes) {
+        console.debug(`[Fleet:debug] Skipped node "${skip.nodeName}" (id=${skip.nodeId}): ${skip.reason}`);
+      }
+    }
    const snapshot = db.getSnapshot(snapshotId);
    res.status(201).json(snapshot);
  } catch (error) {
@@ -2279,6 +2200,11 @@ app.post('/api/fleet/snapshots/:id/restore', authMiddleware, async (req: Request
      return;
    }

+    if (isDebugEnabled()) {
+      const fileNames = files.map(f => f.filename).join(', ');
+      console.debug(`[Fleet:debug] Restore: snapshot=${snapshotId}, node=${nodeId}, stack="${stackName}", files=[${fileNames}], redeploy=${redeploy}`);
+    }
+
    const node = db.getNode(nodeId);
    if (!node) {
      res.status(404).json({ error: 'Target node no longer exists' });
@@ -2372,6 +2298,9 @@ app.delete('/api/fleet/snapshots/:id', authMiddleware, async (req: Request, res:
      res.status(404).json({ error: 'Snapshot not found' });
      return;
    }
+    if (isDebugEnabled()) {
+      console.debug(`[Fleet:debug] Deleting snapshot ${id} (${snapshot.node_count} node(s), ${snapshot.stack_count} stack(s))`);
+    }
    db.deleteSnapshot(id);
    console.log('[Fleet] Snapshot deleted:', id);
    res.json({ message: 'Snapshot deleted' });
@@ -9,6 +9,7 @@ import { ImageUpdateService } from './ImageUpdateService';
 import type { ImageCheckResult } from './ImageUpdateService';
 import { isDebugEnabled } from '../utils/debug';
 import { getErrorMessage } from '../utils/errors';
+import { captureLocalNodeFiles, captureRemoteNodeFiles } from '../utils/snapshot-capture';
 import { NodeRegistry } from './NodeRegistry';
 import { NotificationService } from './NotificationService';

@@ -253,9 +254,9 @@ export class SchedulerService {
        const results = await Promise.allSettled(
            nodes.map(async (node) => {
                if (node.type === 'remote') {
-                    return this.captureRemoteNodeFiles(node);
+                    return captureRemoteNodeFiles(node);
                }
-                return this.captureLocalNodeFiles(node);
+                return captureLocalNodeFiles(node);
            })
        );

@@ -305,85 +306,13 @@ export class SchedulerService {
            db.insertSnapshotFiles(snapshotId, allFiles);
        }

+        if (isDebugEnabled()) {
+            console.debug(`[SchedulerService:debug] Snapshot task ${task.id}: captured ${capturedNodes.length} node(s), ${totalStacks} stack(s), ${allFiles.length} file(s), skipped ${skippedNodes.length}`);
+        }
+
        return `Fleet snapshot created (id=${snapshotId}, ${capturedNodes.length} node(s), ${totalStacks} stack(s)${skippedNodes.length > 0 ? `, ${skippedNodes.length} skipped` : ''})`;
    }

-    private async captureLocalNodeFiles(node: { id: number; name: string }) {
-        const fsService = FileSystemService.getInstance(node.id);
-        const stackNames = await fsService.getStacks();
-        const stacks: Array<{ stackName: string; files: Array<{ filename: string; content: string }> }> = [];
-
-        for (const stackName of stackNames) {
-            const files: Array<{ filename: string; content: string }> = [];
-            try {
-                const composeContent = await fsService.getStackContent(stackName);
-                files.push({ filename: 'compose.yaml', content: composeContent });
-            } catch {
-                continue;
-            }
-            try {
-                const envContent = await fsService.getEnvContent(stackName);
-                files.push({ filename: '.env', content: envContent });
-            } catch {
-                // No .env file
-            }
-            stacks.push({ stackName, files });
-        }
-
-        return { nodeId: node.id, nodeName: node.name, stacks };
-    }
-
-    private async captureRemoteNodeFiles(node: { id: number; name: string; api_url?: string; api_token?: string }) {
-        if (!node.api_url || !node.api_token) {
-            throw new Error('Remote node not configured');
-        }
-
-        const baseUrl = node.api_url.replace(/\/$/, '');
-        const headers = { Authorization: `Bearer ${node.api_token}` };
-
-        const stacksRes = await fetch(`${baseUrl}/api/stacks`, {
-            headers,
-            signal: AbortSignal.timeout(15000),
-        });
-        if (!stacksRes.ok) throw new Error('Failed to fetch stacks from remote node');
-        const stackNames = await stacksRes.json() as string[];
-
-        const stacks: Array<{ stackName: string; files: Array<{ filename: string; content: string }> }> = [];
-
-        for (const stackName of stackNames) {
-            const files: Array<{ filename: string; content: string }> = [];
-            try {
-                const composeRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, {
-                    headers,
-                    signal: AbortSignal.timeout(15000),
-                });
-                if (composeRes.ok) {
-                    const content = await composeRes.text();
-                    files.push({ filename: 'compose.yaml', content });
-                }
-            } catch {
-                continue;
-            }
-            try {
-                const envRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, {
-                    headers,
-                    signal: AbortSignal.timeout(15000),
-                });
-                if (envRes.ok) {
-                    const content = await envRes.text();
-                    files.push({ filename: '.env', content });
-                }
-            } catch {
-                // No .env
-            }
-            if (files.length > 0) {
-                stacks.push({ stackName, files });
-            }
-        }
-
-        return { nodeId: node.id, nodeName: node.name, stacks };
-    }
-
    private async executePrune(task: ScheduledTask): Promise<string> {
        const nodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
        if (task.node_id == null && isDebugEnabled()) {
@@ -0,0 +1,123 @@
+/**
+ * Shared snapshot capture functions used by both the REST API (index.ts)
+ * and the SchedulerService for fleet-wide snapshot operations.
+ */
+
+import { FileSystemService } from '../services/FileSystemService';
+import { isDebugEnabled } from './debug';
+
+export interface SnapshotNodeData {
+  nodeId: number;
+  nodeName: string;
+  stacks: Array<{
+    stackName: string;
+    files: Array<{ filename: string; content: string }>;
+  }>;
+}
+
+/** Minimal node shape accepted by capture functions. */
+export interface CaptureNode {
+  id: number;
+  name: string;
+  api_url?: string;
+  api_token?: string;
+}
+
+/**
+ * Read compose.yaml and .env files for every stack on a local node.
+ * Stacks whose compose file cannot be read are silently skipped.
+ */
+export async function captureLocalNodeFiles(node: CaptureNode): Promise<SnapshotNodeData> {
+  const start = Date.now();
+  const fsService = FileSystemService.getInstance(node.id);
+  const stackNames = await fsService.getStacks();
+  const stacks: SnapshotNodeData['stacks'] = [];
+
+  for (const stackName of stackNames) {
+    const files: Array<{ filename: string; content: string }> = [];
+    try {
+      const composeContent = await fsService.getStackContent(stackName);
+      files.push({ filename: 'compose.yaml', content: composeContent });
+    } catch (e) {
+      console.warn(`[Fleet Snapshot] Could not read compose file for stack "${stackName}", skipping:`, (e as Error).message);
+      continue;
+    }
+    try {
+      const envContent = await fsService.getEnvContent(stackName);
+      files.push({ filename: '.env', content: envContent });
+    } catch {
+      // No .env file - that's fine
+    }
+    stacks.push({ stackName, files });
+  }
+
+  if (isDebugEnabled()) {
+    const fileCount = stacks.reduce((sum, s) => sum + s.files.length, 0);
+    console.debug(`[Fleet:debug] Local capture "${node.name}": ${stacks.length} stack(s), ${fileCount} file(s) in ${Date.now() - start}ms`);
+  }
+
+  return { nodeId: node.id, nodeName: node.name, stacks };
+}
+
+/**
+ * Fetch compose.yaml and .env files for every stack on a remote node
+ * via the Distributed API proxy. Stacks whose compose file cannot be
+ * fetched are silently skipped.
+ */
+export async function captureRemoteNodeFiles(node: CaptureNode): Promise<SnapshotNodeData> {
+  if (!node.api_url || !node.api_token) {
+    throw new Error('Remote node not configured');
+  }
+
+  const start = Date.now();
+  const baseUrl = node.api_url.replace(/\/$/, '');
+  const headers = { Authorization: `Bearer ${node.api_token}` };
+
+  const stacksRes = await fetch(`${baseUrl}/api/stacks`, {
+    headers,
+    signal: AbortSignal.timeout(15000),
+  });
+  if (!stacksRes.ok) throw new Error('Failed to fetch stacks from remote node');
+  const stackNames = await stacksRes.json() as string[];
+
+  const stacks: SnapshotNodeData['stacks'] = [];
+
+  for (const stackName of stackNames) {
+    const files: Array<{ filename: string; content: string }> = [];
+    try {
+      const composeRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, {
+        headers,
+        signal: AbortSignal.timeout(15000),
+      });
+      if (composeRes.ok) {
+        const content = await composeRes.text();
+        files.push({ filename: 'compose.yaml', content });
+      }
+    } catch (e) {
+      console.warn(`[Fleet Snapshot] Failed to fetch remote compose for stack "${stackName}":`, (e as Error).message);
+      continue;
+    }
+    try {
+      const envRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, {
+        headers,
+        signal: AbortSignal.timeout(15000),
+      });
+      if (envRes.ok) {
+        const content = await envRes.text();
+        files.push({ filename: '.env', content });
+      }
+    } catch {
+      // No .env - skip
+    }
+    if (files.length > 0) {
+      stacks.push({ stackName, files });
+    }
+  }
+
+  if (isDebugEnabled()) {
+    const fileCount = stacks.reduce((sum, s) => sum + s.files.length, 0);
+    console.debug(`[Fleet:debug] Remote capture "${node.name}": ${stacks.length} stack(s), ${fileCount} file(s) in ${Date.now() - start}ms`);
+  }
+
+  return { nodeId: node.id, nodeName: node.name, stacks };
+}
@@ -95,3 +95,28 @@ Admins can delete snapshots from the list view by clicking the trash icon on the
 ## Storage

 Snapshots are stored in Sencho's SQLite database. Compose files are typically small (under 10 KB each), so even hundreds of snapshots consume minimal disk space. For very large fleets, consider periodically deleting old snapshots to keep the database lean.
+
+## Troubleshooting
+
+### Snapshot shows skipped nodes
+
+If a remote node is offline, unreachable, or its API token has expired, the node is skipped during snapshot creation. The snapshot list shows a warning icon with a count of skipped nodes. Open the snapshot's detail view to see which nodes were skipped and the reason for each.
+
+Common causes:
+- The remote Sencho instance is stopped or restarting
+- The node's API URL or token was changed after it was added
+- A firewall or network issue is blocking the connection between nodes
+
+To resolve, verify the remote node is running and reachable, then update the node's API URL and token in the Fleet settings if needed. Create a new snapshot after fixing the connectivity issue.
+
+### Restore fails with "Target node no longer exists"
+
+This error occurs when the node recorded in the snapshot has been removed from your fleet since the snapshot was taken. You cannot restore to a node that is no longer registered. Re-add the node first, then retry the restore.
+
+### Restore fails with "No files found for this stack"
+
+The stack you're trying to restore may not have been captured in the snapshot (for example, if the compose file was missing or unreadable at the time the snapshot was taken). Open the snapshot's detail view to verify which stacks and files are available.
+
+### Diagnostic logging
+
+If you need to investigate snapshot operations in detail, enable **Developer Mode** in **Settings > Developer**. This activates diagnostic logging for snapshot creation (per-node capture timing and file counts), restore operations, and scheduled snapshot execution. Diagnostic logs appear in the server's standard output with a `:debug` suffix.
@@ -1,7 +1,7 @@
 import { useState, useEffect, useCallback } from 'react';
 import {
    Camera, ArrowLeft, Server, Layers, FileText, AlertTriangle, Trash2,
-    Eye, ChevronDown, ChevronRight, Plus, Loader2, RotateCcw,
+    Eye, ChevronDown, ChevronLeft, ChevronRight, Plus, Loader2, RotateCcw,
 } from 'lucide-react';
 import { Button } from '@/components/ui/button';
 import { Input } from '@/components/ui/input';
@@ -17,6 +17,7 @@ import {
    AlertDialogDescription, AlertDialogFooter, AlertDialogHeader, AlertDialogTitle,
    AlertDialogTrigger,
 } from '@/components/ui/alert-dialog';
+import { ScrollArea } from '@/components/ui/scroll-area';
 import { apiFetch } from '@/lib/api';
 import { useAuth } from '@/context/AuthContext';
 import { toast } from '@/components/ui/toast-store';
@@ -59,6 +60,8 @@ interface SkippedNode {
    reason: string;
 }

+const PAGE_SIZE = 10;
+
 // --- Main Component ---

 export default function FleetSnapshots() {
@@ -77,6 +80,12 @@ export default function FleetSnapshots() {
    const [previewFiles, setPreviewFiles] = useState<Set<string>>(new Set());
    const [restoringStack, setRestoringStack] = useState<string | null>(null);
    const [deletingId, setDeletingId] = useState<number | null>(null);
+    const [page, setPage] = useState(0);
+
+    const totalPages = Math.max(1, Math.ceil(snapshots.length / PAGE_SIZE));
+    const safePage = Math.min(page, totalPages - 1);
+    const pagedSnapshots = snapshots.slice(safePage * PAGE_SIZE, (safePage + 1) * PAGE_SIZE);
+    const needsPagination = snapshots.length > PAGE_SIZE;

    // --- Data Fetching ---

@@ -104,6 +113,7 @@ export default function FleetSnapshots() {

    const handleCreate = async () => {
        setCreating(true);
+        const loadingId = toast.loading('Creating fleet snapshot...');
        try {
            const res = await apiFetch('/fleet/snapshots', {
                method: 'POST',
@@ -123,6 +133,7 @@ export default function FleetSnapshots() {
            const err = error as Record<string, unknown> | null;
            toast.error(err?.message as string || err?.error as string || 'Something went wrong.');
        } finally {
+            toast.dismiss(loadingId);
            setCreating(false);
        }
    };
@@ -250,12 +261,12 @@ export default function FleetSnapshots() {
                    className="gap-1.5 -ml-2"
                    onClick={() => { setViewMode('list'); setSelectedSnapshot(null); }}
                >
-                    <ArrowLeft className="w-4 h-4" />
+                    <ArrowLeft className="w-4 h-4" strokeWidth={1.5} />
                    Back to Snapshots
                </Button>

                {loadingDetail ? (
-                    <div className="rounded-xl border bg-card p-6 space-y-4">
+                    <div className="rounded-lg border border-card-border border-t-card-border-top bg-card text-card-foreground shadow-card-bevel p-6 space-y-4">
                        <Skeleton className="h-6 w-64" />
                        <Skeleton className="h-4 w-48" />
                        <div className="flex gap-2">
@@ -267,7 +278,7 @@ export default function FleetSnapshots() {
                ) : selectedSnapshot ? (
                    <>
                        {/* Header card */}
-                        <div className="rounded-xl border bg-card p-4 space-y-3">
+                        <div className="rounded-lg border border-card-border border-t-card-border-top bg-card text-card-foreground shadow-card-bevel p-4 space-y-3">
                            <h2 className="text-lg font-semibold">
                                {selectedSnapshot.description || 'Untitled Snapshot'}
                            </h2>
@@ -276,10 +287,10 @@ export default function FleetSnapshots() {
                                {new Date(selectedSnapshot.created_at).toLocaleString()}
                            </p>
                            <div className="flex items-center gap-2">
-                                <Badge variant="secondary">
+                                <Badge variant="secondary" className="font-mono tabular-nums">
                                    {selectedSnapshot.node_count} node{selectedSnapshot.node_count !== 1 ? 's' : ''}
                                </Badge>
-                                <Badge variant="secondary">
+                                <Badge variant="secondary" className="font-mono tabular-nums">
                                    {selectedSnapshot.stack_count} stack{selectedSnapshot.stack_count !== 1 ? 's' : ''}
                                </Badge>
                            </div>
@@ -315,7 +326,7 @@ export default function FleetSnapshots() {
                            {selectedSnapshot.nodes.map(node => {
                                const nodeExpanded = expandedNodes.has(node.nodeId);
                                return (
-                                    <div key={node.nodeId} className="rounded-xl border bg-card overflow-hidden">
+                                    <div key={node.nodeId} className="rounded-lg border border-card-border border-t-card-border-top bg-card text-card-foreground shadow-card-bevel overflow-hidden transition-colors hover:border-t-card-border-hover">
                                        {/* Node header */}
                                        <button
                                            onClick={() => toggleNode(node.nodeId)}
@@ -327,7 +338,7 @@ export default function FleetSnapshots() {
                                            }
                                            <Server className="w-4 h-4 text-muted-foreground shrink-0" />
                                            <span className="text-sm font-medium flex-1 truncate">{node.nodeName}</span>
-                                            <Badge variant="outline" className="text-xs shrink-0">
+                                            <Badge variant="outline" className="text-xs font-mono tabular-nums shrink-0">
                                                {node.stacks.length} stack{node.stacks.length !== 1 ? 's' : ''}
                                            </Badge>
                                        </button>
@@ -349,10 +360,10 @@ export default function FleetSnapshots() {
                                                                    : <ChevronRight className="w-3.5 h-3.5 shrink-0 text-muted-foreground" />
                                                                }
                                                                <Layers className="w-3.5 h-3.5 text-muted-foreground shrink-0" />
-                                                                <span className="text-xs font-medium flex-1 truncate">
+                                                                <span className="text-xs font-mono font-medium flex-1 truncate">
                                                                    {stack.stackName}
                                                                </span>
-                                                                <Badge variant="outline" className="text-[10px] px-1.5 py-0 h-4 shrink-0">
+                                                                <Badge variant="outline" className="text-[10px] font-mono tabular-nums px-1.5 py-0 h-4 shrink-0">
                                                                    {stack.files.length} file{stack.files.length !== 1 ? 's' : ''}
                                                                </Badge>
                                                            </button>
@@ -367,21 +378,23 @@ export default function FleetSnapshots() {
                                                                            <div key={fileKey}>
                                                                                <div className="flex items-center gap-2 px-3 py-1.5 rounded-md hover:bg-muted/50 transition-colors">
                                                                                    <FileText className="w-3.5 h-3.5 text-muted-foreground shrink-0" />
-                                                                                    <span className="text-xs flex-1 truncate">{file.filename}</span>
+                                                                                    <span className="text-xs font-mono flex-1 truncate">{file.filename}</span>
                                                                                    <Button
                                                                                        variant="ghost"
                                                                                        size="sm"
                                                                                        className="h-6 px-2 text-xs"
                                                                                        onClick={() => togglePreview(fileKey)}
                                                                                    >
-                                                                                        <Eye className="w-3 h-3 mr-1" />
+                                                                                        <Eye className="w-3 h-3 mr-1" strokeWidth={1.5} />
                                                                                        {showPreview ? 'Hide' : 'Preview'}
                                                                                    </Button>
                                                                                </div>
                                                                                {showPreview && (
-                                                                                    <pre className="mx-3 mt-1 mb-2 p-3 bg-zinc-950 text-zinc-200 text-xs font-mono rounded-lg overflow-auto max-h-64 whitespace-pre-wrap break-words">
-                                                                                        {file.content}
-                                                                                    </pre>
+                                                                                    <ScrollArea className="mx-3 mt-1 mb-2 max-h-64 rounded-lg bg-background shadow-[inset_0_2px_4px_0_oklch(0_0_0/0.4)]">
+                                                                                        <pre className="p-3 text-xs font-mono text-foreground whitespace-pre-wrap break-words">
+                                                                                            {file.content}
+                                                                                        </pre>
+                                                                                    </ScrollArea>
                                                                                )}
                                                                            </div>
                                                                        );
@@ -421,20 +434,35 @@ export default function FleetSnapshots() {
            {/* Header */}
            <div className="flex items-center justify-between">
                <div className="flex items-center gap-2.5">
-                    <Camera className="w-5 h-5 text-muted-foreground" />
+                    <Camera className="w-5 h-5 text-muted-foreground" strokeWidth={1.5} />
                    <h2 className="text-lg font-semibold">Fleet Snapshots</h2>
                </div>
-                {isAdmin && !showCreateForm && (
-                    <Button size="sm" className="gap-1.5" onClick={() => setShowCreateForm(true)}>
-                        <Plus className="w-4 h-4" />
-                        Create Snapshot
-                    </Button>
-                )}
+                <div className="flex items-center gap-2">
+                    {needsPagination && (
+                        <div className="flex items-center gap-1.5">
+                            <Button variant="ghost" size="icon" className="h-6 w-6" disabled={safePage === 0} onClick={() => setPage(safePage - 1)}>
+                                <ChevronLeft className="h-3.5 w-3.5" strokeWidth={1.5} />
+                            </Button>
+                            <span className="text-xs font-mono tabular-nums text-stat-subtitle min-w-[3rem] text-center">
+                                {safePage + 1} / {totalPages}
+                            </span>
+                            <Button variant="ghost" size="icon" className="h-6 w-6" disabled={safePage >= totalPages - 1} onClick={() => setPage(safePage + 1)}>
+                                <ChevronRight className="h-3.5 w-3.5" strokeWidth={1.5} />
+                            </Button>
+                        </div>
+                    )}
+                    {isAdmin && !showCreateForm && (
+                        <Button size="sm" className="gap-1.5" onClick={() => setShowCreateForm(true)}>
+                            <Plus className="w-4 h-4" strokeWidth={1.5} />
+                            Create Snapshot
+                        </Button>
+                    )}
+                </div>
            </div>

            {/* Create form */}
            {showCreateForm && (
-                <div className="rounded-xl border bg-card p-4 space-y-3">
+                <div className="rounded-lg border border-card-border border-t-card-border-top bg-card text-card-foreground shadow-card-bevel p-4 space-y-3">
                    <Input
                        placeholder="Snapshot description (optional)"
                        value={description}
@@ -461,7 +489,7 @@ export default function FleetSnapshots() {

            {/* Loading state */}
            {loading ? (
-                <div className="rounded-xl border bg-card">
+                <div className="rounded-lg border border-card-border border-t-card-border-top bg-card text-card-foreground shadow-card-bevel">
                    <div className="p-4 space-y-3">
                        {Array.from({ length: 3 }).map((_, i) => (
                            <div key={i} className="flex items-center gap-4">
@@ -484,7 +512,7 @@ export default function FleetSnapshots() {
                </div>
            ) : (
                /* Snapshots table */
-                <div className="rounded-xl border bg-card overflow-hidden">
+                <div className="rounded-lg border border-card-border border-t-card-border-top bg-card text-card-foreground shadow-card-bevel overflow-hidden">
                    <Table>
                        <TableHeader>
                            <TableRow>
@@ -496,12 +524,12 @@ export default function FleetSnapshots() {
                            </TableRow>
                        </TableHeader>
                        <TableBody>
-                            {snapshots.map(snapshot => {
+                            {pagedSnapshots.map(snapshot => {
                                const skipped = parseSkippedNodes(snapshot.skipped_nodes);
                                const skippedNames = skipped.map(s => s.nodeName).join(', ');
                                return (
                                    <TableRow key={snapshot.id}>
-                                        <TableCell className="text-xs whitespace-nowrap">
+                                        <TableCell className="text-xs font-mono tabular-nums whitespace-nowrap">
                                            {new Date(snapshot.created_at).toLocaleString()}
                                        </TableCell>
                                        <TableCell className="text-sm max-w-[300px] truncate">
@@ -511,7 +539,7 @@ export default function FleetSnapshots() {
                                                <span className="italic text-muted-foreground">No description</span>
                                            )}
                                        </TableCell>
-                                        <TableCell className="text-xs text-muted-foreground whitespace-nowrap">
+                                        <TableCell className="text-xs font-mono tabular-nums text-muted-foreground whitespace-nowrap">
                                            {snapshot.node_count} node{snapshot.node_count !== 1 ? 's' : ''}
                                            {' · '}
                                            {snapshot.stack_count} stack{snapshot.stack_count !== 1 ? 's' : ''}
@@ -523,7 +551,7 @@ export default function FleetSnapshots() {
                                                    title={`Skipped: ${skippedNames}`}
                                                >
                                                    <AlertTriangle className="w-3.5 h-3.5" />
-                                                    <span className="text-xs">{skipped.length}</span>
+                                                    <span className="text-xs font-mono tabular-nums">{skipped.length}</span>
                                                </span>
                                            ) : (
                                                <span className="text-xs text-muted-foreground">None</span>
@@ -537,7 +565,7 @@ export default function FleetSnapshots() {
                                                    className="h-7 px-2 text-xs"
                                                    onClick={() => handleViewDetail(snapshot)}
                                                >
-                                                    <Eye className="w-3.5 h-3.5 mr-1" />
+                                                    <Eye className="w-3.5 h-3.5 mr-1" strokeWidth={1.5} />
                                                    View
                                                </Button>
                                                {isAdmin && (
@@ -546,13 +574,13 @@ export default function FleetSnapshots() {
                                                            <Button
                                                                variant="ghost"
                                                                size="sm"
-                                                                className="h-7 px-2 text-xs text-muted-foreground hover:text-red-500 hover:bg-red-500/10"
+                                                                className="h-7 px-2 text-xs text-destructive/60 hover:bg-destructive hover:text-destructive-foreground"
                                                                disabled={deletingId === snapshot.id}
                                                            >
                                                                {deletingId === snapshot.id ? (
                                                                    <Loader2 className="w-3.5 h-3.5 animate-spin" />
                                                                ) : (
-                                                                    <Trash2 className="w-3.5 h-3.5" />
+                                                                    <Trash2 className="w-3.5 h-3.5" strokeWidth={1.5} />
                                                                )}
                                                            </Button>
                                                        </AlertDialogTrigger>
@@ -598,9 +626,10 @@ function RestoreButton({ nodeId, nodeName, stackName, restoring, onRestore }: {
    onRestore: (nodeId: number, stackName: string, redeploy: boolean) => Promise<void>;
 }) {
    const [redeploy, setRedeploy] = useState(false);
+    const [open, setOpen] = useState(false);

    return (
-        <AlertDialog>
+        <AlertDialog open={open} onOpenChange={setOpen}>
            <AlertDialogTrigger asChild>
                <Button
                    variant="outline"
@@ -611,7 +640,7 @@ function RestoreButton({ nodeId, nodeName, stackName, restoring, onRestore }: {
                    {restoring ? (
                        <Loader2 className="w-3 h-3 animate-spin" />
                    ) : (
-                        <RotateCcw className="w-3 h-3" />
+                        <RotateCcw className="w-3 h-3" strokeWidth={1.5} />
                    )}
                    Restore
                </Button>
@@ -640,13 +669,19 @@ function RestoreButton({ nodeId, nodeName, stackName, restoring, onRestore }: {
                </div>
                <AlertDialogFooter>
                    <AlertDialogCancel>Cancel</AlertDialogCancel>
-                    <AlertDialogAction
+                    <Button
                        disabled={restoring}
-                        onClick={() => onRestore(nodeId, stackName, redeploy)}
+                        onClick={async () => {
+                            try {
+                                await onRestore(nodeId, stackName, redeploy);
+                            } finally {
+                                setOpen(false);
+                            }
+                        }}
                    >
                        {restoring && <Loader2 className="w-3.5 h-3.5 animate-spin mr-1.5" />}
                        Restore
-                    </AlertDialogAction>
+                    </Button>
                </AlertDialogFooter>
            </AlertDialogContent>
        </AlertDialog>