diff --git a/backend/src/__tests__/fleet-snapshot-routes.test.ts b/backend/src/__tests__/fleet-snapshot-routes.test.ts index f31509c7..78a11b26 100644 --- a/backend/src/__tests__/fleet-snapshot-routes.test.ts +++ b/backend/src/__tests__/fleet-snapshot-routes.test.ts @@ -4,14 +4,19 @@ * content-at-rest encryption round-trip (file bodies stored as ciphertext, read * back as plaintext so restore and cloud-archive paths stay portable). */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest'; import request from 'supertest'; +import fs from 'fs'; +import path from 'path'; import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; +import * as policyGate from '../helpers/policyGate'; let tmpDir: string; let app: import('express').Express; let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; let CryptoService: typeof import('../services/CryptoService').CryptoService; +let ComposeService: typeof import('../services/ComposeService').ComposeService; +let LicenseService: typeof import('../services/LicenseService').LicenseService; let adminCookie: string; let viewerCookie: string; let snapshotId: number; @@ -24,6 +29,8 @@ beforeAll(async () => { tmpDir = await setupTestDb(); ({ DatabaseService } = await import('../services/DatabaseService')); ({ CryptoService } = await import('../services/CryptoService')); + ({ ComposeService } = await import('../services/ComposeService')); + ({ LicenseService } = await import('../services/LicenseService')); ({ app } = await import('../index')); adminCookie = await loginAsTestAdmin(app); @@ -109,3 +116,231 @@ describe('Snapshot content-at-rest encryption', () => { expect(files[0].content).toBe('plain: text\n'); }); }); + +// The seeded baseline carries a default local node (id 1) whose compose_dir is +// the per-test temp dir, so a local restore writes real files without Docker. +const LOCAL_NODE_ID = 1; +const composePath = (stack: string) => path.join(process.env.COMPOSE_DIR as string, stack, 'compose.yaml'); +const envPath = (stack: string) => path.join(process.env.COMPOSE_DIR as string, stack, '.env'); +// Restore overwrites an existing stack's files; the stack directory is expected +// to already exist (it did at capture time). Seed it to mirror that precondition. +const seedStackDir = (stack: string) => fs.mkdirSync(path.join(process.env.COMPOSE_DIR as string, stack), { recursive: true }); + +describe('Single-stack snapshot restore (behavior lock)', () => { + afterEach(() => vi.restoreAllMocks()); + + it('returns 404 for a missing snapshot', async () => { + const res = await request(app) + .post('/api/fleet/snapshots/999999/restore') + .set('Cookie', adminCookie) + .send({ nodeId: LOCAL_NODE_ID, stackName: 'web' }); + expect(res.status).toBe(404); + }); + + it('returns 404 when the stack has no files in the snapshot', async () => { + const db = DatabaseService.getInstance(); + const id = db.createSnapshot('restore-nofiles', 'admin', 1, 1, '[]', '[]'); + db.insertSnapshotFiles(id, [ + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'present', filename: 'compose.yaml', content: 'services: {}\n' }, + ]); + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore`) + .set('Cookie', adminCookie) + .send({ nodeId: LOCAL_NODE_ID, stackName: 'absent' }); + expect(res.status).toBe(404); + }); + + it('returns 404 when the target node no longer exists', async () => { + const db = DatabaseService.getInstance(); + const id = db.createSnapshot('restore-deadnode', 'admin', 1, 1, '[]', '[]'); + db.insertSnapshotFiles(id, [ + { nodeId: 4242, nodeName: 'gone', stackName: 'orphan', filename: 'compose.yaml', content: 'services: {}\n' }, + ]); + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore`) + .set('Cookie', adminCookie) + .send({ nodeId: 4242, stackName: 'orphan' }); + expect(res.status).toBe(404); + }); + + it('returns 503 when a remote target node has no reachable proxy', async () => { + const db = DatabaseService.getInstance(); + const remoteId = db.addNode({ name: 'unreachable', type: 'remote', api_url: '', api_token: '', compose_dir: '/app/compose', is_default: false }); + const id = db.createSnapshot('restore-remote', 'admin', 1, 1, '[]', '[]'); + db.insertSnapshotFiles(id, [ + { nodeId: remoteId, nodeName: 'unreachable', stackName: 'svc', filename: 'compose.yaml', content: 'services: {}\n' }, + ]); + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore`) + .set('Cookie', adminCookie) + .send({ nodeId: remoteId, stackName: 'svc' }); + expect(res.status).toBe(503); + }); + + it('restores a local stack to disk, including its .env (no redeploy)', async () => { + const db = DatabaseService.getInstance(); + const id = db.createSnapshot('restore-local', 'admin', 1, 1, '[]', '[]'); + db.insertSnapshotFiles(id, [ + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'restore-web', filename: 'compose.yaml', content: 'services:\n app: {}\n' }, + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'restore-web', filename: '.env', content: 'SECRET=restored-value\n' }, + ]); + seedStackDir('restore-web'); + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore`) + .set('Cookie', adminCookie) + .send({ nodeId: LOCAL_NODE_ID, stackName: 'restore-web' }); + expect(res.status).toBe(200); + expect(res.body.redeployed).toBe(false); + expect(fs.readFileSync(composePath('restore-web'), 'utf-8')).toContain('app: {}'); + expect(fs.readFileSync(envPath('restore-web'), 'utf-8')).toContain('SECRET=restored-value'); + }); + + it('returns 409 when the deploy policy blocks the redeploy', async () => { + vi.spyOn(policyGate, 'runPolicyGate').mockImplementation(async (_req, res) => { + res.status(409).json({ error: 'Policy "block-criticals" blocked deploy' }); + return false; + }); + const db = DatabaseService.getInstance(); + const id = db.createSnapshot('restore-409', 'admin', 1, 1, '[]', '[]'); + db.insertSnapshotFiles(id, [ + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'policy-web', filename: 'compose.yaml', content: 'services: {}\n' }, + ]); + seedStackDir('policy-web'); + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore`) + .set('Cookie', adminCookie) + .send({ nodeId: LOCAL_NODE_ID, stackName: 'policy-web', redeploy: true }); + expect(res.status).toBe(409); + }); + + it('redeploys after restore when requested', async () => { + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); + const db = DatabaseService.getInstance(); + const id = db.createSnapshot('restore-redeploy', 'admin', 1, 1, '[]', '[]'); + db.insertSnapshotFiles(id, [ + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'redeploy-web', filename: 'compose.yaml', content: 'services: {}\n' }, + ]); + seedStackDir('redeploy-web'); + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore`) + .set('Cookie', adminCookie) + .send({ nodeId: LOCAL_NODE_ID, stackName: 'redeploy-web', redeploy: true }); + expect(res.status).toBe(200); + expect(res.body.redeployed).toBe(true); + expect(deploySpy).toHaveBeenCalledWith('redeploy-web'); + }); +}); + +describe('Restore-all', () => { + afterEach(() => vi.restoreAllMocks()); + + it('requires authentication', async () => { + const res = await request(app).post(`/api/fleet/snapshots/${snapshotId}/restore-all`).send({}); + expect(res.status).toBe(401); + }); + + it('returns 403 for a non-admin', async () => { + const res = await request(app).post(`/api/fleet/snapshots/${snapshotId}/restore-all`).set('Cookie', viewerCookie).send({}); + expect(res.status).toBe(403); + }); + + it('returns 404 for a missing snapshot', async () => { + const res = await request(app).post('/api/fleet/snapshots/999999/restore-all').set('Cookie', adminCookie).send({}); + expect(res.status).toBe(404); + }); + + it('returns 404 when the snapshot has no files', async () => { + const db = DatabaseService.getInstance(); + const id = db.createSnapshot('restore-all-empty', 'admin', 0, 0, '[]', '[]'); + const res = await request(app).post(`/api/fleet/snapshots/${id}/restore-all`).set('Cookie', adminCookie).send({}); + expect(res.status).toBe(404); + }); + + it('restores every stack and reports the counts', async () => { + const db = DatabaseService.getInstance(); + const id = db.createSnapshot('restore-all-ok', 'admin', 1, 2, '[]', '[]'); + db.insertSnapshotFiles(id, [ + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'all-a', filename: 'compose.yaml', content: 'services:\n a: {}\n' }, + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'all-b', filename: 'compose.yaml', content: 'services:\n b: {}\n' }, + ]); + seedStackDir('all-a'); + seedStackDir('all-b'); + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore-all`) + .set('Cookie', adminCookie) + .send({}); + expect(res.status).toBe(200); + expect(res.body.restored).toBe(2); + expect(res.body.failed).toBe(0); + expect(res.body.results).toHaveLength(2); + expect(fs.readFileSync(composePath('all-a'), 'utf-8')).toContain('a: {}'); + expect(fs.readFileSync(composePath('all-b'), 'utf-8')).toContain('b: {}'); + }); + + it('records a per-stack failure and still restores the rest', async () => { + const db = DatabaseService.getInstance(); + const id = db.createSnapshot('restore-all-partial', 'admin', 2, 2, '[]', '[]'); + db.insertSnapshotFiles(id, [ + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'good', filename: 'compose.yaml', content: 'services: {}\n' }, + { nodeId: 4242, nodeName: 'gone', stackName: 'bad', filename: 'compose.yaml', content: 'services: {}\n' }, + ]); + seedStackDir('good'); + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore-all`) + .set('Cookie', adminCookie) + .send({}); + expect(res.status).toBe(200); + expect(res.body.restored).toBe(1); + expect(res.body.failed).toBe(1); + const bad = (res.body.results as Array<{ stackName: string; success: boolean; error?: string }>).find(r => r.stackName === 'bad'); + expect(bad?.success).toBe(false); + expect(bad?.error).toMatch(/no longer exists/i); + }); + + it('redeploys each restored stack when requested', async () => { + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); + const db = DatabaseService.getInstance(); + const id = db.createSnapshot('restore-all-redeploy', 'admin', 1, 1, '[]', '[]'); + db.insertSnapshotFiles(id, [ + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'redeploy-all-web', filename: 'compose.yaml', content: 'services: {}\n' }, + ]); + seedStackDir('redeploy-all-web'); + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore-all`) + .set('Cookie', adminCookie) + .send({ redeploy: true }); + expect(res.status).toBe(200); + expect(res.body.restored).toBe(1); + expect(res.body.results[0].redeployed).toBe(true); + expect(deploySpy).toHaveBeenCalledWith('redeploy-all-web'); + }); + + it('records a policy-blocked redeploy as a per-stack failure and still restores the rest', async () => { + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); + vi.spyOn(policyGate, 'assertPolicyGateAllows').mockImplementation(async (stackName: string) => { + if (stackName === 'blocked-web') throw new Error('Policy "block-criticals" blocked deploy: 1 image(s) exceed high'); + }); + const db = DatabaseService.getInstance(); + const id = db.createSnapshot('restore-all-policy', 'admin', 1, 2, '[]', '[]'); + db.insertSnapshotFiles(id, [ + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'ok-web', filename: 'compose.yaml', content: 'services: {}\n' }, + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'blocked-web', filename: 'compose.yaml', content: 'services: {}\n' }, + ]); + seedStackDir('ok-web'); + seedStackDir('blocked-web'); + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore-all`) + .set('Cookie', adminCookie) + .send({ redeploy: true }); + expect(res.status).toBe(200); + expect(res.body.restored).toBe(1); + expect(res.body.failed).toBe(1); + const blocked = (res.body.results as Array<{ stackName: string; success: boolean; error?: string }>).find(r => r.stackName === 'blocked-web'); + expect(blocked?.success).toBe(false); + expect(blocked?.error).toMatch(/blocked deploy/i); + expect(deploySpy).toHaveBeenCalledWith('ok-web'); + expect(deploySpy).not.toHaveBeenCalledWith('blocked-web'); + }); +}); diff --git a/backend/src/routes/fleet.ts b/backend/src/routes/fleet.ts index 76ca7821..ef5d2be8 100644 --- a/backend/src/routes/fleet.ts +++ b/backend/src/routes/fleet.ts @@ -16,7 +16,7 @@ import { getSenchoVersion, isValidVersion } from '../services/CapabilityRegistry import { authMiddleware } from '../middleware/auth'; import { requirePaid, requireAdmin, requireNodeProxy } from '../middleware/tierGates'; import { scheduleLocalUpdate } from './license'; -import { runPolicyGate } from '../helpers/policyGate'; +import { runPolicyGate, assertPolicyGateAllows, buildPolicyGateOptions } from '../helpers/policyGate'; import { captureLocalNodeFiles, captureRemoteNodeFiles, type SnapshotNodeData } from '../utils/snapshot-capture'; import { getLatestVersion } from '../utils/version-check'; import { isValidStackName } from '../utils/validation'; @@ -1732,6 +1732,119 @@ fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Resp } }); +// Raised when a remote target node has no reachable proxy address. The +// single-stack restore route maps it to a 503; restore-all records it as a +// per-stack failure instead. +class SnapshotProxyTargetError extends Error { + constructor(message: string) { + super(message); + this.name = 'SnapshotProxyTargetError'; + } +} + +interface RemoteProxyContext { + baseUrl: string; + headers: Record; +} + +// Builds an error from a failed remote response so the thrown message names the +// remote node's actual reason (policy block, validation, write error) instead +// of a generic string. The body is truncated to keep the recorded message bounded. +async function remoteStackError(action: string, res: Awaited>): Promise { + let detail = ''; + try { + detail = (await res.text()).slice(0, 300).trim(); + } catch { + // Remote body unavailable; the status code alone still names the failure. + } + return new Error(`${action} on remote node (${res.status})${detail ? `: ${detail}` : ''}`); +} + +// Builds the base URL + proxy headers for a remote node, or null when the node +// has no reachable target. Tier/variant headers describe the central instance +// and stay unconditional; the Bearer header is gated on a non-empty token +// because pilot-loopback dispatch carries auth via the tunnel. +function buildRemoteProxyContext(node: Node): RemoteProxyContext | null { + const proxyTarget = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!proxyTarget) return null; + const proxyHeaders = LicenseService.getInstance().getProxyHeaders(); + const headers: Record = { + 'Content-Type': 'application/json', + [PROXY_TIER_HEADER]: proxyHeaders.tier, + [PROXY_VARIANT_HEADER]: proxyHeaders.variant ?? '', + }; + if (proxyTarget.apiToken) headers.Authorization = `Bearer ${proxyTarget.apiToken}`; + return { baseUrl: proxyTarget.apiUrl.replace(/\/$/, ''), headers }; +} + +// Writes a snapshot stack's files back to its node: local nodes write to disk +// (backing up any current files first), remote nodes receive them over the +// proxy. Throws SnapshotProxyTargetError when a remote node is unreachable, and +// an Error carrying the remote node's status and reason on a failed remote write. +async function applySnapshotStackFiles( + node: Node, + stackName: string, + files: Array<{ filename: string; content: string }>, +): Promise { + if (node.type === 'local') { + const fsService = FileSystemService.getInstance(node.id); + try { + await fsService.backupStackFiles(stackName); + } catch (e) { + // Stack may not exist yet before first restore; that is ok. + console.warn(`[Fleet Snapshot] Pre-restore backup failed for stack "${stackName}" (may not exist yet):`, getErrorMessage(e, 'unknown')); + } + for (const file of files) { + if (file.filename === 'compose.yaml') { + await fsService.saveStackContent(stackName, file.content); + } else if (file.filename === '.env') { + await fsService.saveEnvContent(stackName, file.content); + } + } + return; + } + + const ctx = buildRemoteProxyContext(node); + if (!ctx) throw new SnapshotProxyTargetError(formatNoTargetError(node)); + for (const file of files) { + if (file.filename === 'compose.yaml') { + const putRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, { + method: 'PUT', + headers: ctx.headers, + body: JSON.stringify({ content: file.content }), + signal: AbortSignal.timeout(15000), + }); + if (!putRes.ok) throw await remoteStackError('Failed to restore compose file', putRes); + } else if (file.filename === '.env') { + const putRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, { + method: 'PUT', + headers: ctx.headers, + body: JSON.stringify({ content: file.content }), + signal: AbortSignal.timeout(15000), + }); + if (!putRes.ok) throw await remoteStackError('Failed to restore env file', putRes); + } + } +} + +// Redeploys a stack after its files are restored. The deploy policy gate stays +// with the caller (local deploys gate centrally; remote deploys are gated by +// the remote node), so this only performs the deploy itself. +async function redeploySnapshotStack(node: Node, stackName: string): Promise { + if (node.type === 'local') { + await ComposeService.getInstance(node.id).deployStack(stackName); + return; + } + const ctx = buildRemoteProxyContext(node); + if (!ctx) throw new SnapshotProxyTargetError(formatNoTargetError(node)); + const deployRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}/deploy`, { + method: 'POST', + headers: ctx.headers, + signal: AbortSignal.timeout(30000), + }); + if (!deployRes.ok) throw await remoteStackError('Failed to redeploy stack', deployRes); +} + fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request, res: Response): Promise => { if (!requireAdmin(req, res)) return; @@ -1773,86 +1886,106 @@ fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request, return; } - if (node.type === 'local') { - const fsService = FileSystemService.getInstance(node.id); + await applySnapshotStackFiles(node, stackName, files); - try { - await fsService.backupStackFiles(stackName); - } catch (e) { - // Stack may not exist yet before first restore; that is ok. - console.warn(`[Fleet Snapshot] Pre-restore backup failed for stack "${stackName}" (may not exist yet):`, getErrorMessage(e, 'unknown')); - } - - for (const file of files) { - if (file.filename === 'compose.yaml') { - await fsService.saveStackContent(stackName, file.content); - } else if (file.filename === '.env') { - await fsService.saveEnvContent(stackName, file.content); - } - } - - if (redeploy) { - if (!(await runPolicyGate(req, res, stackName, node.id))) return; - const composeService = ComposeService.getInstance(node.id); - await composeService.deployStack(stackName); - } - } else { - const proxyTarget = NodeRegistry.getInstance().getProxyTarget(node.id); - if (!proxyTarget) { - res.status(503).json({ error: formatNoTargetError(node) }); - return; - } - - const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); - const proxyHeaders = LicenseService.getInstance().getProxyHeaders(); - // Tier/variant headers describe the central instance and stay - // unconditional; the Bearer header is gated on a non-empty token - // because pilot-loopback dispatch carries auth via the tunnel. - const headers: Record = { - 'Content-Type': 'application/json', - [PROXY_TIER_HEADER]: proxyHeaders.tier, - [PROXY_VARIANT_HEADER]: proxyHeaders.variant ?? '', - }; - if (proxyTarget.apiToken) headers.Authorization = `Bearer ${proxyTarget.apiToken}`; - - for (const file of files) { - if (file.filename === 'compose.yaml') { - const putRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, { - method: 'PUT', - headers, - body: JSON.stringify({ content: file.content }), - signal: AbortSignal.timeout(15000), - }); - if (!putRes.ok) throw new Error('Failed to restore compose file on remote node'); - } else if (file.filename === '.env') { - const putRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, { - method: 'PUT', - headers, - body: JSON.stringify({ content: file.content }), - signal: AbortSignal.timeout(15000), - }); - if (!putRes.ok) throw new Error('Failed to restore env file on remote node'); - } - } - - if (redeploy) { - const deployRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/deploy`, { - method: 'POST', - headers, - signal: AbortSignal.timeout(30000), - }); - if (!deployRes.ok) throw new Error('Failed to redeploy stack on remote node'); - } + if (redeploy) { + // Local deploys are gated centrally here; remote deploys are gated by the + // remote node's own deploy endpoint. + if (node.type === 'local' && !(await runPolicyGate(req, res, stackName, node.id))) return; + await redeploySnapshotStack(node, stackName); } console.log('[Fleet] Snapshot restore: snapshot=%s node=%s stack=%s', snapshotId, sanitizeForLog(nodeId), sanitizeForLog(stackName)); res.json({ message: 'Stack restored successfully', redeployed: redeploy }); } catch (error) { + if (error instanceof SnapshotProxyTargetError) { + res.status(503).json({ error: error.message }); + return; + } console.error('[Fleet Snapshot] Restore error:', error); res.status(500).json({ error: 'Failed to restore stack from snapshot' }); } }); +// One row per (node, stack) in a restore-all run. A failed row carries the +// reason in `error`; a succeeded row reports whether it was also redeployed. +interface SnapshotRestoreResult { + nodeId: number; + nodeName: string; + stackName: string; + success: boolean; + redeployed: boolean; + error?: string; +} + +fleetRouter.post('/snapshots/:id/restore-all', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requireAdmin(req, res)) return; + + try { + const snapshotId = parseIntParam(req, res, 'id', 'snapshot ID'); + if (snapshotId === null) return; + const redeploy: boolean = req.body?.redeploy === true; + + const db = DatabaseService.getInstance(); + const snapshot = db.getSnapshot(snapshotId); + if (!snapshot) { + res.status(404).json({ error: 'Snapshot not found' }); + return; + } + + const files = db.getSnapshotFiles(snapshotId); + if (files.length === 0) { + res.status(404).json({ error: 'Snapshot has no files to restore' }); + return; + } + + // Group the snapshot's files by node + stack, mirroring the detail route. + const groups = new Map }>(); + for (const file of files) { + const key = `${file.node_id}:${file.stack_name}`; + let entry = groups.get(key); + if (!entry) { + entry = { nodeId: file.node_id, nodeName: file.node_name, stackName: file.stack_name, files: [] }; + groups.set(key, entry); + } + entry.files.push({ filename: file.filename, content: file.content }); + } + + const policyOptions = buildPolicyGateOptions(req); + const results: SnapshotRestoreResult[] = []; + + // Restore every stack independently: one stack failing (unreachable node, + // blocked deploy, write error) is recorded and the rest still proceed. + for (const group of groups.values()) { + try { + if (!isValidStackName(group.stackName)) throw new Error('Invalid stack name'); + const node = db.getNode(group.nodeId); + if (!node) throw new Error('Target node no longer exists'); + + await applySnapshotStackFiles(node, group.stackName, group.files); + + let redeployed = false; + if (redeploy) { + if (node.type === 'local') await assertPolicyGateAllows(group.stackName, node.id, policyOptions); + await redeploySnapshotStack(node, group.stackName); + redeployed = true; + } + results.push({ nodeId: group.nodeId, nodeName: group.nodeName, stackName: group.stackName, success: true, redeployed }); + } catch (e) { + results.push({ nodeId: group.nodeId, nodeName: group.nodeName, stackName: group.stackName, success: false, redeployed: false, error: getErrorMessage(e, 'Restore failed') }); + } + } + + const restored = results.filter(r => r.success).length; + const failed = results.length - restored; + console.log('[Fleet] Snapshot restore-all: snapshot=%s restored=%s failed=%s redeploy=%s', snapshotId, restored, failed, redeploy); + res.json({ restored, failed, redeploy, results }); + } catch (error) { + console.error('[Fleet Snapshot] Restore-all error:', error); + res.status(500).json({ error: 'Failed to restore snapshot' }); + } +}); + fleetRouter.delete('/snapshots/:id', authMiddleware, async (req: Request, res: Response): Promise => { if (!requireAdmin(req, res)) return; diff --git a/docs/features/fleet-backups.mdx b/docs/features/fleet-backups.mdx index 0cd97e8c..af94aeba 100644 --- a/docs/features/fleet-backups.mdx +++ b/docs/features/fleet-backups.mdx @@ -50,7 +50,7 @@ Click **View** on any snapshot to open the detail view. Use **Back to Snapshots* The header shows the snapshot's title (or "Untitled Snapshot"), who created it, when, and badge counts for nodes and stacks captured. -Below the header, each node appears as a collapsible card. Expand a node to see its stacks, then expand a stack to see individual files. Each file has a **Preview** button that renders the file contents inline. +Below the header, each node appears as a collapsible card. Expand a node to see its stacks, then expand a stack to see individual files. Each file has a **Preview** button that renders the full file contents inline in a scrollable panel, and a **Download** button that saves that single file to your machine (named `-`, for example `web-compose.yaml`). Snapshot detail view with a node expanded, showing a stack expanded with a compose file, Preview button, and Restore button @@ -80,6 +80,12 @@ Sencho writes the snapshot's files back to the target node: Restoring overwrites the current compose and environment files on the target node. If atomic deployments are enabled, the current files are backed up before restoration. +### Restoring an entire snapshot + +To roll back every captured stack at once, click **Restore all** in the snapshot detail header. A confirmation dialog appears with an optional **Redeploy all stacks after restore** checkbox. On confirm, Sencho restores each stack across the fleet and reports how many succeeded. + +Each stack is restored independently: if one stack cannot be restored (for example, its node has since been removed, or a remote node is offline), that stack is reported as failed and the rest still proceed. The result is summarised as a success, a partial result naming how many failed, or a full failure. + ## Deleting snapshots Admins can delete snapshots from the list view by clicking the trash icon on the right side of each row. A confirmation dialog asks you to confirm before the snapshot is permanently removed. Deleting a snapshot removes all captured file data from the database. This action cannot be undone. @@ -155,6 +161,9 @@ Snapshots are stored in Sencho's SQLite database. Captured file contents, includ The stack was not captured in the snapshot, usually because its compose file was missing or unreadable on disk at the time the snapshot was taken. Open the snapshot's detail view to verify which stacks and files are available, and pick a different snapshot if the one you have is incomplete. + + Restore all applies each stack independently, so a failure on one stack does not stop the others. A stack is reported as failed when its node has been removed from the fleet since the snapshot was taken, when a remote node is offline or unreachable, or when its existing files cannot be written. The successful stacks are fully restored regardless. Resolve the underlying cause (re-add a removed node, bring an offline node back online) and run Restore all again, or restore the remaining stacks individually from the same snapshot. + Double-check the Access Key ID, Secret Access Key, and bucket name; one wrong character is the most common cause. Some providers require S3-compatible API access to be enabled on the bucket separately from the credentials. For MinIO, confirm the user has read/write permission on the target bucket. After correcting the values, click **Test** again before saving. diff --git a/frontend/src/components/FleetSnapshots.tsx b/frontend/src/components/FleetSnapshots.tsx index 271bcc06..0cbf33b0 100644 --- a/frontend/src/components/FleetSnapshots.tsx +++ b/frontend/src/components/FleetSnapshots.tsx @@ -2,7 +2,7 @@ import { useState, useEffect, useCallback } from 'react'; import { Camera, ArrowLeft, Server, Layers, FileText, AlertTriangle, Trash2, Eye, ChevronDown, ChevronLeft, ChevronRight, Plus, Loader2, RotateCcw, - Cloud, CloudUpload, + Cloud, CloudUpload, Download, } from 'lucide-react'; import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; @@ -14,7 +14,6 @@ import { Skeleton } from '@/components/ui/skeleton'; import { Checkbox } from '@/components/ui/checkbox'; import { Label } from '@/components/ui/label'; import { ConfirmModal } from '@/components/ui/modal'; -import { ScrollArea } from '@/components/ui/scroll-area'; import { apiFetch } from '@/lib/api'; import { useAuth } from '@/context/AuthContext'; import { useLicense } from '@/context/LicenseContext'; @@ -93,6 +92,7 @@ export default function FleetSnapshots() { const [expandedStacks, setExpandedStacks] = useState>(new Set()); const [previewFiles, setPreviewFiles] = useState>(new Set()); const [restoringStack, setRestoringStack] = useState(null); + const [restoringAll, setRestoringAll] = useState(false); const [deletingId, setDeletingId] = useState(null); const [confirmDeleteId, setConfirmDeleteId] = useState(null); const [page, setPage] = useState(0); @@ -273,6 +273,65 @@ export default function FleetSnapshots() { } }; + const handleDownloadFile = (stackName: string, file: SnapshotStackFile) => { + try { + const blob = new Blob([file.content], { type: 'text/plain;charset=utf-8' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `${stackName}-${file.filename}`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + setTimeout(() => URL.revokeObjectURL(url), 100); + } catch (error: unknown) { + const err = error as Record | null; + toast.error((err?.message as string) || 'Download failed.'); + } + }; + + const handleRestoreAll = async (redeploy: boolean) => { + if (!selectedSnapshot) return; + setRestoringAll(true); + try { + const res = await apiFetch(`/fleet/snapshots/${selectedSnapshot.id}/restore-all`, { + method: 'POST', + localOnly: true, + body: JSON.stringify({ redeploy }), + }); + if (res.ok) { + const data: { + restored: number; + failed: number; + redeploy: boolean; + results: Array<{ stackName: string; success: boolean; error?: string }>; + } = await res.json(); + const noun = (n: number) => `${n} stack${n === 1 ? '' : 's'}`; + const firstFailed = data.results?.find(r => !r.success); + const failDetail = firstFailed + ? ` First failure: ${firstFailed.stackName} · ${firstFailed.error || 'unknown error'}` + : ''; + if (data.failed === 0) { + toast.success(data.redeploy + ? `Restored and redeployed ${noun(data.restored)}.` + : `Restored ${noun(data.restored)}.`); + } else if (data.restored === 0) { + toast.error(`Restore failed for ${noun(data.failed)}.${failDetail}`); + } else { + toast.warning(`${data.restored} restored, ${data.failed} failed.${failDetail}`); + } + } else { + const err = await res.json().catch(() => null); + toast.error(err?.message || err?.error || err?.data?.error || 'Failed to restore snapshot.'); + } + } catch (error: unknown) { + const err = error as Record | null; + toast.error(err?.message as string || err?.error as string || 'Something went wrong.'); + } finally { + setRestoringAll(false); + } + }; + // --- Toggle helpers --- const toggleNode = (nodeId: number) => { @@ -342,13 +401,20 @@ export default function FleetSnapshots() { <> {/* Header card */}
-

- {selectedSnapshot.description || 'Untitled Snapshot'} -

-

- Created by {selectedSnapshot.created_by} on{' '} - {new Date(selectedSnapshot.created_at).toLocaleString()} -

+
+
+

+ {selectedSnapshot.description || 'Untitled Snapshot'} +

+

+ Created by {selectedSnapshot.created_by} on{' '} + {new Date(selectedSnapshot.created_at).toLocaleString()} +

+
+ {isAdmin && selectedSnapshot.nodes.length > 0 && ( + + )} +
{selectedSnapshot.node_count} node{selectedSnapshot.node_count !== 1 ? 's' : ''} @@ -478,13 +544,22 @@ export default function FleetSnapshots() { {showPreview ? 'Hide' : 'Preview'} +
{showPreview && ( - + 
                                                                                             {file.content}
- +
)}
); @@ -799,3 +874,62 @@ function RestoreButton({ nodeId, nodeName, stackName, restoring, onRestore }: { ); } + +// --- Restore All Button Sub-Component --- + +function RestoreAllButton({ restoring, onRestoreAll }: { + restoring: boolean; + onRestoreAll: (redeploy: boolean) => Promise; +}) { + const [redeploy, setRedeploy] = useState(false); + const [open, setOpen] = useState(false); + + return ( + <> + + { + try { + await onRestoreAll(redeploy); + } finally { + setOpen(false); + } + }} + > +

+ Overwrites the current compose and environment files for every stack on every node in this snapshot. +

+
+ setRedeploy(checked === true)} + /> + +
+ + + ); +}