mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-17 22:17:50 +00:00
feat(fleet-snapshots): add restore-all, per-file download, and scrollable preview (#1276)
Three improvements to the Fleet Snapshots detail view, matching the admin-only access of the existing per-stack restore: - Restore all: a control in the snapshot header restores every captured stack across the fleet in one action, with an optional "redeploy all after restore" checkbox. Each stack is restored independently, so a failure on one (removed node, offline remote, blocked deploy) is reported per stack while the rest still proceed. Backed by POST /api/fleet/snapshots/:id/restore-all. - Per-file download: each compose or .env file in a snapshot can be saved to disk individually from its row. - Scrollable preview: the inline file preview is now a bounded, scrollable panel, so a long compose file can be read in full instead of being clipped. Remote restore and redeploy failures now carry the remote node's status and reason, so a per-stack failure in Restore all is actionable.
This commit is contained in:
@@ -4,14 +4,19 @@
|
||||
* content-at-rest encryption round-trip (file bodies stored as ciphertext, read
|
||||
* back as plaintext so restore and cloud-archive paths stay portable).
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
|
||||
import * as policyGate from '../helpers/policyGate';
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
let CryptoService: typeof import('../services/CryptoService').CryptoService;
|
||||
let ComposeService: typeof import('../services/ComposeService').ComposeService;
|
||||
let LicenseService: typeof import('../services/LicenseService').LicenseService;
|
||||
let adminCookie: string;
|
||||
let viewerCookie: string;
|
||||
let snapshotId: number;
|
||||
@@ -24,6 +29,8 @@ beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
({ CryptoService } = await import('../services/CryptoService'));
|
||||
({ ComposeService } = await import('../services/ComposeService'));
|
||||
({ LicenseService } = await import('../services/LicenseService'));
|
||||
({ app } = await import('../index'));
|
||||
adminCookie = await loginAsTestAdmin(app);
|
||||
|
||||
@@ -109,3 +116,231 @@ describe('Snapshot content-at-rest encryption', () => {
|
||||
expect(files[0].content).toBe('plain: text\n');
|
||||
});
|
||||
});
|
||||
|
||||
// The seeded baseline carries a default local node (id 1) whose compose_dir is
|
||||
// the per-test temp dir, so a local restore writes real files without Docker.
|
||||
const LOCAL_NODE_ID = 1;
|
||||
const composePath = (stack: string) => path.join(process.env.COMPOSE_DIR as string, stack, 'compose.yaml');
|
||||
const envPath = (stack: string) => path.join(process.env.COMPOSE_DIR as string, stack, '.env');
|
||||
// Restore overwrites an existing stack's files; the stack directory is expected
|
||||
// to already exist (it did at capture time). Seed it to mirror that precondition.
|
||||
const seedStackDir = (stack: string) => fs.mkdirSync(path.join(process.env.COMPOSE_DIR as string, stack), { recursive: true });
|
||||
|
||||
describe('Single-stack snapshot restore (behavior lock)', () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it('returns 404 for a missing snapshot', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/snapshots/999999/restore')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: LOCAL_NODE_ID, stackName: 'web' });
|
||||
expect(res.status).toBe(404);
|
||||
});
|
||||
|
||||
it('returns 404 when the stack has no files in the snapshot', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-nofiles', 'admin', 1, 1, '[]', '[]');
|
||||
db.insertSnapshotFiles(id, [
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'present', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
]);
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: LOCAL_NODE_ID, stackName: 'absent' });
|
||||
expect(res.status).toBe(404);
|
||||
});
|
||||
|
||||
it('returns 404 when the target node no longer exists', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-deadnode', 'admin', 1, 1, '[]', '[]');
|
||||
db.insertSnapshotFiles(id, [
|
||||
{ nodeId: 4242, nodeName: 'gone', stackName: 'orphan', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
]);
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: 4242, stackName: 'orphan' });
|
||||
expect(res.status).toBe(404);
|
||||
});
|
||||
|
||||
it('returns 503 when a remote target node has no reachable proxy', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const remoteId = db.addNode({ name: 'unreachable', type: 'remote', api_url: '', api_token: '', compose_dir: '/app/compose', is_default: false });
|
||||
const id = db.createSnapshot('restore-remote', 'admin', 1, 1, '[]', '[]');
|
||||
db.insertSnapshotFiles(id, [
|
||||
{ nodeId: remoteId, nodeName: 'unreachable', stackName: 'svc', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
]);
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: remoteId, stackName: 'svc' });
|
||||
expect(res.status).toBe(503);
|
||||
});
|
||||
|
||||
it('restores a local stack to disk, including its .env (no redeploy)', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-local', 'admin', 1, 1, '[]', '[]');
|
||||
db.insertSnapshotFiles(id, [
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'restore-web', filename: 'compose.yaml', content: 'services:\n app: {}\n' },
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'restore-web', filename: '.env', content: 'SECRET=restored-value\n' },
|
||||
]);
|
||||
seedStackDir('restore-web');
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: LOCAL_NODE_ID, stackName: 'restore-web' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.redeployed).toBe(false);
|
||||
expect(fs.readFileSync(composePath('restore-web'), 'utf-8')).toContain('app: {}');
|
||||
expect(fs.readFileSync(envPath('restore-web'), 'utf-8')).toContain('SECRET=restored-value');
|
||||
});
|
||||
|
||||
it('returns 409 when the deploy policy blocks the redeploy', async () => {
|
||||
vi.spyOn(policyGate, 'runPolicyGate').mockImplementation(async (_req, res) => {
|
||||
res.status(409).json({ error: 'Policy "block-criticals" blocked deploy' });
|
||||
return false;
|
||||
});
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-409', 'admin', 1, 1, '[]', '[]');
|
||||
db.insertSnapshotFiles(id, [
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'policy-web', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
]);
|
||||
seedStackDir('policy-web');
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: LOCAL_NODE_ID, stackName: 'policy-web', redeploy: true });
|
||||
expect(res.status).toBe(409);
|
||||
});
|
||||
|
||||
it('redeploys after restore when requested', async () => {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-redeploy', 'admin', 1, 1, '[]', '[]');
|
||||
db.insertSnapshotFiles(id, [
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'redeploy-web', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
]);
|
||||
seedStackDir('redeploy-web');
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: LOCAL_NODE_ID, stackName: 'redeploy-web', redeploy: true });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.redeployed).toBe(true);
|
||||
expect(deploySpy).toHaveBeenCalledWith('redeploy-web');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Restore-all', () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it('requires authentication', async () => {
|
||||
const res = await request(app).post(`/api/fleet/snapshots/${snapshotId}/restore-all`).send({});
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it('returns 403 for a non-admin', async () => {
|
||||
const res = await request(app).post(`/api/fleet/snapshots/${snapshotId}/restore-all`).set('Cookie', viewerCookie).send({});
|
||||
expect(res.status).toBe(403);
|
||||
});
|
||||
|
||||
it('returns 404 for a missing snapshot', async () => {
|
||||
const res = await request(app).post('/api/fleet/snapshots/999999/restore-all').set('Cookie', adminCookie).send({});
|
||||
expect(res.status).toBe(404);
|
||||
});
|
||||
|
||||
it('returns 404 when the snapshot has no files', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-all-empty', 'admin', 0, 0, '[]', '[]');
|
||||
const res = await request(app).post(`/api/fleet/snapshots/${id}/restore-all`).set('Cookie', adminCookie).send({});
|
||||
expect(res.status).toBe(404);
|
||||
});
|
||||
|
||||
it('restores every stack and reports the counts', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-all-ok', 'admin', 1, 2, '[]', '[]');
|
||||
db.insertSnapshotFiles(id, [
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'all-a', filename: 'compose.yaml', content: 'services:\n a: {}\n' },
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'all-b', filename: 'compose.yaml', content: 'services:\n b: {}\n' },
|
||||
]);
|
||||
seedStackDir('all-a');
|
||||
seedStackDir('all-b');
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore-all`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({});
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.restored).toBe(2);
|
||||
expect(res.body.failed).toBe(0);
|
||||
expect(res.body.results).toHaveLength(2);
|
||||
expect(fs.readFileSync(composePath('all-a'), 'utf-8')).toContain('a: {}');
|
||||
expect(fs.readFileSync(composePath('all-b'), 'utf-8')).toContain('b: {}');
|
||||
});
|
||||
|
||||
it('records a per-stack failure and still restores the rest', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-all-partial', 'admin', 2, 2, '[]', '[]');
|
||||
db.insertSnapshotFiles(id, [
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'good', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
{ nodeId: 4242, nodeName: 'gone', stackName: 'bad', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
]);
|
||||
seedStackDir('good');
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore-all`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({});
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.restored).toBe(1);
|
||||
expect(res.body.failed).toBe(1);
|
||||
const bad = (res.body.results as Array<{ stackName: string; success: boolean; error?: string }>).find(r => r.stackName === 'bad');
|
||||
expect(bad?.success).toBe(false);
|
||||
expect(bad?.error).toMatch(/no longer exists/i);
|
||||
});
|
||||
|
||||
it('redeploys each restored stack when requested', async () => {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-all-redeploy', 'admin', 1, 1, '[]', '[]');
|
||||
db.insertSnapshotFiles(id, [
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'redeploy-all-web', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
]);
|
||||
seedStackDir('redeploy-all-web');
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore-all`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ redeploy: true });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.restored).toBe(1);
|
||||
expect(res.body.results[0].redeployed).toBe(true);
|
||||
expect(deploySpy).toHaveBeenCalledWith('redeploy-all-web');
|
||||
});
|
||||
|
||||
it('records a policy-blocked redeploy as a per-stack failure and still restores the rest', async () => {
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
vi.spyOn(policyGate, 'assertPolicyGateAllows').mockImplementation(async (stackName: string) => {
|
||||
if (stackName === 'blocked-web') throw new Error('Policy "block-criticals" blocked deploy: 1 image(s) exceed high');
|
||||
});
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-all-policy', 'admin', 1, 2, '[]', '[]');
|
||||
db.insertSnapshotFiles(id, [
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'ok-web', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'blocked-web', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
]);
|
||||
seedStackDir('ok-web');
|
||||
seedStackDir('blocked-web');
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore-all`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ redeploy: true });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.restored).toBe(1);
|
||||
expect(res.body.failed).toBe(1);
|
||||
const blocked = (res.body.results as Array<{ stackName: string; success: boolean; error?: string }>).find(r => r.stackName === 'blocked-web');
|
||||
expect(blocked?.success).toBe(false);
|
||||
expect(blocked?.error).toMatch(/blocked deploy/i);
|
||||
expect(deploySpy).toHaveBeenCalledWith('ok-web');
|
||||
expect(deploySpy).not.toHaveBeenCalledWith('blocked-web');
|
||||
});
|
||||
});
|
||||
|
||||
+203
-70
@@ -16,7 +16,7 @@ import { getSenchoVersion, isValidVersion } from '../services/CapabilityRegistry
|
||||
import { authMiddleware } from '../middleware/auth';
|
||||
import { requirePaid, requireAdmin, requireNodeProxy } from '../middleware/tierGates';
|
||||
import { scheduleLocalUpdate } from './license';
|
||||
import { runPolicyGate } from '../helpers/policyGate';
|
||||
import { runPolicyGate, assertPolicyGateAllows, buildPolicyGateOptions } from '../helpers/policyGate';
|
||||
import { captureLocalNodeFiles, captureRemoteNodeFiles, type SnapshotNodeData } from '../utils/snapshot-capture';
|
||||
import { getLatestVersion } from '../utils/version-check';
|
||||
import { isValidStackName } from '../utils/validation';
|
||||
@@ -1732,6 +1732,119 @@ fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Resp
|
||||
}
|
||||
});
|
||||
|
||||
// Raised when a remote target node has no reachable proxy address. The
|
||||
// single-stack restore route maps it to a 503; restore-all records it as a
|
||||
// per-stack failure instead.
|
||||
class SnapshotProxyTargetError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'SnapshotProxyTargetError';
|
||||
}
|
||||
}
|
||||
|
||||
interface RemoteProxyContext {
|
||||
baseUrl: string;
|
||||
headers: Record<string, string>;
|
||||
}
|
||||
|
||||
// Builds an error from a failed remote response so the thrown message names the
|
||||
// remote node's actual reason (policy block, validation, write error) instead
|
||||
// of a generic string. The body is truncated to keep the recorded message bounded.
|
||||
async function remoteStackError(action: string, res: Awaited<ReturnType<typeof fetch>>): Promise<Error> {
|
||||
let detail = '';
|
||||
try {
|
||||
detail = (await res.text()).slice(0, 300).trim();
|
||||
} catch {
|
||||
// Remote body unavailable; the status code alone still names the failure.
|
||||
}
|
||||
return new Error(`${action} on remote node (${res.status})${detail ? `: ${detail}` : ''}`);
|
||||
}
|
||||
|
||||
// Builds the base URL + proxy headers for a remote node, or null when the node
|
||||
// has no reachable target. Tier/variant headers describe the central instance
|
||||
// and stay unconditional; the Bearer header is gated on a non-empty token
|
||||
// because pilot-loopback dispatch carries auth via the tunnel.
|
||||
function buildRemoteProxyContext(node: Node): RemoteProxyContext | null {
|
||||
const proxyTarget = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!proxyTarget) return null;
|
||||
const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
[PROXY_TIER_HEADER]: proxyHeaders.tier,
|
||||
[PROXY_VARIANT_HEADER]: proxyHeaders.variant ?? '',
|
||||
};
|
||||
if (proxyTarget.apiToken) headers.Authorization = `Bearer ${proxyTarget.apiToken}`;
|
||||
return { baseUrl: proxyTarget.apiUrl.replace(/\/$/, ''), headers };
|
||||
}
|
||||
|
||||
// Writes a snapshot stack's files back to its node: local nodes write to disk
|
||||
// (backing up any current files first), remote nodes receive them over the
|
||||
// proxy. Throws SnapshotProxyTargetError when a remote node is unreachable, and
|
||||
// an Error carrying the remote node's status and reason on a failed remote write.
|
||||
async function applySnapshotStackFiles(
|
||||
node: Node,
|
||||
stackName: string,
|
||||
files: Array<{ filename: string; content: string }>,
|
||||
): Promise<void> {
|
||||
if (node.type === 'local') {
|
||||
const fsService = FileSystemService.getInstance(node.id);
|
||||
try {
|
||||
await fsService.backupStackFiles(stackName);
|
||||
} catch (e) {
|
||||
// Stack may not exist yet before first restore; that is ok.
|
||||
console.warn(`[Fleet Snapshot] Pre-restore backup failed for stack "${stackName}" (may not exist yet):`, getErrorMessage(e, 'unknown'));
|
||||
}
|
||||
for (const file of files) {
|
||||
if (file.filename === 'compose.yaml') {
|
||||
await fsService.saveStackContent(stackName, file.content);
|
||||
} else if (file.filename === '.env') {
|
||||
await fsService.saveEnvContent(stackName, file.content);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const ctx = buildRemoteProxyContext(node);
|
||||
if (!ctx) throw new SnapshotProxyTargetError(formatNoTargetError(node));
|
||||
for (const file of files) {
|
||||
if (file.filename === 'compose.yaml') {
|
||||
const putRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, {
|
||||
method: 'PUT',
|
||||
headers: ctx.headers,
|
||||
body: JSON.stringify({ content: file.content }),
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (!putRes.ok) throw await remoteStackError('Failed to restore compose file', putRes);
|
||||
} else if (file.filename === '.env') {
|
||||
const putRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, {
|
||||
method: 'PUT',
|
||||
headers: ctx.headers,
|
||||
body: JSON.stringify({ content: file.content }),
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (!putRes.ok) throw await remoteStackError('Failed to restore env file', putRes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Redeploys a stack after its files are restored. The deploy policy gate stays
|
||||
// with the caller (local deploys gate centrally; remote deploys are gated by
|
||||
// the remote node), so this only performs the deploy itself.
|
||||
async function redeploySnapshotStack(node: Node, stackName: string): Promise<void> {
|
||||
if (node.type === 'local') {
|
||||
await ComposeService.getInstance(node.id).deployStack(stackName);
|
||||
return;
|
||||
}
|
||||
const ctx = buildRemoteProxyContext(node);
|
||||
if (!ctx) throw new SnapshotProxyTargetError(formatNoTargetError(node));
|
||||
const deployRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}/deploy`, {
|
||||
method: 'POST',
|
||||
headers: ctx.headers,
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
if (!deployRes.ok) throw await remoteStackError('Failed to redeploy stack', deployRes);
|
||||
}
|
||||
|
||||
fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
|
||||
@@ -1773,86 +1886,106 @@ fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request,
|
||||
return;
|
||||
}
|
||||
|
||||
if (node.type === 'local') {
|
||||
const fsService = FileSystemService.getInstance(node.id);
|
||||
await applySnapshotStackFiles(node, stackName, files);
|
||||
|
||||
try {
|
||||
await fsService.backupStackFiles(stackName);
|
||||
} catch (e) {
|
||||
// Stack may not exist yet before first restore; that is ok.
|
||||
console.warn(`[Fleet Snapshot] Pre-restore backup failed for stack "${stackName}" (may not exist yet):`, getErrorMessage(e, 'unknown'));
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
if (file.filename === 'compose.yaml') {
|
||||
await fsService.saveStackContent(stackName, file.content);
|
||||
} else if (file.filename === '.env') {
|
||||
await fsService.saveEnvContent(stackName, file.content);
|
||||
}
|
||||
}
|
||||
|
||||
if (redeploy) {
|
||||
if (!(await runPolicyGate(req, res, stackName, node.id))) return;
|
||||
const composeService = ComposeService.getInstance(node.id);
|
||||
await composeService.deployStack(stackName);
|
||||
}
|
||||
} else {
|
||||
const proxyTarget = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!proxyTarget) {
|
||||
res.status(503).json({ error: formatNoTargetError(node) });
|
||||
return;
|
||||
}
|
||||
|
||||
const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
|
||||
const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
|
||||
// Tier/variant headers describe the central instance and stay
|
||||
// unconditional; the Bearer header is gated on a non-empty token
|
||||
// because pilot-loopback dispatch carries auth via the tunnel.
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
[PROXY_TIER_HEADER]: proxyHeaders.tier,
|
||||
[PROXY_VARIANT_HEADER]: proxyHeaders.variant ?? '',
|
||||
};
|
||||
if (proxyTarget.apiToken) headers.Authorization = `Bearer ${proxyTarget.apiToken}`;
|
||||
|
||||
for (const file of files) {
|
||||
if (file.filename === 'compose.yaml') {
|
||||
const putRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, {
|
||||
method: 'PUT',
|
||||
headers,
|
||||
body: JSON.stringify({ content: file.content }),
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (!putRes.ok) throw new Error('Failed to restore compose file on remote node');
|
||||
} else if (file.filename === '.env') {
|
||||
const putRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, {
|
||||
method: 'PUT',
|
||||
headers,
|
||||
body: JSON.stringify({ content: file.content }),
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (!putRes.ok) throw new Error('Failed to restore env file on remote node');
|
||||
}
|
||||
}
|
||||
|
||||
if (redeploy) {
|
||||
const deployRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/deploy`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
if (!deployRes.ok) throw new Error('Failed to redeploy stack on remote node');
|
||||
}
|
||||
if (redeploy) {
|
||||
// Local deploys are gated centrally here; remote deploys are gated by the
|
||||
// remote node's own deploy endpoint.
|
||||
if (node.type === 'local' && !(await runPolicyGate(req, res, stackName, node.id))) return;
|
||||
await redeploySnapshotStack(node, stackName);
|
||||
}
|
||||
|
||||
console.log('[Fleet] Snapshot restore: snapshot=%s node=%s stack=%s', snapshotId, sanitizeForLog(nodeId), sanitizeForLog(stackName));
|
||||
res.json({ message: 'Stack restored successfully', redeployed: redeploy });
|
||||
} catch (error) {
|
||||
if (error instanceof SnapshotProxyTargetError) {
|
||||
res.status(503).json({ error: error.message });
|
||||
return;
|
||||
}
|
||||
console.error('[Fleet Snapshot] Restore error:', error);
|
||||
res.status(500).json({ error: 'Failed to restore stack from snapshot' });
|
||||
}
|
||||
});
|
||||
|
||||
// One row per (node, stack) in a restore-all run. A failed row carries the
|
||||
// reason in `error`; a succeeded row reports whether it was also redeployed.
|
||||
interface SnapshotRestoreResult {
|
||||
nodeId: number;
|
||||
nodeName: string;
|
||||
stackName: string;
|
||||
success: boolean;
|
||||
redeployed: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
fleetRouter.post('/snapshots/:id/restore-all', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
|
||||
try {
|
||||
const snapshotId = parseIntParam(req, res, 'id', 'snapshot ID');
|
||||
if (snapshotId === null) return;
|
||||
const redeploy: boolean = req.body?.redeploy === true;
|
||||
|
||||
const db = DatabaseService.getInstance();
|
||||
const snapshot = db.getSnapshot(snapshotId);
|
||||
if (!snapshot) {
|
||||
res.status(404).json({ error: 'Snapshot not found' });
|
||||
return;
|
||||
}
|
||||
|
||||
const files = db.getSnapshotFiles(snapshotId);
|
||||
if (files.length === 0) {
|
||||
res.status(404).json({ error: 'Snapshot has no files to restore' });
|
||||
return;
|
||||
}
|
||||
|
||||
// Group the snapshot's files by node + stack, mirroring the detail route.
|
||||
const groups = new Map<string, { nodeId: number; nodeName: string; stackName: string; files: Array<{ filename: string; content: string }> }>();
|
||||
for (const file of files) {
|
||||
const key = `${file.node_id}:${file.stack_name}`;
|
||||
let entry = groups.get(key);
|
||||
if (!entry) {
|
||||
entry = { nodeId: file.node_id, nodeName: file.node_name, stackName: file.stack_name, files: [] };
|
||||
groups.set(key, entry);
|
||||
}
|
||||
entry.files.push({ filename: file.filename, content: file.content });
|
||||
}
|
||||
|
||||
const policyOptions = buildPolicyGateOptions(req);
|
||||
const results: SnapshotRestoreResult[] = [];
|
||||
|
||||
// Restore every stack independently: one stack failing (unreachable node,
|
||||
// blocked deploy, write error) is recorded and the rest still proceed.
|
||||
for (const group of groups.values()) {
|
||||
try {
|
||||
if (!isValidStackName(group.stackName)) throw new Error('Invalid stack name');
|
||||
const node = db.getNode(group.nodeId);
|
||||
if (!node) throw new Error('Target node no longer exists');
|
||||
|
||||
await applySnapshotStackFiles(node, group.stackName, group.files);
|
||||
|
||||
let redeployed = false;
|
||||
if (redeploy) {
|
||||
if (node.type === 'local') await assertPolicyGateAllows(group.stackName, node.id, policyOptions);
|
||||
await redeploySnapshotStack(node, group.stackName);
|
||||
redeployed = true;
|
||||
}
|
||||
results.push({ nodeId: group.nodeId, nodeName: group.nodeName, stackName: group.stackName, success: true, redeployed });
|
||||
} catch (e) {
|
||||
results.push({ nodeId: group.nodeId, nodeName: group.nodeName, stackName: group.stackName, success: false, redeployed: false, error: getErrorMessage(e, 'Restore failed') });
|
||||
}
|
||||
}
|
||||
|
||||
const restored = results.filter(r => r.success).length;
|
||||
const failed = results.length - restored;
|
||||
console.log('[Fleet] Snapshot restore-all: snapshot=%s restored=%s failed=%s redeploy=%s', snapshotId, restored, failed, redeploy);
|
||||
res.json({ restored, failed, redeploy, results });
|
||||
} catch (error) {
|
||||
console.error('[Fleet Snapshot] Restore-all error:', error);
|
||||
res.status(500).json({ error: 'Failed to restore snapshot' });
|
||||
}
|
||||
});
|
||||
|
||||
fleetRouter.delete('/snapshots/:id', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user