mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-07-26 11:49:16 +00:00
fix(fleet-snapshots): gate reads on admin role and encrypt content at rest (#1273)
* fix(fleet-snapshots): gate reads on admin role and encrypt content at rest Fleet snapshots capture every node's compose.yaml and .env, so the data is as sensitive as the live stacks. This hardens access and reliability across the snapshot pipeline. - Restrict snapshot reads to administrators. GET /api/fleet/snapshots and /:id now require the admin role, matching create, restore, and delete; the Fleet "Snapshots" tab and its panel render only for admins. Previously any authenticated user could enumerate snapshots and read every node's .env. - Encrypt snapshot file contents at rest with the instance key. Restore and cloud-archive paths decrypt on read, so cloud archives stay portable and a database copy no longer exposes stack secrets in plaintext. Rows written before this change still read back as plaintext. - Surface partial captures. A stack whose compose file cannot be read or fetched, or a file over the 1 MB capture cap, is recorded as a warning and shown on the snapshot instead of being silently dropped, so a snapshot is never mistaken for complete. Remote .env read errors are now distinguished from a genuinely absent .env. Adds route-authz, capture-warning, and encryption round-trip tests; updates the Fleet-Wide Backups feature docs. * fix(fleet-snapshots): gate cloud snapshot reads on admin role The cloud snapshot read routes were guarded by provider/license only, not by role, while their write counterparts (upload, delete) already required admin and the Cloud Backup settings surface is admin-only. Because a downloaded archive contains plaintext compose and .env files, a non-admin could list and download cloud snapshots and read every node's secrets, the same exposure the local snapshot reads were just closed against. - Require admin on GET /api/cloud-backup/snapshots, /status/:id, and /object/:keyB64/download, matching the local snapshot reads and the admin-only Cloud Backup settings section. - When capturing a remote node, treat a 200 response carrying X-Env-Exists: false as a stack with no .env (matching the local ENOENT path) instead of storing an empty .env that restore would later write back. Adds non-admin authorization tests for the cloud read routes and a remote absent-.env capture test.
This commit is contained in:
@@ -122,7 +122,10 @@ describe('POST /api/blueprints/:id/withdraw/:nodeId', () => {
|
||||
expect(fileRows).toHaveLength(1);
|
||||
expect(fileRows[0].stack_name).toBe(bp.name);
|
||||
expect(fileRows[0].filename).toBe('docker-compose.yml');
|
||||
expect(fileRows[0].content).toBe(compose);
|
||||
// Content is encrypted at rest; it decrypts back to the captured compose.
|
||||
const { CryptoService } = await import('../services/CryptoService');
|
||||
expect(CryptoService.getInstance().isEncrypted(fileRows[0].content)).toBe(true);
|
||||
expect(CryptoService.getInstance().decrypt(fileRows[0].content)).toBe(compose);
|
||||
expect(fileRows[0].node_id).toBe(node.id);
|
||||
});
|
||||
|
||||
|
||||
@@ -201,6 +201,47 @@ describe('Cloud backup tier gating', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('Cloud backup read routes are admin-only', () => {
|
||||
// The archive download returns plaintext compose/.env, so listing and
|
||||
// downloading cloud snapshots must be admin-gated like the local snapshot
|
||||
// reads, regardless of provider/tier.
|
||||
let viewerCookie: string;
|
||||
const keyB64 = Buffer.from('sencho/instances/x/snapshots/1.tar.gz').toString('base64url');
|
||||
|
||||
beforeAll(async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const bcrypt = (await import('bcrypt')).default;
|
||||
const hash = await bcrypt.hash('cloud-viewer-pass', 1);
|
||||
try { db.addUser({ username: 'cloud-viewer', password_hash: hash, role: 'viewer' }); } catch { /* may already exist */ }
|
||||
const login = await request(app).post('/api/auth/login').send({ username: 'cloud-viewer', password: 'cloud-viewer-pass' });
|
||||
const cookies = login.headers['set-cookie'] as string | string[];
|
||||
viewerCookie = Array.isArray(cookies) ? cookies[0] : cookies;
|
||||
});
|
||||
|
||||
it('GET /snapshots returns 403 for a non-admin', async () => {
|
||||
const res = await request(app).get('/api/cloud-backup/snapshots').set('Cookie', viewerCookie);
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.body.code).toBe('ADMIN_REQUIRED');
|
||||
});
|
||||
|
||||
it('GET /status/:id returns 403 for a non-admin', async () => {
|
||||
const res = await request(app).get('/api/cloud-backup/status/1').set('Cookie', viewerCookie);
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.body.code).toBe('ADMIN_REQUIRED');
|
||||
});
|
||||
|
||||
it('GET /object/:keyB64/download returns 403 for a non-admin', async () => {
|
||||
const res = await request(app).get(`/api/cloud-backup/object/${keyB64}/download`).set('Cookie', viewerCookie);
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.body.code).toBe('ADMIN_REQUIRED');
|
||||
});
|
||||
|
||||
it('does not block an admin on GET /snapshots', async () => {
|
||||
const res = await request(app).get('/api/cloud-backup/snapshots').set('Cookie', authCookie);
|
||||
expect(res.status).not.toBe(403);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Cloud backup config CRUD', () => {
|
||||
it('redacts secret_key on read; persists encrypted ciphertext', async () => {
|
||||
const putRes = await request(app)
|
||||
|
||||
@@ -192,8 +192,12 @@ describe('CloudBackupService — uploadSnapshot', () => {
|
||||
expect(parsed.instance_id).toBe('test-instance-id');
|
||||
expect(parsed.archive_version).toBe(1);
|
||||
|
||||
expect(entries.find(e => e.name === 'nodes/1_gateway/web/compose.yaml')).toBeDefined();
|
||||
expect(entries.find(e => e.name === 'nodes/1_gateway/web/.env')).toBeDefined();
|
||||
// Content is encrypted at rest but the archive must carry plaintext so a
|
||||
// downloaded snapshot restores on any instance (portability contract).
|
||||
const composeEntry = entries.find(e => e.name === 'nodes/1_gateway/web/compose.yaml');
|
||||
expect(composeEntry?.content).toBe('services: {}\n');
|
||||
const envEntry = entries.find(e => e.name === 'nodes/1_gateway/web/.env');
|
||||
expect(envEntry?.content).toBe('KEY=value\n');
|
||||
|
||||
expect(CloudBackupService.getInstance().getUploadStatus(snapshotId).status).toBe('success');
|
||||
});
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
/**
|
||||
* Fleet snapshot routes: admin-only read enforcement (a non-admin must not be
|
||||
* able to enumerate snapshots or read their secret-bearing .env content) and
|
||||
* content-at-rest encryption round-trip (file bodies stored as ciphertext, read
|
||||
* back as plaintext so restore and cloud-archive paths stay portable).
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
let CryptoService: typeof import('../services/CryptoService').CryptoService;
|
||||
let adminCookie: string;
|
||||
let viewerCookie: string;
|
||||
let snapshotId: number;
|
||||
|
||||
const VIEWER_USER = 'viewer-snap';
|
||||
const VIEWER_PASS = 'viewer-pass-123';
|
||||
const ENV_SECRET = 'DB_PASSWORD=s3cr3t-value\n';
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
({ CryptoService } = await import('../services/CryptoService'));
|
||||
({ app } = await import('../index'));
|
||||
adminCookie = await loginAsTestAdmin(app);
|
||||
|
||||
const db = DatabaseService.getInstance();
|
||||
const bcrypt = (await import('bcrypt')).default;
|
||||
const hash = await bcrypt.hash(VIEWER_PASS, 1);
|
||||
db.addUser({ username: VIEWER_USER, password_hash: hash, role: 'viewer' });
|
||||
const loginRes = await request(app).post('/api/auth/login').send({ username: VIEWER_USER, password: VIEWER_PASS });
|
||||
const cookies = loginRes.headers['set-cookie'] as string | string[];
|
||||
viewerCookie = Array.isArray(cookies) ? cookies[0] : cookies;
|
||||
|
||||
snapshotId = db.createSnapshot('audit-seed', 'admin', 1, 1, '[]', '[]');
|
||||
db.insertSnapshotFiles(snapshotId, [
|
||||
{ nodeId: 1, nodeName: 'local', stackName: 'web', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
{ nodeId: 1, nodeName: 'local', stackName: 'web', filename: '.env', content: ENV_SECRET },
|
||||
]);
|
||||
});
|
||||
|
||||
afterAll(() => cleanupTestDb(tmpDir));
|
||||
|
||||
describe('Fleet snapshot read authorization', () => {
|
||||
it('GET /api/fleet/snapshots requires authentication', async () => {
|
||||
const res = await request(app).get('/api/fleet/snapshots');
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it('GET /api/fleet/snapshots returns 403 for a non-admin', async () => {
|
||||
const res = await request(app).get('/api/fleet/snapshots').set('Cookie', viewerCookie);
|
||||
expect(res.status).toBe(403);
|
||||
});
|
||||
|
||||
it('GET /api/fleet/snapshots returns the list for an admin', async () => {
|
||||
const res = await request(app).get('/api/fleet/snapshots').set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(Array.isArray(res.body.snapshots)).toBe(true);
|
||||
});
|
||||
|
||||
it('GET /api/fleet/snapshots/:id returns 403 for a non-admin', async () => {
|
||||
const res = await request(app).get(`/api/fleet/snapshots/${snapshotId}`).set('Cookie', viewerCookie);
|
||||
expect(res.status).toBe(403);
|
||||
});
|
||||
|
||||
it('GET /api/fleet/snapshots/:id returns decrypted detail for an admin', async () => {
|
||||
const res = await request(app).get(`/api/fleet/snapshots/${snapshotId}`).set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(200);
|
||||
const files = res.body.nodes[0].stacks[0].files as Array<{ filename: string; content: string }>;
|
||||
const envFile = files.find(f => f.filename === '.env');
|
||||
expect(envFile?.content).toBe(ENV_SECRET);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Snapshot content-at-rest encryption', () => {
|
||||
it('stores file content as ciphertext but reads it back as plaintext', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const raw = db.getDb().prepare(
|
||||
"SELECT content FROM fleet_snapshot_files WHERE snapshot_id = ? AND filename = '.env'",
|
||||
).get(snapshotId) as { content: string };
|
||||
|
||||
expect(CryptoService.getInstance().isEncrypted(raw.content)).toBe(true);
|
||||
expect(raw.content).not.toContain('s3cr3t');
|
||||
|
||||
const env = db.getSnapshotFiles(snapshotId).find(f => f.filename === '.env');
|
||||
expect(env?.content).toBe(ENV_SECRET);
|
||||
});
|
||||
|
||||
it('decrypts content on the restore read path (getSnapshotStackFiles)', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const files = db.getSnapshotStackFiles(snapshotId, 1, 'web');
|
||||
const env = files.find(f => f.filename === '.env');
|
||||
expect(env?.content).toBe(ENV_SECRET);
|
||||
});
|
||||
|
||||
it('reads a legacy plaintext row back verbatim (decrypt tolerates non-ciphertext)', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const legacyId = db.createSnapshot('legacy', 'admin', 1, 1, '[]', '[]');
|
||||
// Insert directly, bypassing insertSnapshotFiles' encryption, to simulate
|
||||
// a snapshot written before content-at-rest encryption shipped.
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(legacyId, 1, 'local', 'legacy', 'compose.yaml', 'plain: text\n');
|
||||
|
||||
const files = db.getSnapshotFiles(legacyId);
|
||||
expect(files[0].content).toBe('plain: text\n');
|
||||
});
|
||||
});
|
||||
@@ -504,12 +504,22 @@ describe('Fleet snapshot admin enforcement', () => {
|
||||
expect(res.body.code).toBe('ADMIN_REQUIRED');
|
||||
});
|
||||
|
||||
it('GET /api/fleet/snapshots succeeds for viewer (read-only)', async () => {
|
||||
it('GET /api/fleet/snapshots returns 403 for viewer', async () => {
|
||||
mockTier('paid');
|
||||
const res = await request(app)
|
||||
.get('/api/fleet/snapshots')
|
||||
.set('Authorization', viewerHeader);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.body.code).toBe('ADMIN_REQUIRED');
|
||||
});
|
||||
|
||||
it('GET /api/fleet/snapshots/1 returns 403 for viewer', async () => {
|
||||
mockTier('paid');
|
||||
const res = await request(app)
|
||||
.get('/api/fleet/snapshots/1')
|
||||
.set('Authorization', viewerHeader);
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.body.code).toBe('ADMIN_REQUIRED');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1412,6 +1412,7 @@ describe('SchedulerService - executeSnapshot', () => {
|
||||
1,
|
||||
1,
|
||||
expect.any(String),
|
||||
expect.any(String),
|
||||
);
|
||||
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
||||
1,
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
/**
|
||||
* Unit coverage for the fleet snapshot capture helpers: a stack whose compose
|
||||
* file cannot be captured must be surfaced as a warning rather than silently
|
||||
* dropped, a genuinely-absent .env must NOT warn, a real .env read error must,
|
||||
* and a file over the size cap must be skipped with a warning.
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
|
||||
const mockGetStacks = vi.fn();
|
||||
const mockGetStackContent = vi.fn();
|
||||
const mockGetEnvContent = vi.fn();
|
||||
const mockGetProxyTarget = vi.fn();
|
||||
|
||||
vi.mock('../services/FileSystemService', () => ({
|
||||
FileSystemService: { getInstance: () => ({
|
||||
getStacks: mockGetStacks,
|
||||
getStackContent: mockGetStackContent,
|
||||
getEnvContent: mockGetEnvContent,
|
||||
}) },
|
||||
}));
|
||||
|
||||
vi.mock('../services/NodeRegistry', () => ({
|
||||
NodeRegistry: { getInstance: () => ({ getProxyTarget: mockGetProxyTarget }) },
|
||||
}));
|
||||
|
||||
import {
|
||||
captureLocalNodeFiles,
|
||||
captureRemoteNodeFiles,
|
||||
MAX_SNAPSHOT_FILE_BYTES,
|
||||
} from '../utils/snapshot-capture';
|
||||
|
||||
const localNode = { id: 1, name: 'local', mode: 'proxy' as const };
|
||||
const remoteNode = { id: 2, name: 'remote', mode: 'proxy' as const };
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tok' });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe('captureLocalNodeFiles', () => {
|
||||
it('captures compose and .env with no warnings on the happy path', async () => {
|
||||
mockGetStacks.mockResolvedValue(['web']);
|
||||
mockGetStackContent.mockResolvedValue('services: {}\n');
|
||||
mockGetEnvContent.mockResolvedValue('KEY=value\n');
|
||||
|
||||
const result = await captureLocalNodeFiles(localNode);
|
||||
|
||||
expect(result.stacks).toHaveLength(1);
|
||||
expect(result.stacks[0].files.map(f => f.filename)).toEqual(['compose.yaml', '.env']);
|
||||
expect(result.warnings).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('drops the stack and warns when compose cannot be read', async () => {
|
||||
mockGetStacks.mockResolvedValue(['web']);
|
||||
mockGetStackContent.mockRejectedValue(new Error('EACCES'));
|
||||
|
||||
const result = await captureLocalNodeFiles(localNode);
|
||||
|
||||
expect(result.stacks).toHaveLength(0);
|
||||
expect(result.warnings).toHaveLength(1);
|
||||
expect(result.warnings[0]).toMatchObject({ stackName: 'web' });
|
||||
expect(result.warnings[0].reason).toContain('compose.yaml could not be read');
|
||||
});
|
||||
|
||||
it('does not warn when the .env is simply absent (ENOENT)', async () => {
|
||||
mockGetStacks.mockResolvedValue(['web']);
|
||||
mockGetStackContent.mockResolvedValue('services: {}\n');
|
||||
mockGetEnvContent.mockRejectedValue(Object.assign(new Error('no file'), { code: 'ENOENT' }));
|
||||
|
||||
const result = await captureLocalNodeFiles(localNode);
|
||||
|
||||
expect(result.stacks[0].files.map(f => f.filename)).toEqual(['compose.yaml']);
|
||||
expect(result.warnings).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('warns but still captures the stack when the .env read fails for a non-ENOENT reason', async () => {
|
||||
mockGetStacks.mockResolvedValue(['web']);
|
||||
mockGetStackContent.mockResolvedValue('services: {}\n');
|
||||
mockGetEnvContent.mockRejectedValue(Object.assign(new Error('EACCES'), { code: 'EACCES' }));
|
||||
|
||||
const result = await captureLocalNodeFiles(localNode);
|
||||
|
||||
expect(result.stacks[0].files.map(f => f.filename)).toEqual(['compose.yaml']);
|
||||
expect(result.warnings).toHaveLength(1);
|
||||
expect(result.warnings[0].reason).toContain('.env could not be read');
|
||||
});
|
||||
|
||||
it('skips a compose file that exceeds the size cap and warns', async () => {
|
||||
mockGetStacks.mockResolvedValue(['web']);
|
||||
mockGetStackContent.mockResolvedValue('x'.repeat(MAX_SNAPSHOT_FILE_BYTES + 1));
|
||||
|
||||
const result = await captureLocalNodeFiles(localNode);
|
||||
|
||||
expect(result.stacks).toHaveLength(0);
|
||||
expect(result.warnings[0].reason).toContain('exceeds');
|
||||
});
|
||||
|
||||
it('keeps the stack but warns when the .env exceeds the size cap', async () => {
|
||||
mockGetStacks.mockResolvedValue(['web']);
|
||||
mockGetStackContent.mockResolvedValue('services: {}\n');
|
||||
mockGetEnvContent.mockResolvedValue('x'.repeat(MAX_SNAPSHOT_FILE_BYTES + 1));
|
||||
|
||||
const result = await captureLocalNodeFiles(localNode);
|
||||
|
||||
expect(result.stacks[0].files.map(f => f.filename)).toEqual(['compose.yaml']);
|
||||
expect(result.warnings[0].reason).toContain('exceeds');
|
||||
});
|
||||
});
|
||||
|
||||
describe('captureRemoteNodeFiles', () => {
|
||||
function mockFetchRoutes(routes: Record<string, Partial<Response> & { jsonValue?: unknown; textValue?: string; xEnvExists?: string }>) {
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
const match = Object.keys(routes).find(k => url.endsWith(k));
|
||||
const r = match ? routes[match] : { ok: false, status: 404 };
|
||||
return {
|
||||
ok: r.ok ?? true,
|
||||
status: r.status ?? 200,
|
||||
headers: { get: (name: string) => name.toLowerCase() === 'x-env-exists' ? ((r as { xEnvExists?: string }).xEnvExists ?? null) : null },
|
||||
json: async () => (r as { jsonValue?: unknown }).jsonValue,
|
||||
text: async () => (r as { textValue?: string }).textValue ?? '',
|
||||
} as unknown as Response;
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
return fetchMock;
|
||||
}
|
||||
|
||||
it('captures compose and .env with no warnings on the happy path', async () => {
|
||||
mockFetchRoutes({
|
||||
'/api/stacks': { ok: true, jsonValue: ['web'] },
|
||||
'/api/stacks/web': { ok: true, textValue: 'services: {}\n' },
|
||||
'/api/stacks/web/env': { ok: true, textValue: 'KEY=value\n' },
|
||||
});
|
||||
|
||||
const result = await captureRemoteNodeFiles(remoteNode);
|
||||
|
||||
expect(result.stacks).toHaveLength(1);
|
||||
expect(result.stacks[0].files.map(f => f.filename)).toEqual(['compose.yaml', '.env']);
|
||||
expect(result.warnings).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('drops the stack and warns when the remote compose fetch is not ok', async () => {
|
||||
mockFetchRoutes({
|
||||
'/api/stacks': { ok: true, jsonValue: ['web'] },
|
||||
'/api/stacks/web': { ok: false, status: 500 },
|
||||
});
|
||||
|
||||
const result = await captureRemoteNodeFiles(remoteNode);
|
||||
|
||||
expect(result.stacks).toHaveLength(0);
|
||||
expect(result.warnings[0].reason).toContain('HTTP 500');
|
||||
});
|
||||
|
||||
it('treats a remote .env as absent (no empty file) when X-Env-Exists is false', async () => {
|
||||
mockFetchRoutes({
|
||||
'/api/stacks': { ok: true, jsonValue: ['web'] },
|
||||
'/api/stacks/web': { ok: true, textValue: 'services: {}\n' },
|
||||
'/api/stacks/web/env': { ok: true, textValue: '', xEnvExists: 'false' },
|
||||
});
|
||||
|
||||
const result = await captureRemoteNodeFiles(remoteNode);
|
||||
|
||||
expect(result.stacks[0].files.map(f => f.filename)).toEqual(['compose.yaml']);
|
||||
expect(result.warnings).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('treats a 404 .env as absent without warning', async () => {
|
||||
mockFetchRoutes({
|
||||
'/api/stacks': { ok: true, jsonValue: ['web'] },
|
||||
'/api/stacks/web': { ok: true, textValue: 'services: {}\n' },
|
||||
'/api/stacks/web/env': { ok: false, status: 404 },
|
||||
});
|
||||
|
||||
const result = await captureRemoteNodeFiles(remoteNode);
|
||||
|
||||
expect(result.stacks[0].files.map(f => f.filename)).toEqual(['compose.yaml']);
|
||||
expect(result.warnings).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('warns when the remote .env fetch fails for a non-404 reason', async () => {
|
||||
mockFetchRoutes({
|
||||
'/api/stacks': { ok: true, jsonValue: ['web'] },
|
||||
'/api/stacks/web': { ok: true, textValue: 'services: {}\n' },
|
||||
'/api/stacks/web/env': { ok: false, status: 500 },
|
||||
});
|
||||
|
||||
const result = await captureRemoteNodeFiles(remoteNode);
|
||||
|
||||
expect(result.stacks[0].files.map(f => f.filename)).toEqual(['compose.yaml']);
|
||||
expect(result.warnings[0].reason).toContain('HTTP 500');
|
||||
});
|
||||
|
||||
it('keeps the stack but warns when the remote .env exceeds the size cap', async () => {
|
||||
mockFetchRoutes({
|
||||
'/api/stacks': { ok: true, jsonValue: ['web'] },
|
||||
'/api/stacks/web': { ok: true, textValue: 'services: {}\n' },
|
||||
'/api/stacks/web/env': { ok: true, textValue: 'x'.repeat(MAX_SNAPSHOT_FILE_BYTES + 1) },
|
||||
});
|
||||
|
||||
const result = await captureRemoteNodeFiles(remoteNode);
|
||||
|
||||
expect(result.stacks[0].files.map(f => f.filename)).toEqual(['compose.yaml']);
|
||||
expect(result.warnings[0].reason).toContain('exceeds');
|
||||
});
|
||||
|
||||
it('drops the stack and warns when the remote compose fetch throws', async () => {
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
if (url.endsWith('/api/stacks')) {
|
||||
return { ok: true, status: 200, json: async () => ['web'], text: async () => '' } as Response;
|
||||
}
|
||||
throw new Error('network down');
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const result = await captureRemoteNodeFiles(remoteNode);
|
||||
|
||||
expect(result.stacks).toHaveLength(0);
|
||||
expect(result.warnings[0].reason).toContain('fetch error');
|
||||
});
|
||||
});
|
||||
@@ -186,6 +186,7 @@ cloudBackupRouter.get('/usage', async (req: Request, res: Response): Promise<voi
|
||||
|
||||
cloudBackupRouter.get('/snapshots', async (req: Request, res: Response): Promise<void> => {
|
||||
if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return;
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!gateForCurrentProvider(req, res)) return;
|
||||
try {
|
||||
const entries = await CloudBackupService.getInstance().listCloudSnapshots();
|
||||
@@ -218,6 +219,7 @@ cloudBackupRouter.post('/upload/:id', async (req: Request, res: Response): Promi
|
||||
|
||||
cloudBackupRouter.get('/status/:id', (req: Request, res: Response): void => {
|
||||
if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return;
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!gateForCurrentProvider(req, res)) return;
|
||||
const id = parseSnapshotIdParam(req, res);
|
||||
if (id == null) return;
|
||||
@@ -226,6 +228,7 @@ cloudBackupRouter.get('/status/:id', (req: Request, res: Response): void => {
|
||||
|
||||
cloudBackupRouter.get('/object/:keyB64/download', async (req: Request, res: Response): Promise<void> => {
|
||||
if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return;
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!gateForCurrentProvider(req, res)) return;
|
||||
const objectKey = decodeObjectKey(req, res);
|
||||
if (!objectKey) return;
|
||||
|
||||
@@ -1597,6 +1597,7 @@ fleetRouter.post('/snapshots', authMiddleware, async (req: Request, res: Respons
|
||||
|
||||
let totalStacks = 0;
|
||||
const allFiles: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }> = [];
|
||||
const skippedStacks: Array<{ nodeId: number; nodeName: string; stackName: string; reason: string }> = [];
|
||||
|
||||
for (const nodeData of capturedNodes) {
|
||||
totalStacks += nodeData.stacks.length;
|
||||
@@ -1611,6 +1612,14 @@ fleetRouter.post('/snapshots', authMiddleware, async (req: Request, res: Respons
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const warning of nodeData.warnings) {
|
||||
skippedStacks.push({
|
||||
nodeId: nodeData.nodeId,
|
||||
nodeName: nodeData.nodeName,
|
||||
stackName: warning.stackName,
|
||||
reason: warning.reason,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const snapshotId = db.createSnapshot(
|
||||
@@ -1619,6 +1628,7 @@ fleetRouter.post('/snapshots', authMiddleware, async (req: Request, res: Respons
|
||||
capturedNodes.length,
|
||||
totalStacks,
|
||||
JSON.stringify(skippedNodes),
|
||||
JSON.stringify(skippedStacks),
|
||||
);
|
||||
|
||||
if (allFiles.length > 0) {
|
||||
@@ -1627,21 +1637,30 @@ fleetRouter.post('/snapshots', authMiddleware, async (req: Request, res: Respons
|
||||
|
||||
const cloudSvc = CloudBackupService.getInstance();
|
||||
if (cloudSvc.isEnabled() && cloudSvc.isAutoUploadOn()) {
|
||||
void cloudSvc.uploadSnapshot(snapshotId).catch(uploadErr => {
|
||||
const message = uploadErr instanceof Error ? uploadErr.message : String(uploadErr);
|
||||
console.error('[Fleet Snapshot] Cloud upload failed:', message);
|
||||
void NotificationService.getInstance()
|
||||
.dispatchAlert('error', 'system', `Cloud backup upload failed for snapshot ${snapshotId}: ${message}`)
|
||||
.catch(() => { /* notification dispatch is best-effort */ });
|
||||
});
|
||||
void cloudSvc.uploadSnapshot(snapshotId)
|
||||
.then(() => console.log(`[Fleet Snapshot] Cloud auto-upload OK for snapshot ${snapshotId}`))
|
||||
.catch(uploadErr => {
|
||||
const message = uploadErr instanceof Error ? uploadErr.message : String(uploadErr);
|
||||
console.error('[Fleet Snapshot] Cloud upload failed:', message);
|
||||
void NotificationService.getInstance()
|
||||
.dispatchAlert('error', 'system', `Cloud backup upload failed for snapshot ${snapshotId}: ${message}`)
|
||||
.catch(() => { /* notification dispatch is best-effort */ });
|
||||
});
|
||||
}
|
||||
|
||||
console.log('[Fleet] Snapshot created:', capturedNodes.length, 'nodes,', totalStacks, 'stacks');
|
||||
if (skippedNodes.length > 0 || skippedStacks.length > 0) {
|
||||
console.warn(`[Fleet] Snapshot ${snapshotId} partial: ${capturedNodes.length} node(s), ${totalStacks} stack(s); skipped ${skippedNodes.length} node(s), ${skippedStacks.length} stack(s)`);
|
||||
} else {
|
||||
console.log('[Fleet] Snapshot created:', capturedNodes.length, 'nodes,', totalStacks, 'stacks');
|
||||
}
|
||||
if (isDebugEnabled()) {
|
||||
console.debug(`[Fleet:debug] Snapshot ${snapshotId} capture completed in ${Date.now() - captureStart}ms, ${allFiles.length} file(s) stored`);
|
||||
for (const skip of skippedNodes) {
|
||||
console.debug(`[Fleet:debug] Skipped node "${skip.nodeName}" (id=${skip.nodeId}): ${skip.reason}`);
|
||||
}
|
||||
for (const skip of skippedStacks) {
|
||||
console.debug(`[Fleet:debug] Skipped stack "${skip.stackName}" on "${skip.nodeName}" (id=${skip.nodeId}): ${skip.reason}`);
|
||||
}
|
||||
}
|
||||
const snapshot = db.getSnapshot(snapshotId);
|
||||
res.status(201).json(snapshot);
|
||||
@@ -1652,6 +1671,8 @@ fleetRouter.post('/snapshots', authMiddleware, async (req: Request, res: Respons
|
||||
});
|
||||
|
||||
fleetRouter.get('/snapshots', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
|
||||
try {
|
||||
const limit = Math.min(parseInt(req.query.limit as string, 10) || 50, 100);
|
||||
const offset = parseInt(req.query.offset as string, 10) || 0;
|
||||
@@ -1667,6 +1688,8 @@ fleetRouter.get('/snapshots', authMiddleware, async (req: Request, res: Response
|
||||
});
|
||||
|
||||
fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
|
||||
try {
|
||||
const id = parseIntParam(req, res, 'id', 'snapshot ID');
|
||||
if (id === null) return;
|
||||
|
||||
@@ -232,7 +232,8 @@ export interface FleetSnapshot {
|
||||
created_by: string;
|
||||
node_count: number;
|
||||
stack_count: number;
|
||||
skipped_nodes: string;
|
||||
skipped_nodes: string; // JSON: Array<{ nodeId; nodeName; reason }>
|
||||
skipped_stacks: string; // JSON: Array<{ nodeId; nodeName; stackName; reason }>
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
@@ -821,6 +822,7 @@ export class DatabaseService {
|
||||
node_count INTEGER NOT NULL,
|
||||
stack_count INTEGER NOT NULL,
|
||||
skipped_nodes TEXT NOT NULL DEFAULT '[]',
|
||||
skipped_stacks TEXT NOT NULL DEFAULT '[]',
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
@@ -1196,6 +1198,9 @@ export class DatabaseService {
|
||||
maybeAddCol('nodes', 'pilot_last_seen', 'INTEGER');
|
||||
maybeAddCol('nodes', 'pilot_agent_version', 'TEXT');
|
||||
|
||||
// Fleet snapshot per-stack capture warnings (partial-capture surfacing)
|
||||
maybeAddCol('fleet_snapshots', 'skipped_stacks', "TEXT NOT NULL DEFAULT '[]'");
|
||||
|
||||
// Scheduled operations migrations
|
||||
maybeAddCol('scheduled_task_runs', 'triggered_by', "TEXT NOT NULL DEFAULT 'scheduler'");
|
||||
maybeAddCol('scheduled_tasks', 'prune_targets', 'TEXT DEFAULT NULL');
|
||||
@@ -2882,20 +2887,25 @@ export class DatabaseService {
|
||||
|
||||
// --- Fleet Snapshots ---
|
||||
|
||||
public createSnapshot(description: string, createdBy: string, nodeCount: number, stackCount: number, skippedNodes: string): number {
|
||||
public createSnapshot(description: string, createdBy: string, nodeCount: number, stackCount: number, skippedNodes: string, skippedStacks = '[]'): number {
|
||||
const result = this.db.prepare(
|
||||
'INSERT INTO fleet_snapshots (description, created_by, node_count, stack_count, skipped_nodes, created_at) VALUES (?, ?, ?, ?, ?, ?)'
|
||||
).run(description, createdBy, nodeCount, stackCount, skippedNodes, Date.now());
|
||||
'INSERT INTO fleet_snapshots (description, created_by, node_count, stack_count, skipped_nodes, skipped_stacks, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
|
||||
).run(description, createdBy, nodeCount, stackCount, skippedNodes, skippedStacks, Date.now());
|
||||
return result.lastInsertRowid as number;
|
||||
}
|
||||
|
||||
public insertSnapshotFiles(snapshotId: number, files: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }>): void {
|
||||
// Snapshot file bodies are compose.yaml and .env captures, so they carry
|
||||
// the same secrets as the live stack. Encrypt content at rest with the
|
||||
// instance key; getSnapshotFiles/getSnapshotStackFiles decrypt on read,
|
||||
// so restore and cloud-archive paths see plaintext and stay portable.
|
||||
const crypto = CryptoService.getInstance();
|
||||
const insert = this.db.prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)'
|
||||
);
|
||||
const insertMany = this.db.transaction((rows: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }>) => {
|
||||
for (const row of rows) {
|
||||
insert.run(snapshotId, row.nodeId, row.nodeName, row.stackName, row.filename, row.content);
|
||||
insert.run(snapshotId, row.nodeId, row.nodeName, row.stackName, row.filename, crypto.encrypt(row.content));
|
||||
}
|
||||
});
|
||||
insertMany(files);
|
||||
@@ -2912,15 +2922,21 @@ export class DatabaseService {
|
||||
}
|
||||
|
||||
public getSnapshotFiles(snapshotId: number): FleetSnapshotFile[] {
|
||||
return this.db.prepare(
|
||||
const crypto = CryptoService.getInstance();
|
||||
const rows = this.db.prepare(
|
||||
'SELECT * FROM fleet_snapshot_files WHERE snapshot_id = ? ORDER BY node_name, stack_name'
|
||||
).all(snapshotId) as FleetSnapshotFile[];
|
||||
// decrypt() returns non-ciphertext input unchanged, so rows written
|
||||
// before content-at-rest encryption still read back as plaintext.
|
||||
return rows.map(row => ({ ...row, content: crypto.decrypt(row.content) }));
|
||||
}
|
||||
|
||||
public getSnapshotStackFiles(snapshotId: number, nodeId: number, stackName: string): FleetSnapshotFile[] {
|
||||
return this.db.prepare(
|
||||
const crypto = CryptoService.getInstance();
|
||||
const rows = this.db.prepare(
|
||||
'SELECT * FROM fleet_snapshot_files WHERE snapshot_id = ? AND node_id = ? AND stack_name = ?'
|
||||
).all(snapshotId, nodeId, stackName) as FleetSnapshotFile[];
|
||||
return rows.map(row => ({ ...row, content: crypto.decrypt(row.content) }));
|
||||
}
|
||||
|
||||
public deleteSnapshot(id: number): void {
|
||||
|
||||
@@ -11,7 +11,7 @@ import type { ImageCheckResult } from './ImageUpdateService';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { captureLocalNodeFiles, captureRemoteNodeFiles } from '../utils/snapshot-capture';
|
||||
import { captureLocalNodeFiles, captureRemoteNodeFiles, type SnapshotNodeData } from '../utils/snapshot-capture';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { NotificationService } from './NotificationService';
|
||||
import TrivyService from './TrivyService';
|
||||
@@ -562,7 +562,7 @@ export class SchedulerService {
|
||||
})
|
||||
);
|
||||
|
||||
const capturedNodes: Array<{ nodeId: number; nodeName: string; stacks: Array<{ stackName: string; files: Array<{ filename: string; content: string }> }> }> = [];
|
||||
const capturedNodes: SnapshotNodeData[] = [];
|
||||
const skippedNodes: Array<{ nodeId: number; nodeName: string; reason: string }> = [];
|
||||
|
||||
results.forEach((result, i) => {
|
||||
@@ -579,6 +579,7 @@ export class SchedulerService {
|
||||
|
||||
let totalStacks = 0;
|
||||
const allFiles: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }> = [];
|
||||
const skippedStacks: Array<{ nodeId: number; nodeName: string; stackName: string; reason: string }> = [];
|
||||
|
||||
for (const nodeData of capturedNodes) {
|
||||
totalStacks += nodeData.stacks.length;
|
||||
@@ -593,6 +594,14 @@ export class SchedulerService {
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const warning of nodeData.warnings) {
|
||||
skippedStacks.push({
|
||||
nodeId: nodeData.nodeId,
|
||||
nodeName: nodeData.nodeName,
|
||||
stackName: warning.stackName,
|
||||
reason: warning.reason,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const description = `Scheduled snapshot: ${task.name}`;
|
||||
@@ -602,6 +611,7 @@ export class SchedulerService {
|
||||
capturedNodes.length,
|
||||
totalStacks,
|
||||
JSON.stringify(skippedNodes),
|
||||
JSON.stringify(skippedStacks),
|
||||
);
|
||||
|
||||
if (allFiles.length > 0) {
|
||||
@@ -622,11 +632,18 @@ export class SchedulerService {
|
||||
}
|
||||
}
|
||||
|
||||
if (skippedNodes.length > 0 || skippedStacks.length > 0) {
|
||||
console.warn(`[SchedulerService] Snapshot task ${task.id} partial: skipped ${skippedNodes.length} node(s), ${skippedStacks.length} stack(s)`);
|
||||
}
|
||||
if (isDebugEnabled()) {
|
||||
console.debug(`[SchedulerService:debug] Snapshot task ${task.id}: captured ${capturedNodes.length} node(s), ${totalStacks} stack(s), ${allFiles.length} file(s), skipped ${skippedNodes.length}${cloudUploadNote}`);
|
||||
console.debug(`[SchedulerService:debug] Snapshot task ${task.id}: captured ${capturedNodes.length} node(s), ${totalStacks} stack(s), ${allFiles.length} file(s), skipped ${skippedNodes.length} node(s)/${skippedStacks.length} stack(s)${cloudUploadNote}`);
|
||||
}
|
||||
|
||||
return `Fleet snapshot created (id=${snapshotId}, ${capturedNodes.length} node(s), ${totalStacks} stack(s)${skippedNodes.length > 0 ? `, ${skippedNodes.length} skipped` : ''}${cloudUploadNote})`;
|
||||
const skippedNote = [
|
||||
skippedNodes.length > 0 ? `${skippedNodes.length} node(s)` : '',
|
||||
skippedStacks.length > 0 ? `${skippedStacks.length} stack(s)` : '',
|
||||
].filter(Boolean).join(', ');
|
||||
return `Fleet snapshot created (id=${snapshotId}, ${capturedNodes.length} node(s), ${totalStacks} stack(s)${skippedNote ? `, skipped ${skippedNote}` : ''}${cloudUploadNote})`;
|
||||
}
|
||||
|
||||
private async executePrune(task: ScheduledTask): Promise<string> {
|
||||
|
||||
@@ -16,8 +16,27 @@ export interface SnapshotNodeData {
|
||||
stackName: string;
|
||||
files: Array<{ filename: string; content: string }>;
|
||||
}>;
|
||||
/**
|
||||
* Per-stack capture problems that did not fail the whole node: a stack whose
|
||||
* compose file could not be read (so it is absent from `stacks`), a `.env`
|
||||
* dropped on a read error, or a file skipped for exceeding the size cap.
|
||||
* Surfaced on the snapshot so an operator never mistakes a partial capture
|
||||
* for a complete backup.
|
||||
*/
|
||||
warnings: Array<{ stackName: string; reason: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-file capture ceiling. compose.yaml and .env are normally a few KB; this
|
||||
* generous 1 MB bound stops a pathological or hostile file from bloating the
|
||||
* snapshot DB and the GET-detail payload. An oversize compose.yaml skips the
|
||||
* whole stack; an oversize .env drops only that file and keeps the stack. Both
|
||||
* are recorded as a warning, never silently dropped.
|
||||
*/
|
||||
export const MAX_SNAPSHOT_FILE_BYTES = 1_000_000;
|
||||
/** The cap rendered in MB for operator-facing warning text. */
|
||||
const MAX_SNAPSHOT_FILE_MB = MAX_SNAPSHOT_FILE_BYTES / 1_000_000;
|
||||
|
||||
/**
|
||||
* Minimal node shape accepted by capture functions.
|
||||
* `mode` is required so remote dispatch can emit a tunnel-aware error when
|
||||
@@ -31,44 +50,66 @@ export interface CaptureNode {
|
||||
|
||||
/**
|
||||
* Read compose.yaml and .env files for every stack on a local node.
|
||||
* Stacks whose compose file cannot be read are silently skipped.
|
||||
* A stack whose compose file cannot be read is omitted from `stacks` and
|
||||
* recorded in `warnings`, so a partial capture is never mistaken for complete.
|
||||
*/
|
||||
export async function captureLocalNodeFiles(node: CaptureNode): Promise<SnapshotNodeData> {
|
||||
const start = Date.now();
|
||||
const fsService = FileSystemService.getInstance(node.id);
|
||||
const stackNames = await fsService.getStacks();
|
||||
const stacks: SnapshotNodeData['stacks'] = [];
|
||||
const warnings: SnapshotNodeData['warnings'] = [];
|
||||
|
||||
for (const stackName of stackNames) {
|
||||
const files: Array<{ filename: string; content: string }> = [];
|
||||
|
||||
let composeContent: string;
|
||||
try {
|
||||
const composeContent = await fsService.getStackContent(stackName);
|
||||
files.push({ filename: 'compose.yaml', content: composeContent });
|
||||
composeContent = await fsService.getStackContent(stackName);
|
||||
} catch (e) {
|
||||
console.warn(`[Fleet Snapshot] Could not read compose file for stack "${stackName}", skipping:`, (e as Error).message);
|
||||
const reason = `compose.yaml could not be read: ${(e as Error).message}`;
|
||||
console.warn(`[Fleet Snapshot] Skipping stack "${stackName}" on "${node.name}": ${reason}`);
|
||||
warnings.push({ stackName, reason });
|
||||
continue;
|
||||
}
|
||||
if (Buffer.byteLength(composeContent, 'utf-8') > MAX_SNAPSHOT_FILE_BYTES) {
|
||||
const reason = `compose.yaml exceeds the ${MAX_SNAPSHOT_FILE_MB} MB capture limit; stack skipped`;
|
||||
console.warn(`[Fleet Snapshot] ${reason} ("${stackName}" on "${node.name}")`);
|
||||
warnings.push({ stackName, reason });
|
||||
continue;
|
||||
}
|
||||
files.push({ filename: 'compose.yaml', content: composeContent });
|
||||
|
||||
try {
|
||||
const envContent = await fsService.getEnvContent(stackName);
|
||||
files.push({ filename: '.env', content: envContent });
|
||||
} catch {
|
||||
// No .env file - that's fine
|
||||
if (Buffer.byteLength(envContent, 'utf-8') > MAX_SNAPSHOT_FILE_BYTES) {
|
||||
warnings.push({ stackName, reason: `.env exceeds the ${MAX_SNAPSHOT_FILE_MB} MB capture limit; captured without it` });
|
||||
} else {
|
||||
files.push({ filename: '.env', content: envContent });
|
||||
}
|
||||
} catch (e) {
|
||||
// A missing .env is normal; surface only genuine read errors so a stack
|
||||
// is not silently restored without its secrets.
|
||||
if ((e as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
warnings.push({ stackName, reason: `.env could not be read: ${(e as Error).message}; captured without it` });
|
||||
}
|
||||
}
|
||||
stacks.push({ stackName, files });
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) {
|
||||
const fileCount = stacks.reduce((sum, s) => sum + s.files.length, 0);
|
||||
console.debug(`[Fleet:debug] Local capture "${node.name}": ${stacks.length} stack(s), ${fileCount} file(s) in ${Date.now() - start}ms`);
|
||||
console.debug(`[Fleet:debug] Local capture "${node.name}": ${stacks.length} stack(s), ${fileCount} file(s), ${warnings.length} warning(s) in ${Date.now() - start}ms`);
|
||||
}
|
||||
|
||||
return { nodeId: node.id, nodeName: node.name, stacks };
|
||||
return { nodeId: node.id, nodeName: node.name, stacks, warnings };
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch compose.yaml and .env files for every stack on a remote node
|
||||
* via the Distributed API proxy. Stacks whose compose file cannot be
|
||||
* fetched are silently skipped.
|
||||
* via the Distributed API proxy. A stack whose compose file cannot be
|
||||
* fetched is omitted from `stacks` and recorded in `warnings`, so a partial
|
||||
* capture is never mistaken for complete.
|
||||
*/
|
||||
export async function captureRemoteNodeFiles(node: CaptureNode): Promise<SnapshotNodeData> {
|
||||
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
@@ -89,43 +130,65 @@ export async function captureRemoteNodeFiles(node: CaptureNode): Promise<Snapsho
|
||||
const stackNames = await stacksRes.json() as string[];
|
||||
|
||||
const stacks: SnapshotNodeData['stacks'] = [];
|
||||
const warnings: SnapshotNodeData['warnings'] = [];
|
||||
|
||||
for (const stackName of stackNames) {
|
||||
const files: Array<{ filename: string; content: string }> = [];
|
||||
|
||||
let composeContent: string;
|
||||
try {
|
||||
const composeRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, {
|
||||
headers,
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (composeRes.ok) {
|
||||
const content = await composeRes.text();
|
||||
files.push({ filename: 'compose.yaml', content });
|
||||
if (!composeRes.ok) {
|
||||
const reason = `compose.yaml fetch failed (HTTP ${composeRes.status}); stack skipped`;
|
||||
console.warn(`[Fleet Snapshot] ${reason} ("${stackName}" on "${node.name}")`);
|
||||
warnings.push({ stackName, reason });
|
||||
continue;
|
||||
}
|
||||
composeContent = await composeRes.text();
|
||||
} catch (e) {
|
||||
console.warn(`[Fleet Snapshot] Failed to fetch remote compose for stack "${stackName}":`, (e as Error).message);
|
||||
const reason = `compose.yaml fetch error: ${(e as Error).message}; stack skipped`;
|
||||
console.warn(`[Fleet Snapshot] ${reason} ("${stackName}" on "${node.name}")`);
|
||||
warnings.push({ stackName, reason });
|
||||
continue;
|
||||
}
|
||||
if (Buffer.byteLength(composeContent, 'utf-8') > MAX_SNAPSHOT_FILE_BYTES) {
|
||||
warnings.push({ stackName, reason: `compose.yaml exceeds the ${MAX_SNAPSHOT_FILE_MB} MB capture limit; stack skipped` });
|
||||
continue;
|
||||
}
|
||||
files.push({ filename: 'compose.yaml', content: composeContent });
|
||||
|
||||
try {
|
||||
const envRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, {
|
||||
headers,
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (envRes.ok) {
|
||||
// The remote replies 200 with an empty body and X-Env-Exists: false when a
|
||||
// stack has no .env. Treat that as absent (matching the local ENOENT path)
|
||||
// so restore does not write a spurious empty .env. An older remote that
|
||||
// predates the header falls back to capturing whatever the 200 returned.
|
||||
if (envRes.ok && envRes.headers.get('X-Env-Exists') !== 'false') {
|
||||
const content = await envRes.text();
|
||||
files.push({ filename: '.env', content });
|
||||
if (Buffer.byteLength(content, 'utf-8') > MAX_SNAPSHOT_FILE_BYTES) {
|
||||
warnings.push({ stackName, reason: `.env exceeds the ${MAX_SNAPSHOT_FILE_MB} MB capture limit; captured without it` });
|
||||
} else {
|
||||
files.push({ filename: '.env', content });
|
||||
}
|
||||
} else if (!envRes.ok && envRes.status !== 404) {
|
||||
warnings.push({ stackName, reason: `.env fetch failed (HTTP ${envRes.status}); captured without it` });
|
||||
}
|
||||
} catch {
|
||||
// No .env - skip
|
||||
}
|
||||
if (files.length > 0) {
|
||||
stacks.push({ stackName, files });
|
||||
} catch (e) {
|
||||
warnings.push({ stackName, reason: `.env fetch error: ${(e as Error).message}; captured without it` });
|
||||
}
|
||||
stacks.push({ stackName, files });
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) {
|
||||
const fileCount = stacks.reduce((sum, s) => sum + s.files.length, 0);
|
||||
console.debug(`[Fleet:debug] Remote capture "${node.name}": ${stacks.length} stack(s), ${fileCount} file(s) in ${Date.now() - start}ms`);
|
||||
console.debug(`[Fleet:debug] Remote capture "${node.name}": ${stacks.length} stack(s), ${fileCount} file(s), ${warnings.length} warning(s) in ${Date.now() - start}ms`);
|
||||
}
|
||||
|
||||
return { nodeId: node.id, nodeName: node.name, stacks };
|
||||
return { nodeId: node.id, nodeName: node.name, stacks, warnings };
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ During creation, Sencho connects to each node in parallel:
|
||||
- **Local nodes** are read directly from the compose directory
|
||||
- **Remote nodes** are fetched via the Distributed API proxy using the node's API token
|
||||
|
||||
If a remote node is offline or unreachable, it is **skipped gracefully**. The snapshot still captures data from all reachable nodes, and skipped nodes are recorded with the reason for the failure.
|
||||
If a remote node is offline or unreachable, it is **skipped gracefully**. The snapshot still captures data from all reachable nodes, and skipped nodes are recorded with the reason for the failure. Individual stacks that could not be captured (for example, a compose file that failed to read, or a file too large to store) are recorded the same way, so a snapshot is never silently incomplete.
|
||||
|
||||
### Scheduled snapshots
|
||||
|
||||
@@ -37,7 +37,7 @@ The snapshot list shows each snapshot in a table with the following columns:
|
||||
- **Date** - when the snapshot was taken
|
||||
- **Description** - your optional label, or a prefix like "Scheduled snapshot" for automated ones. If Cloud Backup is configured, an upload icon in this column marks snapshots that have been mirrored off-site.
|
||||
- **Scope** - how many nodes and stacks were captured (e.g. "3 nodes, 21 stacks")
|
||||
- **Warnings** - a warning icon with a count if any nodes were skipped, or "None"
|
||||
- **Warnings** - a warning icon with a count if any nodes or stacks were skipped, or "None"
|
||||
- **Actions** - **View** to open the detail view, a cloud-upload icon for snapshots not yet mirrored to a configured Cloud Backup target, and a delete button for admins
|
||||
|
||||
<Frame>
|
||||
@@ -56,7 +56,7 @@ Below the header, each node appears as a collapsible card. Expand a node to see
|
||||
<img src="/images/fleet-backups/snapshot-detail.png" alt="Snapshot detail view with a node expanded, showing a stack expanded with a compose file, Preview button, and Restore button" />
|
||||
</Frame>
|
||||
|
||||
If any nodes were unreachable during snapshot creation, a warning banner appears at the top of the detail view listing each skipped node and the reason it was skipped.
|
||||
If any nodes were unreachable during snapshot creation, a warning banner appears at the top of the detail view listing each skipped node and the reason it was skipped. A second banner lists any individual stacks that were only partially captured, naming the node, the stack, and the reason.
|
||||
|
||||
## Restoring from a snapshot
|
||||
|
||||
@@ -135,23 +135,13 @@ For in-place rollback, use the **Restore** action on the snapshot detail view as
|
||||
|
||||
## Access control
|
||||
|
||||
| Action | Admin | Node Admin | Deployer | Auditor | Viewer |
|
||||
|--------|-------|------------|----------|---------|--------|
|
||||
| View snapshot list | Yes | Yes | Yes | Yes | Yes |
|
||||
| Browse snapshot contents | Yes | Yes | Yes | Yes | Yes |
|
||||
| Create snapshot | Yes | No | No | No | No |
|
||||
| Restore from snapshot | Yes | No | No | No | No |
|
||||
| Delete snapshot | Yes | No | No | No | No |
|
||||
| Upload snapshot to cloud | Yes | No | No | No | No |
|
||||
| Delete cloud snapshot | Yes | No | No | No | No |
|
||||
Every fleet snapshot action requires the **admin** role: viewing the snapshot list, browsing snapshot contents, creating, restoring, deleting, and uploading to the cloud. Because a snapshot captures the `.env` file of every stack, the snapshot list and detail views are restricted to administrators rather than read-only roles.
|
||||
|
||||
<Note>
|
||||
Cloud backup actions to Sencho Cloud Backup require an Admiral license. Cloud backup actions to a Custom S3-compatible target work on every tier.
|
||||
</Note>
|
||||
Cloud Backup configuration is also admin-only. Mirroring to Sencho Cloud Backup additionally requires an Admiral license; a Custom S3-compatible target works on every tier.
|
||||
|
||||
## Storage
|
||||
|
||||
Snapshots are stored in Sencho's SQLite database. Compose files are typically small (under 10 KB each), so even hundreds of snapshots consume minimal disk space. For very large fleets, consider periodically deleting old snapshots to keep the database lean.
|
||||
Snapshots are stored in Sencho's SQLite database. Captured file contents, including `.env` files, are encrypted at rest with the instance key, so a database copy never exposes stack secrets in plaintext. Compose files are typically small (under 10 KB each), so even hundreds of snapshots consume minimal disk space. Individual files larger than 1 MB are skipped and recorded as a warning to keep snapshots bounded. For very large fleets, consider periodically deleting old snapshots to keep the database lean.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ interface FleetSnapshot {
|
||||
node_count: number;
|
||||
stack_count: number;
|
||||
skipped_nodes: string; // JSON string
|
||||
skipped_stacks: string; // JSON string
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
@@ -58,6 +59,13 @@ interface SkippedNode {
|
||||
reason: string;
|
||||
}
|
||||
|
||||
interface SkippedStack {
|
||||
nodeId: number;
|
||||
nodeName: string;
|
||||
stackName: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
const PAGE_SIZE = 10;
|
||||
|
||||
// --- Main Component ---
|
||||
@@ -294,12 +302,12 @@ export default function FleetSnapshots() {
|
||||
});
|
||||
};
|
||||
|
||||
// --- Parse skipped nodes safely ---
|
||||
// --- Parse JSON-array warning columns safely ---
|
||||
|
||||
function parseSkippedNodes(raw: string): SkippedNode[] {
|
||||
function parseJsonArray<T>(raw: string): T[] {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (Array.isArray(parsed)) return parsed as SkippedNode[];
|
||||
if (Array.isArray(parsed)) return parsed as T[];
|
||||
} catch { /* invalid JSON */ }
|
||||
return [];
|
||||
}
|
||||
@@ -353,7 +361,7 @@ export default function FleetSnapshots() {
|
||||
|
||||
{/* Skipped nodes warning */}
|
||||
{(() => {
|
||||
const skipped = parseSkippedNodes(selectedSnapshot.skipped_nodes);
|
||||
const skipped = parseJsonArray<SkippedNode>(selectedSnapshot.skipped_nodes);
|
||||
if (skipped.length === 0) return null;
|
||||
return (
|
||||
<div className="rounded-xl border border-warning/30 bg-warning/5 p-4">
|
||||
@@ -376,6 +384,33 @@ export default function FleetSnapshots() {
|
||||
);
|
||||
})()}
|
||||
|
||||
{/* Partially captured stacks warning */}
|
||||
{(() => {
|
||||
const skipped = parseJsonArray<SkippedStack>(selectedSnapshot.skipped_stacks);
|
||||
if (skipped.length === 0) return null;
|
||||
return (
|
||||
<div className="rounded-xl border border-warning/30 bg-warning/5 p-4">
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<AlertTriangle className="w-4 h-4 text-warning shrink-0" />
|
||||
<span className="text-sm font-medium text-warning">
|
||||
Some stacks were not fully captured:
|
||||
</span>
|
||||
</div>
|
||||
<ul className="ml-6 space-y-1">
|
||||
{skipped.map((stack, i) => (
|
||||
<li key={`${stack.nodeId}:${stack.stackName}:${i}`} className="text-sm text-muted-foreground">
|
||||
<span className="font-medium">{stack.nodeName}</span>
|
||||
{' / '}
|
||||
<span className="font-mono">{stack.stackName}</span>
|
||||
{' - '}
|
||||
{stack.reason}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
|
||||
{/* Node / Stack / File tree */}
|
||||
<div className="space-y-2">
|
||||
{selectedSnapshot.nodes.map(node => {
|
||||
@@ -580,8 +615,13 @@ export default function FleetSnapshots() {
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{pagedSnapshots.map(snapshot => {
|
||||
const skipped = parseSkippedNodes(snapshot.skipped_nodes);
|
||||
const skippedNames = skipped.map(s => s.nodeName).join(', ');
|
||||
const skippedNodes = parseJsonArray<SkippedNode>(snapshot.skipped_nodes);
|
||||
const skippedStacks = parseJsonArray<SkippedStack>(snapshot.skipped_stacks);
|
||||
const warningCount = skippedNodes.length + skippedStacks.length;
|
||||
const warningTitle = [
|
||||
skippedNodes.length > 0 ? `Nodes: ${skippedNodes.map(s => s.nodeName).join(', ')}` : '',
|
||||
skippedStacks.length > 0 ? `Stacks: ${skippedStacks.map(s => `${s.nodeName}/${s.stackName}`).join(', ')}` : '',
|
||||
].filter(Boolean).join(' · ');
|
||||
return (
|
||||
<TableRow key={snapshot.id}>
|
||||
<TableCell className="text-xs font-mono tabular-nums whitespace-nowrap">
|
||||
@@ -609,13 +649,13 @@ export default function FleetSnapshots() {
|
||||
{snapshot.stack_count} stack{snapshot.stack_count !== 1 ? 's' : ''}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
{skipped.length > 0 ? (
|
||||
{warningCount > 0 ? (
|
||||
<span
|
||||
className="flex items-center gap-1 text-warning"
|
||||
title={`Skipped: ${skippedNames}`}
|
||||
title={warningTitle}
|
||||
>
|
||||
<AlertTriangle className="w-3.5 h-3.5" />
|
||||
<span className="text-xs font-mono tabular-nums">{skipped.length}</span>
|
||||
<span className="text-xs font-mono tabular-nums">{warningCount}</span>
|
||||
</span>
|
||||
) : (
|
||||
<span className="text-xs text-muted-foreground">None</span>
|
||||
|
||||
@@ -78,11 +78,13 @@ export function FleetView({ onNavigateToNode }: FleetViewProps) {
|
||||
<TabsHighlightItem value="overview">
|
||||
<TabsTrigger value="overview">Overview</TabsTrigger>
|
||||
</TabsHighlightItem>
|
||||
<TabsHighlightItem value="snapshots">
|
||||
<TabsTrigger value="snapshots">
|
||||
<Camera className="w-4 h-4 mr-1.5" />Snapshots
|
||||
</TabsTrigger>
|
||||
</TabsHighlightItem>
|
||||
{isAdmin && (
|
||||
<TabsHighlightItem value="snapshots">
|
||||
<TabsTrigger value="snapshots">
|
||||
<Camera className="w-4 h-4 mr-1.5" />Snapshots
|
||||
</TabsTrigger>
|
||||
</TabsHighlightItem>
|
||||
)}
|
||||
<TabsHighlightItem value="configuration">
|
||||
<TabsTrigger value="configuration">
|
||||
<SlidersHorizontal className="w-4 h-4 mr-1.5" />Status
|
||||
@@ -192,9 +194,11 @@ export function FleetView({ onNavigateToNode }: FleetViewProps) {
|
||||
/>
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent value="snapshots">
|
||||
<FleetSnapshots />
|
||||
</TabsContent>
|
||||
{isAdmin && (
|
||||
<TabsContent value="snapshots">
|
||||
<FleetSnapshots />
|
||||
</TabsContent>
|
||||
)}
|
||||
<TabsContent value="configuration">
|
||||
<FleetConfiguration />
|
||||
</TabsContent>
|
||||
|
||||
Reference in New Issue
Block a user