mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-28 11:17:07 +00:00
fix(fleet): isolate corrupt snapshot file decrypt failures (#1650)
* fix(fleet): isolate corrupt snapshot file decrypt failures A single damaged encrypted fleet-snapshot row no longer fails detail, restore, or off-site upload for the whole snapshot. Unavailable members are marked, restore is blocked before mutation, and cloud upload fails closed with no PutObject. * fix(fleet): fail closed on damaged enc snapshot envelopes Unrecognized enc: payloads no longer fall through as usable plaintext. Only clear legacy prose stays readable; delimiter-byte and similar envelope damage stays unavailable through restore and cloud upload. * fix(fleet): subordinate legacy enc prose to envelope shape Legacy exceptions no longer trigger from = or whitespace alone. Encryption-shaped payloads (length and hex density) stay unavailable through restore and cloud upload, while short genuine prose such as enc:hello remains usable. * fix(fleet): preserve non-envelope enc legacy plaintext Any non-empty enc: payload that is not encryption-shaped is kept verbatim, including punctuation forms such as enc:hello-world, while envelope-shaped damage remains unavailable.
This commit is contained in:
@@ -260,6 +260,72 @@ describe('CloudBackupService — uploadSnapshot', () => {
|
||||
expect(status.error).toContain('bad creds');
|
||||
});
|
||||
|
||||
it('fails closed with no PutObject when a snapshot file is unavailable', async () => {
|
||||
seedCustomProvider();
|
||||
const db = DatabaseService.getInstance();
|
||||
const snapshotId = db.createSnapshot('Corrupt member', 'admin', 1, 1, '[]');
|
||||
const cipher = CryptoService.getInstance().encrypt('services: {}\n');
|
||||
const payload = cipher.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(snapshotId, 1, 'gateway', 'web', 'compose.yaml', `enc:${iv}:${tag}:${ct.slice(0, 3)}`);
|
||||
|
||||
sentSpy.mockClear();
|
||||
await expect(CloudBackupService.getInstance().uploadSnapshot(snapshotId)).rejects.toThrow(/could not be decrypted/i);
|
||||
|
||||
const putCalls = sentSpy.mock.calls.filter(c => c[0].name === 'PutObjectCommand');
|
||||
expect(putCalls).toHaveLength(0);
|
||||
const status = CloudBackupService.getInstance().getUploadStatus(snapshotId);
|
||||
expect(status.status).toBe('failed');
|
||||
expect(status.error).toMatch(/could not be decrypted/i);
|
||||
});
|
||||
|
||||
it('fails closed with no PutObject when a healthy sibling file exists alongside an unavailable one', async () => {
|
||||
seedCustomProvider();
|
||||
const db = DatabaseService.getInstance();
|
||||
const snapshotId = db.createSnapshot('Mixed members', 'admin', 1, 1, '[]');
|
||||
const good = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
|
||||
const bad = CryptoService.getInstance().encrypt('SECRET=long-enough-to-truncate\n');
|
||||
const payload = bad.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(snapshotId, 1, 'gateway', 'web', 'compose.yaml', good);
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(snapshotId, 1, 'gateway', 'web', '.env', `enc:${iv}:${tag}:${ct.slice(0, 3)}`);
|
||||
|
||||
sentSpy.mockClear();
|
||||
await expect(CloudBackupService.getInstance().uploadSnapshot(snapshotId)).rejects.toThrow(/could not be decrypted/i);
|
||||
|
||||
const putCalls = sentSpy.mock.calls.filter(c => c[0].name === 'PutObjectCommand');
|
||||
expect(putCalls).toHaveLength(0);
|
||||
const status = CloudBackupService.getInstance().getUploadStatus(snapshotId);
|
||||
expect(status.status).toBe('failed');
|
||||
expect(status.error).toMatch(/could not be decrypted/i);
|
||||
});
|
||||
|
||||
it('fails closed with no PutObject when a delimiter byte is corrupted', async () => {
|
||||
seedCustomProvider();
|
||||
const db = DatabaseService.getInstance();
|
||||
const snapshotId = db.createSnapshot('Delim damage', 'admin', 1, 1, '[]');
|
||||
const cipher = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
|
||||
const payload = cipher.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(snapshotId, 1, 'gateway', 'web', 'compose.yaml', `enc:${iv}=${tag}:${ct}`);
|
||||
|
||||
sentSpy.mockClear();
|
||||
await expect(CloudBackupService.getInstance().uploadSnapshot(snapshotId)).rejects.toThrow(/could not be decrypted/i);
|
||||
|
||||
const putCalls = sentSpy.mock.calls.filter(c => c[0].name === 'PutObjectCommand');
|
||||
expect(putCalls).toHaveLength(0);
|
||||
const status = CloudBackupService.getInstance().getUploadStatus(snapshotId);
|
||||
expect(status.status).toBe('failed');
|
||||
});
|
||||
|
||||
it('throws when no provider is configured', async () => {
|
||||
await expect(CloudBackupService.getInstance().uploadSnapshot(999)).rejects.toThrow(/not configured/i);
|
||||
});
|
||||
|
||||
@@ -101,14 +101,16 @@ describe('Snapshot content-at-rest encryption', () => {
|
||||
expect(raw.content).not.toContain('s3cr3t');
|
||||
|
||||
const env = db.getSnapshotFiles(snapshotId).find(f => f.filename === '.env');
|
||||
expect(env?.content).toBe(ENV_SECRET);
|
||||
expect(env?.available).toBe(true);
|
||||
if (env?.available) expect(env.content).toBe(ENV_SECRET);
|
||||
});
|
||||
|
||||
it('decrypts content on the restore read path (getSnapshotStackFiles)', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const files = db.getSnapshotStackFiles(snapshotId, 1, 'web');
|
||||
const env = files.find(f => f.filename === '.env');
|
||||
expect(env?.content).toBe(ENV_SECRET);
|
||||
expect(env?.available).toBe(true);
|
||||
if (env?.available) expect(env.content).toBe(ENV_SECRET);
|
||||
});
|
||||
|
||||
it('reads a legacy plaintext row back verbatim (decrypt tolerates non-ciphertext)', () => {
|
||||
@@ -121,7 +123,57 @@ describe('Snapshot content-at-rest encryption', () => {
|
||||
).run(legacyId, 1, 'local', 'legacy', 'compose.yaml', 'plain: text\n');
|
||||
|
||||
const files = db.getSnapshotFiles(legacyId);
|
||||
expect(files[0].content).toBe('plain: text\n');
|
||||
expect(files[0].available).toBe(true);
|
||||
if (files[0].available) expect(files[0].content).toBe('plain: text\n');
|
||||
});
|
||||
|
||||
it('reads punctuated legacy enc: plaintext rows back verbatim', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const legacyId = db.createSnapshot('legacy-punct', 'admin', 1, 1, '[]', '[]');
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(legacyId, 1, 'local', 'legacy', 'compose.yaml', 'enc:hello-world');
|
||||
|
||||
const files = db.getSnapshotFiles(legacyId);
|
||||
expect(files[0].available).toBe(true);
|
||||
if (files[0].available) expect(files[0].content).toBe('enc:hello-world');
|
||||
});
|
||||
|
||||
it('isolates a corrupt encrypted sibling without failing the read', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('partial-corrupt', 'admin', 1, 2, '[]', '[]');
|
||||
const good = CryptoService.getInstance().encrypt('services: {}\n');
|
||||
const bad = CryptoService.getInstance().encrypt('SECRET=x\n');
|
||||
const payload = bad.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
const damaged = `enc:${iv}:${tag}:${ct.slice(0, 3)}`;
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(id, 1, 'local', 'good', 'compose.yaml', good);
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(id, 1, 'local', 'bad', 'compose.yaml', damaged);
|
||||
|
||||
const files = db.getSnapshotFiles(id);
|
||||
expect(files).toHaveLength(2);
|
||||
const goodFile = files.find(f => f.stack_name === 'good');
|
||||
const badFile = files.find(f => f.stack_name === 'bad');
|
||||
expect(goodFile?.available).toBe(true);
|
||||
if (goodFile?.available) expect(goodFile.content).toBe('services: {}\n');
|
||||
expect(badFile?.available).toBe(false);
|
||||
});
|
||||
|
||||
it('preserves a valid empty file as available empty content', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('empty-env', 'admin', 1, 1, '[]', '[]');
|
||||
db.insertSnapshotFiles(id, [
|
||||
{ nodeId: 1, nodeName: 'local', stackName: 'web', filename: '.env', content: '' },
|
||||
{ nodeId: 1, nodeName: 'local', stackName: 'web', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
]);
|
||||
const files = db.getSnapshotFiles(id);
|
||||
const env = files.find(f => f.filename === '.env');
|
||||
expect(env?.available).toBe(true);
|
||||
if (env?.available) expect(env.content).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -221,6 +273,118 @@ describe('Single-stack snapshot restore (behavior lock)', () => {
|
||||
expect(res.status).toBe(409);
|
||||
});
|
||||
|
||||
it('returns 409 SNAPSHOT_FILE_UNAVAILABLE and does not mutate live files', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-corrupt', 'admin', 1, 1, '[]', '[]');
|
||||
const cipher = CryptoService.getInstance().encrypt('services: {}\n');
|
||||
const payload = cipher.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
const damaged = `enc:${iv}:${tag}:${ct.slice(0, 3)}`;
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(id, LOCAL_NODE_ID, 'local', 'corrupt-web', 'compose.yaml', damaged);
|
||||
seedStackDir('corrupt-web');
|
||||
const beforeCompose = 'services:\n keep: {}\n';
|
||||
const beforeEnv = 'KEEP=1\n';
|
||||
fs.writeFileSync(composePath('corrupt-web'), beforeCompose);
|
||||
fs.writeFileSync(envPath('corrupt-web'), beforeEnv);
|
||||
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: LOCAL_NODE_ID, stackName: 'corrupt-web', redeploy: true, restoreNotes: true });
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.code).toBe('SNAPSHOT_FILE_UNAVAILABLE');
|
||||
expect(fs.readFileSync(composePath('corrupt-web'), 'utf-8')).toBe(beforeCompose);
|
||||
expect(fs.readFileSync(envPath('corrupt-web'), 'utf-8')).toBe(beforeEnv);
|
||||
expect(deploySpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns 409 for mixed available and unavailable files in the same stack without writing', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-mixed', 'admin', 1, 1, '[]', '[]');
|
||||
const good = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
|
||||
const bad = CryptoService.getInstance().encrypt('SECRET=long-enough-to-truncate\n');
|
||||
const payload = bad.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(id, LOCAL_NODE_ID, 'local', 'mixed-web', 'compose.yaml', good);
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(id, LOCAL_NODE_ID, 'local', 'mixed-web', '.env', `enc:${iv}:${tag}:${ct.slice(0, 3)}`);
|
||||
seedStackDir('mixed-web');
|
||||
const beforeCompose = 'services:\n keep: {}\n';
|
||||
const beforeEnv = 'KEEP=1\n';
|
||||
fs.writeFileSync(composePath('mixed-web'), beforeCompose);
|
||||
fs.writeFileSync(envPath('mixed-web'), beforeEnv);
|
||||
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: LOCAL_NODE_ID, stackName: 'mixed-web', redeploy: true });
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.code).toBe('SNAPSHOT_FILE_UNAVAILABLE');
|
||||
expect(fs.readFileSync(composePath('mixed-web'), 'utf-8')).toBe(beforeCompose);
|
||||
expect(fs.readFileSync(envPath('mixed-web'), 'utf-8')).toBe(beforeEnv);
|
||||
expect(deploySpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns 409 when a delimiter byte is corrupted and does not write ciphertext', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-delim', 'admin', 1, 1, '[]', '[]');
|
||||
const cipher = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
|
||||
const payload = cipher.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
const damaged = `enc:${iv}=${tag}:${ct}`;
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(id, LOCAL_NODE_ID, 'local', 'delim-web', 'compose.yaml', damaged);
|
||||
seedStackDir('delim-web');
|
||||
const beforeCompose = 'services:\n keep: {}\n';
|
||||
fs.writeFileSync(composePath('delim-web'), beforeCompose);
|
||||
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: LOCAL_NODE_ID, stackName: 'delim-web', redeploy: true });
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.code).toBe('SNAPSHOT_FILE_UNAVAILABLE');
|
||||
expect(fs.readFileSync(composePath('delim-web'), 'utf-8')).toBe(beforeCompose);
|
||||
expect(deploySpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('detail returns 200 with unavailable marker and intact sibling content', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('detail-corrupt', 'admin', 1, 2, '[]', '[]');
|
||||
const good = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
|
||||
const bad = CryptoService.getInstance().encrypt('SECRET=long-enough-to-truncate\n');
|
||||
const payload = bad.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(id, LOCAL_NODE_ID, 'local', 'ok', 'compose.yaml', good);
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(id, LOCAL_NODE_ID, 'local', 'bad', 'compose.yaml', `enc:${iv}:${tag}:${ct.slice(0, 3)}`);
|
||||
|
||||
const res = await request(app).get(`/api/fleet/snapshots/${id}`).set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.fileDecryptWarnings).toEqual([
|
||||
expect.objectContaining({ stackName: 'bad', filename: 'compose.yaml' }),
|
||||
]);
|
||||
const okStack = res.body.nodes[0].stacks.find((s: { stackName: string }) => s.stackName === 'ok');
|
||||
const badStack = res.body.nodes[0].stacks.find((s: { stackName: string }) => s.stackName === 'bad');
|
||||
expect(okStack.files[0].content).toBe('services: { ok: {} }\n');
|
||||
expect(okStack.files[0].unavailable).toBeUndefined();
|
||||
expect(badStack.files[0].unavailable).toBe(true);
|
||||
expect(badStack.files[0].content).toBeUndefined();
|
||||
expect(JSON.stringify(res.body)).not.toContain('enc:');
|
||||
});
|
||||
|
||||
it('redeploys after restore when requested', async () => {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
@@ -583,6 +747,83 @@ describe('Restore-all', () => {
|
||||
expect(bad?.error).toMatch(/no longer exists/i);
|
||||
});
|
||||
|
||||
it('isolates corrupt decrypt stacks before any mutation with notes and redeploy requested', async () => {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-all-corrupt', 'admin', 1, 2, '[]', '[]');
|
||||
const good = CryptoService.getInstance().encrypt('services:\n app: {}\n');
|
||||
const bad = CryptoService.getInstance().encrypt('SECRET=x\n');
|
||||
const payload = bad.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
const damaged = `enc:${iv}:${tag}:${ct.slice(0, 3)}`;
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(id, LOCAL_NODE_ID, 'local', 'healthy', 'compose.yaml', good);
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(id, LOCAL_NODE_ID, 'local', 'corrupt', 'compose.yaml', damaged);
|
||||
seedStackDir('healthy');
|
||||
seedStackDir('corrupt');
|
||||
fs.writeFileSync(composePath('corrupt'), 'services:\n keep: {}\n');
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore-all`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ redeploy: true, restoreNotes: true });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.restored).toBe(1);
|
||||
expect(res.body.failed).toBe(1);
|
||||
const corrupt = (res.body.results as Array<{ stackName: string; success: boolean; error?: string; redeployed: boolean }>)
|
||||
.find(r => r.stackName === 'corrupt');
|
||||
expect(corrupt?.success).toBe(false);
|
||||
expect(corrupt?.error).toMatch(/could not be decrypted/i);
|
||||
expect(corrupt?.redeployed).toBe(false);
|
||||
expect(fs.readFileSync(composePath('corrupt'), 'utf-8')).toContain('keep: {}');
|
||||
expect(fs.readFileSync(composePath('healthy'), 'utf-8')).toContain('app: {}');
|
||||
expect(deploySpy).toHaveBeenCalledTimes(1);
|
||||
expect(deploySpy).toHaveBeenCalledWith('healthy', undefined, undefined, {
|
||||
source: 'fleet_snapshot',
|
||||
actor: 'system:fleet-snapshot',
|
||||
});
|
||||
expect(deploySpy.mock.calls.every(call => call[0] !== 'corrupt')).toBe(true);
|
||||
});
|
||||
|
||||
it('isolates delimiter-byte corruption before restore-all mutation', async () => {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-all-delim', 'admin', 1, 2, '[]', '[]');
|
||||
const good = CryptoService.getInstance().encrypt('services:\n app: {}\n');
|
||||
const bad = CryptoService.getInstance().encrypt('SECRET=long-enough-to-mutate\n');
|
||||
const payload = bad.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
const damaged = `enc:${iv} ${tag}:${ct}`;
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(id, LOCAL_NODE_ID, 'local', 'healthy', 'compose.yaml', good);
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(id, LOCAL_NODE_ID, 'local', 'corrupt', 'compose.yaml', damaged);
|
||||
seedStackDir('healthy');
|
||||
seedStackDir('corrupt');
|
||||
fs.writeFileSync(composePath('corrupt'), 'services:\n keep: {}\n');
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore-all`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ redeploy: true });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.restored).toBe(1);
|
||||
expect(res.body.failed).toBe(1);
|
||||
const corrupt = (res.body.results as Array<{ stackName: string; success: boolean; error?: string }>)
|
||||
.find(r => r.stackName === 'corrupt');
|
||||
expect(corrupt?.success).toBe(false);
|
||||
expect(fs.readFileSync(composePath('corrupt'), 'utf-8')).toContain('keep: {}');
|
||||
expect(deploySpy).toHaveBeenCalledTimes(1);
|
||||
expect(deploySpy.mock.calls.every(call => call[0] !== 'corrupt')).toBe(true);
|
||||
});
|
||||
|
||||
it('redeploys each restored stack when requested', async () => {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { CryptoService } from '../services/CryptoService';
|
||||
import {
|
||||
classifySnapshotFileContent,
|
||||
isEnvelopeLikeDamage,
|
||||
isEnvelopeShapedPayload,
|
||||
isStructurallyValidSnapshotEnvelope,
|
||||
} from '../helpers/snapshotFileDecrypt';
|
||||
|
||||
describe('snapshotFileDecrypt classification', () => {
|
||||
const encrypt = (plain: string) => CryptoService.getInstance().encrypt(plain);
|
||||
|
||||
it('returns plaintext for non-enc values including empty string', () => {
|
||||
expect(classifySnapshotFileContent('')).toEqual({ kind: 'usable', content: '' });
|
||||
expect(classifySnapshotFileContent('services:\n web:\n')).toEqual({
|
||||
kind: 'usable',
|
||||
content: 'services:\n web:\n',
|
||||
});
|
||||
});
|
||||
|
||||
it('decrypts a structurally valid envelope', () => {
|
||||
const cipher = encrypt('SECRET=1\n');
|
||||
expect(isStructurallyValidSnapshotEnvelope(cipher)).toBe(true);
|
||||
expect(classifySnapshotFileContent(cipher)).toEqual({ kind: 'usable', content: 'SECRET=1\n' });
|
||||
});
|
||||
|
||||
it('marks auth-failing valid envelopes unavailable', () => {
|
||||
const cipher = encrypt('ok');
|
||||
// Flip last hex nibble of ciphertext to keep structure valid but break auth
|
||||
const parts = cipher.split(':');
|
||||
const last = parts[parts.length - 1];
|
||||
const flipped = (last.slice(0, -1) + (last.endsWith('0') ? '1' : '0'));
|
||||
const tampered = [...parts.slice(0, -1), flipped].join(':');
|
||||
expect(isStructurallyValidSnapshotEnvelope(tampered)).toBe(true);
|
||||
expect(classifySnapshotFileContent(tampered)).toEqual(
|
||||
expect.objectContaining({ kind: 'unavailable', reason: 'decrypt_failed' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('preserves clearly non-envelope legacy enc: prose including punctuation', () => {
|
||||
const legacyValues = [
|
||||
'enc:hello',
|
||||
'enc:FOO_BAR=baz',
|
||||
'enc: path with spaces',
|
||||
'enc:hello-world',
|
||||
'enc:hello/world',
|
||||
'enc:{legacy}',
|
||||
'enc:hello.world',
|
||||
'enc:hello!',
|
||||
];
|
||||
for (const value of legacyValues) {
|
||||
expect(isEnvelopeShapedPayload(value.slice('enc:'.length)), value).toBe(false);
|
||||
expect(classifySnapshotFileContent(value), value).toEqual({ kind: 'usable', content: value });
|
||||
expect(isEnvelopeLikeDamage(value), value).toBe(false);
|
||||
}
|
||||
expect(isEnvelopeLikeDamage('enc:deadbeef')).toBe(true);
|
||||
expect(classifySnapshotFileContent('enc:deadbeef')).toEqual({
|
||||
kind: 'unavailable',
|
||||
reason: 'envelope_damage',
|
||||
});
|
||||
});
|
||||
|
||||
describe('encrypt-then-corrupt detectable family', () => {
|
||||
let good: string;
|
||||
beforeEach(() => {
|
||||
good = encrypt('compose content\n');
|
||||
});
|
||||
|
||||
const envelopeDamage = { kind: 'unavailable' as const, reason: 'envelope_damage' as const };
|
||||
|
||||
it('fails closed on empty truncation to enc:', () => {
|
||||
expect(classifySnapshotFileContent('enc:')).toEqual(envelopeDamage);
|
||||
});
|
||||
|
||||
it('fails closed on short hex-only truncation', () => {
|
||||
expect(classifySnapshotFileContent('enc:deadbeef')).toEqual(envelopeDamage);
|
||||
});
|
||||
|
||||
it('fails closed when IV is truncated', () => {
|
||||
const payload = good.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
const damaged = `enc:${iv.slice(0, 10)}:${tag}:${ct}`;
|
||||
expect(classifySnapshotFileContent(damaged)).toEqual(envelopeDamage);
|
||||
});
|
||||
|
||||
it('fails closed when tag is truncated', () => {
|
||||
const payload = good.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
const damaged = `enc:${iv}:${tag.slice(0, 8)}:${ct}`;
|
||||
expect(classifySnapshotFileContent(damaged)).toEqual(envelopeDamage);
|
||||
});
|
||||
|
||||
it('fails closed when ciphertext is truncated or odd-length', () => {
|
||||
const payload = good.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
expect(classifySnapshotFileContent(`enc:${iv}:${tag}:${ct.slice(0, -1)}`)).toEqual(envelopeDamage);
|
||||
expect(classifySnapshotFileContent(`enc:${iv}:${tag}:${ct.slice(0, 3)}`)).toEqual(envelopeDamage);
|
||||
});
|
||||
|
||||
it('fails closed on non-hex mutation in IV, tag, and ciphertext', () => {
|
||||
const payload = good.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
const ivDamaged = `enc:${iv.slice(0, 5)}g${iv.slice(6)}:${tag}:${ct}`;
|
||||
const tagDamaged = `enc:${iv}:${tag.slice(0, 5)}g${tag.slice(6)}:${ct}`;
|
||||
const ctDamaged = `enc:${iv}:${tag}:${ct.slice(0, 5)}g${ct.slice(6)}`;
|
||||
expect(classifySnapshotFileContent(ivDamaged)).toEqual(envelopeDamage);
|
||||
expect(classifySnapshotFileContent(tagDamaged)).toEqual(envelopeDamage);
|
||||
expect(classifySnapshotFileContent(ctDamaged)).toEqual(envelopeDamage);
|
||||
});
|
||||
|
||||
it('fails closed when a delimiter is removed or added', () => {
|
||||
const payload = good.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
expect(classifySnapshotFileContent(`enc:${iv}:${tag}${ct}`)).toEqual(envelopeDamage);
|
||||
expect(classifySnapshotFileContent(`enc:${iv}:${tag}:${ct}:00`)).toEqual(envelopeDamage);
|
||||
});
|
||||
|
||||
it('fails closed for single-character delimiter substitutions including = and whitespace', () => {
|
||||
const payload = good.slice('enc:'.length);
|
||||
const colonIdx = payload.indexOf(':');
|
||||
expect(colonIdx).toBeGreaterThan(0);
|
||||
const mutants = ['Z', '=', ' ', '\t', '|', '/', '+', '@', '.', ',', ';', '_', '-'];
|
||||
for (const ch of mutants) {
|
||||
const damaged = `enc:${payload.slice(0, colonIdx)}${ch}${payload.slice(colonIdx + 1)}`;
|
||||
expect(isStructurallyValidSnapshotEnvelope(damaged), `delim=${JSON.stringify(ch)}`).toBe(false);
|
||||
expect(isEnvelopeLikeDamage(damaged), `delim=${JSON.stringify(ch)}`).toBe(true);
|
||||
expect(classifySnapshotFileContent(damaged), `delim=${JSON.stringify(ch)}`).toEqual(envelopeDamage);
|
||||
}
|
||||
});
|
||||
|
||||
it('fails closed when = or whitespace is inserted into IV, tag, or ciphertext fields', () => {
|
||||
const payload = good.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
const cases = [
|
||||
`enc:${iv.slice(0, 8)}=${iv.slice(8)}:${tag}:${ct}`,
|
||||
`enc:${iv}:${tag.slice(0, 8)} ${tag.slice(8)}:${ct}`,
|
||||
`enc:${iv}:${tag}:${ct.slice(0, 4)}=${ct.slice(4)}`,
|
||||
`enc:${iv}:${tag}:${ct.slice(0, 4)} ${ct.slice(4)}`,
|
||||
];
|
||||
for (const damaged of cases) {
|
||||
expect(classifySnapshotFileContent(damaged)).toEqual(envelopeDamage);
|
||||
}
|
||||
});
|
||||
|
||||
it('fails closed when a non-hex extra field is appended', () => {
|
||||
const payload = good.slice('enc:'.length);
|
||||
const [iv, tag, ct] = payload.split(':');
|
||||
const extraField = `enc:${iv}:${tag}:${ct}:oops`;
|
||||
expect(isEnvelopeLikeDamage(extraField)).toBe(true);
|
||||
expect(classifySnapshotFileContent(extraField)).toEqual(envelopeDamage);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,148 @@
|
||||
import { CryptoService } from '../services/CryptoService';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
|
||||
const ENCRYPTED_PREFIX = 'enc:';
|
||||
const HEX_RE = /^[0-9a-fA-F]+$/;
|
||||
/**
|
||||
* Producer envelopes are at least an IV worth of hex (24) plus delimiters /
|
||||
* ciphertext. Genuine legacy prose such as enc:hello is much shorter.
|
||||
*/
|
||||
const MIN_ENVELOPE_LIKE_LENGTH = 24;
|
||||
/** Damaged encrypt() payloads remain mostly hex even after a one-byte mutation. */
|
||||
const ENVELOPE_HEX_DENSITY = 0.75;
|
||||
|
||||
/** Database row shape for fleet_snapshot_files (content still ciphertext or legacy plaintext). */
|
||||
export interface SnapshotFileRow {
|
||||
node_id: number;
|
||||
node_name: string;
|
||||
stack_name: string;
|
||||
filename: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
type SnapshotFileMeta = Omit<SnapshotFileRow, 'content'>;
|
||||
|
||||
/**
|
||||
* Decrypted snapshot file read result. Unavailable rows carry no content so
|
||||
* callers cannot accidentally forward a placeholder into restore or archives.
|
||||
*/
|
||||
export type SnapshotFileReadResult =
|
||||
| (SnapshotFileMeta & { available: true; content: string })
|
||||
| (SnapshotFileMeta & { available: false });
|
||||
|
||||
export type AvailableSnapshotFile = Extract<SnapshotFileReadResult, { available: true }>;
|
||||
|
||||
export function isAvailableSnapshotFile(file: SnapshotFileReadResult): file is AvailableSnapshotFile {
|
||||
return file.available;
|
||||
}
|
||||
|
||||
export function isStructurallyValidSnapshotEnvelope(value: string): boolean {
|
||||
if (!value.startsWith(ENCRYPTED_PREFIX)) return false;
|
||||
const parts = value.slice(ENCRYPTED_PREFIX.length).split(':');
|
||||
if (parts.length !== 3) return false;
|
||||
const [ivHex, authTagHex, encryptedHex] = parts;
|
||||
return (
|
||||
HEX_RE.test(ivHex) && ivHex.length === 24 &&
|
||||
HEX_RE.test(authTagHex) && authTagHex.length === 32 &&
|
||||
!!encryptedHex && encryptedHex.length % 2 === 0 && HEX_RE.test(encryptedHex)
|
||||
);
|
||||
}
|
||||
|
||||
function hexDensity(payload: string): number {
|
||||
if (payload.length === 0) return 0;
|
||||
const hexChars = payload.match(/[0-9a-fA-F]/g)?.length ?? 0;
|
||||
return hexChars / payload.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the payload still looks like CryptoService.encrypt output after
|
||||
* truncation or one-byte field/delimiter corruption (high length + hex density),
|
||||
* or is pure hex of any length.
|
||||
*/
|
||||
export function isEnvelopeShapedPayload(payload: string): boolean {
|
||||
if (payload === '') return true;
|
||||
if (HEX_RE.test(payload)) return true;
|
||||
return payload.length >= MIN_ENVELOPE_LIKE_LENGTH && hexDensity(payload) >= ENVELOPE_HEX_DENSITY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Non-envelope legacy plaintext that happens to start with enc:.
|
||||
* Any non-empty payload that is not encryption-shaped is preserved verbatim
|
||||
* (SEN-213). Envelope-shaped damage never qualifies, even with = or whitespace.
|
||||
*/
|
||||
export function isClearlyLegacyEncProse(value: string): boolean {
|
||||
if (!value.startsWith(ENCRYPTED_PREFIX)) return false;
|
||||
const payload = value.slice(ENCRYPTED_PREFIX.length);
|
||||
if (payload === '') return false;
|
||||
return !isEnvelopeShapedPayload(payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* Producer-envelope damage (no DB provenance). Any enc: payload that is not
|
||||
* a structurally valid envelope and not clearly legacy prose is treated as
|
||||
* damage so delimiter substitution and similar corruption cannot fall through
|
||||
* as writable plaintext.
|
||||
*/
|
||||
export function isEnvelopeLikeDamage(value: string): boolean {
|
||||
if (!value.startsWith(ENCRYPTED_PREFIX)) return false;
|
||||
if (isStructurallyValidSnapshotEnvelope(value)) return false;
|
||||
if (isClearlyLegacyEncProse(value)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
export type SnapshotContentClass =
|
||||
| { kind: 'usable'; content: string }
|
||||
| { kind: 'unavailable'; reason: 'decrypt_failed' | 'envelope_damage'; detail?: string };
|
||||
|
||||
/**
|
||||
* Classify a stored snapshot file body. Without a provenance marker, enc:
|
||||
* values that are not valid envelopes and not clearly legacy prose fail closed.
|
||||
*/
|
||||
export function classifySnapshotFileContent(raw: string): SnapshotContentClass {
|
||||
if (!raw.startsWith(ENCRYPTED_PREFIX)) {
|
||||
return { kind: 'usable', content: raw };
|
||||
}
|
||||
|
||||
if (isStructurallyValidSnapshotEnvelope(raw)) {
|
||||
try {
|
||||
return { kind: 'usable', content: CryptoService.getInstance().decrypt(raw) };
|
||||
} catch (err) {
|
||||
return {
|
||||
kind: 'unavailable',
|
||||
reason: 'decrypt_failed',
|
||||
detail: getErrorMessage(err, 'decrypt failed'),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (isClearlyLegacyEncProse(raw)) {
|
||||
return { kind: 'usable', content: raw };
|
||||
}
|
||||
|
||||
return { kind: 'unavailable', reason: 'envelope_damage' };
|
||||
}
|
||||
|
||||
export function readSnapshotFileRow(
|
||||
row: SnapshotFileRow,
|
||||
snapshotId: number,
|
||||
): SnapshotFileReadResult {
|
||||
const { content: raw, ...meta } = row;
|
||||
const classified = classifySnapshotFileContent(raw);
|
||||
|
||||
if (classified.kind === 'usable') {
|
||||
return { ...meta, available: true, content: classified.content };
|
||||
}
|
||||
|
||||
const reasonText = classified.detail
|
||||
? `${classified.reason}: ${classified.detail}`
|
||||
: classified.reason;
|
||||
console.error(
|
||||
`[snapshotFileDecrypt] Failed to decrypt snapshot file ` +
|
||||
`snapshot=${sanitizeForLog(snapshotId)} ` +
|
||||
`node=${sanitizeForLog(meta.node_id)} ` +
|
||||
`stack=${sanitizeForLog(meta.stack_name)} ` +
|
||||
`file=${sanitizeForLog(meta.filename)}: ${reasonText}`,
|
||||
);
|
||||
return { ...meta, available: false };
|
||||
}
|
||||
+56
-10
@@ -2575,8 +2575,13 @@ fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Resp
|
||||
|
||||
const files = db.getSnapshotFiles(id);
|
||||
|
||||
// Group files by node and stack.
|
||||
const nodesMap = new Map<number, { nodeId: number; nodeName: string; stacks: Map<string, Array<{ filename: string; content: string }>> }>();
|
||||
// Group files by node and stack. Unavailable decrypts keep attribution but
|
||||
// never expose ciphertext or fabricated placeholders as content.
|
||||
type DetailFile =
|
||||
| { filename: string; content: string }
|
||||
| { filename: string; unavailable: true };
|
||||
const fileDecryptWarnings: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string }> = [];
|
||||
const nodesMap = new Map<number, { nodeId: number; nodeName: string; stacks: Map<string, DetailFile[]> }>();
|
||||
for (const file of files) {
|
||||
if (!nodesMap.has(file.node_id)) {
|
||||
nodesMap.set(file.node_id, { nodeId: file.node_id, nodeName: file.node_name, stacks: new Map() });
|
||||
@@ -2585,7 +2590,17 @@ fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Resp
|
||||
if (!nodeEntry.stacks.has(file.stack_name)) {
|
||||
nodeEntry.stacks.set(file.stack_name, []);
|
||||
}
|
||||
nodeEntry.stacks.get(file.stack_name)!.push({ filename: file.filename, content: file.content });
|
||||
if (file.available) {
|
||||
nodeEntry.stacks.get(file.stack_name)!.push({ filename: file.filename, content: file.content });
|
||||
} else {
|
||||
nodeEntry.stacks.get(file.stack_name)!.push({ filename: file.filename, unavailable: true });
|
||||
fileDecryptWarnings.push({
|
||||
nodeId: file.node_id,
|
||||
nodeName: file.node_name,
|
||||
stackName: file.stack_name,
|
||||
filename: file.filename,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const nodes = Array.from(nodesMap.values()).map(n => ({
|
||||
@@ -2620,7 +2635,7 @@ fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Resp
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) console.debug('[Fleet:debug] Snapshot detail:', id, files.length, 'files');
|
||||
res.json({ ...snapshot, nodes, documentation });
|
||||
res.json({ ...snapshot, nodes, documentation, fileDecryptWarnings });
|
||||
} catch (error) {
|
||||
console.error('[Fleet Snapshot] Detail error:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch snapshot details' });
|
||||
@@ -2827,9 +2842,19 @@ fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request,
|
||||
res.status(404).json({ error: 'No files found for this stack in the snapshot' });
|
||||
return;
|
||||
}
|
||||
if (files.some(f => !f.available)) {
|
||||
res.status(409).json({
|
||||
error: 'One or more snapshot files could not be decrypted',
|
||||
code: 'SNAPSHOT_FILE_UNAVAILABLE',
|
||||
});
|
||||
return;
|
||||
}
|
||||
const writableFiles = files
|
||||
.filter((f): f is Extract<typeof f, { available: true }> => f.available)
|
||||
.map(f => ({ filename: f.filename, content: f.content }));
|
||||
|
||||
if (isDebugEnabled()) {
|
||||
const fileNames = files.map(f => f.filename).join(', ');
|
||||
const fileNames = writableFiles.map(f => f.filename).join(', ');
|
||||
console.debug('[Fleet:debug] Restore: snapshot=%s, node=%s, stack="%s", files=[%s], redeploy=%s', sanitizeForLog(snapshotId), sanitizeForLog(nodeId), sanitizeForLog(stackName), sanitizeForLog(fileNames), sanitizeForLog(redeploy));
|
||||
}
|
||||
|
||||
@@ -2839,7 +2864,7 @@ fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request,
|
||||
return;
|
||||
}
|
||||
|
||||
await applySnapshotStackFiles(node, stackName, files);
|
||||
await applySnapshotStackFiles(node, stackName, writableFiles);
|
||||
|
||||
// Dossier notes are restored only on explicit opt-in, so a routine file
|
||||
// restore never clobbers the operator's current notes. The note write is
|
||||
@@ -2915,8 +2940,14 @@ fleetRouter.post('/snapshots/:id/restore-all', authMiddleware, async (req: Reque
|
||||
return;
|
||||
}
|
||||
|
||||
// Group the snapshot's files by node + stack, mirroring the detail route.
|
||||
const groups = new Map<string, { nodeId: number; nodeName: string; stackName: string; files: Array<{ filename: string; content: string }> }>();
|
||||
// Group the snapshot's files by node + stack, retaining availability so
|
||||
// unavailable rows are rejected before any content is written.
|
||||
const groups = new Map<string, {
|
||||
nodeId: number;
|
||||
nodeName: string;
|
||||
stackName: string;
|
||||
files: typeof files;
|
||||
}>();
|
||||
for (const file of files) {
|
||||
const key = `${file.node_id}:${file.stack_name}`;
|
||||
let entry = groups.get(key);
|
||||
@@ -2924,7 +2955,7 @@ fleetRouter.post('/snapshots/:id/restore-all', authMiddleware, async (req: Reque
|
||||
entry = { nodeId: file.node_id, nodeName: file.node_name, stackName: file.stack_name, files: [] };
|
||||
groups.set(key, entry);
|
||||
}
|
||||
entry.files.push({ filename: file.filename, content: file.content });
|
||||
entry.files.push(file);
|
||||
}
|
||||
|
||||
const policyOptions = buildPolicyGateOptions(req);
|
||||
@@ -2935,10 +2966,25 @@ fleetRouter.post('/snapshots/:id/restore-all', authMiddleware, async (req: Reque
|
||||
for (const group of groups.values()) {
|
||||
try {
|
||||
if (!isValidStackName(group.stackName)) throw new Error('Invalid stack name');
|
||||
if (group.files.some(f => !f.available)) {
|
||||
results.push({
|
||||
nodeId: group.nodeId,
|
||||
nodeName: group.nodeName,
|
||||
stackName: group.stackName,
|
||||
success: false,
|
||||
redeployed: false,
|
||||
notesRestored: false,
|
||||
error: 'One or more snapshot files could not be decrypted',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const node = db.getNode(group.nodeId);
|
||||
if (!node) throw new Error('Target node no longer exists');
|
||||
|
||||
await applySnapshotStackFiles(node, group.stackName, group.files);
|
||||
const writableFiles = group.files
|
||||
.filter((f): f is Extract<typeof f, { available: true }> => f.available)
|
||||
.map(f => ({ filename: f.filename, content: f.content }));
|
||||
await applySnapshotStackFiles(node, group.stackName, writableFiles);
|
||||
|
||||
// Files are restored; a notes failure is recorded but does not fail the
|
||||
// stack (and must not block the redeploy below).
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* CloudBackupService — off-site replication for fleet snapshots.
|
||||
* CloudBackupService: off-site replication for fleet snapshots.
|
||||
*
|
||||
* Two providers share the same S3-compatible code path:
|
||||
* - 'sencho' : managed Sencho Cloud Backup. Credentials provisioned by
|
||||
@@ -15,9 +15,13 @@ import { Readable } from 'stream';
|
||||
import * as zlib from 'zlib';
|
||||
import * as tar from 'tar-stream';
|
||||
import axios from 'axios';
|
||||
import { DatabaseService, type FleetSnapshotFile } from './DatabaseService';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import { CryptoService } from './CryptoService';
|
||||
import { LicenseService } from './LicenseService';
|
||||
import {
|
||||
isAvailableSnapshotFile,
|
||||
type AvailableSnapshotFile,
|
||||
} from '../helpers/snapshotFileDecrypt';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
|
||||
@@ -255,9 +259,16 @@ export class CloudBackupService {
|
||||
const documentation = db.getSnapshotDocumentation(snapshotId);
|
||||
const objectKey = this.buildObjectKey(cfg, snapshot.id, snapshot.description, snapshot.created_at);
|
||||
|
||||
const availableFiles = files.filter(isAvailableSnapshotFile);
|
||||
if (availableFiles.length !== files.length) {
|
||||
const message = 'One or more snapshot files could not be decrypted';
|
||||
this.setStatus(snapshotId, { status: 'failed', objectKey, error: message, updatedAt: Date.now() });
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
this.setStatus(snapshotId, { status: 'uploading', objectKey, updatedAt: Date.now() });
|
||||
try {
|
||||
const archive = await this.buildArchive(snapshot, files, documentation);
|
||||
const archive = await this.buildArchive(snapshot, availableFiles, documentation);
|
||||
const { client, sdk } = await this.buildS3Client(cfg);
|
||||
await client.send(new sdk.PutObjectCommand({
|
||||
Bucket: cfg.bucket,
|
||||
@@ -364,7 +375,7 @@ export class CloudBackupService {
|
||||
|
||||
private async buildArchive(
|
||||
snapshot: { id: number; description: string; created_by: string; node_count: number; stack_count: number; skipped_nodes: string; created_at: number },
|
||||
files: FleetSnapshotFile[],
|
||||
files: AvailableSnapshotFile[],
|
||||
documentation = '',
|
||||
): Promise<Buffer> {
|
||||
const pack = tar.pack();
|
||||
|
||||
@@ -10,6 +10,9 @@ import { EXPOSURE_INTENTS, type ExposureIntent } from './network/types';
|
||||
import { HIGH_EPSS_THRESHOLD } from './securityPosture';
|
||||
import type { BackendScheduledAction } from './scheduledActionRegistry';
|
||||
import { stackPatternMatches } from '../helpers/stackPattern';
|
||||
import { readSnapshotFileRow, type SnapshotFileReadResult, type SnapshotFileRow } from '../helpers/snapshotFileDecrypt';
|
||||
|
||||
export type { SnapshotFileReadResult } from '../helpers/snapshotFileDecrypt';
|
||||
|
||||
function isPilotMode(): boolean {
|
||||
return process.env.SENCHO_MODE === 'pilot';
|
||||
@@ -4594,8 +4597,9 @@ export class DatabaseService {
|
||||
public insertSnapshotFiles(snapshotId: number, files: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }>): void {
|
||||
// Snapshot file bodies are compose.yaml and .env captures, so they carry
|
||||
// the same secrets as the live stack. Encrypt content at rest with the
|
||||
// instance key; getSnapshotFiles/getSnapshotStackFiles decrypt on read,
|
||||
// so restore and cloud-archive paths see plaintext and stay portable.
|
||||
// instance key. Getters classify and decrypt per row (see
|
||||
// snapshotFileDecrypt.ts); unavailable rows omit content so callers
|
||||
// cannot treat damage as usable plaintext.
|
||||
const crypto = CryptoService.getInstance();
|
||||
const insert = this.db.prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)'
|
||||
@@ -4630,7 +4634,7 @@ export class DatabaseService {
|
||||
const row = this.db.prepare('SELECT documentation FROM fleet_snapshots WHERE id = ?').get(id) as { documentation: string } | undefined;
|
||||
if (!row || row.documentation === '') return '';
|
||||
try {
|
||||
// decrypt() returns non-ciphertext input unchanged, mirroring the file path.
|
||||
// decrypt() returns non-ciphertext input unchanged.
|
||||
return CryptoService.getInstance().decrypt(row.documentation);
|
||||
} catch (e) {
|
||||
// A corrupt blob or a key rotation must not break the primary backup
|
||||
@@ -4641,22 +4645,24 @@ export class DatabaseService {
|
||||
}
|
||||
}
|
||||
|
||||
public getSnapshotFiles(snapshotId: number): FleetSnapshotFile[] {
|
||||
const crypto = CryptoService.getInstance();
|
||||
// Per-row classification isolates corrupt encrypted rows so intact stacks
|
||||
// remain readable. See helpers/snapshotFileDecrypt.ts.
|
||||
public getSnapshotFiles(snapshotId: number): SnapshotFileReadResult[] {
|
||||
const rows = this.db.prepare(
|
||||
'SELECT * FROM fleet_snapshot_files WHERE snapshot_id = ? ORDER BY node_name, stack_name'
|
||||
).all(snapshotId) as FleetSnapshotFile[];
|
||||
// decrypt() returns non-ciphertext input unchanged, so rows written
|
||||
// before content-at-rest encryption still read back as plaintext.
|
||||
return rows.map(row => ({ ...row, content: crypto.decrypt(row.content) }));
|
||||
'SELECT node_id, node_name, stack_name, filename, content FROM fleet_snapshot_files WHERE snapshot_id = ? ORDER BY node_name, stack_name'
|
||||
).all(snapshotId) as SnapshotFileRow[];
|
||||
return this.mapSnapshotFileRows(rows, snapshotId);
|
||||
}
|
||||
|
||||
public getSnapshotStackFiles(snapshotId: number, nodeId: number, stackName: string): FleetSnapshotFile[] {
|
||||
const crypto = CryptoService.getInstance();
|
||||
public getSnapshotStackFiles(snapshotId: number, nodeId: number, stackName: string): SnapshotFileReadResult[] {
|
||||
const rows = this.db.prepare(
|
||||
'SELECT * FROM fleet_snapshot_files WHERE snapshot_id = ? AND node_id = ? AND stack_name = ?'
|
||||
).all(snapshotId, nodeId, stackName) as FleetSnapshotFile[];
|
||||
return rows.map(row => ({ ...row, content: crypto.decrypt(row.content) }));
|
||||
'SELECT node_id, node_name, stack_name, filename, content FROM fleet_snapshot_files WHERE snapshot_id = ? AND node_id = ? AND stack_name = ?'
|
||||
).all(snapshotId, nodeId, stackName) as SnapshotFileRow[];
|
||||
return this.mapSnapshotFileRows(rows, snapshotId);
|
||||
}
|
||||
|
||||
private mapSnapshotFileRows(rows: SnapshotFileRow[], snapshotId: number): SnapshotFileReadResult[] {
|
||||
return rows.map(row => readSnapshotFileRow(row, snapshotId));
|
||||
}
|
||||
|
||||
/** Created-at of the most recent fleet snapshot covering a stack, or null. */
|
||||
|
||||
Reference in New Issue
Block a user