fix(fleet): isolate corrupt snapshot file decrypt failures (#1650)

* fix(fleet): isolate corrupt snapshot file decrypt failures

A single damaged encrypted fleet-snapshot row no longer fails detail, restore, or off-site upload for the whole snapshot. Unavailable members are marked, restore is blocked before mutation, and cloud upload fails closed with no PutObject.

* fix(fleet): fail closed on damaged enc snapshot envelopes

Unrecognized enc: payloads no longer fall through as usable plaintext. Only clear legacy prose stays readable; delimiter-byte and similar envelope damage stays unavailable through restore and cloud upload.

* fix(fleet): subordinate legacy enc prose to envelope shape

Legacy exceptions no longer trigger from = or whitespace alone. Encryption-shaped payloads (length and hex density) stay unavailable through restore and cloud upload, while short genuine prose such as enc:hello remains usable.

* fix(fleet): preserve non-envelope enc legacy plaintext

Any non-empty enc: payload that is not encryption-shaped is kept verbatim, including punctuation forms such as enc:hello-world, while envelope-shaped damage remains unavailable.
This commit is contained in:
Anso
2026-07-19 20:08:31 -04:00
committed by GitHub
parent d94e586af3
commit 3b027957c4
11 changed files with 935 additions and 48 deletions
@@ -260,6 +260,72 @@ describe('CloudBackupService — uploadSnapshot', () => {
expect(status.error).toContain('bad creds');
});
it('fails closed with no PutObject when a snapshot file is unavailable', async () => {
seedCustomProvider();
const db = DatabaseService.getInstance();
const snapshotId = db.createSnapshot('Corrupt member', 'admin', 1, 1, '[]');
const cipher = CryptoService.getInstance().encrypt('services: {}\n');
const payload = cipher.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(snapshotId, 1, 'gateway', 'web', 'compose.yaml', `enc:${iv}:${tag}:${ct.slice(0, 3)}`);
sentSpy.mockClear();
await expect(CloudBackupService.getInstance().uploadSnapshot(snapshotId)).rejects.toThrow(/could not be decrypted/i);
const putCalls = sentSpy.mock.calls.filter(c => c[0].name === 'PutObjectCommand');
expect(putCalls).toHaveLength(0);
const status = CloudBackupService.getInstance().getUploadStatus(snapshotId);
expect(status.status).toBe('failed');
expect(status.error).toMatch(/could not be decrypted/i);
});
it('fails closed with no PutObject when a healthy sibling file exists alongside an unavailable one', async () => {
seedCustomProvider();
const db = DatabaseService.getInstance();
const snapshotId = db.createSnapshot('Mixed members', 'admin', 1, 1, '[]');
const good = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
const bad = CryptoService.getInstance().encrypt('SECRET=long-enough-to-truncate\n');
const payload = bad.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(snapshotId, 1, 'gateway', 'web', 'compose.yaml', good);
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(snapshotId, 1, 'gateway', 'web', '.env', `enc:${iv}:${tag}:${ct.slice(0, 3)}`);
sentSpy.mockClear();
await expect(CloudBackupService.getInstance().uploadSnapshot(snapshotId)).rejects.toThrow(/could not be decrypted/i);
const putCalls = sentSpy.mock.calls.filter(c => c[0].name === 'PutObjectCommand');
expect(putCalls).toHaveLength(0);
const status = CloudBackupService.getInstance().getUploadStatus(snapshotId);
expect(status.status).toBe('failed');
expect(status.error).toMatch(/could not be decrypted/i);
});
it('fails closed with no PutObject when a delimiter byte is corrupted', async () => {
seedCustomProvider();
const db = DatabaseService.getInstance();
const snapshotId = db.createSnapshot('Delim damage', 'admin', 1, 1, '[]');
const cipher = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
const payload = cipher.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(snapshotId, 1, 'gateway', 'web', 'compose.yaml', `enc:${iv}=${tag}:${ct}`);
sentSpy.mockClear();
await expect(CloudBackupService.getInstance().uploadSnapshot(snapshotId)).rejects.toThrow(/could not be decrypted/i);
const putCalls = sentSpy.mock.calls.filter(c => c[0].name === 'PutObjectCommand');
expect(putCalls).toHaveLength(0);
const status = CloudBackupService.getInstance().getUploadStatus(snapshotId);
expect(status.status).toBe('failed');
});
it('throws when no provider is configured', async () => {
await expect(CloudBackupService.getInstance().uploadSnapshot(999)).rejects.toThrow(/not configured/i);
});
@@ -101,14 +101,16 @@ describe('Snapshot content-at-rest encryption', () => {
expect(raw.content).not.toContain('s3cr3t');
const env = db.getSnapshotFiles(snapshotId).find(f => f.filename === '.env');
expect(env?.content).toBe(ENV_SECRET);
expect(env?.available).toBe(true);
if (env?.available) expect(env.content).toBe(ENV_SECRET);
});
it('decrypts content on the restore read path (getSnapshotStackFiles)', () => {
const db = DatabaseService.getInstance();
const files = db.getSnapshotStackFiles(snapshotId, 1, 'web');
const env = files.find(f => f.filename === '.env');
expect(env?.content).toBe(ENV_SECRET);
expect(env?.available).toBe(true);
if (env?.available) expect(env.content).toBe(ENV_SECRET);
});
it('reads a legacy plaintext row back verbatim (decrypt tolerates non-ciphertext)', () => {
@@ -121,7 +123,57 @@ describe('Snapshot content-at-rest encryption', () => {
).run(legacyId, 1, 'local', 'legacy', 'compose.yaml', 'plain: text\n');
const files = db.getSnapshotFiles(legacyId);
expect(files[0].content).toBe('plain: text\n');
expect(files[0].available).toBe(true);
if (files[0].available) expect(files[0].content).toBe('plain: text\n');
});
it('reads punctuated legacy enc: plaintext rows back verbatim', () => {
const db = DatabaseService.getInstance();
const legacyId = db.createSnapshot('legacy-punct', 'admin', 1, 1, '[]', '[]');
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(legacyId, 1, 'local', 'legacy', 'compose.yaml', 'enc:hello-world');
const files = db.getSnapshotFiles(legacyId);
expect(files[0].available).toBe(true);
if (files[0].available) expect(files[0].content).toBe('enc:hello-world');
});
it('isolates a corrupt encrypted sibling without failing the read', () => {
const db = DatabaseService.getInstance();
const id = db.createSnapshot('partial-corrupt', 'admin', 1, 2, '[]', '[]');
const good = CryptoService.getInstance().encrypt('services: {}\n');
const bad = CryptoService.getInstance().encrypt('SECRET=x\n');
const payload = bad.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const damaged = `enc:${iv}:${tag}:${ct.slice(0, 3)}`;
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, 1, 'local', 'good', 'compose.yaml', good);
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, 1, 'local', 'bad', 'compose.yaml', damaged);
const files = db.getSnapshotFiles(id);
expect(files).toHaveLength(2);
const goodFile = files.find(f => f.stack_name === 'good');
const badFile = files.find(f => f.stack_name === 'bad');
expect(goodFile?.available).toBe(true);
if (goodFile?.available) expect(goodFile.content).toBe('services: {}\n');
expect(badFile?.available).toBe(false);
});
it('preserves a valid empty file as available empty content', () => {
const db = DatabaseService.getInstance();
const id = db.createSnapshot('empty-env', 'admin', 1, 1, '[]', '[]');
db.insertSnapshotFiles(id, [
{ nodeId: 1, nodeName: 'local', stackName: 'web', filename: '.env', content: '' },
{ nodeId: 1, nodeName: 'local', stackName: 'web', filename: 'compose.yaml', content: 'services: {}\n' },
]);
const files = db.getSnapshotFiles(id);
const env = files.find(f => f.filename === '.env');
expect(env?.available).toBe(true);
if (env?.available) expect(env.content).toBe('');
});
});
@@ -221,6 +273,118 @@ describe('Single-stack snapshot restore (behavior lock)', () => {
expect(res.status).toBe(409);
});
it('returns 409 SNAPSHOT_FILE_UNAVAILABLE and does not mutate live files', async () => {
const db = DatabaseService.getInstance();
const id = db.createSnapshot('restore-corrupt', 'admin', 1, 1, '[]', '[]');
const cipher = CryptoService.getInstance().encrypt('services: {}\n');
const payload = cipher.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const damaged = `enc:${iv}:${tag}:${ct.slice(0, 3)}`;
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'corrupt-web', 'compose.yaml', damaged);
seedStackDir('corrupt-web');
const beforeCompose = 'services:\n keep: {}\n';
const beforeEnv = 'KEEP=1\n';
fs.writeFileSync(composePath('corrupt-web'), beforeCompose);
fs.writeFileSync(envPath('corrupt-web'), beforeEnv);
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
const res = await request(app)
.post(`/api/fleet/snapshots/${id}/restore`)
.set('Cookie', adminCookie)
.send({ nodeId: LOCAL_NODE_ID, stackName: 'corrupt-web', redeploy: true, restoreNotes: true });
expect(res.status).toBe(409);
expect(res.body.code).toBe('SNAPSHOT_FILE_UNAVAILABLE');
expect(fs.readFileSync(composePath('corrupt-web'), 'utf-8')).toBe(beforeCompose);
expect(fs.readFileSync(envPath('corrupt-web'), 'utf-8')).toBe(beforeEnv);
expect(deploySpy).not.toHaveBeenCalled();
});
it('returns 409 for mixed available and unavailable files in the same stack without writing', async () => {
const db = DatabaseService.getInstance();
const id = db.createSnapshot('restore-mixed', 'admin', 1, 1, '[]', '[]');
const good = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
const bad = CryptoService.getInstance().encrypt('SECRET=long-enough-to-truncate\n');
const payload = bad.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'mixed-web', 'compose.yaml', good);
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'mixed-web', '.env', `enc:${iv}:${tag}:${ct.slice(0, 3)}`);
seedStackDir('mixed-web');
const beforeCompose = 'services:\n keep: {}\n';
const beforeEnv = 'KEEP=1\n';
fs.writeFileSync(composePath('mixed-web'), beforeCompose);
fs.writeFileSync(envPath('mixed-web'), beforeEnv);
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
const res = await request(app)
.post(`/api/fleet/snapshots/${id}/restore`)
.set('Cookie', adminCookie)
.send({ nodeId: LOCAL_NODE_ID, stackName: 'mixed-web', redeploy: true });
expect(res.status).toBe(409);
expect(res.body.code).toBe('SNAPSHOT_FILE_UNAVAILABLE');
expect(fs.readFileSync(composePath('mixed-web'), 'utf-8')).toBe(beforeCompose);
expect(fs.readFileSync(envPath('mixed-web'), 'utf-8')).toBe(beforeEnv);
expect(deploySpy).not.toHaveBeenCalled();
});
it('returns 409 when a delimiter byte is corrupted and does not write ciphertext', async () => {
const db = DatabaseService.getInstance();
const id = db.createSnapshot('restore-delim', 'admin', 1, 1, '[]', '[]');
const cipher = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
const payload = cipher.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const damaged = `enc:${iv}=${tag}:${ct}`;
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'delim-web', 'compose.yaml', damaged);
seedStackDir('delim-web');
const beforeCompose = 'services:\n keep: {}\n';
fs.writeFileSync(composePath('delim-web'), beforeCompose);
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
const res = await request(app)
.post(`/api/fleet/snapshots/${id}/restore`)
.set('Cookie', adminCookie)
.send({ nodeId: LOCAL_NODE_ID, stackName: 'delim-web', redeploy: true });
expect(res.status).toBe(409);
expect(res.body.code).toBe('SNAPSHOT_FILE_UNAVAILABLE');
expect(fs.readFileSync(composePath('delim-web'), 'utf-8')).toBe(beforeCompose);
expect(deploySpy).not.toHaveBeenCalled();
});
it('detail returns 200 with unavailable marker and intact sibling content', async () => {
const db = DatabaseService.getInstance();
const id = db.createSnapshot('detail-corrupt', 'admin', 1, 2, '[]', '[]');
const good = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
const bad = CryptoService.getInstance().encrypt('SECRET=long-enough-to-truncate\n');
const payload = bad.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'ok', 'compose.yaml', good);
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'bad', 'compose.yaml', `enc:${iv}:${tag}:${ct.slice(0, 3)}`);
const res = await request(app).get(`/api/fleet/snapshots/${id}`).set('Cookie', adminCookie);
expect(res.status).toBe(200);
expect(res.body.fileDecryptWarnings).toEqual([
expect.objectContaining({ stackName: 'bad', filename: 'compose.yaml' }),
]);
const okStack = res.body.nodes[0].stacks.find((s: { stackName: string }) => s.stackName === 'ok');
const badStack = res.body.nodes[0].stacks.find((s: { stackName: string }) => s.stackName === 'bad');
expect(okStack.files[0].content).toBe('services: { ok: {} }\n');
expect(okStack.files[0].unavailable).toBeUndefined();
expect(badStack.files[0].unavailable).toBe(true);
expect(badStack.files[0].content).toBeUndefined();
expect(JSON.stringify(res.body)).not.toContain('enc:');
});
it('redeploys after restore when requested', async () => {
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
@@ -583,6 +747,83 @@ describe('Restore-all', () => {
expect(bad?.error).toMatch(/no longer exists/i);
});
it('isolates corrupt decrypt stacks before any mutation with notes and redeploy requested', async () => {
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
const db = DatabaseService.getInstance();
const id = db.createSnapshot('restore-all-corrupt', 'admin', 1, 2, '[]', '[]');
const good = CryptoService.getInstance().encrypt('services:\n app: {}\n');
const bad = CryptoService.getInstance().encrypt('SECRET=x\n');
const payload = bad.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const damaged = `enc:${iv}:${tag}:${ct.slice(0, 3)}`;
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'healthy', 'compose.yaml', good);
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'corrupt', 'compose.yaml', damaged);
seedStackDir('healthy');
seedStackDir('corrupt');
fs.writeFileSync(composePath('corrupt'), 'services:\n keep: {}\n');
const res = await request(app)
.post(`/api/fleet/snapshots/${id}/restore-all`)
.set('Cookie', adminCookie)
.send({ redeploy: true, restoreNotes: true });
expect(res.status).toBe(200);
expect(res.body.restored).toBe(1);
expect(res.body.failed).toBe(1);
const corrupt = (res.body.results as Array<{ stackName: string; success: boolean; error?: string; redeployed: boolean }>)
.find(r => r.stackName === 'corrupt');
expect(corrupt?.success).toBe(false);
expect(corrupt?.error).toMatch(/could not be decrypted/i);
expect(corrupt?.redeployed).toBe(false);
expect(fs.readFileSync(composePath('corrupt'), 'utf-8')).toContain('keep: {}');
expect(fs.readFileSync(composePath('healthy'), 'utf-8')).toContain('app: {}');
expect(deploySpy).toHaveBeenCalledTimes(1);
expect(deploySpy).toHaveBeenCalledWith('healthy', undefined, undefined, {
source: 'fleet_snapshot',
actor: 'system:fleet-snapshot',
});
expect(deploySpy.mock.calls.every(call => call[0] !== 'corrupt')).toBe(true);
});
it('isolates delimiter-byte corruption before restore-all mutation', async () => {
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
const db = DatabaseService.getInstance();
const id = db.createSnapshot('restore-all-delim', 'admin', 1, 2, '[]', '[]');
const good = CryptoService.getInstance().encrypt('services:\n app: {}\n');
const bad = CryptoService.getInstance().encrypt('SECRET=long-enough-to-mutate\n');
const payload = bad.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const damaged = `enc:${iv} ${tag}:${ct}`;
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'healthy', 'compose.yaml', good);
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'corrupt', 'compose.yaml', damaged);
seedStackDir('healthy');
seedStackDir('corrupt');
fs.writeFileSync(composePath('corrupt'), 'services:\n keep: {}\n');
const res = await request(app)
.post(`/api/fleet/snapshots/${id}/restore-all`)
.set('Cookie', adminCookie)
.send({ redeploy: true });
expect(res.status).toBe(200);
expect(res.body.restored).toBe(1);
expect(res.body.failed).toBe(1);
const corrupt = (res.body.results as Array<{ stackName: string; success: boolean; error?: string }>)
.find(r => r.stackName === 'corrupt');
expect(corrupt?.success).toBe(false);
expect(fs.readFileSync(composePath('corrupt'), 'utf-8')).toContain('keep: {}');
expect(deploySpy).toHaveBeenCalledTimes(1);
expect(deploySpy.mock.calls.every(call => call[0] !== 'corrupt')).toBe(true);
});
it('redeploys each restored stack when requested', async () => {
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
@@ -0,0 +1,153 @@
import { describe, it, expect, beforeEach } from 'vitest';
import { CryptoService } from '../services/CryptoService';
import {
classifySnapshotFileContent,
isEnvelopeLikeDamage,
isEnvelopeShapedPayload,
isStructurallyValidSnapshotEnvelope,
} from '../helpers/snapshotFileDecrypt';
describe('snapshotFileDecrypt classification', () => {
const encrypt = (plain: string) => CryptoService.getInstance().encrypt(plain);
it('returns plaintext for non-enc values including empty string', () => {
expect(classifySnapshotFileContent('')).toEqual({ kind: 'usable', content: '' });
expect(classifySnapshotFileContent('services:\n web:\n')).toEqual({
kind: 'usable',
content: 'services:\n web:\n',
});
});
it('decrypts a structurally valid envelope', () => {
const cipher = encrypt('SECRET=1\n');
expect(isStructurallyValidSnapshotEnvelope(cipher)).toBe(true);
expect(classifySnapshotFileContent(cipher)).toEqual({ kind: 'usable', content: 'SECRET=1\n' });
});
it('marks auth-failing valid envelopes unavailable', () => {
const cipher = encrypt('ok');
// Flip last hex nibble of ciphertext to keep structure valid but break auth
const parts = cipher.split(':');
const last = parts[parts.length - 1];
const flipped = (last.slice(0, -1) + (last.endsWith('0') ? '1' : '0'));
const tampered = [...parts.slice(0, -1), flipped].join(':');
expect(isStructurallyValidSnapshotEnvelope(tampered)).toBe(true);
expect(classifySnapshotFileContent(tampered)).toEqual(
expect.objectContaining({ kind: 'unavailable', reason: 'decrypt_failed' }),
);
});
it('preserves clearly non-envelope legacy enc: prose including punctuation', () => {
const legacyValues = [
'enc:hello',
'enc:FOO_BAR=baz',
'enc: path with spaces',
'enc:hello-world',
'enc:hello/world',
'enc:{legacy}',
'enc:hello.world',
'enc:hello!',
];
for (const value of legacyValues) {
expect(isEnvelopeShapedPayload(value.slice('enc:'.length)), value).toBe(false);
expect(classifySnapshotFileContent(value), value).toEqual({ kind: 'usable', content: value });
expect(isEnvelopeLikeDamage(value), value).toBe(false);
}
expect(isEnvelopeLikeDamage('enc:deadbeef')).toBe(true);
expect(classifySnapshotFileContent('enc:deadbeef')).toEqual({
kind: 'unavailable',
reason: 'envelope_damage',
});
});
describe('encrypt-then-corrupt detectable family', () => {
let good: string;
beforeEach(() => {
good = encrypt('compose content\n');
});
const envelopeDamage = { kind: 'unavailable' as const, reason: 'envelope_damage' as const };
it('fails closed on empty truncation to enc:', () => {
expect(classifySnapshotFileContent('enc:')).toEqual(envelopeDamage);
});
it('fails closed on short hex-only truncation', () => {
expect(classifySnapshotFileContent('enc:deadbeef')).toEqual(envelopeDamage);
});
it('fails closed when IV is truncated', () => {
const payload = good.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const damaged = `enc:${iv.slice(0, 10)}:${tag}:${ct}`;
expect(classifySnapshotFileContent(damaged)).toEqual(envelopeDamage);
});
it('fails closed when tag is truncated', () => {
const payload = good.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const damaged = `enc:${iv}:${tag.slice(0, 8)}:${ct}`;
expect(classifySnapshotFileContent(damaged)).toEqual(envelopeDamage);
});
it('fails closed when ciphertext is truncated or odd-length', () => {
const payload = good.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
expect(classifySnapshotFileContent(`enc:${iv}:${tag}:${ct.slice(0, -1)}`)).toEqual(envelopeDamage);
expect(classifySnapshotFileContent(`enc:${iv}:${tag}:${ct.slice(0, 3)}`)).toEqual(envelopeDamage);
});
it('fails closed on non-hex mutation in IV, tag, and ciphertext', () => {
const payload = good.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const ivDamaged = `enc:${iv.slice(0, 5)}g${iv.slice(6)}:${tag}:${ct}`;
const tagDamaged = `enc:${iv}:${tag.slice(0, 5)}g${tag.slice(6)}:${ct}`;
const ctDamaged = `enc:${iv}:${tag}:${ct.slice(0, 5)}g${ct.slice(6)}`;
expect(classifySnapshotFileContent(ivDamaged)).toEqual(envelopeDamage);
expect(classifySnapshotFileContent(tagDamaged)).toEqual(envelopeDamage);
expect(classifySnapshotFileContent(ctDamaged)).toEqual(envelopeDamage);
});
it('fails closed when a delimiter is removed or added', () => {
const payload = good.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
expect(classifySnapshotFileContent(`enc:${iv}:${tag}${ct}`)).toEqual(envelopeDamage);
expect(classifySnapshotFileContent(`enc:${iv}:${tag}:${ct}:00`)).toEqual(envelopeDamage);
});
it('fails closed for single-character delimiter substitutions including = and whitespace', () => {
const payload = good.slice('enc:'.length);
const colonIdx = payload.indexOf(':');
expect(colonIdx).toBeGreaterThan(0);
const mutants = ['Z', '=', ' ', '\t', '|', '/', '+', '@', '.', ',', ';', '_', '-'];
for (const ch of mutants) {
const damaged = `enc:${payload.slice(0, colonIdx)}${ch}${payload.slice(colonIdx + 1)}`;
expect(isStructurallyValidSnapshotEnvelope(damaged), `delim=${JSON.stringify(ch)}`).toBe(false);
expect(isEnvelopeLikeDamage(damaged), `delim=${JSON.stringify(ch)}`).toBe(true);
expect(classifySnapshotFileContent(damaged), `delim=${JSON.stringify(ch)}`).toEqual(envelopeDamage);
}
});
it('fails closed when = or whitespace is inserted into IV, tag, or ciphertext fields', () => {
const payload = good.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const cases = [
`enc:${iv.slice(0, 8)}=${iv.slice(8)}:${tag}:${ct}`,
`enc:${iv}:${tag.slice(0, 8)} ${tag.slice(8)}:${ct}`,
`enc:${iv}:${tag}:${ct.slice(0, 4)}=${ct.slice(4)}`,
`enc:${iv}:${tag}:${ct.slice(0, 4)} ${ct.slice(4)}`,
];
for (const damaged of cases) {
expect(classifySnapshotFileContent(damaged)).toEqual(envelopeDamage);
}
});
it('fails closed when a non-hex extra field is appended', () => {
const payload = good.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const extraField = `enc:${iv}:${tag}:${ct}:oops`;
expect(isEnvelopeLikeDamage(extraField)).toBe(true);
expect(classifySnapshotFileContent(extraField)).toEqual(envelopeDamage);
});
});
});
+148
View File
@@ -0,0 +1,148 @@
import { CryptoService } from '../services/CryptoService';
import { getErrorMessage } from '../utils/errors';
import { sanitizeForLog } from '../utils/safeLog';
const ENCRYPTED_PREFIX = 'enc:';
const HEX_RE = /^[0-9a-fA-F]+$/;
/**
* Producer envelopes are at least an IV worth of hex (24) plus delimiters /
* ciphertext. Genuine legacy prose such as enc:hello is much shorter.
*/
const MIN_ENVELOPE_LIKE_LENGTH = 24;
/** Damaged encrypt() payloads remain mostly hex even after a one-byte mutation. */
const ENVELOPE_HEX_DENSITY = 0.75;
/** Database row shape for fleet_snapshot_files (content still ciphertext or legacy plaintext). */
export interface SnapshotFileRow {
node_id: number;
node_name: string;
stack_name: string;
filename: string;
content: string;
}
type SnapshotFileMeta = Omit<SnapshotFileRow, 'content'>;
/**
* Decrypted snapshot file read result. Unavailable rows carry no content so
* callers cannot accidentally forward a placeholder into restore or archives.
*/
export type SnapshotFileReadResult =
| (SnapshotFileMeta & { available: true; content: string })
| (SnapshotFileMeta & { available: false });
export type AvailableSnapshotFile = Extract<SnapshotFileReadResult, { available: true }>;
export function isAvailableSnapshotFile(file: SnapshotFileReadResult): file is AvailableSnapshotFile {
return file.available;
}
export function isStructurallyValidSnapshotEnvelope(value: string): boolean {
if (!value.startsWith(ENCRYPTED_PREFIX)) return false;
const parts = value.slice(ENCRYPTED_PREFIX.length).split(':');
if (parts.length !== 3) return false;
const [ivHex, authTagHex, encryptedHex] = parts;
return (
HEX_RE.test(ivHex) && ivHex.length === 24 &&
HEX_RE.test(authTagHex) && authTagHex.length === 32 &&
!!encryptedHex && encryptedHex.length % 2 === 0 && HEX_RE.test(encryptedHex)
);
}
function hexDensity(payload: string): number {
if (payload.length === 0) return 0;
const hexChars = payload.match(/[0-9a-fA-F]/g)?.length ?? 0;
return hexChars / payload.length;
}
/**
* True when the payload still looks like CryptoService.encrypt output after
* truncation or one-byte field/delimiter corruption (high length + hex density),
* or is pure hex of any length.
*/
export function isEnvelopeShapedPayload(payload: string): boolean {
if (payload === '') return true;
if (HEX_RE.test(payload)) return true;
return payload.length >= MIN_ENVELOPE_LIKE_LENGTH && hexDensity(payload) >= ENVELOPE_HEX_DENSITY;
}
/**
* Non-envelope legacy plaintext that happens to start with enc:.
* Any non-empty payload that is not encryption-shaped is preserved verbatim
* (SEN-213). Envelope-shaped damage never qualifies, even with = or whitespace.
*/
export function isClearlyLegacyEncProse(value: string): boolean {
if (!value.startsWith(ENCRYPTED_PREFIX)) return false;
const payload = value.slice(ENCRYPTED_PREFIX.length);
if (payload === '') return false;
return !isEnvelopeShapedPayload(payload);
}
/**
* Producer-envelope damage (no DB provenance). Any enc: payload that is not
* a structurally valid envelope and not clearly legacy prose is treated as
* damage so delimiter substitution and similar corruption cannot fall through
* as writable plaintext.
*/
export function isEnvelopeLikeDamage(value: string): boolean {
if (!value.startsWith(ENCRYPTED_PREFIX)) return false;
if (isStructurallyValidSnapshotEnvelope(value)) return false;
if (isClearlyLegacyEncProse(value)) return false;
return true;
}
export type SnapshotContentClass =
| { kind: 'usable'; content: string }
| { kind: 'unavailable'; reason: 'decrypt_failed' | 'envelope_damage'; detail?: string };
/**
* Classify a stored snapshot file body. Without a provenance marker, enc:
* values that are not valid envelopes and not clearly legacy prose fail closed.
*/
export function classifySnapshotFileContent(raw: string): SnapshotContentClass {
if (!raw.startsWith(ENCRYPTED_PREFIX)) {
return { kind: 'usable', content: raw };
}
if (isStructurallyValidSnapshotEnvelope(raw)) {
try {
return { kind: 'usable', content: CryptoService.getInstance().decrypt(raw) };
} catch (err) {
return {
kind: 'unavailable',
reason: 'decrypt_failed',
detail: getErrorMessage(err, 'decrypt failed'),
};
}
}
if (isClearlyLegacyEncProse(raw)) {
return { kind: 'usable', content: raw };
}
return { kind: 'unavailable', reason: 'envelope_damage' };
}
export function readSnapshotFileRow(
row: SnapshotFileRow,
snapshotId: number,
): SnapshotFileReadResult {
const { content: raw, ...meta } = row;
const classified = classifySnapshotFileContent(raw);
if (classified.kind === 'usable') {
return { ...meta, available: true, content: classified.content };
}
const reasonText = classified.detail
? `${classified.reason}: ${classified.detail}`
: classified.reason;
console.error(
`[snapshotFileDecrypt] Failed to decrypt snapshot file ` +
`snapshot=${sanitizeForLog(snapshotId)} ` +
`node=${sanitizeForLog(meta.node_id)} ` +
`stack=${sanitizeForLog(meta.stack_name)} ` +
`file=${sanitizeForLog(meta.filename)}: ${reasonText}`,
);
return { ...meta, available: false };
}
+56 -10
View File
@@ -2575,8 +2575,13 @@ fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Resp
const files = db.getSnapshotFiles(id);
// Group files by node and stack.
const nodesMap = new Map<number, { nodeId: number; nodeName: string; stacks: Map<string, Array<{ filename: string; content: string }>> }>();
// Group files by node and stack. Unavailable decrypts keep attribution but
// never expose ciphertext or fabricated placeholders as content.
type DetailFile =
| { filename: string; content: string }
| { filename: string; unavailable: true };
const fileDecryptWarnings: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string }> = [];
const nodesMap = new Map<number, { nodeId: number; nodeName: string; stacks: Map<string, DetailFile[]> }>();
for (const file of files) {
if (!nodesMap.has(file.node_id)) {
nodesMap.set(file.node_id, { nodeId: file.node_id, nodeName: file.node_name, stacks: new Map() });
@@ -2585,7 +2590,17 @@ fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Resp
if (!nodeEntry.stacks.has(file.stack_name)) {
nodeEntry.stacks.set(file.stack_name, []);
}
nodeEntry.stacks.get(file.stack_name)!.push({ filename: file.filename, content: file.content });
if (file.available) {
nodeEntry.stacks.get(file.stack_name)!.push({ filename: file.filename, content: file.content });
} else {
nodeEntry.stacks.get(file.stack_name)!.push({ filename: file.filename, unavailable: true });
fileDecryptWarnings.push({
nodeId: file.node_id,
nodeName: file.node_name,
stackName: file.stack_name,
filename: file.filename,
});
}
}
const nodes = Array.from(nodesMap.values()).map(n => ({
@@ -2620,7 +2635,7 @@ fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Resp
}
if (isDebugEnabled()) console.debug('[Fleet:debug] Snapshot detail:', id, files.length, 'files');
res.json({ ...snapshot, nodes, documentation });
res.json({ ...snapshot, nodes, documentation, fileDecryptWarnings });
} catch (error) {
console.error('[Fleet Snapshot] Detail error:', error);
res.status(500).json({ error: 'Failed to fetch snapshot details' });
@@ -2827,9 +2842,19 @@ fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request,
res.status(404).json({ error: 'No files found for this stack in the snapshot' });
return;
}
if (files.some(f => !f.available)) {
res.status(409).json({
error: 'One or more snapshot files could not be decrypted',
code: 'SNAPSHOT_FILE_UNAVAILABLE',
});
return;
}
const writableFiles = files
.filter((f): f is Extract<typeof f, { available: true }> => f.available)
.map(f => ({ filename: f.filename, content: f.content }));
if (isDebugEnabled()) {
const fileNames = files.map(f => f.filename).join(', ');
const fileNames = writableFiles.map(f => f.filename).join(', ');
console.debug('[Fleet:debug] Restore: snapshot=%s, node=%s, stack="%s", files=[%s], redeploy=%s', sanitizeForLog(snapshotId), sanitizeForLog(nodeId), sanitizeForLog(stackName), sanitizeForLog(fileNames), sanitizeForLog(redeploy));
}
@@ -2839,7 +2864,7 @@ fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request,
return;
}
await applySnapshotStackFiles(node, stackName, files);
await applySnapshotStackFiles(node, stackName, writableFiles);
// Dossier notes are restored only on explicit opt-in, so a routine file
// restore never clobbers the operator's current notes. The note write is
@@ -2915,8 +2940,14 @@ fleetRouter.post('/snapshots/:id/restore-all', authMiddleware, async (req: Reque
return;
}
// Group the snapshot's files by node + stack, mirroring the detail route.
const groups = new Map<string, { nodeId: number; nodeName: string; stackName: string; files: Array<{ filename: string; content: string }> }>();
// Group the snapshot's files by node + stack, retaining availability so
// unavailable rows are rejected before any content is written.
const groups = new Map<string, {
nodeId: number;
nodeName: string;
stackName: string;
files: typeof files;
}>();
for (const file of files) {
const key = `${file.node_id}:${file.stack_name}`;
let entry = groups.get(key);
@@ -2924,7 +2955,7 @@ fleetRouter.post('/snapshots/:id/restore-all', authMiddleware, async (req: Reque
entry = { nodeId: file.node_id, nodeName: file.node_name, stackName: file.stack_name, files: [] };
groups.set(key, entry);
}
entry.files.push({ filename: file.filename, content: file.content });
entry.files.push(file);
}
const policyOptions = buildPolicyGateOptions(req);
@@ -2935,10 +2966,25 @@ fleetRouter.post('/snapshots/:id/restore-all', authMiddleware, async (req: Reque
for (const group of groups.values()) {
try {
if (!isValidStackName(group.stackName)) throw new Error('Invalid stack name');
if (group.files.some(f => !f.available)) {
results.push({
nodeId: group.nodeId,
nodeName: group.nodeName,
stackName: group.stackName,
success: false,
redeployed: false,
notesRestored: false,
error: 'One or more snapshot files could not be decrypted',
});
continue;
}
const node = db.getNode(group.nodeId);
if (!node) throw new Error('Target node no longer exists');
await applySnapshotStackFiles(node, group.stackName, group.files);
const writableFiles = group.files
.filter((f): f is Extract<typeof f, { available: true }> => f.available)
.map(f => ({ filename: f.filename, content: f.content }));
await applySnapshotStackFiles(node, group.stackName, writableFiles);
// Files are restored; a notes failure is recorded but does not fail the
// stack (and must not block the redeploy below).
+15 -4
View File
@@ -1,5 +1,5 @@
/**
* CloudBackupService off-site replication for fleet snapshots.
* CloudBackupService: off-site replication for fleet snapshots.
*
* Two providers share the same S3-compatible code path:
* - 'sencho' : managed Sencho Cloud Backup. Credentials provisioned by
@@ -15,9 +15,13 @@ import { Readable } from 'stream';
import * as zlib from 'zlib';
import * as tar from 'tar-stream';
import axios from 'axios';
import { DatabaseService, type FleetSnapshotFile } from './DatabaseService';
import { DatabaseService } from './DatabaseService';
import { CryptoService } from './CryptoService';
import { LicenseService } from './LicenseService';
import {
isAvailableSnapshotFile,
type AvailableSnapshotFile,
} from '../helpers/snapshotFileDecrypt';
import { getErrorMessage } from '../utils/errors';
import { isDebugEnabled } from '../utils/debug';
@@ -255,9 +259,16 @@ export class CloudBackupService {
const documentation = db.getSnapshotDocumentation(snapshotId);
const objectKey = this.buildObjectKey(cfg, snapshot.id, snapshot.description, snapshot.created_at);
const availableFiles = files.filter(isAvailableSnapshotFile);
if (availableFiles.length !== files.length) {
const message = 'One or more snapshot files could not be decrypted';
this.setStatus(snapshotId, { status: 'failed', objectKey, error: message, updatedAt: Date.now() });
throw new Error(message);
}
this.setStatus(snapshotId, { status: 'uploading', objectKey, updatedAt: Date.now() });
try {
const archive = await this.buildArchive(snapshot, files, documentation);
const archive = await this.buildArchive(snapshot, availableFiles, documentation);
const { client, sdk } = await this.buildS3Client(cfg);
await client.send(new sdk.PutObjectCommand({
Bucket: cfg.bucket,
@@ -364,7 +375,7 @@ export class CloudBackupService {
private async buildArchive(
snapshot: { id: number; description: string; created_by: string; node_count: number; stack_count: number; skipped_nodes: string; created_at: number },
files: FleetSnapshotFile[],
files: AvailableSnapshotFile[],
documentation = '',
): Promise<Buffer> {
const pack = tar.pack();
+21 -15
View File
@@ -10,6 +10,9 @@ import { EXPOSURE_INTENTS, type ExposureIntent } from './network/types';
import { HIGH_EPSS_THRESHOLD } from './securityPosture';
import type { BackendScheduledAction } from './scheduledActionRegistry';
import { stackPatternMatches } from '../helpers/stackPattern';
import { readSnapshotFileRow, type SnapshotFileReadResult, type SnapshotFileRow } from '../helpers/snapshotFileDecrypt';
export type { SnapshotFileReadResult } from '../helpers/snapshotFileDecrypt';
function isPilotMode(): boolean {
return process.env.SENCHO_MODE === 'pilot';
@@ -4594,8 +4597,9 @@ export class DatabaseService {
public insertSnapshotFiles(snapshotId: number, files: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }>): void {
// Snapshot file bodies are compose.yaml and .env captures, so they carry
// the same secrets as the live stack. Encrypt content at rest with the
// instance key; getSnapshotFiles/getSnapshotStackFiles decrypt on read,
// so restore and cloud-archive paths see plaintext and stay portable.
// instance key. Getters classify and decrypt per row (see
// snapshotFileDecrypt.ts); unavailable rows omit content so callers
// cannot treat damage as usable plaintext.
const crypto = CryptoService.getInstance();
const insert = this.db.prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)'
@@ -4630,7 +4634,7 @@ export class DatabaseService {
const row = this.db.prepare('SELECT documentation FROM fleet_snapshots WHERE id = ?').get(id) as { documentation: string } | undefined;
if (!row || row.documentation === '') return '';
try {
// decrypt() returns non-ciphertext input unchanged, mirroring the file path.
// decrypt() returns non-ciphertext input unchanged.
return CryptoService.getInstance().decrypt(row.documentation);
} catch (e) {
// A corrupt blob or a key rotation must not break the primary backup
@@ -4641,22 +4645,24 @@ export class DatabaseService {
}
}
public getSnapshotFiles(snapshotId: number): FleetSnapshotFile[] {
const crypto = CryptoService.getInstance();
// Per-row classification isolates corrupt encrypted rows so intact stacks
// remain readable. See helpers/snapshotFileDecrypt.ts.
public getSnapshotFiles(snapshotId: number): SnapshotFileReadResult[] {
const rows = this.db.prepare(
'SELECT * FROM fleet_snapshot_files WHERE snapshot_id = ? ORDER BY node_name, stack_name'
).all(snapshotId) as FleetSnapshotFile[];
// decrypt() returns non-ciphertext input unchanged, so rows written
// before content-at-rest encryption still read back as plaintext.
return rows.map(row => ({ ...row, content: crypto.decrypt(row.content) }));
'SELECT node_id, node_name, stack_name, filename, content FROM fleet_snapshot_files WHERE snapshot_id = ? ORDER BY node_name, stack_name'
).all(snapshotId) as SnapshotFileRow[];
return this.mapSnapshotFileRows(rows, snapshotId);
}
public getSnapshotStackFiles(snapshotId: number, nodeId: number, stackName: string): FleetSnapshotFile[] {
const crypto = CryptoService.getInstance();
public getSnapshotStackFiles(snapshotId: number, nodeId: number, stackName: string): SnapshotFileReadResult[] {
const rows = this.db.prepare(
'SELECT * FROM fleet_snapshot_files WHERE snapshot_id = ? AND node_id = ? AND stack_name = ?'
).all(snapshotId, nodeId, stackName) as FleetSnapshotFile[];
return rows.map(row => ({ ...row, content: crypto.decrypt(row.content) }));
'SELECT node_id, node_name, stack_name, filename, content FROM fleet_snapshot_files WHERE snapshot_id = ? AND node_id = ? AND stack_name = ?'
).all(snapshotId, nodeId, stackName) as SnapshotFileRow[];
return this.mapSnapshotFileRows(rows, snapshotId);
}
private mapSnapshotFileRows(rows: SnapshotFileRow[], snapshotId: number): SnapshotFileReadResult[] {
return rows.map(row => readSnapshotFileRow(row, snapshotId));
}
/** Created-at of the most recent fleet snapshot covering a stack, or null. */