fix(fleet): isolate corrupt snapshot file decrypt failures (#1650)

* fix(fleet): isolate corrupt snapshot file decrypt failures

A single damaged encrypted fleet-snapshot row no longer fails detail, restore, or off-site upload for the whole snapshot. Unavailable members are marked, restore is blocked before mutation, and cloud upload fails closed with no PutObject.

* fix(fleet): fail closed on damaged enc snapshot envelopes

Unrecognized enc: payloads no longer fall through as usable plaintext. Only clear legacy prose stays readable; delimiter-byte and similar envelope damage stays unavailable through restore and cloud upload.

* fix(fleet): subordinate legacy enc prose to envelope shape

Legacy exceptions no longer trigger from = or whitespace alone. Encryption-shaped payloads (length and hex density) stay unavailable through restore and cloud upload, while short genuine prose such as enc:hello remains usable.

* fix(fleet): preserve non-envelope enc legacy plaintext

Any non-empty enc: payload that is not encryption-shaped is kept verbatim, including punctuation forms such as enc:hello-world, while envelope-shaped damage remains unavailable.
This commit is contained in:
Anso
2026-07-19 20:08:31 -04:00
committed by GitHub
parent d94e586af3
commit 3b027957c4
11 changed files with 935 additions and 48 deletions
@@ -260,6 +260,72 @@ describe('CloudBackupService — uploadSnapshot', () => {
expect(status.error).toContain('bad creds');
});
it('fails closed with no PutObject when a snapshot file is unavailable', async () => {
seedCustomProvider();
const db = DatabaseService.getInstance();
const snapshotId = db.createSnapshot('Corrupt member', 'admin', 1, 1, '[]');
const cipher = CryptoService.getInstance().encrypt('services: {}\n');
const payload = cipher.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(snapshotId, 1, 'gateway', 'web', 'compose.yaml', `enc:${iv}:${tag}:${ct.slice(0, 3)}`);
sentSpy.mockClear();
await expect(CloudBackupService.getInstance().uploadSnapshot(snapshotId)).rejects.toThrow(/could not be decrypted/i);
const putCalls = sentSpy.mock.calls.filter(c => c[0].name === 'PutObjectCommand');
expect(putCalls).toHaveLength(0);
const status = CloudBackupService.getInstance().getUploadStatus(snapshotId);
expect(status.status).toBe('failed');
expect(status.error).toMatch(/could not be decrypted/i);
});
it('fails closed with no PutObject when a healthy sibling file exists alongside an unavailable one', async () => {
seedCustomProvider();
const db = DatabaseService.getInstance();
const snapshotId = db.createSnapshot('Mixed members', 'admin', 1, 1, '[]');
const good = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
const bad = CryptoService.getInstance().encrypt('SECRET=long-enough-to-truncate\n');
const payload = bad.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(snapshotId, 1, 'gateway', 'web', 'compose.yaml', good);
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(snapshotId, 1, 'gateway', 'web', '.env', `enc:${iv}:${tag}:${ct.slice(0, 3)}`);
sentSpy.mockClear();
await expect(CloudBackupService.getInstance().uploadSnapshot(snapshotId)).rejects.toThrow(/could not be decrypted/i);
const putCalls = sentSpy.mock.calls.filter(c => c[0].name === 'PutObjectCommand');
expect(putCalls).toHaveLength(0);
const status = CloudBackupService.getInstance().getUploadStatus(snapshotId);
expect(status.status).toBe('failed');
expect(status.error).toMatch(/could not be decrypted/i);
});
it('fails closed with no PutObject when a delimiter byte is corrupted', async () => {
seedCustomProvider();
const db = DatabaseService.getInstance();
const snapshotId = db.createSnapshot('Delim damage', 'admin', 1, 1, '[]');
const cipher = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
const payload = cipher.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(snapshotId, 1, 'gateway', 'web', 'compose.yaml', `enc:${iv}=${tag}:${ct}`);
sentSpy.mockClear();
await expect(CloudBackupService.getInstance().uploadSnapshot(snapshotId)).rejects.toThrow(/could not be decrypted/i);
const putCalls = sentSpy.mock.calls.filter(c => c[0].name === 'PutObjectCommand');
expect(putCalls).toHaveLength(0);
const status = CloudBackupService.getInstance().getUploadStatus(snapshotId);
expect(status.status).toBe('failed');
});
it('throws when no provider is configured', async () => {
await expect(CloudBackupService.getInstance().uploadSnapshot(999)).rejects.toThrow(/not configured/i);
});
@@ -101,14 +101,16 @@ describe('Snapshot content-at-rest encryption', () => {
expect(raw.content).not.toContain('s3cr3t');
const env = db.getSnapshotFiles(snapshotId).find(f => f.filename === '.env');
expect(env?.content).toBe(ENV_SECRET);
expect(env?.available).toBe(true);
if (env?.available) expect(env.content).toBe(ENV_SECRET);
});
it('decrypts content on the restore read path (getSnapshotStackFiles)', () => {
const db = DatabaseService.getInstance();
const files = db.getSnapshotStackFiles(snapshotId, 1, 'web');
const env = files.find(f => f.filename === '.env');
expect(env?.content).toBe(ENV_SECRET);
expect(env?.available).toBe(true);
if (env?.available) expect(env.content).toBe(ENV_SECRET);
});
it('reads a legacy plaintext row back verbatim (decrypt tolerates non-ciphertext)', () => {
@@ -121,7 +123,57 @@ describe('Snapshot content-at-rest encryption', () => {
).run(legacyId, 1, 'local', 'legacy', 'compose.yaml', 'plain: text\n');
const files = db.getSnapshotFiles(legacyId);
expect(files[0].content).toBe('plain: text\n');
expect(files[0].available).toBe(true);
if (files[0].available) expect(files[0].content).toBe('plain: text\n');
});
it('reads punctuated legacy enc: plaintext rows back verbatim', () => {
const db = DatabaseService.getInstance();
const legacyId = db.createSnapshot('legacy-punct', 'admin', 1, 1, '[]', '[]');
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(legacyId, 1, 'local', 'legacy', 'compose.yaml', 'enc:hello-world');
const files = db.getSnapshotFiles(legacyId);
expect(files[0].available).toBe(true);
if (files[0].available) expect(files[0].content).toBe('enc:hello-world');
});
it('isolates a corrupt encrypted sibling without failing the read', () => {
const db = DatabaseService.getInstance();
const id = db.createSnapshot('partial-corrupt', 'admin', 1, 2, '[]', '[]');
const good = CryptoService.getInstance().encrypt('services: {}\n');
const bad = CryptoService.getInstance().encrypt('SECRET=x\n');
const payload = bad.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const damaged = `enc:${iv}:${tag}:${ct.slice(0, 3)}`;
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, 1, 'local', 'good', 'compose.yaml', good);
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, 1, 'local', 'bad', 'compose.yaml', damaged);
const files = db.getSnapshotFiles(id);
expect(files).toHaveLength(2);
const goodFile = files.find(f => f.stack_name === 'good');
const badFile = files.find(f => f.stack_name === 'bad');
expect(goodFile?.available).toBe(true);
if (goodFile?.available) expect(goodFile.content).toBe('services: {}\n');
expect(badFile?.available).toBe(false);
});
it('preserves a valid empty file as available empty content', () => {
const db = DatabaseService.getInstance();
const id = db.createSnapshot('empty-env', 'admin', 1, 1, '[]', '[]');
db.insertSnapshotFiles(id, [
{ nodeId: 1, nodeName: 'local', stackName: 'web', filename: '.env', content: '' },
{ nodeId: 1, nodeName: 'local', stackName: 'web', filename: 'compose.yaml', content: 'services: {}\n' },
]);
const files = db.getSnapshotFiles(id);
const env = files.find(f => f.filename === '.env');
expect(env?.available).toBe(true);
if (env?.available) expect(env.content).toBe('');
});
});
@@ -221,6 +273,118 @@ describe('Single-stack snapshot restore (behavior lock)', () => {
expect(res.status).toBe(409);
});
it('returns 409 SNAPSHOT_FILE_UNAVAILABLE and does not mutate live files', async () => {
const db = DatabaseService.getInstance();
const id = db.createSnapshot('restore-corrupt', 'admin', 1, 1, '[]', '[]');
const cipher = CryptoService.getInstance().encrypt('services: {}\n');
const payload = cipher.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const damaged = `enc:${iv}:${tag}:${ct.slice(0, 3)}`;
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'corrupt-web', 'compose.yaml', damaged);
seedStackDir('corrupt-web');
const beforeCompose = 'services:\n keep: {}\n';
const beforeEnv = 'KEEP=1\n';
fs.writeFileSync(composePath('corrupt-web'), beforeCompose);
fs.writeFileSync(envPath('corrupt-web'), beforeEnv);
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
const res = await request(app)
.post(`/api/fleet/snapshots/${id}/restore`)
.set('Cookie', adminCookie)
.send({ nodeId: LOCAL_NODE_ID, stackName: 'corrupt-web', redeploy: true, restoreNotes: true });
expect(res.status).toBe(409);
expect(res.body.code).toBe('SNAPSHOT_FILE_UNAVAILABLE');
expect(fs.readFileSync(composePath('corrupt-web'), 'utf-8')).toBe(beforeCompose);
expect(fs.readFileSync(envPath('corrupt-web'), 'utf-8')).toBe(beforeEnv);
expect(deploySpy).not.toHaveBeenCalled();
});
it('returns 409 for mixed available and unavailable files in the same stack without writing', async () => {
const db = DatabaseService.getInstance();
const id = db.createSnapshot('restore-mixed', 'admin', 1, 1, '[]', '[]');
const good = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
const bad = CryptoService.getInstance().encrypt('SECRET=long-enough-to-truncate\n');
const payload = bad.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'mixed-web', 'compose.yaml', good);
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'mixed-web', '.env', `enc:${iv}:${tag}:${ct.slice(0, 3)}`);
seedStackDir('mixed-web');
const beforeCompose = 'services:\n keep: {}\n';
const beforeEnv = 'KEEP=1\n';
fs.writeFileSync(composePath('mixed-web'), beforeCompose);
fs.writeFileSync(envPath('mixed-web'), beforeEnv);
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
const res = await request(app)
.post(`/api/fleet/snapshots/${id}/restore`)
.set('Cookie', adminCookie)
.send({ nodeId: LOCAL_NODE_ID, stackName: 'mixed-web', redeploy: true });
expect(res.status).toBe(409);
expect(res.body.code).toBe('SNAPSHOT_FILE_UNAVAILABLE');
expect(fs.readFileSync(composePath('mixed-web'), 'utf-8')).toBe(beforeCompose);
expect(fs.readFileSync(envPath('mixed-web'), 'utf-8')).toBe(beforeEnv);
expect(deploySpy).not.toHaveBeenCalled();
});
it('returns 409 when a delimiter byte is corrupted and does not write ciphertext', async () => {
const db = DatabaseService.getInstance();
const id = db.createSnapshot('restore-delim', 'admin', 1, 1, '[]', '[]');
const cipher = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
const payload = cipher.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const damaged = `enc:${iv}=${tag}:${ct}`;
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'delim-web', 'compose.yaml', damaged);
seedStackDir('delim-web');
const beforeCompose = 'services:\n keep: {}\n';
fs.writeFileSync(composePath('delim-web'), beforeCompose);
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
const res = await request(app)
.post(`/api/fleet/snapshots/${id}/restore`)
.set('Cookie', adminCookie)
.send({ nodeId: LOCAL_NODE_ID, stackName: 'delim-web', redeploy: true });
expect(res.status).toBe(409);
expect(res.body.code).toBe('SNAPSHOT_FILE_UNAVAILABLE');
expect(fs.readFileSync(composePath('delim-web'), 'utf-8')).toBe(beforeCompose);
expect(deploySpy).not.toHaveBeenCalled();
});
it('detail returns 200 with unavailable marker and intact sibling content', async () => {
const db = DatabaseService.getInstance();
const id = db.createSnapshot('detail-corrupt', 'admin', 1, 2, '[]', '[]');
const good = CryptoService.getInstance().encrypt('services: { ok: {} }\n');
const bad = CryptoService.getInstance().encrypt('SECRET=long-enough-to-truncate\n');
const payload = bad.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'ok', 'compose.yaml', good);
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'bad', 'compose.yaml', `enc:${iv}:${tag}:${ct.slice(0, 3)}`);
const res = await request(app).get(`/api/fleet/snapshots/${id}`).set('Cookie', adminCookie);
expect(res.status).toBe(200);
expect(res.body.fileDecryptWarnings).toEqual([
expect.objectContaining({ stackName: 'bad', filename: 'compose.yaml' }),
]);
const okStack = res.body.nodes[0].stacks.find((s: { stackName: string }) => s.stackName === 'ok');
const badStack = res.body.nodes[0].stacks.find((s: { stackName: string }) => s.stackName === 'bad');
expect(okStack.files[0].content).toBe('services: { ok: {} }\n');
expect(okStack.files[0].unavailable).toBeUndefined();
expect(badStack.files[0].unavailable).toBe(true);
expect(badStack.files[0].content).toBeUndefined();
expect(JSON.stringify(res.body)).not.toContain('enc:');
});
it('redeploys after restore when requested', async () => {
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
@@ -583,6 +747,83 @@ describe('Restore-all', () => {
expect(bad?.error).toMatch(/no longer exists/i);
});
it('isolates corrupt decrypt stacks before any mutation with notes and redeploy requested', async () => {
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
const db = DatabaseService.getInstance();
const id = db.createSnapshot('restore-all-corrupt', 'admin', 1, 2, '[]', '[]');
const good = CryptoService.getInstance().encrypt('services:\n app: {}\n');
const bad = CryptoService.getInstance().encrypt('SECRET=x\n');
const payload = bad.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const damaged = `enc:${iv}:${tag}:${ct.slice(0, 3)}`;
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'healthy', 'compose.yaml', good);
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'corrupt', 'compose.yaml', damaged);
seedStackDir('healthy');
seedStackDir('corrupt');
fs.writeFileSync(composePath('corrupt'), 'services:\n keep: {}\n');
const res = await request(app)
.post(`/api/fleet/snapshots/${id}/restore-all`)
.set('Cookie', adminCookie)
.send({ redeploy: true, restoreNotes: true });
expect(res.status).toBe(200);
expect(res.body.restored).toBe(1);
expect(res.body.failed).toBe(1);
const corrupt = (res.body.results as Array<{ stackName: string; success: boolean; error?: string; redeployed: boolean }>)
.find(r => r.stackName === 'corrupt');
expect(corrupt?.success).toBe(false);
expect(corrupt?.error).toMatch(/could not be decrypted/i);
expect(corrupt?.redeployed).toBe(false);
expect(fs.readFileSync(composePath('corrupt'), 'utf-8')).toContain('keep: {}');
expect(fs.readFileSync(composePath('healthy'), 'utf-8')).toContain('app: {}');
expect(deploySpy).toHaveBeenCalledTimes(1);
expect(deploySpy).toHaveBeenCalledWith('healthy', undefined, undefined, {
source: 'fleet_snapshot',
actor: 'system:fleet-snapshot',
});
expect(deploySpy.mock.calls.every(call => call[0] !== 'corrupt')).toBe(true);
});
it('isolates delimiter-byte corruption before restore-all mutation', async () => {
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
const db = DatabaseService.getInstance();
const id = db.createSnapshot('restore-all-delim', 'admin', 1, 2, '[]', '[]');
const good = CryptoService.getInstance().encrypt('services:\n app: {}\n');
const bad = CryptoService.getInstance().encrypt('SECRET=long-enough-to-mutate\n');
const payload = bad.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const damaged = `enc:${iv} ${tag}:${ct}`;
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'healthy', 'compose.yaml', good);
db.getDb().prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
).run(id, LOCAL_NODE_ID, 'local', 'corrupt', 'compose.yaml', damaged);
seedStackDir('healthy');
seedStackDir('corrupt');
fs.writeFileSync(composePath('corrupt'), 'services:\n keep: {}\n');
const res = await request(app)
.post(`/api/fleet/snapshots/${id}/restore-all`)
.set('Cookie', adminCookie)
.send({ redeploy: true });
expect(res.status).toBe(200);
expect(res.body.restored).toBe(1);
expect(res.body.failed).toBe(1);
const corrupt = (res.body.results as Array<{ stackName: string; success: boolean; error?: string }>)
.find(r => r.stackName === 'corrupt');
expect(corrupt?.success).toBe(false);
expect(fs.readFileSync(composePath('corrupt'), 'utf-8')).toContain('keep: {}');
expect(deploySpy).toHaveBeenCalledTimes(1);
expect(deploySpy.mock.calls.every(call => call[0] !== 'corrupt')).toBe(true);
});
it('redeploys each restored stack when requested', async () => {
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
@@ -0,0 +1,153 @@
import { describe, it, expect, beforeEach } from 'vitest';
import { CryptoService } from '../services/CryptoService';
import {
classifySnapshotFileContent,
isEnvelopeLikeDamage,
isEnvelopeShapedPayload,
isStructurallyValidSnapshotEnvelope,
} from '../helpers/snapshotFileDecrypt';
describe('snapshotFileDecrypt classification', () => {
const encrypt = (plain: string) => CryptoService.getInstance().encrypt(plain);
it('returns plaintext for non-enc values including empty string', () => {
expect(classifySnapshotFileContent('')).toEqual({ kind: 'usable', content: '' });
expect(classifySnapshotFileContent('services:\n web:\n')).toEqual({
kind: 'usable',
content: 'services:\n web:\n',
});
});
it('decrypts a structurally valid envelope', () => {
const cipher = encrypt('SECRET=1\n');
expect(isStructurallyValidSnapshotEnvelope(cipher)).toBe(true);
expect(classifySnapshotFileContent(cipher)).toEqual({ kind: 'usable', content: 'SECRET=1\n' });
});
it('marks auth-failing valid envelopes unavailable', () => {
const cipher = encrypt('ok');
// Flip last hex nibble of ciphertext to keep structure valid but break auth
const parts = cipher.split(':');
const last = parts[parts.length - 1];
const flipped = (last.slice(0, -1) + (last.endsWith('0') ? '1' : '0'));
const tampered = [...parts.slice(0, -1), flipped].join(':');
expect(isStructurallyValidSnapshotEnvelope(tampered)).toBe(true);
expect(classifySnapshotFileContent(tampered)).toEqual(
expect.objectContaining({ kind: 'unavailable', reason: 'decrypt_failed' }),
);
});
it('preserves clearly non-envelope legacy enc: prose including punctuation', () => {
const legacyValues = [
'enc:hello',
'enc:FOO_BAR=baz',
'enc: path with spaces',
'enc:hello-world',
'enc:hello/world',
'enc:{legacy}',
'enc:hello.world',
'enc:hello!',
];
for (const value of legacyValues) {
expect(isEnvelopeShapedPayload(value.slice('enc:'.length)), value).toBe(false);
expect(classifySnapshotFileContent(value), value).toEqual({ kind: 'usable', content: value });
expect(isEnvelopeLikeDamage(value), value).toBe(false);
}
expect(isEnvelopeLikeDamage('enc:deadbeef')).toBe(true);
expect(classifySnapshotFileContent('enc:deadbeef')).toEqual({
kind: 'unavailable',
reason: 'envelope_damage',
});
});
describe('encrypt-then-corrupt detectable family', () => {
let good: string;
beforeEach(() => {
good = encrypt('compose content\n');
});
const envelopeDamage = { kind: 'unavailable' as const, reason: 'envelope_damage' as const };
it('fails closed on empty truncation to enc:', () => {
expect(classifySnapshotFileContent('enc:')).toEqual(envelopeDamage);
});
it('fails closed on short hex-only truncation', () => {
expect(classifySnapshotFileContent('enc:deadbeef')).toEqual(envelopeDamage);
});
it('fails closed when IV is truncated', () => {
const payload = good.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const damaged = `enc:${iv.slice(0, 10)}:${tag}:${ct}`;
expect(classifySnapshotFileContent(damaged)).toEqual(envelopeDamage);
});
it('fails closed when tag is truncated', () => {
const payload = good.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const damaged = `enc:${iv}:${tag.slice(0, 8)}:${ct}`;
expect(classifySnapshotFileContent(damaged)).toEqual(envelopeDamage);
});
it('fails closed when ciphertext is truncated or odd-length', () => {
const payload = good.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
expect(classifySnapshotFileContent(`enc:${iv}:${tag}:${ct.slice(0, -1)}`)).toEqual(envelopeDamage);
expect(classifySnapshotFileContent(`enc:${iv}:${tag}:${ct.slice(0, 3)}`)).toEqual(envelopeDamage);
});
it('fails closed on non-hex mutation in IV, tag, and ciphertext', () => {
const payload = good.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const ivDamaged = `enc:${iv.slice(0, 5)}g${iv.slice(6)}:${tag}:${ct}`;
const tagDamaged = `enc:${iv}:${tag.slice(0, 5)}g${tag.slice(6)}:${ct}`;
const ctDamaged = `enc:${iv}:${tag}:${ct.slice(0, 5)}g${ct.slice(6)}`;
expect(classifySnapshotFileContent(ivDamaged)).toEqual(envelopeDamage);
expect(classifySnapshotFileContent(tagDamaged)).toEqual(envelopeDamage);
expect(classifySnapshotFileContent(ctDamaged)).toEqual(envelopeDamage);
});
it('fails closed when a delimiter is removed or added', () => {
const payload = good.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
expect(classifySnapshotFileContent(`enc:${iv}:${tag}${ct}`)).toEqual(envelopeDamage);
expect(classifySnapshotFileContent(`enc:${iv}:${tag}:${ct}:00`)).toEqual(envelopeDamage);
});
it('fails closed for single-character delimiter substitutions including = and whitespace', () => {
const payload = good.slice('enc:'.length);
const colonIdx = payload.indexOf(':');
expect(colonIdx).toBeGreaterThan(0);
const mutants = ['Z', '=', ' ', '\t', '|', '/', '+', '@', '.', ',', ';', '_', '-'];
for (const ch of mutants) {
const damaged = `enc:${payload.slice(0, colonIdx)}${ch}${payload.slice(colonIdx + 1)}`;
expect(isStructurallyValidSnapshotEnvelope(damaged), `delim=${JSON.stringify(ch)}`).toBe(false);
expect(isEnvelopeLikeDamage(damaged), `delim=${JSON.stringify(ch)}`).toBe(true);
expect(classifySnapshotFileContent(damaged), `delim=${JSON.stringify(ch)}`).toEqual(envelopeDamage);
}
});
it('fails closed when = or whitespace is inserted into IV, tag, or ciphertext fields', () => {
const payload = good.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const cases = [
`enc:${iv.slice(0, 8)}=${iv.slice(8)}:${tag}:${ct}`,
`enc:${iv}:${tag.slice(0, 8)} ${tag.slice(8)}:${ct}`,
`enc:${iv}:${tag}:${ct.slice(0, 4)}=${ct.slice(4)}`,
`enc:${iv}:${tag}:${ct.slice(0, 4)} ${ct.slice(4)}`,
];
for (const damaged of cases) {
expect(classifySnapshotFileContent(damaged)).toEqual(envelopeDamage);
}
});
it('fails closed when a non-hex extra field is appended', () => {
const payload = good.slice('enc:'.length);
const [iv, tag, ct] = payload.split(':');
const extraField = `enc:${iv}:${tag}:${ct}:oops`;
expect(isEnvelopeLikeDamage(extraField)).toBe(true);
expect(classifySnapshotFileContent(extraField)).toEqual(envelopeDamage);
});
});
});
+148
View File
@@ -0,0 +1,148 @@
import { CryptoService } from '../services/CryptoService';
import { getErrorMessage } from '../utils/errors';
import { sanitizeForLog } from '../utils/safeLog';
const ENCRYPTED_PREFIX = 'enc:';
const HEX_RE = /^[0-9a-fA-F]+$/;
/**
* Producer envelopes are at least an IV worth of hex (24) plus delimiters /
* ciphertext. Genuine legacy prose such as enc:hello is much shorter.
*/
const MIN_ENVELOPE_LIKE_LENGTH = 24;
/** Damaged encrypt() payloads remain mostly hex even after a one-byte mutation. */
const ENVELOPE_HEX_DENSITY = 0.75;
/** Database row shape for fleet_snapshot_files (content still ciphertext or legacy plaintext). */
export interface SnapshotFileRow {
node_id: number;
node_name: string;
stack_name: string;
filename: string;
content: string;
}
type SnapshotFileMeta = Omit<SnapshotFileRow, 'content'>;
/**
* Decrypted snapshot file read result. Unavailable rows carry no content so
* callers cannot accidentally forward a placeholder into restore or archives.
*/
export type SnapshotFileReadResult =
| (SnapshotFileMeta & { available: true; content: string })
| (SnapshotFileMeta & { available: false });
export type AvailableSnapshotFile = Extract<SnapshotFileReadResult, { available: true }>;
export function isAvailableSnapshotFile(file: SnapshotFileReadResult): file is AvailableSnapshotFile {
return file.available;
}
export function isStructurallyValidSnapshotEnvelope(value: string): boolean {
if (!value.startsWith(ENCRYPTED_PREFIX)) return false;
const parts = value.slice(ENCRYPTED_PREFIX.length).split(':');
if (parts.length !== 3) return false;
const [ivHex, authTagHex, encryptedHex] = parts;
return (
HEX_RE.test(ivHex) && ivHex.length === 24 &&
HEX_RE.test(authTagHex) && authTagHex.length === 32 &&
!!encryptedHex && encryptedHex.length % 2 === 0 && HEX_RE.test(encryptedHex)
);
}
function hexDensity(payload: string): number {
if (payload.length === 0) return 0;
const hexChars = payload.match(/[0-9a-fA-F]/g)?.length ?? 0;
return hexChars / payload.length;
}
/**
* True when the payload still looks like CryptoService.encrypt output after
* truncation or one-byte field/delimiter corruption (high length + hex density),
* or is pure hex of any length.
*/
export function isEnvelopeShapedPayload(payload: string): boolean {
if (payload === '') return true;
if (HEX_RE.test(payload)) return true;
return payload.length >= MIN_ENVELOPE_LIKE_LENGTH && hexDensity(payload) >= ENVELOPE_HEX_DENSITY;
}
/**
* Non-envelope legacy plaintext that happens to start with enc:.
* Any non-empty payload that is not encryption-shaped is preserved verbatim
* (SEN-213). Envelope-shaped damage never qualifies, even with = or whitespace.
*/
export function isClearlyLegacyEncProse(value: string): boolean {
if (!value.startsWith(ENCRYPTED_PREFIX)) return false;
const payload = value.slice(ENCRYPTED_PREFIX.length);
if (payload === '') return false;
return !isEnvelopeShapedPayload(payload);
}
/**
* Producer-envelope damage (no DB provenance). Any enc: payload that is not
* a structurally valid envelope and not clearly legacy prose is treated as
* damage so delimiter substitution and similar corruption cannot fall through
* as writable plaintext.
*/
export function isEnvelopeLikeDamage(value: string): boolean {
if (!value.startsWith(ENCRYPTED_PREFIX)) return false;
if (isStructurallyValidSnapshotEnvelope(value)) return false;
if (isClearlyLegacyEncProse(value)) return false;
return true;
}
export type SnapshotContentClass =
| { kind: 'usable'; content: string }
| { kind: 'unavailable'; reason: 'decrypt_failed' | 'envelope_damage'; detail?: string };
/**
* Classify a stored snapshot file body. Without a provenance marker, enc:
* values that are not valid envelopes and not clearly legacy prose fail closed.
*/
export function classifySnapshotFileContent(raw: string): SnapshotContentClass {
if (!raw.startsWith(ENCRYPTED_PREFIX)) {
return { kind: 'usable', content: raw };
}
if (isStructurallyValidSnapshotEnvelope(raw)) {
try {
return { kind: 'usable', content: CryptoService.getInstance().decrypt(raw) };
} catch (err) {
return {
kind: 'unavailable',
reason: 'decrypt_failed',
detail: getErrorMessage(err, 'decrypt failed'),
};
}
}
if (isClearlyLegacyEncProse(raw)) {
return { kind: 'usable', content: raw };
}
return { kind: 'unavailable', reason: 'envelope_damage' };
}
export function readSnapshotFileRow(
row: SnapshotFileRow,
snapshotId: number,
): SnapshotFileReadResult {
const { content: raw, ...meta } = row;
const classified = classifySnapshotFileContent(raw);
if (classified.kind === 'usable') {
return { ...meta, available: true, content: classified.content };
}
const reasonText = classified.detail
? `${classified.reason}: ${classified.detail}`
: classified.reason;
console.error(
`[snapshotFileDecrypt] Failed to decrypt snapshot file ` +
`snapshot=${sanitizeForLog(snapshotId)} ` +
`node=${sanitizeForLog(meta.node_id)} ` +
`stack=${sanitizeForLog(meta.stack_name)} ` +
`file=${sanitizeForLog(meta.filename)}: ${reasonText}`,
);
return { ...meta, available: false };
}
+56 -10
View File
@@ -2575,8 +2575,13 @@ fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Resp
const files = db.getSnapshotFiles(id);
// Group files by node and stack.
const nodesMap = new Map<number, { nodeId: number; nodeName: string; stacks: Map<string, Array<{ filename: string; content: string }>> }>();
// Group files by node and stack. Unavailable decrypts keep attribution but
// never expose ciphertext or fabricated placeholders as content.
type DetailFile =
| { filename: string; content: string }
| { filename: string; unavailable: true };
const fileDecryptWarnings: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string }> = [];
const nodesMap = new Map<number, { nodeId: number; nodeName: string; stacks: Map<string, DetailFile[]> }>();
for (const file of files) {
if (!nodesMap.has(file.node_id)) {
nodesMap.set(file.node_id, { nodeId: file.node_id, nodeName: file.node_name, stacks: new Map() });
@@ -2585,7 +2590,17 @@ fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Resp
if (!nodeEntry.stacks.has(file.stack_name)) {
nodeEntry.stacks.set(file.stack_name, []);
}
nodeEntry.stacks.get(file.stack_name)!.push({ filename: file.filename, content: file.content });
if (file.available) {
nodeEntry.stacks.get(file.stack_name)!.push({ filename: file.filename, content: file.content });
} else {
nodeEntry.stacks.get(file.stack_name)!.push({ filename: file.filename, unavailable: true });
fileDecryptWarnings.push({
nodeId: file.node_id,
nodeName: file.node_name,
stackName: file.stack_name,
filename: file.filename,
});
}
}
const nodes = Array.from(nodesMap.values()).map(n => ({
@@ -2620,7 +2635,7 @@ fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Resp
}
if (isDebugEnabled()) console.debug('[Fleet:debug] Snapshot detail:', id, files.length, 'files');
res.json({ ...snapshot, nodes, documentation });
res.json({ ...snapshot, nodes, documentation, fileDecryptWarnings });
} catch (error) {
console.error('[Fleet Snapshot] Detail error:', error);
res.status(500).json({ error: 'Failed to fetch snapshot details' });
@@ -2827,9 +2842,19 @@ fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request,
res.status(404).json({ error: 'No files found for this stack in the snapshot' });
return;
}
if (files.some(f => !f.available)) {
res.status(409).json({
error: 'One or more snapshot files could not be decrypted',
code: 'SNAPSHOT_FILE_UNAVAILABLE',
});
return;
}
const writableFiles = files
.filter((f): f is Extract<typeof f, { available: true }> => f.available)
.map(f => ({ filename: f.filename, content: f.content }));
if (isDebugEnabled()) {
const fileNames = files.map(f => f.filename).join(', ');
const fileNames = writableFiles.map(f => f.filename).join(', ');
console.debug('[Fleet:debug] Restore: snapshot=%s, node=%s, stack="%s", files=[%s], redeploy=%s', sanitizeForLog(snapshotId), sanitizeForLog(nodeId), sanitizeForLog(stackName), sanitizeForLog(fileNames), sanitizeForLog(redeploy));
}
@@ -2839,7 +2864,7 @@ fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request,
return;
}
await applySnapshotStackFiles(node, stackName, files);
await applySnapshotStackFiles(node, stackName, writableFiles);
// Dossier notes are restored only on explicit opt-in, so a routine file
// restore never clobbers the operator's current notes. The note write is
@@ -2915,8 +2940,14 @@ fleetRouter.post('/snapshots/:id/restore-all', authMiddleware, async (req: Reque
return;
}
// Group the snapshot's files by node + stack, mirroring the detail route.
const groups = new Map<string, { nodeId: number; nodeName: string; stackName: string; files: Array<{ filename: string; content: string }> }>();
// Group the snapshot's files by node + stack, retaining availability so
// unavailable rows are rejected before any content is written.
const groups = new Map<string, {
nodeId: number;
nodeName: string;
stackName: string;
files: typeof files;
}>();
for (const file of files) {
const key = `${file.node_id}:${file.stack_name}`;
let entry = groups.get(key);
@@ -2924,7 +2955,7 @@ fleetRouter.post('/snapshots/:id/restore-all', authMiddleware, async (req: Reque
entry = { nodeId: file.node_id, nodeName: file.node_name, stackName: file.stack_name, files: [] };
groups.set(key, entry);
}
entry.files.push({ filename: file.filename, content: file.content });
entry.files.push(file);
}
const policyOptions = buildPolicyGateOptions(req);
@@ -2935,10 +2966,25 @@ fleetRouter.post('/snapshots/:id/restore-all', authMiddleware, async (req: Reque
for (const group of groups.values()) {
try {
if (!isValidStackName(group.stackName)) throw new Error('Invalid stack name');
if (group.files.some(f => !f.available)) {
results.push({
nodeId: group.nodeId,
nodeName: group.nodeName,
stackName: group.stackName,
success: false,
redeployed: false,
notesRestored: false,
error: 'One or more snapshot files could not be decrypted',
});
continue;
}
const node = db.getNode(group.nodeId);
if (!node) throw new Error('Target node no longer exists');
await applySnapshotStackFiles(node, group.stackName, group.files);
const writableFiles = group.files
.filter((f): f is Extract<typeof f, { available: true }> => f.available)
.map(f => ({ filename: f.filename, content: f.content }));
await applySnapshotStackFiles(node, group.stackName, writableFiles);
// Files are restored; a notes failure is recorded but does not fail the
// stack (and must not block the redeploy below).
+15 -4
View File
@@ -1,5 +1,5 @@
/**
* CloudBackupService off-site replication for fleet snapshots.
* CloudBackupService: off-site replication for fleet snapshots.
*
* Two providers share the same S3-compatible code path:
* - 'sencho' : managed Sencho Cloud Backup. Credentials provisioned by
@@ -15,9 +15,13 @@ import { Readable } from 'stream';
import * as zlib from 'zlib';
import * as tar from 'tar-stream';
import axios from 'axios';
import { DatabaseService, type FleetSnapshotFile } from './DatabaseService';
import { DatabaseService } from './DatabaseService';
import { CryptoService } from './CryptoService';
import { LicenseService } from './LicenseService';
import {
isAvailableSnapshotFile,
type AvailableSnapshotFile,
} from '../helpers/snapshotFileDecrypt';
import { getErrorMessage } from '../utils/errors';
import { isDebugEnabled } from '../utils/debug';
@@ -255,9 +259,16 @@ export class CloudBackupService {
const documentation = db.getSnapshotDocumentation(snapshotId);
const objectKey = this.buildObjectKey(cfg, snapshot.id, snapshot.description, snapshot.created_at);
const availableFiles = files.filter(isAvailableSnapshotFile);
if (availableFiles.length !== files.length) {
const message = 'One or more snapshot files could not be decrypted';
this.setStatus(snapshotId, { status: 'failed', objectKey, error: message, updatedAt: Date.now() });
throw new Error(message);
}
this.setStatus(snapshotId, { status: 'uploading', objectKey, updatedAt: Date.now() });
try {
const archive = await this.buildArchive(snapshot, files, documentation);
const archive = await this.buildArchive(snapshot, availableFiles, documentation);
const { client, sdk } = await this.buildS3Client(cfg);
await client.send(new sdk.PutObjectCommand({
Bucket: cfg.bucket,
@@ -364,7 +375,7 @@ export class CloudBackupService {
private async buildArchive(
snapshot: { id: number; description: string; created_by: string; node_count: number; stack_count: number; skipped_nodes: string; created_at: number },
files: FleetSnapshotFile[],
files: AvailableSnapshotFile[],
documentation = '',
): Promise<Buffer> {
const pack = tar.pack();
+21 -15
View File
@@ -10,6 +10,9 @@ import { EXPOSURE_INTENTS, type ExposureIntent } from './network/types';
import { HIGH_EPSS_THRESHOLD } from './securityPosture';
import type { BackendScheduledAction } from './scheduledActionRegistry';
import { stackPatternMatches } from '../helpers/stackPattern';
import { readSnapshotFileRow, type SnapshotFileReadResult, type SnapshotFileRow } from '../helpers/snapshotFileDecrypt';
export type { SnapshotFileReadResult } from '../helpers/snapshotFileDecrypt';
function isPilotMode(): boolean {
return process.env.SENCHO_MODE === 'pilot';
@@ -4594,8 +4597,9 @@ export class DatabaseService {
public insertSnapshotFiles(snapshotId: number, files: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }>): void {
// Snapshot file bodies are compose.yaml and .env captures, so they carry
// the same secrets as the live stack. Encrypt content at rest with the
// instance key; getSnapshotFiles/getSnapshotStackFiles decrypt on read,
// so restore and cloud-archive paths see plaintext and stay portable.
// instance key. Getters classify and decrypt per row (see
// snapshotFileDecrypt.ts); unavailable rows omit content so callers
// cannot treat damage as usable plaintext.
const crypto = CryptoService.getInstance();
const insert = this.db.prepare(
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)'
@@ -4630,7 +4634,7 @@ export class DatabaseService {
const row = this.db.prepare('SELECT documentation FROM fleet_snapshots WHERE id = ?').get(id) as { documentation: string } | undefined;
if (!row || row.documentation === '') return '';
try {
// decrypt() returns non-ciphertext input unchanged, mirroring the file path.
// decrypt() returns non-ciphertext input unchanged.
return CryptoService.getInstance().decrypt(row.documentation);
} catch (e) {
// A corrupt blob or a key rotation must not break the primary backup
@@ -4641,22 +4645,24 @@ export class DatabaseService {
}
}
public getSnapshotFiles(snapshotId: number): FleetSnapshotFile[] {
const crypto = CryptoService.getInstance();
// Per-row classification isolates corrupt encrypted rows so intact stacks
// remain readable. See helpers/snapshotFileDecrypt.ts.
public getSnapshotFiles(snapshotId: number): SnapshotFileReadResult[] {
const rows = this.db.prepare(
'SELECT * FROM fleet_snapshot_files WHERE snapshot_id = ? ORDER BY node_name, stack_name'
).all(snapshotId) as FleetSnapshotFile[];
// decrypt() returns non-ciphertext input unchanged, so rows written
// before content-at-rest encryption still read back as plaintext.
return rows.map(row => ({ ...row, content: crypto.decrypt(row.content) }));
'SELECT node_id, node_name, stack_name, filename, content FROM fleet_snapshot_files WHERE snapshot_id = ? ORDER BY node_name, stack_name'
).all(snapshotId) as SnapshotFileRow[];
return this.mapSnapshotFileRows(rows, snapshotId);
}
public getSnapshotStackFiles(snapshotId: number, nodeId: number, stackName: string): FleetSnapshotFile[] {
const crypto = CryptoService.getInstance();
public getSnapshotStackFiles(snapshotId: number, nodeId: number, stackName: string): SnapshotFileReadResult[] {
const rows = this.db.prepare(
'SELECT * FROM fleet_snapshot_files WHERE snapshot_id = ? AND node_id = ? AND stack_name = ?'
).all(snapshotId, nodeId, stackName) as FleetSnapshotFile[];
return rows.map(row => ({ ...row, content: crypto.decrypt(row.content) }));
'SELECT node_id, node_name, stack_name, filename, content FROM fleet_snapshot_files WHERE snapshot_id = ? AND node_id = ? AND stack_name = ?'
).all(snapshotId, nodeId, stackName) as SnapshotFileRow[];
return this.mapSnapshotFileRows(rows, snapshotId);
}
private mapSnapshotFileRows(rows: SnapshotFileRow[], snapshotId: number): SnapshotFileReadResult[] {
return rows.map(row => readSnapshotFileRow(row, snapshotId));
}
/** Created-at of the most recent fleet snapshot covering a stack, or null. */
+6 -1
View File
@@ -158,9 +158,14 @@ Recovery Vault configuration is also admin-only. Mirroring to Recovery Vault add
Snapshots are stored in Sencho's SQLite database. Captured file contents, including `.env` files, are encrypted at rest with the instance key, so a database copy never exposes stack secrets in plaintext. Compose files are typically small (under 10 KB each), so even hundreds of snapshots consume minimal disk space. Individual files larger than 1 MB are skipped and recorded as a warning to keep snapshots bounded. For very large fleets, consider periodically deleting old snapshots to keep the database lean.
If a stored encrypted file cannot be decrypted (for example after on-disk corruption, including damaged ciphertext that still looks encryption-shaped), the snapshot detail view marks that file unavailable and lists it in a warning. Restore for that stack is blocked so live compose and environment files are never overwritten with that damaged data. Configured off-site upload (Recovery Vault or Custom S3) also refuses to publish an archive that would omit or fabricate those files. Intact stacks in the same snapshot remain readable and restorable. Only short, clearly non-ciphertext values that happen to start with `enc:` (for example `enc:hello`, `enc:hello-world`, or `enc:FOO_BAR=baz`) stay readable as legacy plaintext. Encryption-shaped damage remains unavailable.
## Troubleshooting
<AccordionGroup>
<Accordion title="A snapshot file shows as Unavailable">
The stored ciphertext for that file could not be decrypted. Sencho keeps the rest of the snapshot readable and blocks restore for the affected stack so live compose and `.env` files are not overwritten. Configured off-site upload (Recovery Vault or Custom S3) also refuses that snapshot. Create a fresh snapshot from healthy nodes if you need a complete archive again.
</Accordion>
<Accordion title="A snapshot shows skipped nodes">
If a remote node is offline, unreachable, or its API token has expired, the node is skipped during snapshot creation. The list shows a warning icon with a count of skipped nodes, and the snapshot's detail view names each one along with the reason. Common causes are the remote Sencho instance being stopped or restarting, the node's API URL or token having been changed after it was added, or a firewall or network issue blocking the connection. Verify the remote is running and reachable, update the node's API URL and token in **Settings → Infrastructure → Nodes** if needed, then create a new snapshot.
</Accordion>
@@ -171,7 +176,7 @@ Snapshots are stored in Sencho's SQLite database. Captured file contents, includ
The stack was not captured in the snapshot, usually because its compose file was missing or unreadable on disk at the time the snapshot was taken. Open the snapshot's detail view to verify which stacks and files are available, and pick a different snapshot if the one you have is incomplete.
</Accordion>
<Accordion title="Restore all reports that some stacks failed">
Restore all applies each stack independently, so a failure on one stack does not stop the others. A stack is reported as failed when its node has been removed from the fleet since the snapshot was taken, when a remote node is offline or unreachable, or when its existing files cannot be written. The successful stacks are fully restored regardless. Resolve the underlying cause (re-add a removed node, bring an offline node back online) and run Restore all again, or restore the remaining stacks individually from the same snapshot.
Restore all applies each stack independently, so a failure on one stack does not stop the others. A stack is reported as failed when its node has been removed from the fleet since the snapshot was taken, when a remote node is offline or unreachable, when a snapshot file for that stack could not be decrypted, or when its existing files cannot be written. The successful stacks are fully restored regardless. Resolve the underlying cause (re-add a removed node, bring an offline node back online, or pick a healthy snapshot) and run Restore all again, or restore the remaining stacks individually from the same snapshot.
</Accordion>
<Accordion title="Recovery Vault Test reports an authentication error">
Double-check the Access Key ID, Secret Access Key, and bucket name; one wrong character is the most common cause. Some providers require S3-compatible API access to be enabled on the bucket separately from the credentials. For MinIO, confirm the user has read/write permission on the target bucket. After correcting the values, click **Test** again before saving.
+45 -8
View File
@@ -492,6 +492,21 @@ components:
- $ref: "#/components/schemas/FleetSnapshot"
- type: object
properties:
fileDecryptWarnings:
type: array
description: Non-sensitive identifiers for snapshot files that could not be decrypted.
items:
type: object
required: [nodeId, nodeName, stackName, filename]
properties:
nodeId:
type: integer
nodeName:
type: string
stackName:
type: string
filename:
type: string
nodes:
type: array
items:
@@ -511,12 +526,23 @@ components:
files:
type: array
items:
type: object
properties:
filename:
type: string
content:
type: string
oneOf:
- type: object
required: [filename, content]
properties:
filename:
type: string
content:
type: string
description: Decrypted file body; may be an empty string.
- type: object
required: [filename, unavailable]
properties:
filename:
type: string
unavailable:
type: boolean
enum: [true]
ScheduledTask:
type: object
@@ -3397,10 +3423,21 @@ paths:
$ref: "#/components/responses/Forbidden"
"404":
$ref: "#/components/responses/NotFound"
"409":
description: One or more snapshot files for the stack could not be decrypted.
content:
application/json:
schema:
type: object
required: [error, code]
properties:
error:
type: string
code:
type: string
enum: [SNAPSHOT_FILE_UNAVAILABLE]
"500":
$ref: "#/components/responses/InternalError"
# ── Scheduled Tasks ─────────────────────────────────────
/api/scheduled-tasks:
get:
operationId: listScheduledTasks
+64 -7
View File
@@ -35,9 +35,14 @@ interface FleetSnapshot {
has_documentation?: number;
}
interface SnapshotStackFile {
filename: string;
content: string;
type SnapshotStackFile =
| { filename: string; content: string }
| { filename: string; unavailable: true };
function isUnavailableSnapshotFile(
file: SnapshotStackFile,
): file is { filename: string; unavailable: true } {
return 'unavailable' in file;
}
interface SnapshotStack {
@@ -70,6 +75,7 @@ interface SnapshotDocumentation {
interface FleetSnapshotDetail extends FleetSnapshot {
nodes: SnapshotNode[];
documentation?: SnapshotDocumentation;
fileDecryptWarnings?: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string }>;
}
// Ordered labels for the read-only dossier block; only non-empty fields render.
@@ -110,7 +116,7 @@ export default function FleetSnapshots() {
// Cloud-upload affordance is reachable when the saved provider is custom
// (every tier) or sencho on a paid license. A downgraded admin whose
// saved provider is still 'sencho' sees no upload button they cannot
// saved provider is still 'sencho' sees no upload button; they cannot
// call POST /cloud-backup/upload/:id because gateForCurrentProvider would
// 403 anyway, so the UI must not advertise an action that is gated away.
const [cloudEnabled, setCloudEnabled] = useState(false);
@@ -303,7 +309,11 @@ export default function FleetSnapshots() {
}
} else {
const err = await res.json().catch(() => null);
toast.error(err?.message || err?.error || err?.data?.error || 'Failed to restore stack.');
if (res.status === 409 && err?.code === 'SNAPSHOT_FILE_UNAVAILABLE') {
toast.error(err?.error || 'One or more snapshot files could not be decrypted.');
} else {
toast.error(err?.message || err?.error || err?.data?.error || 'Failed to restore stack.');
}
}
} catch (error: unknown) {
const err = error as Record<string, unknown> | null;
@@ -314,6 +324,10 @@ export default function FleetSnapshots() {
};
const handleDownloadFile = (stackName: string, file: SnapshotStackFile) => {
if (isUnavailableSnapshotFile(file)) {
toast.error('This snapshot file could not be decrypted.');
return;
}
try {
const blob = new Blob([file.content], { type: 'text/plain;charset=utf-8' });
const url = URL.createObjectURL(blob);
@@ -505,6 +519,32 @@ export default function FleetSnapshots() {
);
})()}
{(() => {
const warnings = selectedSnapshot.fileDecryptWarnings ?? [];
if (warnings.length === 0) return null;
return (
<div className="rounded-xl border border-warning/30 bg-warning/5 p-4">
<div className="flex items-center gap-2 mb-2">
<AlertTriangle className="w-4 h-4 text-warning shrink-0" />
<span className="text-sm font-medium text-warning">
Some snapshot files could not be decrypted:
</span>
</div>
<ul className="ml-6 space-y-1">
{warnings.map((w, i) => (
<li key={`${w.nodeId}:${w.stackName}:${w.filename}:${i}`} className="text-sm text-muted-foreground">
<span className="font-medium">{w.nodeName}</span>
{' / '}
<span className="font-mono">{w.stackName}</span>
{' / '}
<span className="font-mono">{w.filename}</span>
</li>
))}
</ul>
</div>
);
})()}
{/* Partially captured stacks warning */}
{(() => {
const skipped = parseJsonArray<SkippedStack>(selectedSnapshot.skipped_stacks);
@@ -587,6 +627,7 @@ export default function FleetSnapshots() {
{node.stacks.map(stack => {
const stackKey = `${node.nodeId}:${stack.stackName}`;
const stackExpanded = expandedStacks.has(stackKey);
const stackHasUnavailable = stack.files.some(isUnavailableSnapshotFile);
const dossier = selectedSnapshot.documentation?.stacks
.find(s => s.nodeId === node.nodeId && s.stackName === stack.stackName)?.dossier;
return (
@@ -615,6 +656,7 @@ export default function FleetSnapshots() {
stackName={stack.stackName}
hasDossier={!!dossier}
restoring={restoringStack === `${node.nodeId}:${stack.stackName}`}
disabled={stackHasUnavailable}
onRestore={handleRestore}
/>
)}
@@ -626,6 +668,19 @@ export default function FleetSnapshots() {
{stack.files.map(file => {
const fileKey = `${stackKey}:${file.filename}`;
const showPreview = previewFiles.has(fileKey);
if (isUnavailableSnapshotFile(file)) {
return (
<div key={fileKey}>
<div className="flex items-center gap-2 px-3 py-1.5 rounded-md hover:bg-muted/50 transition-colors">
<FileText className="w-3.5 h-3.5 text-muted-foreground shrink-0" />
<span className="text-xs font-mono flex-1 truncate">{file.filename}</span>
<Badge variant="outline" className="text-[10px] text-warning border-warning/40">
Unavailable
</Badge>
</div>
</div>
);
}
return (
<div key={fileKey}>
<div className="flex items-center gap-2 px-3 py-1.5 rounded-md hover:bg-muted/50 transition-colors">
@@ -911,12 +966,13 @@ export default function FleetSnapshots() {
// --- Restore Button Sub-Component ---
function RestoreButton({ nodeId, nodeName, stackName, hasDossier, restoring, onRestore }: {
function RestoreButton({ nodeId, nodeName, stackName, hasDossier, restoring, disabled, onRestore }: {
nodeId: number;
nodeName: string;
stackName: string;
hasDossier: boolean;
restoring: boolean;
disabled?: boolean;
onRestore: (nodeId: number, stackName: string, redeploy: boolean, restoreNotes: boolean) => Promise<void>;
}) {
const [redeploy, setRedeploy] = useState(false);
@@ -929,7 +985,8 @@ function RestoreButton({ nodeId, nodeName, stackName, hasDossier, restoring, onR
variant="ghost"
size="sm"
className="h-6 px-2 text-xs"
disabled={restoring}
disabled={restoring || disabled}
title={disabled ? 'One or more files in this stack could not be decrypted' : undefined}
onClick={() => setOpen(true)}
>
{restoring ? (
@@ -0,0 +1,117 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { render, screen, waitFor, fireEvent } from '@testing-library/react';
import FleetSnapshots from '../FleetSnapshots';
vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() }));
import { apiFetch } from '@/lib/api';
vi.mock('@/context/AuthContext', () => ({
useAuth: () => ({ user: { role: 'admin' }, isAdmin: true }),
}));
vi.mock('@/context/LicenseContext', () => ({
useLicense: () => ({ isPaid: true }),
}));
vi.mock('@/components/ui/toast-store', () => ({
toast: { success: vi.fn(), error: vi.fn(), warning: vi.fn(), loading: vi.fn(), dismiss: vi.fn() },
}));
const mockedFetch = vi.mocked(apiFetch);
describe('FleetSnapshots unavailable files', () => {
beforeEach(() => {
mockedFetch.mockReset();
mockedFetch.mockImplementation(async (path: string) => {
if (path === '/fleet/snapshots') {
return {
ok: true,
json: async () => ({
snapshots: [{
id: 7,
description: 'fleet-snap-test',
created_by: 'admin',
node_count: 1,
stack_count: 2,
skipped_nodes: '[]',
skipped_stacks: '[]',
created_at: Date.now(),
has_documentation: 0,
}],
total: 1,
}),
} as Response;
}
if (path === '/fleet/snapshots/7') {
return {
ok: true,
json: async () => ({
id: 7,
description: 'fleet-snap-test',
created_by: 'admin',
node_count: 1,
stack_count: 2,
skipped_nodes: '[]',
skipped_stacks: '[]',
created_at: Date.now(),
fileDecryptWarnings: [
{ nodeId: 1, nodeName: 'local', stackName: 'bad', filename: 'compose.yaml' },
],
nodes: [{
nodeId: 1,
nodeName: 'local',
stacks: [
{
stackName: 'good',
files: [{ filename: '.env', content: '' }],
},
{
stackName: 'bad',
files: [{ filename: 'compose.yaml', unavailable: true }],
},
],
}],
}),
} as Response;
}
if (path === '/cloud-backup/config') {
return { ok: true, json: async () => ({ provider: 'disabled' }) } as Response;
}
if (path === '/cloud-backup/snapshots') {
return { ok: true, json: async () => [] } as Response;
}
return { ok: true, json: async () => ({}) } as Response;
});
});
it('shows a decrypt warning and disables restore only for unavailable stacks', async () => {
render(<FleetSnapshots />);
await waitFor(() => expect(screen.getByText('fleet-snap-test')).toBeInTheDocument());
fireEvent.click(screen.getByRole('button', { name: /View/i }));
await waitFor(() =>
expect(screen.getByText(/Some snapshot files could not be decrypted/i)).toBeInTheDocument(),
);
expect(screen.getByText('compose.yaml')).toBeInTheDocument();
fireEvent.click(screen.getByRole('button', { name: /local/i }));
await waitFor(() => expect(screen.getByText('good')).toBeInTheDocument());
const restoreButtons = screen.getAllByRole('button', { name: /^Restore$/i });
expect(restoreButtons).toHaveLength(2);
expect((restoreButtons[0] as HTMLButtonElement).disabled).toBe(false);
expect((restoreButtons[1] as HTMLButtonElement).disabled).toBe(true);
const stackButtons = screen.getAllByRole('button').filter(btn =>
btn.textContent?.includes('good') && btn.textContent?.includes('file'),
);
fireEvent.click(stackButtons[0]);
await waitFor(() => expect(screen.getByText('Preview')).toBeInTheDocument());
expect(screen.getByText('Download')).toBeInTheDocument();
const badStackButtons = screen.getAllByRole('button').filter(btn =>
btn.textContent?.includes('bad') && btn.textContent?.includes('file'),
);
fireEvent.click(badStackButtons[0]);
await waitFor(() => expect(screen.getByText('Unavailable')).toBeInTheDocument());
});
});