diff --git a/backend/src/__tests__/filesystem-backup.test.ts b/backend/src/__tests__/filesystem-backup.test.ts index ecc1d76a..bfadfbf9 100644 --- a/backend/src/__tests__/filesystem-backup.test.ts +++ b/backend/src/__tests__/filesystem-backup.test.ts @@ -9,6 +9,7 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import path from 'path'; import os from 'os'; import { promises as fsPromises } from 'fs'; +import { createHash } from 'crypto'; // Mutable state the mocked NodeRegistry reads. Each test rewrites these // before instantiating FileSystemService. @@ -297,4 +298,189 @@ describe('FileSystemService backup location', () => { const kept = await fsPromises.readFile(path.join(stackDir, 'notes.txt'), 'utf-8'); expect(kept).toBe('keep me\n'); }); + + // Integrity guard: a backup carries a .checksums manifest, and a restore + // verifies each backed-up file against it before touching the live stack, so a + // truncated or corrupted backup is rejected with a clear error instead of being + // copied back silently. These reuse the outer mkdtemp/DATA_DIR harness. + describe('backup integrity checksum', () => { + // Independent oracle for the manifest hashes. Production hashes the raw file + // Buffer; for the UTF-8 text fixtures used here that yields the same digest as + // hashing Buffer.from(s, 'utf-8'). + const sha = (s: string) => createHash('sha256').update(Buffer.from(s, 'utf-8')).digest('hex'); + + it('writes a .checksums manifest with the SHA-256 of each backed-up file', async () => { + const stackName = 'sums'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + const composeBody = 'services:\n web: {}\n'; + const envBody = 'FOO=bar\n'; + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), composeBody, 'utf-8'); + await fsPromises.writeFile(path.join(stackDir, '.env'), envBody, 'utf-8'); + + await FileSystemService.getInstance().backupStackFiles(stackName); + + const backupDir = path.join(dataDir, 'backups', '1', stackName); + const manifest = JSON.parse(await fsPromises.readFile(path.join(backupDir, '.checksums'), 'utf-8')); + expect(manifest['compose.yaml']).toBe(sha(composeBody)); + expect(manifest['.env']).toBe(sha(envBody)); + }); + + it('aborts the restore and leaves the live file unchanged when a backed-up file is corrupt', async () => { + const stackName = 'corrupt'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: good\n', 'utf-8'); + + const service = FileSystemService.getInstance(); + await service.backupStackFiles(stackName); + + // Corrupt the backed-up copy on disk (truncation or bit-rot after a clean backup). + const backupDir = path.join(dataDir, 'backups', '1', stackName); + await fsPromises.writeFile(path.join(backupDir, 'compose.yaml'), 'name: go', 'utf-8'); + + // The live file is the post-deploy state a rollback would revert. + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: current\n', 'utf-8'); + + await expect(service.restoreStackFiles(stackName)).rejects.toThrow(/integrity|corrupt/i); + + // The live file must be untouched, not overwritten with the corrupt bytes. + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe('name: current\n'); + // The manifest marker must never leak into the stack directory. + await expect(fsPromises.access(path.join(stackDir, '.checksums'))).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('verifies before removing orphans, so a corrupt backup mutates nothing', async () => { + const stackName = 'atomic'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: good\n', 'utf-8'); + + const service = FileSystemService.getInstance(); + await service.backupStackFiles(stackName); // backup has compose.yaml, no .env + + const backupDir = path.join(dataDir, 'backups', '1', stackName); + await fsPromises.writeFile(path.join(backupDir, 'compose.yaml'), 'trunc', 'utf-8'); + + // A post-backup deploy added a .env: a faithful restore would remove it as an + // orphan. The integrity check must run first, so the corrupt backup leaves both + // the orphan and the live compose exactly as they are. + await fsPromises.writeFile(path.join(stackDir, '.env'), 'SECRET=x\n', 'utf-8'); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: current\n', 'utf-8'); + + await expect(service.restoreStackFiles(stackName)).rejects.toThrow(/integrity|corrupt/i); + + expect(await fsPromises.readFile(path.join(stackDir, '.env'), 'utf-8')).toBe('SECRET=x\n'); + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe('name: current\n'); + }); + + it('restores faithfully when no .checksums manifest exists (pre-feature backup)', async () => { + const stackName = 'legacy'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: original\n', 'utf-8'); + + const service = FileSystemService.getInstance(); + await service.backupStackFiles(stackName); + + // A backup taken before the integrity feature existed has no manifest. + const backupDir = path.join(dataDir, 'backups', '1', stackName); + await fsPromises.rm(path.join(backupDir, '.checksums')); + + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: mutated\n', 'utf-8'); + await service.restoreStackFiles(stackName); + + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe('name: original\n'); + }); + + it('restores faithfully when the manifest is unparseable (degrades to no verification)', async () => { + const stackName = 'garbled'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: original\n', 'utf-8'); + + const service = FileSystemService.getInstance(); + const backupDir = path.join(dataDir, 'backups', '1', stackName); + + // Both garbage JSON and an empty file are unparseable; neither proves the + // data files are bad, so a needed rollback must still proceed. + for (const garbage of ['not json', '']) { + await service.backupStackFiles(stackName); + await fsPromises.writeFile(path.join(backupDir, '.checksums'), garbage, 'utf-8'); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: mutated\n', 'utf-8'); + await service.restoreStackFiles(stackName); + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe('name: original\n'); + } + }); + + it('copies a backup file that has no checksum entry without verifying it', async () => { + const stackName = 'partial'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: original\n', 'utf-8'); + + const service = FileSystemService.getInstance(); + await service.backupStackFiles(stackName); // manifest records compose.yaml only + + // A managed file present in the backup slot but absent from the manifest + // (e.g. its read failed during backup, so it was never recorded). It must + // still be restored: the check is never stricter than what was recorded. + const backupDir = path.join(dataDir, 'backups', '1', stackName); + await fsPromises.writeFile(path.join(backupDir, '.env'), 'TOKEN=restored\n', 'utf-8'); + + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: mutated\n', 'utf-8'); + await service.restoreStackFiles(stackName); + + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe('name: original\n'); + expect(await fsPromises.readFile(path.join(stackDir, '.env'), 'utf-8')).toBe('TOKEN=restored\n'); + }); + + it('aborts the restore when the backed-up .env is corrupt, leaving the live .env intact', async () => { + const stackName = 'envcorrupt'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: good\n', 'utf-8'); + await fsPromises.writeFile(path.join(stackDir, '.env'), 'TOKEN=good\n', 'utf-8'); + + const service = FileSystemService.getInstance(); + await service.backupStackFiles(stackName); // manifest covers compose.yaml and .env + + // Corrupt only the backed-up .env; compose.yaml stays valid. The .env's own + // manifest entry must be checked, so the restore aborts on it. + const backupDir = path.join(dataDir, 'backups', '1', stackName); + await fsPromises.writeFile(path.join(backupDir, '.env'), 'TOKEN=go', 'utf-8'); + + await fsPromises.writeFile(path.join(stackDir, '.env'), 'TOKEN=current\n', 'utf-8'); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: current\n', 'utf-8'); + + await expect(service.restoreStackFiles(stackName)).rejects.toThrow(/integrity|corrupt/i); + + // The abort must touch nothing: the live .env keeps its current contents, and + // the valid-but-unrestored compose.yaml is left as-is rather than reverted to + // the backup, proving the .env mismatch halts before any file is copied back. + expect(await fsPromises.readFile(path.join(stackDir, '.env'), 'utf-8')).toBe('TOKEN=current\n'); + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe('name: current\n'); + }); + + it('restores faithfully when a manifest entry is not a string (degrades to unverified)', async () => { + const stackName = 'nonstring'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: original\n', 'utf-8'); + + const service = FileSystemService.getInstance(); + await service.backupStackFiles(stackName); + + // A parseable manifest whose value is not a hex string (tampered or + // wrong-typed) must not block a rollback whose data files are intact: a + // manifest problem is not proof the backup is corrupt. + const backupDir = path.join(dataDir, 'backups', '1', stackName); + await fsPromises.writeFile(path.join(backupDir, '.checksums'), JSON.stringify({ 'compose.yaml': 123 }), 'utf-8'); + + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: mutated\n', 'utf-8'); + await service.restoreStackFiles(stackName); + + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe('name: original\n'); + }); + }); }); diff --git a/backend/src/services/FileSystemService.ts b/backend/src/services/FileSystemService.ts index 3261adf0..49e30622 100644 --- a/backend/src/services/FileSystemService.ts +++ b/backend/src/services/FileSystemService.ts @@ -9,6 +9,7 @@ import { NodeRegistry } from './NodeRegistry'; import { isPathWithinBase, isValidStackName } from '../utils/validation'; import { isBinaryBuffer } from '../utils/binaryDetect'; import { sanitizeForLog } from '../utils/safeLog'; +import { sha256HexBuffer } from '../utils/hashing'; export interface FileEntry { name: string; @@ -56,6 +57,11 @@ const PROTECTED_STACK_FILES = new Set([ '.env', ]); +// Bookkeeping markers Sencho writes into the backup slot. They are never copied +// back into the stack directory on restore: `.timestamp` records when the backup +// was taken; `.checksums` is the integrity manifest verified before a restore. +const BACKUP_MARKER_FILES = new Set(['.timestamp', '.checksums']); + // Compose filenames Sencho recognizes, in resolution-priority order. Mirrors the // list FileSystemService uses elsewhere; named here for the import scan. const IMPORT_COMPOSE_FILENAMES = ['compose.yaml', 'compose.yml', 'docker-compose.yaml', 'docker-compose.yml'] as const; @@ -967,13 +973,29 @@ export class FileSystemService { } } - // Copy compose file + // Copy each managed file by reading it into memory, writing it to the backup + // slot, and recording the SHA-256 of the source bytes. Hashing the source + // (not the destination) means a truncated copy is caught when restore re-hashes + // the backup and finds it no longer matches. Managed files (compose, .env) are + // small; revisit this read-into-memory if a large file is ever added to + // PROTECTED_STACK_FILES. + // Canonical js/path-injection barrier inline with each source read sink: + // resolve against the compose base and confirm containment, mirroring + // snapshotStackFiles. stackDir is already validated by resolveStackDir, but + // re-establishing containment at the readFile sink itself lets static analysis + // credit the barrier, which it does not through the helper. + const baseResolved = path.resolve(this.baseDir); + const checksums: Record = {}; const composeFiles = ['compose.yaml', 'compose.yml', 'docker-compose.yaml', 'docker-compose.yml']; for (const file of composeFiles) { - const src = path.join(stackDir, file); + const src = path.resolve(baseResolved, path.join(stackDir, file)); + if (!src.startsWith(baseResolved + path.sep)) { + throw Object.assign(new Error('Path escapes compose directory'), { code: 'INVALID_PATH' }); + } try { - await fsPromises.access(src); - await fsPromises.copyFile(src, path.join(backupDir, file)); + const buf = await fsPromises.readFile(src); + await fsPromises.writeFile(path.join(backupDir, file), buf); + checksums[file] = sha256HexBuffer(buf); } catch (e: unknown) { const code = (e as NodeJS.ErrnoException)?.code; if (code !== 'ENOENT') { @@ -982,11 +1004,15 @@ export class FileSystemService { } } - // Copy .env if it exists - const envSrc = path.join(stackDir, '.env'); + // Copy .env if it exists (same inline containment barrier as above). + const envSrc = path.resolve(baseResolved, path.join(stackDir, '.env')); + if (!envSrc.startsWith(baseResolved + path.sep)) { + throw Object.assign(new Error('Path escapes compose directory'), { code: 'INVALID_PATH' }); + } try { - await fsPromises.access(envSrc); - await fsPromises.copyFile(envSrc, path.join(backupDir, '.env')); + const buf = await fsPromises.readFile(envSrc); + await fsPromises.writeFile(path.join(backupDir, '.env'), buf); + checksums['.env'] = sha256HexBuffer(buf); } catch (e: unknown) { const code = (e as NodeJS.ErrnoException)?.code; if (code !== 'ENOENT') { @@ -994,6 +1020,13 @@ export class FileSystemService { } } + // Write the integrity manifest before the timestamp marker, so a crash + // between the two leaves the checksums present (a backup that restore can + // verify) rather than a timestamp with no integrity data. Only files that + // were actually written above appear here, so the manifest never claims a + // file the slot does not hold. + await fsPromises.writeFile(path.join(backupDir, '.checksums'), JSON.stringify(checksums), 'utf-8'); + // Write timestamp marker await fsPromises.writeFile(path.join(backupDir, '.timestamp'), Date.now().toString(), 'utf-8'); if (debug) console.debug(`[FileSystemService:debug] Backup completed in ${Date.now() - t0}ms`, { stackName }); @@ -1019,6 +1052,44 @@ export class FileSystemService { const items = await fsPromises.readdir(backupDir); const backedUp = new Set(items); + // Verify the backup's integrity before mutating the stack, so a corrupt or + // truncated backup is rejected rather than copied back silently. Each backed-up + // file is re-hashed and compared to the .checksums manifest written at backup + // time. This runs before the orphan removal and copy below, so a failed check + // leaves the live stack exactly as it was. A backup with no manifest (taken + // before this guard existed) or a file with no recorded checksum is left + // unverified: the check is never stricter than what the backup recorded, so it + // cannot block a rollback the backup can still serve. + let checksums: Record | null = null; + try { + const parsed: unknown = JSON.parse(await fsPromises.readFile(path.join(backupDir, '.checksums'), 'utf-8')); + if (parsed !== null && typeof parsed === 'object') { + checksums = parsed as Record; + } + } catch (e: unknown) { + // ENOENT is a pre-feature backup with no manifest: restore unverified. A + // present but unreadable or malformed manifest does not prove the data files + // are bad, and blocking would deny a needed rollback, so warn and proceed + // unverified rather than fail. + if ((e as NodeJS.ErrnoException)?.code !== 'ENOENT') { + console.warn('[FileSystemService] Backup checksum manifest unreadable, restoring without integrity check:', (e as Error).message); + } + } + if (checksums) { + for (const item of items) { + if (BACKUP_MARKER_FILES.has(item)) continue; + const expected = checksums[item]; + // A missing or non-string entry (a tampered or malformed-but-parseable + // manifest) is treated as no recorded checksum: skip rather than fail, so a + // manifest problem never blocks a rollback whose data files are intact. + if (typeof expected !== 'string') continue; + const actual = sha256HexBuffer(await fsPromises.readFile(path.join(backupDir, item))); + if (actual !== expected) { + throw new Error(`Rollback aborted: the backup of ${item} is corrupt (integrity check failed); the stack files were not changed.`); + } + } + } + // Remove managed files the backup does not contain before copying, so a // rollback is a faithful revert rather than an additive overlay. If the // failed deploy switched compose variants (e.g. compose.yaml -> @@ -1056,10 +1127,10 @@ export class FileSystemService { } for (const item of items) { - if (item === '.timestamp') continue; + if (BACKUP_MARKER_FILES.has(item)) continue; await fsPromises.copyFile(path.join(backupDir, item), path.join(stackDir, item)); } - if (debug) console.debug(`[FileSystemService:debug] Restore completed in ${Date.now() - t0}ms`, { stackName, restored: items.filter(i => i !== '.timestamp').length, removedOrphans }); + if (debug) console.debug(`[FileSystemService:debug] Restore completed in ${Date.now() - t0}ms`, { stackName, restored: items.filter(i => !BACKUP_MARKER_FILES.has(i)).length, removedOrphans }); } /** diff --git a/backend/src/utils/hashing.ts b/backend/src/utils/hashing.ts index 5c5d4476..a42a2d85 100644 --- a/backend/src/utils/hashing.ts +++ b/backend/src/utils/hashing.ts @@ -4,3 +4,8 @@ import { createHash } from 'crypto'; export function sha256Hex(content: string): string { return createHash('sha256').update(content, 'utf8').digest('hex'); } + +/** Hex-encoded SHA-256 of raw bytes. Binary-safe; use for file-content integrity. */ +export function sha256HexBuffer(data: Buffer): string { + return createHash('sha256').update(data).digest('hex'); +} diff --git a/docs/features/atomic-deployments.mdx b/docs/features/atomic-deployments.mdx index 573e99f8..2433343f 100644 --- a/docs/features/atomic-deployments.mdx +++ b/docs/features/atomic-deployments.mdx @@ -14,7 +14,7 @@ The same backup also powers the **Rollback** action in the stack editor, so you 3. **Health probe.** Sencho waits 3 seconds, then lists every container with the `com.docker.compose.project=` label and checks each one for a non-zero exit code. Any container that has exited with a non-zero status counts as a crash. 4. **Auto-rollback on failure.** When a crash is detected, Sencho streams `=== Deployment failed - restoring previous compose and env files ===`, restores the backed-up files, and re-runs `docker compose up -d` with the restored configuration. On success it streams `=== Restored previous compose and env files ===`. Restoring the files reverts the compose and `.env` configuration; an image referenced by a moving tag (such as `latest`) is not reverted, because the local tag still resolves to the newly pulled digest. The original deploy error is preserved and reported as the deploy result, so a failed-then-rolled-back deploy still registers as a failure in the deploy progress modal. -If the rollback itself fails (for example, the re-deploy step cannot pull a previously available image, or the file restore is blocked by filesystem permissions), Sencho streams `=== Rollback failed - manual intervention may be required ===`. The backup files remain at `/backups//` so you can copy them back manually. +If the rollback itself fails (for example, the re-deploy step cannot pull a previously available image, or the file restore is blocked by filesystem permissions), Sencho streams `=== Rollback failed - manual intervention may be required ===`. The backup files remain at `/backups///` so you can copy them back manually. ## Which operations are protected @@ -39,12 +39,14 @@ The menu entry is hidden when no backup exists for the stack, for example on a f ## Where backups are stored -Backups live under `/backups//`, in the same writable volume Sencho uses for its database and other persisted state. They are intentionally kept outside the user's compose folder, so the operation works even when a container has chowned its bind-mounted stack directory to root. +Backups live under `/backups///`, in the same writable volume Sencho uses for its database and other persisted state. They are intentionally kept outside the user's compose folder, so the operation works even when a container has chowned its bind-mounted stack directory to root. -Each backup is a flat copy of the compose file Sencho found, plus `.env` if it exists, plus a `.timestamp` marker that records when the backup was taken. There is one backup slot per stack: every protected deploy or update overwrites the previous backup, so the **Rollback** menu always reverts to the configuration that was on disk immediately before the most recent run. +Each backup is a flat copy of the compose file Sencho found, plus `.env` if it exists, plus two markers: a `.timestamp` recording when the backup was taken and a `.checksums` integrity manifest holding a SHA-256 for each backed-up file. There is one backup slot per stack: every protected deploy or update overwrites the previous backup, so the **Rollback** menu always reverts to the configuration that was on disk immediately before the most recent run. A restore is a faithful revert, not an overlay. Sencho replaces the compose file and `.env` with the backed-up copies and removes any compose variant or `.env` that was added after the backup was taken, so the stack returns to exactly the file set it had before the run. For example, if a deploy switched the stack from `compose.yaml` to `docker-compose.yml` or introduced a new `.env`, a rollback undoes both. Files Sencho does not manage are left untouched. +Before a restore overwrites anything, Sencho re-hashes each backed-up file and compares it against the `.checksums` manifest. If a file no longer matches (for example, a backup truncated by an out-of-disk write), Sencho aborts the restore with a clear error and leaves the stack exactly as it was, rather than copying the corrupt content back over a working configuration. + ## Troubleshooting @@ -57,13 +59,18 @@ A restore is a faithful revert, not an overlay. Sencho replaces the compose file The health probe is a 3-second window after `docker compose up -d` returns. Crashes that happen after that window are out of scope for atomic rollback by design, because Sencho cannot tell whether a late exit is a real failure or a normal restart. The [health gate](/features/health-gated-updates) covers exactly this period: it observes the stack for a configurable window after the update, records a verdict on the stack timeline, and offers a manual rollback when containers do not stay healthy. For ongoing health beyond that, use **Auto-Heal Policies** to restart unhealthy containers automatically and **Alert Rules** to page you when a container exits unexpectedly. - This message means the auto-rollback attempted to restore the backup and re-deploy, but the restore step or the re-deploy itself errored out. The backup files are still at `/backups//`. To recover: + This message means the auto-rollback attempted to restore the backup and re-deploy, but the restore step or the re-deploy itself errored out. The backup files are still at `/backups///`. To recover: - 1. Copy `compose.yaml` (or the variant Sencho backed up) and `.env` from `/backups//` back into the stack directory. + 1. Copy `compose.yaml` (or the variant Sencho backed up) and `.env` from `/backups///` back into the stack directory. 2. Open the stack in the editor and click **Deploy** to re-run with the restored configuration. The most common causes are filesystem permissions on the stack directory and a missing image in a private registry that the original deploy could not pull. + + The backup slot holds a file whose contents no longer match the checksum recorded when the backup was taken, usually because the disk filled up or the write was interrupted while the backup was being written. Sencho refuses to copy that file back, so your live stack is left untouched rather than overwritten with corrupt content. + + Edit the compose file or `.env` directly in the editor to the configuration you want, then click **Deploy**. The next protected deploy writes a fresh, verified backup, and **Rollback** works again from that point. + Sencho keeps a single backup per stack. If you ran two atomic deploys back to back, the second deploy overwrote the first backup with the broken configuration before it failed. **Rollback** then restores that broken configuration, because as far as Sencho is concerned it is the most recent known state.