feat(snapshots): preserve stack dossiers with fleet snapshots (#1339)

* feat(snapshots): preserve stack dossiers with fleet snapshots

Fleet snapshots can now optionally capture each stack's Dossier notes
alongside its compose and .env files, so a recovery restores the
operational knowledge around a stack, not just its configuration.

- Opt-in global setting "snapshot_documentation" (default off), toggled
  from the renamed Fleet settings section.
- Capture reads local dossiers from the database and remote dossiers over
  the Distributed API proxy; only stacks with notes are recorded, and
  secret values are never included.
- Captured notes are stored encrypted at rest in a new fleet_snapshots
  column and surfaced in the snapshot detail view behind a badge.
- Cloud and downloaded archives gain a documentation.json (archive_version 2).
- Restore stays conservative: dossier notes are written back only when the
  operator explicitly opts in, on both single-stack and restore-all paths.
- Existing snapshots and archives remain valid; behavior is unchanged when
  the setting is off.

* fix(snapshots): harden dossier-notes restore against bad input and partial failures

Address review findings on the documentation-snapshots restore path:

- Parse `restoreNotes` strictly (=== true) on single-stack restore, matching
  restore-all, so a stray non-boolean can never opt in to overwriting notes.
- Guard findSnapshotDossier: require an array of stacks and real dossier
  content, so a malformed or all-blank entry can't clobber current notes.
- Make the dossier-notes write non-fatal relative to the file restore: a notes
  failure (e.g. a remote dossier PUT) is caught, reported via `notesError`, and
  no longer 500s the single restore or fails the stack in restore-all once the
  files are already written.
- Surface the partial outcome in the UI: a warning toast on single restore, a
  summary note on restore-all, and gate the "Documentation captured" badge and
  restore-all notes control on captured stacks while rendering capture warnings.

Adds tests for strict parsing, malformed/blank blobs, remote notes restore
(success + non-fatal failure, single and bulk), and scheduled capture-on.

* fix(snapshots): drop unused binding in restore-all remote notes test

The restore-all remote notes test destructured a node id it never uses
(restore-all is driven by snapshot id alone), tripping no-unused-vars and
failing the lint step. Bind only the snapshot id.
This commit is contained in:
Anso
2026-06-08 08:44:59 -04:00
committed by GitHub
parent 842ee7dd0c
commit 710647a44f
16 changed files with 1020 additions and 55 deletions
+12 -2
View File
@@ -252,11 +252,12 @@ export class CloudBackupService {
const snapshot = db.getSnapshot(snapshotId);
if (!snapshot) throw new Error(`Snapshot ${snapshotId} not found.`);
const files = db.getSnapshotFiles(snapshotId);
const documentation = db.getSnapshotDocumentation(snapshotId);
const objectKey = this.buildObjectKey(cfg, snapshot.id, snapshot.description, snapshot.created_at);
this.setStatus(snapshotId, { status: 'uploading', objectKey, updatedAt: Date.now() });
try {
const archive = await this.buildArchive(snapshot, files);
const archive = await this.buildArchive(snapshot, files, documentation);
const { client, sdk } = await this.buildS3Client(cfg);
await client.send(new sdk.PutObjectCommand({
Bucket: cfg.bucket,
@@ -364,6 +365,7 @@ export class CloudBackupService {
private async buildArchive(
snapshot: { id: number; description: string; created_by: string; node_count: number; stack_count: number; skipped_nodes: string; created_at: number },
files: FleetSnapshotFile[],
documentation = '',
): Promise<Buffer> {
const pack = tar.pack();
const metadata = {
@@ -374,11 +376,19 @@ export class CloudBackupService {
node_count: snapshot.node_count,
stack_count: snapshot.stack_count,
skipped_nodes: safeParseJson(snapshot.skipped_nodes, []),
has_documentation: documentation !== '',
instance_id: this.getInstanceId(),
archive_version: 1,
// Version 2 adds the optional documentation.json entry; readers of
// version 1 archives simply will not find that file.
archive_version: 2,
};
pack.entry({ name: 'metadata.json' }, JSON.stringify(metadata, null, 2));
// Captured Stack Dossier metadata travels with the archive when present.
if (documentation !== '') {
pack.entry({ name: 'documentation.json' }, documentation);
}
for (const file of files) {
const safeNodeName = sanitizePathSegment(file.node_name);
const safeStackName = sanitizePathSegment(file.stack_name);
+39 -5
View File
@@ -278,6 +278,10 @@ export interface FleetSnapshot {
skipped_nodes: string; // JSON: Array<{ nodeId; nodeName; reason }>
skipped_stacks: string; // JSON: Array<{ nodeId; nodeName; stackName; reason }>
created_at: number;
/** 1 when the snapshot captured Stack Dossier metadata, 0 otherwise. The
* encrypted blob itself is never projected into list/detail rows; read it
* with getSnapshotDocumentation(). */
has_documentation: number;
}
export interface FleetSnapshotFile {
@@ -867,6 +871,7 @@ export class DatabaseService {
stack_count INTEGER NOT NULL,
skipped_nodes TEXT NOT NULL DEFAULT '[]',
skipped_stacks TEXT NOT NULL DEFAULT '[]',
documentation TEXT NOT NULL DEFAULT '',
created_at INTEGER NOT NULL
);
@@ -1280,6 +1285,8 @@ export class DatabaseService {
// Fleet snapshot per-stack capture warnings (partial-capture surfacing)
maybeAddCol('fleet_snapshots', 'skipped_stacks', "TEXT NOT NULL DEFAULT '[]'");
// Captured Stack Dossier metadata (opt-in documentation snapshots)
maybeAddCol('fleet_snapshots', 'documentation', "TEXT NOT NULL DEFAULT ''");
// Scheduled operations migrations
maybeAddCol('scheduled_task_runs', 'triggered_by', "TEXT NOT NULL DEFAULT 'scheduler'");
@@ -3084,10 +3091,14 @@ export class DatabaseService {
// --- Fleet Snapshots ---
public createSnapshot(description: string, createdBy: string, nodeCount: number, stackCount: number, skippedNodes: string, skippedStacks = '[]'): number {
public createSnapshot(description: string, createdBy: string, nodeCount: number, stackCount: number, skippedNodes: string, skippedStacks = '[]', documentation = ''): number {
// Dossier metadata can carry operational notes (static IPs, firewall
// rules); encrypt it at rest with the same instance key as the file
// bodies. An empty string means the snapshot captured no documentation.
const storedDocs = documentation === '' ? '' : CryptoService.getInstance().encrypt(documentation);
const result = this.db.prepare(
'INSERT INTO fleet_snapshots (description, created_by, node_count, stack_count, skipped_nodes, skipped_stacks, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
).run(description, createdBy, nodeCount, stackCount, skippedNodes, skippedStacks, Date.now());
'INSERT INTO fleet_snapshots (description, created_by, node_count, stack_count, skipped_nodes, skipped_stacks, documentation, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
).run(description, createdBy, nodeCount, stackCount, skippedNodes, skippedStacks, storedDocs, Date.now());
return result.lastInsertRowid as number;
}
@@ -3108,14 +3119,37 @@ export class DatabaseService {
insertMany(files);
}
// The encrypted `documentation` blob is deliberately excluded from these
// projections (it can be large and is decrypted only on demand). Callers
// get a cheap `has_documentation` flag; read the blob with
// getSnapshotDocumentation().
private static readonly SNAPSHOT_COLUMNS =
"id, description, created_by, node_count, stack_count, skipped_nodes, skipped_stacks, created_at, (documentation != '') AS has_documentation";
public getSnapshots(limit = 50, offset = 0): FleetSnapshot[] {
return this.db.prepare(
'SELECT * FROM fleet_snapshots ORDER BY created_at DESC LIMIT ? OFFSET ?'
`SELECT ${DatabaseService.SNAPSHOT_COLUMNS} FROM fleet_snapshots ORDER BY created_at DESC LIMIT ? OFFSET ?`
).all(limit, offset) as FleetSnapshot[];
}
public getSnapshot(id: number): FleetSnapshot | undefined {
return this.db.prepare('SELECT * FROM fleet_snapshots WHERE id = ?').get(id) as FleetSnapshot | undefined;
return this.db.prepare(`SELECT ${DatabaseService.SNAPSHOT_COLUMNS} FROM fleet_snapshots WHERE id = ?`).get(id) as FleetSnapshot | undefined;
}
/** Decrypted Stack Dossier metadata JSON captured with the snapshot, or '' when none. */
public getSnapshotDocumentation(id: number): string {
const row = this.db.prepare('SELECT documentation FROM fleet_snapshots WHERE id = ?').get(id) as { documentation: string } | undefined;
if (!row || row.documentation === '') return '';
try {
// decrypt() returns non-ciphertext input unchanged, mirroring the file path.
return CryptoService.getInstance().decrypt(row.documentation);
} catch (e) {
// A corrupt blob or a key rotation must not break the primary backup
// flows: documentation is an optional side payload, so degrade to
// "no documentation" rather than failing upload/detail/restore.
console.error(`[DatabaseService] Failed to decrypt documentation for snapshot ${id}:`, (e as Error).message);
return '';
}
}
public getSnapshotFiles(snapshotId: number): FleetSnapshotFile[] {
+9 -3
View File
@@ -11,7 +11,7 @@ import type { ImageCheckResult } from './ImageUpdateService';
import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
import { sanitizeForLog } from '../utils/safeLog';
import { captureLocalNodeFiles, captureRemoteNodeFiles, type SnapshotNodeData } from '../utils/snapshot-capture';
import { captureLocalNodeFiles, captureRemoteNodeFiles, buildSnapshotDocumentation, type SnapshotNodeData } from '../utils/snapshot-capture';
import { NodeRegistry } from './NodeRegistry';
import { NotificationService } from './NotificationService';
import TrivyService from './TrivyService';
@@ -533,13 +533,14 @@ export class SchedulerService {
private async executeSnapshot(task: ScheduledTask): Promise<string> {
const db = DatabaseService.getInstance();
const nodes = db.getNodes();
const captureDocs = db.getGlobalSettings().snapshot_documentation === '1';
const results = await Promise.allSettled(
nodes.map(async (node) => {
if (node.type === 'remote') {
return captureRemoteNodeFiles(node);
return captureRemoteNodeFiles(node, captureDocs);
}
return captureLocalNodeFiles(node);
return captureLocalNodeFiles(node, captureDocs);
})
);
@@ -585,6 +586,10 @@ export class SchedulerService {
}
}
const documentation = captureDocs
? buildSnapshotDocumentation(capturedNodes, new Date().toISOString())
: null;
const description = `Scheduled snapshot: ${task.name}`;
const snapshotId = db.createSnapshot(
description,
@@ -593,6 +598,7 @@ export class SchedulerService {
totalStacks,
JSON.stringify(skippedNodes),
JSON.stringify(skippedStacks),
documentation ? JSON.stringify(documentation) : '',
);
if (allFiles.length > 0) {