fix(fleet): isolate corrupt snapshot file decrypt failures (#1650)

* fix(fleet): isolate corrupt snapshot file decrypt failures

A single damaged encrypted fleet-snapshot row no longer fails detail, restore, or off-site upload for the whole snapshot. Unavailable members are marked, restore is blocked before mutation, and cloud upload fails closed with no PutObject.

* fix(fleet): fail closed on damaged enc snapshot envelopes

Unrecognized enc: payloads no longer fall through as usable plaintext. Only clear legacy prose stays readable; delimiter-byte and similar envelope damage stays unavailable through restore and cloud upload.

* fix(fleet): subordinate legacy enc prose to envelope shape

Legacy exceptions no longer trigger from = or whitespace alone. Encryption-shaped payloads (length and hex density) stay unavailable through restore and cloud upload, while short genuine prose such as enc:hello remains usable.

* fix(fleet): preserve non-envelope enc legacy plaintext

Any non-empty enc: payload that is not encryption-shaped is kept verbatim, including punctuation forms such as enc:hello-world, while envelope-shaped damage remains unavailable.

backend/src/services/CloudBackupService.ts

@@ -1,5 +1,5 @@
 /**
- * CloudBackupService — off-site replication for fleet snapshots.
+ * CloudBackupService: off-site replication for fleet snapshots.
  *
  * Two providers share the same S3-compatible code path:
  *   - 'sencho' : managed Sencho Cloud Backup. Credentials provisioned by
@@ -15,9 +15,13 @@ import { Readable } from 'stream';
 import * as zlib from 'zlib';
 import * as tar from 'tar-stream';
 import axios from 'axios';
-import { DatabaseService, type FleetSnapshotFile } from './DatabaseService';
+import { DatabaseService } from './DatabaseService';
 import { CryptoService } from './CryptoService';
 import { LicenseService } from './LicenseService';
+import {
+    isAvailableSnapshotFile,
+    type AvailableSnapshotFile,
+} from '../helpers/snapshotFileDecrypt';
 import { getErrorMessage } from '../utils/errors';
 import { isDebugEnabled } from '../utils/debug';
 
@@ -255,9 +259,16 @@ export class CloudBackupService {
         const documentation = db.getSnapshotDocumentation(snapshotId);
         const objectKey = this.buildObjectKey(cfg, snapshot.id, snapshot.description, snapshot.created_at);
 
+        const availableFiles = files.filter(isAvailableSnapshotFile);
+        if (availableFiles.length !== files.length) {
+            const message = 'One or more snapshot files could not be decrypted';
+            this.setStatus(snapshotId, { status: 'failed', objectKey, error: message, updatedAt: Date.now() });
+            throw new Error(message);
+        }
+
         this.setStatus(snapshotId, { status: 'uploading', objectKey, updatedAt: Date.now() });
         try {
-            const archive = await this.buildArchive(snapshot, files, documentation);
+            const archive = await this.buildArchive(snapshot, availableFiles, documentation);
             const { client, sdk } = await this.buildS3Client(cfg);
             await client.send(new sdk.PutObjectCommand({
                 Bucket: cfg.bucket,
@@ -364,7 +375,7 @@ export class CloudBackupService {
 
     private async buildArchive(
         snapshot: { id: number; description: string; created_by: string; node_count: number; stack_count: number; skipped_nodes: string; created_at: number },
-        files: FleetSnapshotFile[],
+        files: AvailableSnapshotFile[],
         documentation = '',
     ): Promise<Buffer> {
         const pack = tar.pack();

backend/src/services/DatabaseService.ts

@@ -10,6 +10,9 @@ import { EXPOSURE_INTENTS, type ExposureIntent } from './network/types';
 import { HIGH_EPSS_THRESHOLD } from './securityPosture';
 import type { BackendScheduledAction } from './scheduledActionRegistry';
 import { stackPatternMatches } from '../helpers/stackPattern';
+import { readSnapshotFileRow, type SnapshotFileReadResult, type SnapshotFileRow } from '../helpers/snapshotFileDecrypt';
+
+export type { SnapshotFileReadResult } from '../helpers/snapshotFileDecrypt';
 
 function isPilotMode(): boolean {
     return process.env.SENCHO_MODE === 'pilot';
@@ -4594,8 +4597,9 @@ export class DatabaseService {
     public insertSnapshotFiles(snapshotId: number, files: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }>): void {
         // Snapshot file bodies are compose.yaml and .env captures, so they carry
         // the same secrets as the live stack. Encrypt content at rest with the
-        // instance key; getSnapshotFiles/getSnapshotStackFiles decrypt on read,
-        // so restore and cloud-archive paths see plaintext and stay portable.
+        // instance key. Getters classify and decrypt per row (see
+        // snapshotFileDecrypt.ts); unavailable rows omit content so callers
+        // cannot treat damage as usable plaintext.
         const crypto = CryptoService.getInstance();
         const insert = this.db.prepare(
             'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)'
@@ -4630,7 +4634,7 @@ export class DatabaseService {
         const row = this.db.prepare('SELECT documentation FROM fleet_snapshots WHERE id = ?').get(id) as { documentation: string } | undefined;
         if (!row || row.documentation === '') return '';
         try {
-            // decrypt() returns non-ciphertext input unchanged, mirroring the file path.
+            // decrypt() returns non-ciphertext input unchanged.
             return CryptoService.getInstance().decrypt(row.documentation);
         } catch (e) {
             // A corrupt blob or a key rotation must not break the primary backup
@@ -4641,22 +4645,24 @@ export class DatabaseService {
         }
     }
 
-    public getSnapshotFiles(snapshotId: number): FleetSnapshotFile[] {
-        const crypto = CryptoService.getInstance();
+    // Per-row classification isolates corrupt encrypted rows so intact stacks
+    // remain readable. See helpers/snapshotFileDecrypt.ts.
+    public getSnapshotFiles(snapshotId: number): SnapshotFileReadResult[] {
         const rows = this.db.prepare(
-            'SELECT * FROM fleet_snapshot_files WHERE snapshot_id = ? ORDER BY node_name, stack_name'
-        ).all(snapshotId) as FleetSnapshotFile[];
-        // decrypt() returns non-ciphertext input unchanged, so rows written
-        // before content-at-rest encryption still read back as plaintext.
-        return rows.map(row => ({ ...row, content: crypto.decrypt(row.content) }));
+            'SELECT node_id, node_name, stack_name, filename, content FROM fleet_snapshot_files WHERE snapshot_id = ? ORDER BY node_name, stack_name'
+        ).all(snapshotId) as SnapshotFileRow[];
+        return this.mapSnapshotFileRows(rows, snapshotId);
     }
 
-    public getSnapshotStackFiles(snapshotId: number, nodeId: number, stackName: string): FleetSnapshotFile[] {
-        const crypto = CryptoService.getInstance();
+    public getSnapshotStackFiles(snapshotId: number, nodeId: number, stackName: string): SnapshotFileReadResult[] {
         const rows = this.db.prepare(
-            'SELECT * FROM fleet_snapshot_files WHERE snapshot_id = ? AND node_id = ? AND stack_name = ?'
-        ).all(snapshotId, nodeId, stackName) as FleetSnapshotFile[];
-        return rows.map(row => ({ ...row, content: crypto.decrypt(row.content) }));
+            'SELECT node_id, node_name, stack_name, filename, content FROM fleet_snapshot_files WHERE snapshot_id = ? AND node_id = ? AND stack_name = ?'
+        ).all(snapshotId, nodeId, stackName) as SnapshotFileRow[];
+        return this.mapSnapshotFileRows(rows, snapshotId);
+    }
+
+    private mapSnapshotFileRows(rows: SnapshotFileRow[], snapshotId: number): SnapshotFileReadResult[] {
+        return rows.map(row => readSnapshotFileRow(row, snapshotId));
     }
 
     /** Created-at of the most recent fleet snapshot covering a stack, or null. */