mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-09 18:32:52 +00:00
feat(cloud-backup): mirror fleet snapshots to S3-compatible storage (#782)
* feat(cloud-backup): mirror fleet snapshots to S3-compatible storage
Add an Admiral-tier Cloud Backup feature that replicates every fleet
snapshot to off-site storage, with two provider modes that share the
same `@aws-sdk/client-s3` code path:
- Sencho Cloud Backup: zero-config, 500 MB allowance backed by
Cloudflare R2, provisioned via the sencho.io worker against the
user's Lemon Squeezy license.
- Custom S3 (BYOB): any S3-compatible bucket (AWS, MinIO, Backblaze
B2, Wasabi, R2 with own keys), with credentials encrypted via
`CryptoService` before storage.
API-triggered snapshots upload fire-and-forget so the UI returns
immediately; scheduled snapshots block on the upload so the task's
success/failure reflects cloud durability. Object keys include the
instance_id segment to prevent collisions when the same Admiral
license is activated on multiple Sencho instances.
* fix(cloud-backup): drop ES2022-only Error cause arg breaking ES2020 build
The backend tsconfig pins lib to ES2020. The two-argument
`Error(message, { cause })` form requires ES2022, so tsc rejected it
with TS2554. Revert to single-argument throw to match the
convention used elsewhere in the backend services.
This commit is contained in:
Anso
@@ -0,0 +1,400 @@
|
||||
/**
|
||||
* CloudBackupService — off-site replication for fleet snapshots.
|
||||
*
|
||||
* Two providers share the same S3-compatible code path:
|
||||
* - 'sencho' : managed Sencho Cloud Backup. Credentials provisioned by
|
||||
* sencho.io/api/cloud-backup/provision and stored in system_state.
|
||||
* - 'custom' : bring-your-own-bucket. Credentials user-configured in global_settings.
|
||||
*
|
||||
* Routes call this service after a snapshot is persisted to SQLite. The service
|
||||
* reads snapshot rows from DatabaseService, packs them into a tar.gz archive,
|
||||
* and uploads via @aws-sdk/client-s3.
|
||||
*/
|
||||
|
||||
import {
|
||||
S3Client,
|
||||
PutObjectCommand,
|
||||
GetObjectCommand,
|
||||
ListObjectsV2Command,
|
||||
DeleteObjectCommand,
|
||||
HeadBucketCommand,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { Readable } from 'stream';
|
||||
import * as zlib from 'zlib';
|
||||
import * as tar from 'tar-stream';
|
||||
import axios from 'axios';
|
||||
import { DatabaseService, type FleetSnapshotFile } from './DatabaseService';
|
||||
import { CryptoService } from './CryptoService';
|
||||
import { LicenseService } from './LicenseService';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
|
||||
export type CloudProvider = 'disabled' | 'sencho' | 'custom';
|
||||
export type UploadStatus = 'idle' | 'uploading' | 'success' | 'failed';
|
||||
|
||||
export interface ResolvedCloudConfig {
|
||||
provider: 'sencho' | 'custom';
|
||||
endpoint: string;
|
||||
region: string;
|
||||
bucket: string;
|
||||
accessKey: string;
|
||||
secretKey: string;
|
||||
pathPrefix: string;
|
||||
autoUpload: boolean;
|
||||
quotaBytes?: number;
|
||||
}
|
||||
|
||||
export interface CloudSnapshotEntry {
|
||||
objectKey: string;
|
||||
sizeBytes: number;
|
||||
lastModified: string | null;
|
||||
snapshotId: number | null;
|
||||
}
|
||||
|
||||
export interface ProvisionResult {
|
||||
success: boolean;
|
||||
quotaBytes?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface UploadStatusEntry {
|
||||
status: UploadStatus;
|
||||
objectKey?: string;
|
||||
error?: string;
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
const SENCHO_CLOUD_BACKUP_API_DEFAULT = 'https://sencho.io';
|
||||
const PROVIDER_KEY = 'cloud_backup_provider';
|
||||
const SUCCESS_STATUS_TTL_MS = 5 * 60 * 1000;
|
||||
|
||||
export class CloudBackupService {
|
||||
private static instance: CloudBackupService;
|
||||
private uploadStatus = new Map<number, UploadStatusEntry>();
|
||||
private statusGcTimer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
private constructor() {
|
||||
this.statusGcTimer = setInterval(() => this.gcUploadStatus(), 60 * 1000);
|
||||
if (typeof this.statusGcTimer.unref === 'function') this.statusGcTimer.unref();
|
||||
}
|
||||
|
||||
public static getInstance(): CloudBackupService {
|
||||
if (!CloudBackupService.instance) {
|
||||
CloudBackupService.instance = new CloudBackupService();
|
||||
}
|
||||
return CloudBackupService.instance;
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
if (this.statusGcTimer) {
|
||||
clearInterval(this.statusGcTimer);
|
||||
this.statusGcTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Configuration ─────────────────────────────────────────────────────────
|
||||
|
||||
public getProvider(): CloudProvider {
|
||||
const value = DatabaseService.getInstance().getGlobalSettings()[PROVIDER_KEY];
|
||||
if (value === 'sencho' || value === 'custom') return value;
|
||||
return 'disabled';
|
||||
}
|
||||
|
||||
public isEnabled(): boolean {
|
||||
return this.getResolvedConfig() !== null;
|
||||
}
|
||||
|
||||
public isAutoUploadOn(): boolean {
|
||||
const cfg = this.getResolvedConfig();
|
||||
return cfg?.autoUpload === true;
|
||||
}
|
||||
|
||||
public getResolvedConfig(): ResolvedCloudConfig | null {
|
||||
const provider = this.getProvider();
|
||||
if (provider === 'disabled') return null;
|
||||
const db = DatabaseService.getInstance();
|
||||
const crypto = CryptoService.getInstance();
|
||||
|
||||
if (provider === 'sencho') {
|
||||
const endpoint = db.getSystemState('sencho_cloud_backup_endpoint');
|
||||
const bucket = db.getSystemState('sencho_cloud_backup_bucket');
|
||||
const accessKey = db.getSystemState('sencho_cloud_backup_access_key');
|
||||
const secretRaw = db.getSystemState('sencho_cloud_backup_secret_key');
|
||||
const pathPrefix = db.getSystemState('sencho_cloud_backup_path_prefix') || '';
|
||||
const quotaRaw = db.getSystemState('sencho_cloud_backup_quota_bytes');
|
||||
if (!endpoint || !bucket || !accessKey || !secretRaw) return null;
|
||||
return {
|
||||
provider: 'sencho',
|
||||
endpoint,
|
||||
region: 'auto',
|
||||
bucket,
|
||||
accessKey,
|
||||
secretKey: crypto.decrypt(secretRaw),
|
||||
pathPrefix,
|
||||
autoUpload: true,
|
||||
quotaBytes: quotaRaw ? parseInt(quotaRaw, 10) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
const settings = db.getGlobalSettings();
|
||||
const endpoint = settings.cloud_backup_endpoint;
|
||||
const region = settings.cloud_backup_region;
|
||||
const bucket = settings.cloud_backup_bucket;
|
||||
const accessKey = settings.cloud_backup_access_key;
|
||||
const secretRaw = settings.cloud_backup_secret_key;
|
||||
const pathPrefix = settings.cloud_backup_path_prefix || 'sencho/';
|
||||
if (!endpoint || !bucket || !accessKey || !secretRaw) return null;
|
||||
return {
|
||||
provider: 'custom',
|
||||
endpoint,
|
||||
region: region || 'us-east-1',
|
||||
bucket,
|
||||
accessKey,
|
||||
secretKey: crypto.decrypt(secretRaw),
|
||||
pathPrefix,
|
||||
autoUpload: settings.cloud_backup_auto_upload === '1',
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Sencho Cloud Backup lifecycle ─────────────────────────────────────────
|
||||
|
||||
public async provisionSenchoCloudBackup(): Promise<ProvisionResult> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const crypto = CryptoService.getInstance();
|
||||
const licenseKey = db.getSystemState('license_key');
|
||||
if (!licenseKey) return { success: false, error: 'No license key found. Activate an Admiral license first.' };
|
||||
|
||||
const variant = LicenseService.getInstance().getVariant();
|
||||
if (variant !== 'admiral') return { success: false, error: 'Sencho Cloud Backup requires the Admiral tier.' };
|
||||
|
||||
const apiBase = process.env.SENCHO_CLOUD_BACKUP_API || SENCHO_CLOUD_BACKUP_API_DEFAULT;
|
||||
try {
|
||||
const res = await axios.post(`${apiBase}/api/cloud-backup/provision`, { license_key: licenseKey }, { timeout: 15000 });
|
||||
const data = res.data as {
|
||||
endpoint: string;
|
||||
region?: string;
|
||||
bucket: string;
|
||||
access_key: string;
|
||||
secret_key: string;
|
||||
path_prefix: string;
|
||||
quota_bytes: number;
|
||||
};
|
||||
db.setSystemState('sencho_cloud_backup_endpoint', data.endpoint);
|
||||
db.setSystemState('sencho_cloud_backup_bucket', data.bucket);
|
||||
db.setSystemState('sencho_cloud_backup_access_key', data.access_key);
|
||||
db.setSystemState('sencho_cloud_backup_secret_key', crypto.encrypt(data.secret_key));
|
||||
db.setSystemState('sencho_cloud_backup_path_prefix', data.path_prefix);
|
||||
db.setSystemState('sencho_cloud_backup_quota_bytes', String(data.quota_bytes));
|
||||
db.setSystemState('sencho_cloud_backup_provisioned_at', new Date().toISOString());
|
||||
db.updateGlobalSetting(PROVIDER_KEY, 'sencho');
|
||||
return { success: true, quotaBytes: data.quota_bytes };
|
||||
} catch (err) {
|
||||
const responseError = (err as { response?: { data?: { error?: string } } }).response?.data?.error;
|
||||
return { success: false, error: responseError || getErrorMessage(err, 'Failed to provision Sencho Cloud Backup.') };
|
||||
}
|
||||
}
|
||||
|
||||
public async refreshSenchoCloudBackupCredentials(): Promise<void> {
|
||||
const result = await this.provisionSenchoCloudBackup();
|
||||
if (!result.success) throw new Error(result.error || 'Failed to refresh Sencho Cloud Backup credentials.');
|
||||
}
|
||||
|
||||
public async getSenchoCloudBackupUsage(): Promise<{ used_bytes: number; quota_bytes: number; object_count: number }> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const licenseKey = db.getSystemState('license_key');
|
||||
if (!licenseKey) throw new Error('No license key found.');
|
||||
const apiBase = process.env.SENCHO_CLOUD_BACKUP_API || SENCHO_CLOUD_BACKUP_API_DEFAULT;
|
||||
const res = await axios.post(`${apiBase}/api/cloud-backup/usage`, { license_key: licenseKey }, { timeout: 15000 });
|
||||
const data = res.data as { used_bytes: number; quota_bytes: number; object_count: number };
|
||||
return data;
|
||||
}
|
||||
|
||||
// ─── S3 operations ─────────────────────────────────────────────────────────
|
||||
|
||||
public async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
const cfg = this.getResolvedConfig();
|
||||
if (!cfg) return { success: false, error: 'No cloud backup configuration is active.' };
|
||||
try {
|
||||
const client = this.buildS3Client(cfg);
|
||||
await client.send(new HeadBucketCommand({ Bucket: cfg.bucket }));
|
||||
return { success: true };
|
||||
} catch (err) {
|
||||
return { success: false, error: getErrorMessage(err, 'Connection test failed.') };
|
||||
}
|
||||
}
|
||||
|
||||
public async uploadSnapshot(snapshotId: number): Promise<void> {
|
||||
const cfg = this.getResolvedConfig();
|
||||
if (!cfg) throw new Error('Cloud backup is not configured.');
|
||||
const db = DatabaseService.getInstance();
|
||||
const snapshot = db.getSnapshot(snapshotId);
|
||||
if (!snapshot) throw new Error(`Snapshot ${snapshotId} not found.`);
|
||||
const files = db.getSnapshotFiles(snapshotId);
|
||||
const objectKey = this.buildObjectKey(cfg, snapshot.id, snapshot.description, snapshot.created_at);
|
||||
|
||||
this.setStatus(snapshotId, { status: 'uploading', objectKey, updatedAt: Date.now() });
|
||||
try {
|
||||
const archive = await this.buildArchive(snapshot, files);
|
||||
const client = this.buildS3Client(cfg);
|
||||
await client.send(new PutObjectCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Key: objectKey,
|
||||
Body: archive,
|
||||
ContentType: 'application/gzip',
|
||||
}));
|
||||
this.setStatus(snapshotId, { status: 'success', objectKey, updatedAt: Date.now() });
|
||||
if (isDebugEnabled()) {
|
||||
console.log(`[CloudBackup:debug] Uploaded snapshot ${snapshotId} (${archive.byteLength} bytes) to ${cfg.bucket}/${objectKey}`);
|
||||
}
|
||||
} catch (err) {
|
||||
const message = getErrorMessage(err, 'Cloud upload failed.');
|
||||
this.setStatus(snapshotId, { status: 'failed', objectKey, error: message, updatedAt: Date.now() });
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
public async downloadSnapshot(objectKey: string): Promise<Buffer> {
|
||||
const cfg = this.getResolvedConfig();
|
||||
if (!cfg) throw new Error('Cloud backup is not configured.');
|
||||
const client = this.buildS3Client(cfg);
|
||||
const result = await client.send(new GetObjectCommand({ Bucket: cfg.bucket, Key: objectKey }));
|
||||
const body = result.Body as Readable | undefined;
|
||||
if (!body) throw new Error('Empty response body.');
|
||||
return await streamToBuffer(body);
|
||||
}
|
||||
|
||||
public async listCloudSnapshots(): Promise<CloudSnapshotEntry[]> {
|
||||
const cfg = this.getResolvedConfig();
|
||||
if (!cfg) return [];
|
||||
const client = this.buildS3Client(cfg);
|
||||
const prefix = `${cfg.pathPrefix}instances/${this.getInstanceId()}/snapshots/`;
|
||||
const result = await client.send(new ListObjectsV2Command({
|
||||
Bucket: cfg.bucket,
|
||||
Prefix: prefix,
|
||||
MaxKeys: 1000,
|
||||
}));
|
||||
const objects = result.Contents || [];
|
||||
return objects
|
||||
.filter(o => !!o.Key)
|
||||
.map(o => {
|
||||
const key = o.Key as string;
|
||||
const basename = key.split('/').pop() || key;
|
||||
const idMatch = basename.match(/^(\d+)_/);
|
||||
return {
|
||||
objectKey: key,
|
||||
sizeBytes: o.Size ?? 0,
|
||||
lastModified: o.LastModified ? o.LastModified.toISOString() : null,
|
||||
snapshotId: idMatch ? parseInt(idMatch[1], 10) : null,
|
||||
};
|
||||
})
|
||||
.sort((a, b) => (b.lastModified || '').localeCompare(a.lastModified || ''));
|
||||
}
|
||||
|
||||
public async deleteCloudSnapshot(objectKey: string): Promise<void> {
|
||||
const cfg = this.getResolvedConfig();
|
||||
if (!cfg) throw new Error('Cloud backup is not configured.');
|
||||
const client = this.buildS3Client(cfg);
|
||||
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: objectKey }));
|
||||
}
|
||||
|
||||
// ─── Status tracking ───────────────────────────────────────────────────────
|
||||
|
||||
public getUploadStatus(snapshotId: number): UploadStatusEntry {
|
||||
return this.uploadStatus.get(snapshotId) || { status: 'idle', updatedAt: 0 };
|
||||
}
|
||||
|
||||
private setStatus(snapshotId: number, entry: UploadStatusEntry): void {
|
||||
this.uploadStatus.set(snapshotId, entry);
|
||||
}
|
||||
|
||||
private gcUploadStatus(): void {
|
||||
const now = Date.now();
|
||||
for (const [id, entry] of this.uploadStatus.entries()) {
|
||||
if (entry.status === 'success' && now - entry.updatedAt > SUCCESS_STATUS_TTL_MS) {
|
||||
this.uploadStatus.delete(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Internals ─────────────────────────────────────────────────────────────
|
||||
|
||||
private buildS3Client(cfg: ResolvedCloudConfig): S3Client {
|
||||
return new S3Client({
|
||||
endpoint: cfg.endpoint,
|
||||
region: cfg.region,
|
||||
credentials: { accessKeyId: cfg.accessKey, secretAccessKey: cfg.secretKey },
|
||||
forcePathStyle: true,
|
||||
});
|
||||
}
|
||||
|
||||
private getInstanceId(): string {
|
||||
return DatabaseService.getInstance().getSystemState('instance_id') || 'unknown';
|
||||
}
|
||||
|
||||
private buildObjectKey(cfg: ResolvedCloudConfig, snapshotId: number, description: string, createdAt: number): string {
|
||||
const ts = new Date(createdAt).toISOString().replace(/[:.]/g, '-');
|
||||
const slug = description.slice(0, 40).replace(/[^a-z0-9]+/gi, '-').replace(/^-+|-+$/g, '').toLowerCase() || 'snapshot';
|
||||
return `${cfg.pathPrefix}instances/${this.getInstanceId()}/snapshots/${snapshotId}_${ts}_${slug}.tar.gz`;
|
||||
}
|
||||
|
||||
private async buildArchive(
|
||||
snapshot: { id: number; description: string; created_by: string; node_count: number; stack_count: number; skipped_nodes: string; created_at: number },
|
||||
files: FleetSnapshotFile[],
|
||||
): Promise<Buffer> {
|
||||
const pack = tar.pack();
|
||||
const metadata = {
|
||||
id: snapshot.id,
|
||||
description: snapshot.description,
|
||||
created_by: snapshot.created_by,
|
||||
created_at: snapshot.created_at,
|
||||
node_count: snapshot.node_count,
|
||||
stack_count: snapshot.stack_count,
|
||||
skipped_nodes: safeParseJson(snapshot.skipped_nodes, []),
|
||||
instance_id: this.getInstanceId(),
|
||||
archive_version: 1,
|
||||
};
|
||||
pack.entry({ name: 'metadata.json' }, JSON.stringify(metadata, null, 2));
|
||||
|
||||
for (const file of files) {
|
||||
const safeNodeName = sanitizePathSegment(file.node_name);
|
||||
const safeStackName = sanitizePathSegment(file.stack_name);
|
||||
const safeFilename = sanitizePathSegment(file.filename);
|
||||
const entryPath = `nodes/${file.node_id}_${safeNodeName}/${safeStackName}/${safeFilename}`;
|
||||
pack.entry({ name: entryPath }, file.content);
|
||||
}
|
||||
pack.finalize();
|
||||
|
||||
const gzip = zlib.createGzip();
|
||||
const chunks: Buffer[] = [];
|
||||
return await new Promise<Buffer>((resolve, reject) => {
|
||||
gzip.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
gzip.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
gzip.on('error', reject);
|
||||
pack.on('error', reject);
|
||||
pack.pipe(gzip);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function streamToBuffer(stream: Readable): Promise<Buffer> {
|
||||
return new Promise<Buffer>((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
stream.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
stream.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function sanitizePathSegment(input: string): string {
|
||||
// Preserve dotfiles like `.env` while neutralising traversal (`..`) and path separators.
|
||||
const cleaned = input.replace(/\.\./g, '_').replace(/[\\/]/g, '_');
|
||||
if (/^\.+$/.test(cleaned)) return '_';
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
function safeParseJson<T>(value: string | null | undefined, fallback: T): T {
|
||||
if (!value) return fallback;
|
||||
try { return JSON.parse(value) as T; } catch { return fallback; }
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import { NotificationService } from './NotificationService';
|
||||
import TrivyService from './TrivyService';
|
||||
import type { ScanAllNodeImagesResult } from './TrivyService';
|
||||
import TrivyInstaller from './TrivyInstaller';
|
||||
import { CloudBackupService } from './CloudBackupService';
|
||||
|
||||
const TRIVY_UPDATE_CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
||||
const TRIVY_UPDATE_CHECK_STARTUP_DELAY_MS = 5 * 60 * 1000;
|
||||
@@ -501,11 +502,25 @@ export class SchedulerService {
|
||||
db.insertSnapshotFiles(snapshotId, allFiles);
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) {
|
||||
console.debug(`[SchedulerService:debug] Snapshot task ${task.id}: captured ${capturedNodes.length} node(s), ${totalStacks} stack(s), ${allFiles.length} file(s), skipped ${skippedNodes.length}`);
|
||||
let cloudUploadNote = '';
|
||||
const cloudSvc = CloudBackupService.getInstance();
|
||||
if (cloudSvc.isEnabled() && cloudSvc.isAutoUploadOn()) {
|
||||
try {
|
||||
await cloudSvc.uploadSnapshot(snapshotId);
|
||||
cloudUploadNote = ', cloud upload OK';
|
||||
} catch (err) {
|
||||
const message = getErrorMessage(err, 'Cloud upload failed');
|
||||
console.error('[SchedulerService] Cloud upload failed:', message);
|
||||
this.safeDispatch('warning', 'system', `Cloud backup failed for scheduled snapshot ${snapshotId}: ${message}`);
|
||||
cloudUploadNote = ', cloud upload FAILED';
|
||||
}
|
||||
}
|
||||
|
||||
return `Fleet snapshot created (id=${snapshotId}, ${capturedNodes.length} node(s), ${totalStacks} stack(s)${skippedNodes.length > 0 ? `, ${skippedNodes.length} skipped` : ''})`;
|
||||
if (isDebugEnabled()) {
|
||||
console.debug(`[SchedulerService:debug] Snapshot task ${task.id}: captured ${capturedNodes.length} node(s), ${totalStacks} stack(s), ${allFiles.length} file(s), skipped ${skippedNodes.length}${cloudUploadNote}`);
|
||||
}
|
||||
|
||||
return `Fleet snapshot created (id=${snapshotId}, ${capturedNodes.length} node(s), ${totalStacks} stack(s)${skippedNodes.length > 0 ? `, ${skippedNodes.length} skipped` : ''}${cloudUploadNote})`;
|
||||
}
|
||||
|
||||
private async executePrune(task: ScheduledTask): Promise<string> {
|
||||
|
||||
Reference in New Issue
Block a user