feat(cloud-backup): mirror fleet snapshots to S3-compatible storage (#782)

* feat(cloud-backup): mirror fleet snapshots to S3-compatible storage

Add an Admiral-tier Cloud Backup feature that replicates every fleet
snapshot to off-site storage, with two provider modes that share the
same `@aws-sdk/client-s3` code path:

- Sencho Cloud Backup: zero-config, 500 MB allowance backed by
  Cloudflare R2, provisioned via the sencho.io worker against the
  user's Lemon Squeezy license.
- Custom S3 (BYOB): any S3-compatible bucket (AWS, MinIO, Backblaze
  B2, Wasabi, R2 with own keys), with credentials encrypted via
  `CryptoService` before storage.

API-triggered snapshots upload fire-and-forget so the UI returns
immediately; scheduled snapshots block on the upload so the task's
success/failure reflects cloud durability. Object keys include the
instance_id segment to prevent collisions when the same Admiral
license is activated on multiple Sencho instances.

* fix(cloud-backup): drop ES2022-only Error cause arg breaking ES2020 build

The backend tsconfig pins lib to ES2020. The two-argument
`Error(message, { cause })` form requires ES2022, so tsc rejected it
with TS2554. Revert to single-argument throw to match the
convention used elsewhere in the backend services.
This commit is contained in:
Anso
2026-04-26 15:42:21 -04:00
committed by GitHub
parent 801a098a5b
commit 03f91cd5bb
20 changed files with 2805 additions and 401 deletions
@@ -0,0 +1,193 @@
/**
* Tests for /api/cloud-backup routes — tier gating (community/skipper/admiral),
* admin gating, config CRUD round-trip with secret encryption, audit logging.
* The S3 SDK is mocked at the module level so no network calls happen.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
import request from 'supertest';
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
const sentSpy = vi.fn();
vi.mock('@aws-sdk/client-s3', () => {
class S3Client { async send(cmd: { name: string; input: Record<string, unknown> }) { return sentSpy(cmd); } }
class PutObjectCommand { name = 'PutObjectCommand'; constructor(public input: Record<string, unknown>) {} }
class GetObjectCommand { name = 'GetObjectCommand'; constructor(public input: Record<string, unknown>) {} }
class ListObjectsV2Command { name = 'ListObjectsV2Command'; constructor(public input: Record<string, unknown>) {} }
class DeleteObjectCommand { name = 'DeleteObjectCommand'; constructor(public input: Record<string, unknown>) {} }
class HeadBucketCommand { name = 'HeadBucketCommand'; constructor(public input: Record<string, unknown>) {} }
return { S3Client, PutObjectCommand, GetObjectCommand, ListObjectsV2Command, DeleteObjectCommand, HeadBucketCommand };
});
let tmpDir: string;
let app: import('express').Express;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let LicenseService: typeof import('../services/LicenseService').LicenseService;
let authCookie: string;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
({ LicenseService } = await import('../services/LicenseService'));
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
vi.spyOn(LicenseService.getInstance(), 'getVariant').mockReturnValue('admiral');
({ app } = await import('../index'));
authCookie = await loginAsTestAdmin(app);
});
afterAll(() => {
cleanupTestDb(tmpDir);
});
beforeEach(() => {
sentSpy.mockReset();
const db = DatabaseService.getInstance();
for (const k of [
'cloud_backup_provider',
'cloud_backup_endpoint',
'cloud_backup_region',
'cloud_backup_bucket',
'cloud_backup_access_key',
'cloud_backup_secret_key',
'cloud_backup_path_prefix',
'cloud_backup_auto_upload',
]) {
db.updateGlobalSetting(k, '');
}
});
describe('Cloud backup tier gating', () => {
it('rejects community tier with PAID_REQUIRED', async () => {
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValueOnce('community');
const res = await request(app).get('/api/cloud-backup/config').set('Cookie', authCookie);
expect(res.status).toBe(403);
expect(res.body.code).toBe('PAID_REQUIRED');
});
it('rejects skipper tier with ADMIRAL_REQUIRED', async () => {
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValueOnce('paid');
vi.spyOn(LicenseService.getInstance(), 'getVariant').mockReturnValueOnce('skipper');
const res = await request(app).get('/api/cloud-backup/config').set('Cookie', authCookie);
expect(res.status).toBe(403);
expect(res.body.code).toBe('ADMIRAL_REQUIRED');
});
it('admiral tier reaches the handler', async () => {
const res = await request(app).get('/api/cloud-backup/config').set('Cookie', authCookie);
expect(res.status).toBe(200);
expect(res.body).toHaveProperty('provider', 'disabled');
});
});
describe('Cloud backup config CRUD', () => {
it('redacts secret_key on read; persists encrypted ciphertext', async () => {
const putRes = await request(app)
.put('/api/cloud-backup/config')
.set('Cookie', authCookie)
.send({
provider: 'custom',
custom: {
endpoint: 'https://s3.example.com',
region: 'us-east-1',
bucket: 'b',
access_key: 'AKIA1234',
secret_key: 'super-secret',
path_prefix: 'sencho/',
auto_upload: true,
},
});
expect(putRes.status).toBe(204);
const stored = DatabaseService.getInstance().getGlobalSettings().cloud_backup_secret_key;
expect(stored.startsWith('enc:')).toBe(true);
expect(stored.includes('super-secret')).toBe(false);
const getRes = await request(app).get('/api/cloud-backup/config').set('Cookie', authCookie);
expect(getRes.status).toBe(200);
expect(getRes.body.provider).toBe('custom');
expect(getRes.body.custom.secret_key).toBe('***');
expect(getRes.body.custom.bucket).toBe('b');
});
it('preserves saved secret when client sends "***"', async () => {
const db = DatabaseService.getInstance();
await request(app)
.put('/api/cloud-backup/config')
.set('Cookie', authCookie)
.send({
provider: 'custom',
custom: { endpoint: 'https://e', region: 'r', bucket: 'b', access_key: 'a', secret_key: 'first-secret', path_prefix: 's/', auto_upload: false },
});
const firstStored = db.getGlobalSettings().cloud_backup_secret_key;
await request(app)
.put('/api/cloud-backup/config')
.set('Cookie', authCookie)
.send({
provider: 'custom',
custom: { endpoint: 'https://e', region: 'r2', bucket: 'b', access_key: 'a', secret_key: '***', path_prefix: 's/', auto_upload: true },
});
const secondStored = db.getGlobalSettings().cloud_backup_secret_key;
expect(secondStored).toBe(firstStored);
expect(db.getGlobalSettings().cloud_backup_region).toBe('r2');
expect(db.getGlobalSettings().cloud_backup_auto_upload).toBe('1');
});
it('rejects invalid provider value', async () => {
const res = await request(app)
.put('/api/cloud-backup/config')
.set('Cookie', authCookie)
.send({ provider: 'bogus' });
expect(res.status).toBe(400);
});
it('rejects custom config missing required fields', async () => {
const res = await request(app)
.put('/api/cloud-backup/config')
.set('Cookie', authCookie)
.send({ provider: 'custom', custom: { endpoint: '', bucket: '', access_key: '' } });
expect(res.status).toBe(400);
});
});
describe('Cloud backup audit log', () => {
it('writes audit row with the cloud-backup summary on PUT /config', async () => {
await request(app)
.put('/api/cloud-backup/config')
.set('Cookie', authCookie)
.send({
provider: 'custom',
custom: { endpoint: 'https://s3.example.com', region: 'r', bucket: 'b', access_key: 'a', secret_key: 's', path_prefix: 'p/', auto_upload: false },
});
const { entries } = DatabaseService.getInstance().getAuditLogs({ limit: 50 });
const cloudEntry = entries.find(e => e.path.includes('/cloud-backup/config') && e.method === 'PUT');
expect(cloudEntry).toBeDefined();
expect(cloudEntry!.summary).toBe('Updated cloud backup config');
});
});
describe('Cloud backup test endpoint', () => {
it('reports failure when no provider is configured', async () => {
const res = await request(app).post('/api/cloud-backup/test').set('Cookie', authCookie).send({});
expect(res.status).toBe(200);
expect(res.body.success).toBe(false);
});
it('reports success when HeadBucketCommand resolves', async () => {
const db = DatabaseService.getInstance();
const { CryptoService } = await import('../services/CryptoService');
db.updateGlobalSetting('cloud_backup_provider', 'custom');
db.updateGlobalSetting('cloud_backup_endpoint', 'https://s3.example.com');
db.updateGlobalSetting('cloud_backup_region', 'us-east-1');
db.updateGlobalSetting('cloud_backup_bucket', 'b');
db.updateGlobalSetting('cloud_backup_access_key', 'a');
db.updateGlobalSetting('cloud_backup_secret_key', CryptoService.getInstance().encrypt('s'));
sentSpy.mockResolvedValueOnce({});
const res = await request(app).post('/api/cloud-backup/test').set('Cookie', authCookie).send({});
expect(res.status).toBe(200);
expect(res.body.success).toBe(true);
});
});
@@ -0,0 +1,242 @@
/**
* Tests for CloudBackupService — provider resolution, encryption round-trip,
* archive format, and S3 client invocation. The S3 SDK is mocked at the
* module level so no network calls happen.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
import * as zlib from 'zlib';
import * as tar from 'tar-stream';
import { Readable } from 'stream';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
const sentSpy = vi.fn();
const s3ClientCtorSpy = vi.fn();
vi.mock('@aws-sdk/client-s3', () => {
class S3Client {
constructor(opts: unknown) { s3ClientCtorSpy(opts); }
async send(cmd: { name: string; input: Record<string, unknown> }) { return sentSpy(cmd); }
}
class PutObjectCommand { name = 'PutObjectCommand'; constructor(public input: Record<string, unknown>) {} }
class GetObjectCommand { name = 'GetObjectCommand'; constructor(public input: Record<string, unknown>) {} }
class ListObjectsV2Command { name = 'ListObjectsV2Command'; constructor(public input: Record<string, unknown>) {} }
class DeleteObjectCommand { name = 'DeleteObjectCommand'; constructor(public input: Record<string, unknown>) {} }
class HeadBucketCommand { name = 'HeadBucketCommand'; constructor(public input: Record<string, unknown>) {} }
return { S3Client, PutObjectCommand, GetObjectCommand, ListObjectsV2Command, DeleteObjectCommand, HeadBucketCommand };
});
let tmpDir: string;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let CryptoService: typeof import('../services/CryptoService').CryptoService;
let CloudBackupService: typeof import('../services/CloudBackupService').CloudBackupService;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
({ CryptoService } = await import('../services/CryptoService'));
({ CloudBackupService } = await import('../services/CloudBackupService'));
});
afterAll(() => {
CloudBackupService.getInstance().stop();
cleanupTestDb(tmpDir);
});
beforeEach(() => {
sentSpy.mockReset();
s3ClientCtorSpy.mockReset();
const db = DatabaseService.getInstance();
// Reset all cloud-backup-related settings between tests.
for (const k of [
'cloud_backup_provider',
'cloud_backup_endpoint',
'cloud_backup_region',
'cloud_backup_bucket',
'cloud_backup_access_key',
'cloud_backup_secret_key',
'cloud_backup_path_prefix',
'cloud_backup_auto_upload',
]) {
db.updateGlobalSetting(k, '');
}
for (const k of [
'sencho_cloud_backup_endpoint',
'sencho_cloud_backup_bucket',
'sencho_cloud_backup_access_key',
'sencho_cloud_backup_secret_key',
'sencho_cloud_backup_path_prefix',
'sencho_cloud_backup_quota_bytes',
'sencho_cloud_backup_provisioned_at',
]) {
db.setSystemState(k, '');
}
db.setSystemState('instance_id', 'test-instance-id');
});
describe('CloudBackupService — provider resolution', () => {
it('returns "disabled" when no provider is set', () => {
expect(CloudBackupService.getInstance().getProvider()).toBe('disabled');
expect(CloudBackupService.getInstance().isEnabled()).toBe(false);
expect(CloudBackupService.getInstance().getResolvedConfig()).toBeNull();
});
it('returns null config for custom provider when fields are missing', () => {
const db = DatabaseService.getInstance();
db.updateGlobalSetting('cloud_backup_provider', 'custom');
db.updateGlobalSetting('cloud_backup_endpoint', 'https://s3.example.com');
// bucket, access_key, secret_key still missing
expect(CloudBackupService.getInstance().getResolvedConfig()).toBeNull();
});
it('decrypts custom secret_key on read', () => {
const db = DatabaseService.getInstance();
const crypto = CryptoService.getInstance();
db.updateGlobalSetting('cloud_backup_provider', 'custom');
db.updateGlobalSetting('cloud_backup_endpoint', 'https://s3.example.com');
db.updateGlobalSetting('cloud_backup_region', 'us-east-1');
db.updateGlobalSetting('cloud_backup_bucket', 'my-bucket');
db.updateGlobalSetting('cloud_backup_access_key', 'AKIA1234');
db.updateGlobalSetting('cloud_backup_secret_key', crypto.encrypt('plaintext-secret'));
db.updateGlobalSetting('cloud_backup_auto_upload', '1');
const cfg = CloudBackupService.getInstance().getResolvedConfig();
expect(cfg).not.toBeNull();
expect(cfg!.provider).toBe('custom');
expect(cfg!.secretKey).toBe('plaintext-secret');
expect(cfg!.autoUpload).toBe(true);
});
it('resolves sencho provider from system_state and forces auto_upload on', () => {
const db = DatabaseService.getInstance();
const crypto = CryptoService.getInstance();
db.updateGlobalSetting('cloud_backup_provider', 'sencho');
db.setSystemState('sencho_cloud_backup_endpoint', 'https://r2.example.com');
db.setSystemState('sencho_cloud_backup_bucket', 'sencho-cloud-backups');
db.setSystemState('sencho_cloud_backup_access_key', 'R2-ACCESS');
db.setSystemState('sencho_cloud_backup_secret_key', crypto.encrypt('R2-SECRET'));
db.setSystemState('sencho_cloud_backup_path_prefix', 'tenants/123/');
db.setSystemState('sencho_cloud_backup_quota_bytes', '524288000');
const cfg = CloudBackupService.getInstance().getResolvedConfig();
expect(cfg!.provider).toBe('sencho');
expect(cfg!.region).toBe('auto');
expect(cfg!.secretKey).toBe('R2-SECRET');
expect(cfg!.autoUpload).toBe(true);
expect(cfg!.quotaBytes).toBe(524_288_000);
});
});
describe('CloudBackupService — uploadSnapshot', () => {
function seedCustomProvider() {
const db = DatabaseService.getInstance();
const crypto = CryptoService.getInstance();
db.updateGlobalSetting('cloud_backup_provider', 'custom');
db.updateGlobalSetting('cloud_backup_endpoint', 'https://s3.example.com');
db.updateGlobalSetting('cloud_backup_region', 'us-east-1');
db.updateGlobalSetting('cloud_backup_bucket', 'my-bucket');
db.updateGlobalSetting('cloud_backup_access_key', 'AKIA1234');
db.updateGlobalSetting('cloud_backup_secret_key', crypto.encrypt('test-secret'));
db.updateGlobalSetting('cloud_backup_path_prefix', 'sencho/');
db.updateGlobalSetting('cloud_backup_auto_upload', '1');
}
it('uploads a snapshot with correct object key and gzipped tar archive', async () => {
seedCustomProvider();
const db = DatabaseService.getInstance();
const snapshotId = db.createSnapshot('Test backup', 'admin', 1, 1, '[]');
db.insertSnapshotFiles(snapshotId, [
{ nodeId: 1, nodeName: 'gateway', stackName: 'web', filename: 'compose.yaml', content: 'services: {}\n' },
{ nodeId: 1, nodeName: 'gateway', stackName: 'web', filename: '.env', content: 'KEY=value\n' },
]);
sentSpy.mockResolvedValue({});
await CloudBackupService.getInstance().uploadSnapshot(snapshotId);
expect(s3ClientCtorSpy).toHaveBeenCalledWith(expect.objectContaining({
endpoint: 'https://s3.example.com',
region: 'us-east-1',
forcePathStyle: true,
credentials: { accessKeyId: 'AKIA1234', secretAccessKey: 'test-secret' },
}));
const putCall = sentSpy.mock.calls.find(c => c[0].name === 'PutObjectCommand');
expect(putCall).toBeDefined();
const input = putCall![0].input as { Bucket: string; Key: string; Body: Buffer; ContentType: string };
expect(input.Bucket).toBe('my-bucket');
expect(input.Key).toContain('sencho/instances/test-instance-id/snapshots/');
expect(input.Key).toMatch(/\.tar\.gz$/);
expect(input.ContentType).toBe('application/gzip');
expect(Buffer.isBuffer(input.Body)).toBe(true);
expect(input.Body.byteLength).toBeGreaterThan(0);
const decompressed = zlib.gunzipSync(input.Body);
const entries: Array<{ name: string; content: string }> = await new Promise((resolve, reject) => {
const extract = tar.extract();
const list: Array<{ name: string; content: string }> = [];
extract.on('entry', (header, stream, next) => {
const chunks: Buffer[] = [];
stream.on('data', (c: Buffer) => chunks.push(c));
stream.on('end', () => { list.push({ name: header.name, content: Buffer.concat(chunks).toString('utf-8') }); next(); });
stream.resume();
});
extract.on('finish', () => resolve(list));
extract.on('error', reject);
Readable.from(decompressed).pipe(extract);
});
const meta = entries.find(e => e.name === 'metadata.json');
expect(meta).toBeDefined();
const parsed = JSON.parse(meta!.content);
expect(parsed.id).toBe(snapshotId);
expect(parsed.instance_id).toBe('test-instance-id');
expect(parsed.archive_version).toBe(1);
expect(entries.find(e => e.name === 'nodes/1_gateway/web/compose.yaml')).toBeDefined();
expect(entries.find(e => e.name === 'nodes/1_gateway/web/.env')).toBeDefined();
expect(CloudBackupService.getInstance().getUploadStatus(snapshotId).status).toBe('success');
});
it('records failure status when upload throws', async () => {
seedCustomProvider();
const db = DatabaseService.getInstance();
const snapshotId = db.createSnapshot('Failing', 'admin', 0, 0, '[]');
sentSpy.mockRejectedValueOnce(new Error('AccessDenied: bad creds'));
await expect(CloudBackupService.getInstance().uploadSnapshot(snapshotId)).rejects.toThrow(/bad creds/);
const status = CloudBackupService.getInstance().getUploadStatus(snapshotId);
expect(status.status).toBe('failed');
expect(status.error).toContain('bad creds');
});
it('throws when no provider is configured', async () => {
await expect(CloudBackupService.getInstance().uploadSnapshot(999)).rejects.toThrow(/not configured/i);
});
});
describe('CloudBackupService — listCloudSnapshots', () => {
it('parses snapshot ID from object key and sorts by lastModified desc', async () => {
const db = DatabaseService.getInstance();
const crypto = CryptoService.getInstance();
db.updateGlobalSetting('cloud_backup_provider', 'custom');
db.updateGlobalSetting('cloud_backup_endpoint', 'https://s3.example.com');
db.updateGlobalSetting('cloud_backup_region', 'us-east-1');
db.updateGlobalSetting('cloud_backup_bucket', 'b');
db.updateGlobalSetting('cloud_backup_access_key', 'a');
db.updateGlobalSetting('cloud_backup_secret_key', crypto.encrypt('s'));
db.updateGlobalSetting('cloud_backup_path_prefix', 'sencho/');
sentSpy.mockResolvedValueOnce({
Contents: [
{ Key: 'sencho/instances/test-instance-id/snapshots/3_2026-01-01_a.tar.gz', Size: 100, LastModified: new Date('2026-01-01T00:00:00Z') },
{ Key: 'sencho/instances/test-instance-id/snapshots/7_2026-04-01_b.tar.gz', Size: 200, LastModified: new Date('2026-04-01T00:00:00Z') },
],
});
const list = await CloudBackupService.getInstance().listCloudSnapshots();
expect(list).toHaveLength(2);
expect(list[0].snapshotId).toBe(7);
expect(list[1].snapshotId).toBe(3);
});
});
@@ -145,6 +145,16 @@ vi.mock('../services/NotificationService', () => ({
},
}));
vi.mock('../services/CloudBackupService', () => ({
CloudBackupService: {
getInstance: () => ({
isEnabled: () => false,
isAutoUploadOn: () => false,
uploadSnapshot: vi.fn().mockResolvedValue(undefined),
}),
},
}));
vi.mock('../services/NodeRegistry', () => ({
NodeRegistry: {
getInstance: () => ({