mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-07 01:14:14 +00:00
feat(cloud-backup): mirror fleet snapshots to S3-compatible storage (#782)
* feat(cloud-backup): mirror fleet snapshots to S3-compatible storage
Add an Admiral-tier Cloud Backup feature that replicates every fleet
snapshot to off-site storage, with two provider modes that share the
same `@aws-sdk/client-s3` code path:
- Sencho Cloud Backup: zero-config, 500 MB allowance backed by
Cloudflare R2, provisioned via the sencho.io worker against the
user's Lemon Squeezy license.
- Custom S3 (BYOB): any S3-compatible bucket (AWS, MinIO, Backblaze
B2, Wasabi, R2 with own keys), with credentials encrypted via
`CryptoService` before storage.
API-triggered snapshots upload fire-and-forget so the UI returns
immediately; scheduled snapshots block on the upload so the task's
success/failure reflects cloud durability. Object keys include the
instance_id segment to prevent collisions when the same Admiral
license is activated on multiple Sencho instances.
* fix(cloud-backup): drop ES2022-only Error cause arg breaking ES2020 build
The backend tsconfig pins lib to ES2020. The two-argument
`Error(message, { cause })` form requires ES2022, so tsc rejected it
with TS2554. Revert to single-argument throw to match the
convention used elsewhere in the backend services.
This commit is contained in:
Anso
@@ -0,0 +1,193 @@
|
||||
/**
|
||||
* Tests for /api/cloud-backup routes — tier gating (community/skipper/admiral),
|
||||
* admin gating, config CRUD round-trip with secret encryption, audit logging.
|
||||
* The S3 SDK is mocked at the module level so no network calls happen.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
|
||||
|
||||
const sentSpy = vi.fn();
|
||||
|
||||
vi.mock('@aws-sdk/client-s3', () => {
|
||||
class S3Client { async send(cmd: { name: string; input: Record<string, unknown> }) { return sentSpy(cmd); } }
|
||||
class PutObjectCommand { name = 'PutObjectCommand'; constructor(public input: Record<string, unknown>) {} }
|
||||
class GetObjectCommand { name = 'GetObjectCommand'; constructor(public input: Record<string, unknown>) {} }
|
||||
class ListObjectsV2Command { name = 'ListObjectsV2Command'; constructor(public input: Record<string, unknown>) {} }
|
||||
class DeleteObjectCommand { name = 'DeleteObjectCommand'; constructor(public input: Record<string, unknown>) {} }
|
||||
class HeadBucketCommand { name = 'HeadBucketCommand'; constructor(public input: Record<string, unknown>) {} }
|
||||
return { S3Client, PutObjectCommand, GetObjectCommand, ListObjectsV2Command, DeleteObjectCommand, HeadBucketCommand };
|
||||
});
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
let LicenseService: typeof import('../services/LicenseService').LicenseService;
|
||||
let authCookie: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
({ LicenseService } = await import('../services/LicenseService'));
|
||||
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
vi.spyOn(LicenseService.getInstance(), 'getVariant').mockReturnValue('admiral');
|
||||
|
||||
({ app } = await import('../index'));
|
||||
authCookie = await loginAsTestAdmin(app);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
sentSpy.mockReset();
|
||||
const db = DatabaseService.getInstance();
|
||||
for (const k of [
|
||||
'cloud_backup_provider',
|
||||
'cloud_backup_endpoint',
|
||||
'cloud_backup_region',
|
||||
'cloud_backup_bucket',
|
||||
'cloud_backup_access_key',
|
||||
'cloud_backup_secret_key',
|
||||
'cloud_backup_path_prefix',
|
||||
'cloud_backup_auto_upload',
|
||||
]) {
|
||||
db.updateGlobalSetting(k, '');
|
||||
}
|
||||
});
|
||||
|
||||
describe('Cloud backup tier gating', () => {
|
||||
it('rejects community tier with PAID_REQUIRED', async () => {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValueOnce('community');
|
||||
const res = await request(app).get('/api/cloud-backup/config').set('Cookie', authCookie);
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.body.code).toBe('PAID_REQUIRED');
|
||||
});
|
||||
|
||||
it('rejects skipper tier with ADMIRAL_REQUIRED', async () => {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValueOnce('paid');
|
||||
vi.spyOn(LicenseService.getInstance(), 'getVariant').mockReturnValueOnce('skipper');
|
||||
const res = await request(app).get('/api/cloud-backup/config').set('Cookie', authCookie);
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.body.code).toBe('ADMIRAL_REQUIRED');
|
||||
});
|
||||
|
||||
it('admiral tier reaches the handler', async () => {
|
||||
const res = await request(app).get('/api/cloud-backup/config').set('Cookie', authCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toHaveProperty('provider', 'disabled');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Cloud backup config CRUD', () => {
|
||||
it('redacts secret_key on read; persists encrypted ciphertext', async () => {
|
||||
const putRes = await request(app)
|
||||
.put('/api/cloud-backup/config')
|
||||
.set('Cookie', authCookie)
|
||||
.send({
|
||||
provider: 'custom',
|
||||
custom: {
|
||||
endpoint: 'https://s3.example.com',
|
||||
region: 'us-east-1',
|
||||
bucket: 'b',
|
||||
access_key: 'AKIA1234',
|
||||
secret_key: 'super-secret',
|
||||
path_prefix: 'sencho/',
|
||||
auto_upload: true,
|
||||
},
|
||||
});
|
||||
expect(putRes.status).toBe(204);
|
||||
|
||||
const stored = DatabaseService.getInstance().getGlobalSettings().cloud_backup_secret_key;
|
||||
expect(stored.startsWith('enc:')).toBe(true);
|
||||
expect(stored.includes('super-secret')).toBe(false);
|
||||
|
||||
const getRes = await request(app).get('/api/cloud-backup/config').set('Cookie', authCookie);
|
||||
expect(getRes.status).toBe(200);
|
||||
expect(getRes.body.provider).toBe('custom');
|
||||
expect(getRes.body.custom.secret_key).toBe('***');
|
||||
expect(getRes.body.custom.bucket).toBe('b');
|
||||
});
|
||||
|
||||
it('preserves saved secret when client sends "***"', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
await request(app)
|
||||
.put('/api/cloud-backup/config')
|
||||
.set('Cookie', authCookie)
|
||||
.send({
|
||||
provider: 'custom',
|
||||
custom: { endpoint: 'https://e', region: 'r', bucket: 'b', access_key: 'a', secret_key: 'first-secret', path_prefix: 's/', auto_upload: false },
|
||||
});
|
||||
const firstStored = db.getGlobalSettings().cloud_backup_secret_key;
|
||||
|
||||
await request(app)
|
||||
.put('/api/cloud-backup/config')
|
||||
.set('Cookie', authCookie)
|
||||
.send({
|
||||
provider: 'custom',
|
||||
custom: { endpoint: 'https://e', region: 'r2', bucket: 'b', access_key: 'a', secret_key: '***', path_prefix: 's/', auto_upload: true },
|
||||
});
|
||||
const secondStored = db.getGlobalSettings().cloud_backup_secret_key;
|
||||
expect(secondStored).toBe(firstStored);
|
||||
expect(db.getGlobalSettings().cloud_backup_region).toBe('r2');
|
||||
expect(db.getGlobalSettings().cloud_backup_auto_upload).toBe('1');
|
||||
});
|
||||
|
||||
it('rejects invalid provider value', async () => {
|
||||
const res = await request(app)
|
||||
.put('/api/cloud-backup/config')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ provider: 'bogus' });
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects custom config missing required fields', async () => {
|
||||
const res = await request(app)
|
||||
.put('/api/cloud-backup/config')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ provider: 'custom', custom: { endpoint: '', bucket: '', access_key: '' } });
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Cloud backup audit log', () => {
|
||||
it('writes audit row with the cloud-backup summary on PUT /config', async () => {
|
||||
await request(app)
|
||||
.put('/api/cloud-backup/config')
|
||||
.set('Cookie', authCookie)
|
||||
.send({
|
||||
provider: 'custom',
|
||||
custom: { endpoint: 'https://s3.example.com', region: 'r', bucket: 'b', access_key: 'a', secret_key: 's', path_prefix: 'p/', auto_upload: false },
|
||||
});
|
||||
const { entries } = DatabaseService.getInstance().getAuditLogs({ limit: 50 });
|
||||
const cloudEntry = entries.find(e => e.path.includes('/cloud-backup/config') && e.method === 'PUT');
|
||||
expect(cloudEntry).toBeDefined();
|
||||
expect(cloudEntry!.summary).toBe('Updated cloud backup config');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Cloud backup test endpoint', () => {
|
||||
it('reports failure when no provider is configured', async () => {
|
||||
const res = await request(app).post('/api/cloud-backup/test').set('Cookie', authCookie).send({});
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.success).toBe(false);
|
||||
});
|
||||
|
||||
it('reports success when HeadBucketCommand resolves', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const { CryptoService } = await import('../services/CryptoService');
|
||||
db.updateGlobalSetting('cloud_backup_provider', 'custom');
|
||||
db.updateGlobalSetting('cloud_backup_endpoint', 'https://s3.example.com');
|
||||
db.updateGlobalSetting('cloud_backup_region', 'us-east-1');
|
||||
db.updateGlobalSetting('cloud_backup_bucket', 'b');
|
||||
db.updateGlobalSetting('cloud_backup_access_key', 'a');
|
||||
db.updateGlobalSetting('cloud_backup_secret_key', CryptoService.getInstance().encrypt('s'));
|
||||
|
||||
sentSpy.mockResolvedValueOnce({});
|
||||
const res = await request(app).post('/api/cloud-backup/test').set('Cookie', authCookie).send({});
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.success).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,242 @@
|
||||
/**
|
||||
* Tests for CloudBackupService — provider resolution, encryption round-trip,
|
||||
* archive format, and S3 client invocation. The S3 SDK is mocked at the
|
||||
* module level so no network calls happen.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
|
||||
import * as zlib from 'zlib';
|
||||
import * as tar from 'tar-stream';
|
||||
import { Readable } from 'stream';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
|
||||
const sentSpy = vi.fn();
|
||||
const s3ClientCtorSpy = vi.fn();
|
||||
|
||||
vi.mock('@aws-sdk/client-s3', () => {
|
||||
class S3Client {
|
||||
constructor(opts: unknown) { s3ClientCtorSpy(opts); }
|
||||
async send(cmd: { name: string; input: Record<string, unknown> }) { return sentSpy(cmd); }
|
||||
}
|
||||
class PutObjectCommand { name = 'PutObjectCommand'; constructor(public input: Record<string, unknown>) {} }
|
||||
class GetObjectCommand { name = 'GetObjectCommand'; constructor(public input: Record<string, unknown>) {} }
|
||||
class ListObjectsV2Command { name = 'ListObjectsV2Command'; constructor(public input: Record<string, unknown>) {} }
|
||||
class DeleteObjectCommand { name = 'DeleteObjectCommand'; constructor(public input: Record<string, unknown>) {} }
|
||||
class HeadBucketCommand { name = 'HeadBucketCommand'; constructor(public input: Record<string, unknown>) {} }
|
||||
return { S3Client, PutObjectCommand, GetObjectCommand, ListObjectsV2Command, DeleteObjectCommand, HeadBucketCommand };
|
||||
});
|
||||
|
||||
let tmpDir: string;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
let CryptoService: typeof import('../services/CryptoService').CryptoService;
|
||||
let CloudBackupService: typeof import('../services/CloudBackupService').CloudBackupService;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
({ CryptoService } = await import('../services/CryptoService'));
|
||||
({ CloudBackupService } = await import('../services/CloudBackupService'));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
CloudBackupService.getInstance().stop();
|
||||
cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
sentSpy.mockReset();
|
||||
s3ClientCtorSpy.mockReset();
|
||||
const db = DatabaseService.getInstance();
|
||||
// Reset all cloud-backup-related settings between tests.
|
||||
for (const k of [
|
||||
'cloud_backup_provider',
|
||||
'cloud_backup_endpoint',
|
||||
'cloud_backup_region',
|
||||
'cloud_backup_bucket',
|
||||
'cloud_backup_access_key',
|
||||
'cloud_backup_secret_key',
|
||||
'cloud_backup_path_prefix',
|
||||
'cloud_backup_auto_upload',
|
||||
]) {
|
||||
db.updateGlobalSetting(k, '');
|
||||
}
|
||||
for (const k of [
|
||||
'sencho_cloud_backup_endpoint',
|
||||
'sencho_cloud_backup_bucket',
|
||||
'sencho_cloud_backup_access_key',
|
||||
'sencho_cloud_backup_secret_key',
|
||||
'sencho_cloud_backup_path_prefix',
|
||||
'sencho_cloud_backup_quota_bytes',
|
||||
'sencho_cloud_backup_provisioned_at',
|
||||
]) {
|
||||
db.setSystemState(k, '');
|
||||
}
|
||||
db.setSystemState('instance_id', 'test-instance-id');
|
||||
});
|
||||
|
||||
describe('CloudBackupService — provider resolution', () => {
|
||||
it('returns "disabled" when no provider is set', () => {
|
||||
expect(CloudBackupService.getInstance().getProvider()).toBe('disabled');
|
||||
expect(CloudBackupService.getInstance().isEnabled()).toBe(false);
|
||||
expect(CloudBackupService.getInstance().getResolvedConfig()).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null config for custom provider when fields are missing', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
db.updateGlobalSetting('cloud_backup_provider', 'custom');
|
||||
db.updateGlobalSetting('cloud_backup_endpoint', 'https://s3.example.com');
|
||||
// bucket, access_key, secret_key still missing
|
||||
expect(CloudBackupService.getInstance().getResolvedConfig()).toBeNull();
|
||||
});
|
||||
|
||||
it('decrypts custom secret_key on read', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const crypto = CryptoService.getInstance();
|
||||
db.updateGlobalSetting('cloud_backup_provider', 'custom');
|
||||
db.updateGlobalSetting('cloud_backup_endpoint', 'https://s3.example.com');
|
||||
db.updateGlobalSetting('cloud_backup_region', 'us-east-1');
|
||||
db.updateGlobalSetting('cloud_backup_bucket', 'my-bucket');
|
||||
db.updateGlobalSetting('cloud_backup_access_key', 'AKIA1234');
|
||||
db.updateGlobalSetting('cloud_backup_secret_key', crypto.encrypt('plaintext-secret'));
|
||||
db.updateGlobalSetting('cloud_backup_auto_upload', '1');
|
||||
|
||||
const cfg = CloudBackupService.getInstance().getResolvedConfig();
|
||||
expect(cfg).not.toBeNull();
|
||||
expect(cfg!.provider).toBe('custom');
|
||||
expect(cfg!.secretKey).toBe('plaintext-secret');
|
||||
expect(cfg!.autoUpload).toBe(true);
|
||||
});
|
||||
|
||||
it('resolves sencho provider from system_state and forces auto_upload on', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const crypto = CryptoService.getInstance();
|
||||
db.updateGlobalSetting('cloud_backup_provider', 'sencho');
|
||||
db.setSystemState('sencho_cloud_backup_endpoint', 'https://r2.example.com');
|
||||
db.setSystemState('sencho_cloud_backup_bucket', 'sencho-cloud-backups');
|
||||
db.setSystemState('sencho_cloud_backup_access_key', 'R2-ACCESS');
|
||||
db.setSystemState('sencho_cloud_backup_secret_key', crypto.encrypt('R2-SECRET'));
|
||||
db.setSystemState('sencho_cloud_backup_path_prefix', 'tenants/123/');
|
||||
db.setSystemState('sencho_cloud_backup_quota_bytes', '524288000');
|
||||
|
||||
const cfg = CloudBackupService.getInstance().getResolvedConfig();
|
||||
expect(cfg!.provider).toBe('sencho');
|
||||
expect(cfg!.region).toBe('auto');
|
||||
expect(cfg!.secretKey).toBe('R2-SECRET');
|
||||
expect(cfg!.autoUpload).toBe(true);
|
||||
expect(cfg!.quotaBytes).toBe(524_288_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('CloudBackupService — uploadSnapshot', () => {
|
||||
function seedCustomProvider() {
|
||||
const db = DatabaseService.getInstance();
|
||||
const crypto = CryptoService.getInstance();
|
||||
db.updateGlobalSetting('cloud_backup_provider', 'custom');
|
||||
db.updateGlobalSetting('cloud_backup_endpoint', 'https://s3.example.com');
|
||||
db.updateGlobalSetting('cloud_backup_region', 'us-east-1');
|
||||
db.updateGlobalSetting('cloud_backup_bucket', 'my-bucket');
|
||||
db.updateGlobalSetting('cloud_backup_access_key', 'AKIA1234');
|
||||
db.updateGlobalSetting('cloud_backup_secret_key', crypto.encrypt('test-secret'));
|
||||
db.updateGlobalSetting('cloud_backup_path_prefix', 'sencho/');
|
||||
db.updateGlobalSetting('cloud_backup_auto_upload', '1');
|
||||
}
|
||||
|
||||
it('uploads a snapshot with correct object key and gzipped tar archive', async () => {
|
||||
seedCustomProvider();
|
||||
const db = DatabaseService.getInstance();
|
||||
|
||||
const snapshotId = db.createSnapshot('Test backup', 'admin', 1, 1, '[]');
|
||||
db.insertSnapshotFiles(snapshotId, [
|
||||
{ nodeId: 1, nodeName: 'gateway', stackName: 'web', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
{ nodeId: 1, nodeName: 'gateway', stackName: 'web', filename: '.env', content: 'KEY=value\n' },
|
||||
]);
|
||||
|
||||
sentSpy.mockResolvedValue({});
|
||||
await CloudBackupService.getInstance().uploadSnapshot(snapshotId);
|
||||
|
||||
expect(s3ClientCtorSpy).toHaveBeenCalledWith(expect.objectContaining({
|
||||
endpoint: 'https://s3.example.com',
|
||||
region: 'us-east-1',
|
||||
forcePathStyle: true,
|
||||
credentials: { accessKeyId: 'AKIA1234', secretAccessKey: 'test-secret' },
|
||||
}));
|
||||
|
||||
const putCall = sentSpy.mock.calls.find(c => c[0].name === 'PutObjectCommand');
|
||||
expect(putCall).toBeDefined();
|
||||
const input = putCall![0].input as { Bucket: string; Key: string; Body: Buffer; ContentType: string };
|
||||
expect(input.Bucket).toBe('my-bucket');
|
||||
expect(input.Key).toContain('sencho/instances/test-instance-id/snapshots/');
|
||||
expect(input.Key).toMatch(/\.tar\.gz$/);
|
||||
expect(input.ContentType).toBe('application/gzip');
|
||||
expect(Buffer.isBuffer(input.Body)).toBe(true);
|
||||
expect(input.Body.byteLength).toBeGreaterThan(0);
|
||||
|
||||
const decompressed = zlib.gunzipSync(input.Body);
|
||||
const entries: Array<{ name: string; content: string }> = await new Promise((resolve, reject) => {
|
||||
const extract = tar.extract();
|
||||
const list: Array<{ name: string; content: string }> = [];
|
||||
extract.on('entry', (header, stream, next) => {
|
||||
const chunks: Buffer[] = [];
|
||||
stream.on('data', (c: Buffer) => chunks.push(c));
|
||||
stream.on('end', () => { list.push({ name: header.name, content: Buffer.concat(chunks).toString('utf-8') }); next(); });
|
||||
stream.resume();
|
||||
});
|
||||
extract.on('finish', () => resolve(list));
|
||||
extract.on('error', reject);
|
||||
Readable.from(decompressed).pipe(extract);
|
||||
});
|
||||
|
||||
const meta = entries.find(e => e.name === 'metadata.json');
|
||||
expect(meta).toBeDefined();
|
||||
const parsed = JSON.parse(meta!.content);
|
||||
expect(parsed.id).toBe(snapshotId);
|
||||
expect(parsed.instance_id).toBe('test-instance-id');
|
||||
expect(parsed.archive_version).toBe(1);
|
||||
|
||||
expect(entries.find(e => e.name === 'nodes/1_gateway/web/compose.yaml')).toBeDefined();
|
||||
expect(entries.find(e => e.name === 'nodes/1_gateway/web/.env')).toBeDefined();
|
||||
|
||||
expect(CloudBackupService.getInstance().getUploadStatus(snapshotId).status).toBe('success');
|
||||
});
|
||||
|
||||
it('records failure status when upload throws', async () => {
|
||||
seedCustomProvider();
|
||||
const db = DatabaseService.getInstance();
|
||||
const snapshotId = db.createSnapshot('Failing', 'admin', 0, 0, '[]');
|
||||
|
||||
sentSpy.mockRejectedValueOnce(new Error('AccessDenied: bad creds'));
|
||||
await expect(CloudBackupService.getInstance().uploadSnapshot(snapshotId)).rejects.toThrow(/bad creds/);
|
||||
|
||||
const status = CloudBackupService.getInstance().getUploadStatus(snapshotId);
|
||||
expect(status.status).toBe('failed');
|
||||
expect(status.error).toContain('bad creds');
|
||||
});
|
||||
|
||||
it('throws when no provider is configured', async () => {
|
||||
await expect(CloudBackupService.getInstance().uploadSnapshot(999)).rejects.toThrow(/not configured/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe('CloudBackupService — listCloudSnapshots', () => {
|
||||
it('parses snapshot ID from object key and sorts by lastModified desc', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const crypto = CryptoService.getInstance();
|
||||
db.updateGlobalSetting('cloud_backup_provider', 'custom');
|
||||
db.updateGlobalSetting('cloud_backup_endpoint', 'https://s3.example.com');
|
||||
db.updateGlobalSetting('cloud_backup_region', 'us-east-1');
|
||||
db.updateGlobalSetting('cloud_backup_bucket', 'b');
|
||||
db.updateGlobalSetting('cloud_backup_access_key', 'a');
|
||||
db.updateGlobalSetting('cloud_backup_secret_key', crypto.encrypt('s'));
|
||||
db.updateGlobalSetting('cloud_backup_path_prefix', 'sencho/');
|
||||
|
||||
sentSpy.mockResolvedValueOnce({
|
||||
Contents: [
|
||||
{ Key: 'sencho/instances/test-instance-id/snapshots/3_2026-01-01_a.tar.gz', Size: 100, LastModified: new Date('2026-01-01T00:00:00Z') },
|
||||
{ Key: 'sencho/instances/test-instance-id/snapshots/7_2026-04-01_b.tar.gz', Size: 200, LastModified: new Date('2026-04-01T00:00:00Z') },
|
||||
],
|
||||
});
|
||||
const list = await CloudBackupService.getInstance().listCloudSnapshots();
|
||||
expect(list).toHaveLength(2);
|
||||
expect(list[0].snapshotId).toBe(7);
|
||||
expect(list[1].snapshotId).toBe(3);
|
||||
});
|
||||
});
|
||||
@@ -145,6 +145,16 @@ vi.mock('../services/NotificationService', () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/CloudBackupService', () => ({
|
||||
CloudBackupService: {
|
||||
getInstance: () => ({
|
||||
isEnabled: () => false,
|
||||
isAutoUploadOn: () => false,
|
||||
uploadSnapshot: vi.fn().mockResolvedValue(undefined),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/NodeRegistry', () => ({
|
||||
NodeRegistry: {
|
||||
getInstance: () => ({
|
||||
|
||||
@@ -18,6 +18,7 @@ import { webhooksRouter } from './routes/webhooks';
|
||||
import { usersRouter } from './routes/users';
|
||||
import { gitSourcesRouter, stackGitSourceRouter } from './routes/gitSources';
|
||||
import { fleetRouter } from './routes/fleet';
|
||||
import { cloudBackupRouter } from './routes/cloudBackup';
|
||||
import { permissionsRouter } from './routes/permissions';
|
||||
import { convertRouter } from './routes/convert';
|
||||
import { alertsRouter } from './routes/alerts';
|
||||
@@ -91,6 +92,7 @@ app.use('/api/stacks', stackLabelsRouter);
|
||||
app.use('/api/api-tokens', apiTokensRouter);
|
||||
app.use('/api/audit-log', auditLogRouter);
|
||||
app.use('/api/fleet', fleetRouter);
|
||||
app.use('/api/cloud-backup', cloudBackupRouter);
|
||||
app.use('/api/webhooks', webhooksRouter);
|
||||
app.use('/api/users', usersRouter);
|
||||
app.use('/api/git-sources', gitSourcesRouter);
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
import { Router, type Request, type Response } from 'express';
|
||||
import { CloudBackupService } from '../services/CloudBackupService';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { CryptoService } from '../services/CryptoService';
|
||||
import { requireAdmin, requireAdmiral } from '../middleware/tierGates';
|
||||
import { rejectApiTokenScope } from '../middleware/apiTokenScope';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
|
||||
const SCOPE_MESSAGE = 'API tokens cannot manage cloud backup configuration.';
|
||||
const SECRET_REDACTED = '***';
|
||||
const VALID_PROVIDERS = new Set(['disabled', 'sencho', 'custom']);
|
||||
|
||||
function parseSnapshotIdParam(req: Request, res: Response): number | null {
|
||||
const raw = req.params.id as string | undefined;
|
||||
const parsed = parseInt(raw ?? '', 10);
|
||||
if (isNaN(parsed) || parsed <= 0) {
|
||||
res.status(400).json({ error: 'Invalid snapshot ID' });
|
||||
return null;
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function decodeObjectKey(req: Request, res: Response): string | null {
|
||||
const raw = req.params.keyB64 as string | undefined;
|
||||
if (!raw) {
|
||||
res.status(400).json({ error: 'Missing object key' });
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const decoded = Buffer.from(raw, 'base64url').toString('utf-8');
|
||||
if (!decoded || decoded.includes('..') || decoded.startsWith('/')) {
|
||||
res.status(400).json({ error: 'Invalid object key' });
|
||||
return null;
|
||||
}
|
||||
return decoded;
|
||||
} catch {
|
||||
res.status(400).json({ error: 'Invalid object key encoding' });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const cloudBackupRouter = Router();
|
||||
|
||||
cloudBackupRouter.get('/config', (req: Request, res: Response): void => {
|
||||
if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const settings = db.getGlobalSettings();
|
||||
const provider = CloudBackupService.getInstance().getProvider();
|
||||
const senchoProvisioned = !!db.getSystemState('sencho_cloud_backup_provisioned_at');
|
||||
res.json({
|
||||
provider,
|
||||
sencho_provisioned: senchoProvisioned,
|
||||
sencho_provisioned_at: db.getSystemState('sencho_cloud_backup_provisioned_at'),
|
||||
custom: {
|
||||
endpoint: settings.cloud_backup_endpoint || '',
|
||||
region: settings.cloud_backup_region || '',
|
||||
bucket: settings.cloud_backup_bucket || '',
|
||||
access_key: settings.cloud_backup_access_key || '',
|
||||
secret_key: settings.cloud_backup_secret_key ? SECRET_REDACTED : '',
|
||||
path_prefix: settings.cloud_backup_path_prefix || 'sencho/',
|
||||
auto_upload: settings.cloud_backup_auto_upload === '1',
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[CloudBackup] config get error:', error);
|
||||
res.status(500).json({ error: 'Failed to load cloud backup config' });
|
||||
}
|
||||
});
|
||||
|
||||
cloudBackupRouter.put('/config', (req: Request, res: Response): void => {
|
||||
if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return;
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
try {
|
||||
const body = req.body ?? {};
|
||||
const provider = body.provider as string | undefined;
|
||||
if (!provider || !VALID_PROVIDERS.has(provider)) {
|
||||
res.status(400).json({ error: 'provider must be one of: disabled, sencho, custom' });
|
||||
return;
|
||||
}
|
||||
const db = DatabaseService.getInstance();
|
||||
const crypto = CryptoService.getInstance();
|
||||
db.updateGlobalSetting('cloud_backup_provider', provider);
|
||||
|
||||
if (provider === 'custom') {
|
||||
const c = body.custom ?? {};
|
||||
const endpoint = typeof c.endpoint === 'string' ? c.endpoint.trim() : '';
|
||||
const region = typeof c.region === 'string' ? c.region.trim() : '';
|
||||
const bucket = typeof c.bucket === 'string' ? c.bucket.trim() : '';
|
||||
const accessKey = typeof c.access_key === 'string' ? c.access_key.trim() : '';
|
||||
const pathPrefix = typeof c.path_prefix === 'string' ? c.path_prefix.trim() : 'sencho/';
|
||||
const autoUpload = c.auto_upload === true || c.auto_upload === '1' ? '1' : '0';
|
||||
|
||||
if (!endpoint || !bucket || !accessKey) {
|
||||
res.status(400).json({ error: 'endpoint, bucket, and access_key are required for custom S3.' });
|
||||
return;
|
||||
}
|
||||
if (!/^https?:\/\//i.test(endpoint)) {
|
||||
res.status(400).json({ error: 'endpoint must start with http:// or https://' });
|
||||
return;
|
||||
}
|
||||
|
||||
db.updateGlobalSetting('cloud_backup_endpoint', endpoint);
|
||||
db.updateGlobalSetting('cloud_backup_region', region);
|
||||
db.updateGlobalSetting('cloud_backup_bucket', bucket);
|
||||
db.updateGlobalSetting('cloud_backup_access_key', accessKey);
|
||||
db.updateGlobalSetting('cloud_backup_path_prefix', pathPrefix);
|
||||
db.updateGlobalSetting('cloud_backup_auto_upload', autoUpload);
|
||||
|
||||
const incomingSecret = typeof c.secret_key === 'string' ? c.secret_key : '';
|
||||
if (incomingSecret && incomingSecret !== SECRET_REDACTED) {
|
||||
db.updateGlobalSetting('cloud_backup_secret_key', crypto.encrypt(incomingSecret));
|
||||
}
|
||||
}
|
||||
|
||||
res.status(204).send();
|
||||
} catch (error) {
|
||||
console.error('[CloudBackup] config update error:', error);
|
||||
res.status(500).json({ error: 'Failed to save cloud backup config' });
|
||||
}
|
||||
});
|
||||
|
||||
cloudBackupRouter.post('/test', async (req: Request, res: Response): Promise<void> => {
|
||||
if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return;
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
try {
|
||||
const result = await CloudBackupService.getInstance().testConnection();
|
||||
res.json(result);
|
||||
} catch (error) {
|
||||
console.error('[CloudBackup] test error:', error);
|
||||
res.status(500).json({ success: false, error: getErrorMessage(error, 'Connection test failed') });
|
||||
}
|
||||
});
|
||||
|
||||
cloudBackupRouter.post('/provision', async (req: Request, res: Response): Promise<void> => {
|
||||
if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return;
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
try {
|
||||
const result = await CloudBackupService.getInstance().provisionSenchoCloudBackup();
|
||||
if (!result.success) {
|
||||
res.status(400).json({ error: result.error || 'Provisioning failed' });
|
||||
return;
|
||||
}
|
||||
res.json({ success: true, quota_bytes: result.quotaBytes });
|
||||
} catch (error) {
|
||||
console.error('[CloudBackup] provision error:', error);
|
||||
res.status(500).json({ error: 'Failed to provision Sencho Cloud Backup' });
|
||||
}
|
||||
});
|
||||
|
||||
cloudBackupRouter.get('/usage', async (req: Request, res: Response): Promise<void> => {
|
||||
if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
try {
|
||||
const svc = CloudBackupService.getInstance();
|
||||
if (svc.getProvider() !== 'sencho') {
|
||||
res.status(400).json({ error: 'Usage is only available for Sencho Cloud Backup' });
|
||||
return;
|
||||
}
|
||||
const usage = await svc.getSenchoCloudBackupUsage();
|
||||
res.json(usage);
|
||||
} catch (error) {
|
||||
console.error('[CloudBackup] usage error:', error);
|
||||
res.status(502).json({ error: getErrorMessage(error, 'Failed to fetch usage') });
|
||||
}
|
||||
});
|
||||
|
||||
cloudBackupRouter.get('/snapshots', async (req: Request, res: Response): Promise<void> => {
|
||||
if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
try {
|
||||
const entries = await CloudBackupService.getInstance().listCloudSnapshots();
|
||||
res.json(entries);
|
||||
} catch (error) {
|
||||
console.error('[CloudBackup] list error:', error);
|
||||
res.status(502).json({ error: getErrorMessage(error, 'Failed to list cloud snapshots') });
|
||||
}
|
||||
});
|
||||
|
||||
cloudBackupRouter.post('/upload/:id', async (req: Request, res: Response): Promise<void> => {
|
||||
if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return;
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
const id = parseSnapshotIdParam(req, res);
|
||||
if (id == null) return;
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
if (!db.getSnapshot(id)) {
|
||||
res.status(404).json({ error: 'Snapshot not found' });
|
||||
return;
|
||||
}
|
||||
await CloudBackupService.getInstance().uploadSnapshot(id);
|
||||
res.status(202).json({ status: 'success', snapshot_id: id });
|
||||
} catch (error) {
|
||||
console.error('[CloudBackup] upload error:', error);
|
||||
res.status(502).json({ error: getErrorMessage(error, 'Cloud upload failed') });
|
||||
}
|
||||
});
|
||||
|
||||
cloudBackupRouter.get('/status/:id', (req: Request, res: Response): void => {
|
||||
if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
const id = parseSnapshotIdParam(req, res);
|
||||
if (id == null) return;
|
||||
res.json(CloudBackupService.getInstance().getUploadStatus(id));
|
||||
});
|
||||
|
||||
cloudBackupRouter.get('/object/:keyB64/download', async (req: Request, res: Response): Promise<void> => {
|
||||
if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
const objectKey = decodeObjectKey(req, res);
|
||||
if (!objectKey) return;
|
||||
try {
|
||||
const buffer = await CloudBackupService.getInstance().downloadSnapshot(objectKey);
|
||||
const filename = objectKey.split('/').pop() || 'snapshot.tar.gz';
|
||||
res.setHeader('Content-Type', 'application/gzip');
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
|
||||
res.setHeader('Content-Length', String(buffer.byteLength));
|
||||
res.end(buffer);
|
||||
} catch (error) {
|
||||
console.error('[CloudBackup] download error:', error);
|
||||
res.status(502).json({ error: getErrorMessage(error, 'Failed to download cloud snapshot') });
|
||||
}
|
||||
});
|
||||
|
||||
cloudBackupRouter.delete('/object/:keyB64', async (req: Request, res: Response): Promise<void> => {
|
||||
if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return;
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
const objectKey = decodeObjectKey(req, res);
|
||||
if (!objectKey) return;
|
||||
try {
|
||||
await CloudBackupService.getInstance().deleteCloudSnapshot(objectKey);
|
||||
res.status(204).send();
|
||||
} catch (error) {
|
||||
console.error('[CloudBackup] delete error:', error);
|
||||
res.status(502).json({ error: getErrorMessage(error, 'Failed to delete cloud snapshot') });
|
||||
}
|
||||
});
|
||||
@@ -20,6 +20,8 @@ import { getLatestVersion } from '../utils/version-check';
|
||||
import { isValidStackName } from '../utils/validation';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { CloudBackupService } from '../services/CloudBackupService';
|
||||
import { NotificationService } from '../services/NotificationService';
|
||||
|
||||
const updateTracker = FleetUpdateTrackerService.getInstance();
|
||||
const UPDATE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
|
||||
@@ -819,6 +821,17 @@ fleetRouter.post('/snapshots', authMiddleware, async (req: Request, res: Respons
|
||||
db.insertSnapshotFiles(snapshotId, allFiles);
|
||||
}
|
||||
|
||||
const cloudSvc = CloudBackupService.getInstance();
|
||||
if (cloudSvc.isEnabled() && cloudSvc.isAutoUploadOn()) {
|
||||
void cloudSvc.uploadSnapshot(snapshotId).catch(uploadErr => {
|
||||
const message = uploadErr instanceof Error ? uploadErr.message : String(uploadErr);
|
||||
console.error('[Fleet Snapshot] Cloud upload failed:', message);
|
||||
void NotificationService.getInstance()
|
||||
.dispatchAlert('error', 'system', `Cloud backup upload failed for snapshot ${snapshotId}: ${message}`)
|
||||
.catch(() => { /* notification dispatch is best-effort */ });
|
||||
});
|
||||
}
|
||||
|
||||
console.log('[Fleet] Snapshot created:', capturedNodes.length, 'nodes,', totalStacks, 'stacks');
|
||||
if (isDebugEnabled()) {
|
||||
console.debug(`[Fleet:debug] Snapshot ${snapshotId} capture completed in ${Date.now() - captureStart}ms, ${allFiles.length} file(s) stored`);
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
/**
|
||||
* CloudBackupService — off-site replication for fleet snapshots.
|
||||
*
|
||||
* Two providers share the same S3-compatible code path:
|
||||
* - 'sencho' : managed Sencho Cloud Backup. Credentials provisioned by
|
||||
* sencho.io/api/cloud-backup/provision and stored in system_state.
|
||||
* - 'custom' : bring-your-own-bucket. Credentials user-configured in global_settings.
|
||||
*
|
||||
* Routes call this service after a snapshot is persisted to SQLite. The service
|
||||
* reads snapshot rows from DatabaseService, packs them into a tar.gz archive,
|
||||
* and uploads via @aws-sdk/client-s3.
|
||||
*/
|
||||
|
||||
import {
|
||||
S3Client,
|
||||
PutObjectCommand,
|
||||
GetObjectCommand,
|
||||
ListObjectsV2Command,
|
||||
DeleteObjectCommand,
|
||||
HeadBucketCommand,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { Readable } from 'stream';
|
||||
import * as zlib from 'zlib';
|
||||
import * as tar from 'tar-stream';
|
||||
import axios from 'axios';
|
||||
import { DatabaseService, type FleetSnapshotFile } from './DatabaseService';
|
||||
import { CryptoService } from './CryptoService';
|
||||
import { LicenseService } from './LicenseService';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
|
||||
export type CloudProvider = 'disabled' | 'sencho' | 'custom';
|
||||
export type UploadStatus = 'idle' | 'uploading' | 'success' | 'failed';
|
||||
|
||||
export interface ResolvedCloudConfig {
|
||||
provider: 'sencho' | 'custom';
|
||||
endpoint: string;
|
||||
region: string;
|
||||
bucket: string;
|
||||
accessKey: string;
|
||||
secretKey: string;
|
||||
pathPrefix: string;
|
||||
autoUpload: boolean;
|
||||
quotaBytes?: number;
|
||||
}
|
||||
|
||||
export interface CloudSnapshotEntry {
|
||||
objectKey: string;
|
||||
sizeBytes: number;
|
||||
lastModified: string | null;
|
||||
snapshotId: number | null;
|
||||
}
|
||||
|
||||
export interface ProvisionResult {
|
||||
success: boolean;
|
||||
quotaBytes?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface UploadStatusEntry {
|
||||
status: UploadStatus;
|
||||
objectKey?: string;
|
||||
error?: string;
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
const SENCHO_CLOUD_BACKUP_API_DEFAULT = 'https://sencho.io';
|
||||
const PROVIDER_KEY = 'cloud_backup_provider';
|
||||
const SUCCESS_STATUS_TTL_MS = 5 * 60 * 1000;
|
||||
|
||||
export class CloudBackupService {
|
||||
private static instance: CloudBackupService;
|
||||
private uploadStatus = new Map<number, UploadStatusEntry>();
|
||||
private statusGcTimer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
private constructor() {
|
||||
this.statusGcTimer = setInterval(() => this.gcUploadStatus(), 60 * 1000);
|
||||
if (typeof this.statusGcTimer.unref === 'function') this.statusGcTimer.unref();
|
||||
}
|
||||
|
||||
public static getInstance(): CloudBackupService {
|
||||
if (!CloudBackupService.instance) {
|
||||
CloudBackupService.instance = new CloudBackupService();
|
||||
}
|
||||
return CloudBackupService.instance;
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
if (this.statusGcTimer) {
|
||||
clearInterval(this.statusGcTimer);
|
||||
this.statusGcTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Configuration ─────────────────────────────────────────────────────────
|
||||
|
||||
public getProvider(): CloudProvider {
|
||||
const value = DatabaseService.getInstance().getGlobalSettings()[PROVIDER_KEY];
|
||||
if (value === 'sencho' || value === 'custom') return value;
|
||||
return 'disabled';
|
||||
}
|
||||
|
||||
public isEnabled(): boolean {
|
||||
return this.getResolvedConfig() !== null;
|
||||
}
|
||||
|
||||
public isAutoUploadOn(): boolean {
|
||||
const cfg = this.getResolvedConfig();
|
||||
return cfg?.autoUpload === true;
|
||||
}
|
||||
|
||||
public getResolvedConfig(): ResolvedCloudConfig | null {
|
||||
const provider = this.getProvider();
|
||||
if (provider === 'disabled') return null;
|
||||
const db = DatabaseService.getInstance();
|
||||
const crypto = CryptoService.getInstance();
|
||||
|
||||
if (provider === 'sencho') {
|
||||
const endpoint = db.getSystemState('sencho_cloud_backup_endpoint');
|
||||
const bucket = db.getSystemState('sencho_cloud_backup_bucket');
|
||||
const accessKey = db.getSystemState('sencho_cloud_backup_access_key');
|
||||
const secretRaw = db.getSystemState('sencho_cloud_backup_secret_key');
|
||||
const pathPrefix = db.getSystemState('sencho_cloud_backup_path_prefix') || '';
|
||||
const quotaRaw = db.getSystemState('sencho_cloud_backup_quota_bytes');
|
||||
if (!endpoint || !bucket || !accessKey || !secretRaw) return null;
|
||||
return {
|
||||
provider: 'sencho',
|
||||
endpoint,
|
||||
region: 'auto',
|
||||
bucket,
|
||||
accessKey,
|
||||
secretKey: crypto.decrypt(secretRaw),
|
||||
pathPrefix,
|
||||
autoUpload: true,
|
||||
quotaBytes: quotaRaw ? parseInt(quotaRaw, 10) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
const settings = db.getGlobalSettings();
|
||||
const endpoint = settings.cloud_backup_endpoint;
|
||||
const region = settings.cloud_backup_region;
|
||||
const bucket = settings.cloud_backup_bucket;
|
||||
const accessKey = settings.cloud_backup_access_key;
|
||||
const secretRaw = settings.cloud_backup_secret_key;
|
||||
const pathPrefix = settings.cloud_backup_path_prefix || 'sencho/';
|
||||
if (!endpoint || !bucket || !accessKey || !secretRaw) return null;
|
||||
return {
|
||||
provider: 'custom',
|
||||
endpoint,
|
||||
region: region || 'us-east-1',
|
||||
bucket,
|
||||
accessKey,
|
||||
secretKey: crypto.decrypt(secretRaw),
|
||||
pathPrefix,
|
||||
autoUpload: settings.cloud_backup_auto_upload === '1',
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Sencho Cloud Backup lifecycle ─────────────────────────────────────────
|
||||
|
||||
public async provisionSenchoCloudBackup(): Promise<ProvisionResult> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const crypto = CryptoService.getInstance();
|
||||
const licenseKey = db.getSystemState('license_key');
|
||||
if (!licenseKey) return { success: false, error: 'No license key found. Activate an Admiral license first.' };
|
||||
|
||||
const variant = LicenseService.getInstance().getVariant();
|
||||
if (variant !== 'admiral') return { success: false, error: 'Sencho Cloud Backup requires the Admiral tier.' };
|
||||
|
||||
const apiBase = process.env.SENCHO_CLOUD_BACKUP_API || SENCHO_CLOUD_BACKUP_API_DEFAULT;
|
||||
try {
|
||||
const res = await axios.post(`${apiBase}/api/cloud-backup/provision`, { license_key: licenseKey }, { timeout: 15000 });
|
||||
const data = res.data as {
|
||||
endpoint: string;
|
||||
region?: string;
|
||||
bucket: string;
|
||||
access_key: string;
|
||||
secret_key: string;
|
||||
path_prefix: string;
|
||||
quota_bytes: number;
|
||||
};
|
||||
db.setSystemState('sencho_cloud_backup_endpoint', data.endpoint);
|
||||
db.setSystemState('sencho_cloud_backup_bucket', data.bucket);
|
||||
db.setSystemState('sencho_cloud_backup_access_key', data.access_key);
|
||||
db.setSystemState('sencho_cloud_backup_secret_key', crypto.encrypt(data.secret_key));
|
||||
db.setSystemState('sencho_cloud_backup_path_prefix', data.path_prefix);
|
||||
db.setSystemState('sencho_cloud_backup_quota_bytes', String(data.quota_bytes));
|
||||
db.setSystemState('sencho_cloud_backup_provisioned_at', new Date().toISOString());
|
||||
db.updateGlobalSetting(PROVIDER_KEY, 'sencho');
|
||||
return { success: true, quotaBytes: data.quota_bytes };
|
||||
} catch (err) {
|
||||
const responseError = (err as { response?: { data?: { error?: string } } }).response?.data?.error;
|
||||
return { success: false, error: responseError || getErrorMessage(err, 'Failed to provision Sencho Cloud Backup.') };
|
||||
}
|
||||
}
|
||||
|
||||
public async refreshSenchoCloudBackupCredentials(): Promise<void> {
|
||||
const result = await this.provisionSenchoCloudBackup();
|
||||
if (!result.success) throw new Error(result.error || 'Failed to refresh Sencho Cloud Backup credentials.');
|
||||
}
|
||||
|
||||
public async getSenchoCloudBackupUsage(): Promise<{ used_bytes: number; quota_bytes: number; object_count: number }> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const licenseKey = db.getSystemState('license_key');
|
||||
if (!licenseKey) throw new Error('No license key found.');
|
||||
const apiBase = process.env.SENCHO_CLOUD_BACKUP_API || SENCHO_CLOUD_BACKUP_API_DEFAULT;
|
||||
const res = await axios.post(`${apiBase}/api/cloud-backup/usage`, { license_key: licenseKey }, { timeout: 15000 });
|
||||
const data = res.data as { used_bytes: number; quota_bytes: number; object_count: number };
|
||||
return data;
|
||||
}
|
||||
|
||||
// ─── S3 operations ─────────────────────────────────────────────────────────
|
||||
|
||||
public async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
const cfg = this.getResolvedConfig();
|
||||
if (!cfg) return { success: false, error: 'No cloud backup configuration is active.' };
|
||||
try {
|
||||
const client = this.buildS3Client(cfg);
|
||||
await client.send(new HeadBucketCommand({ Bucket: cfg.bucket }));
|
||||
return { success: true };
|
||||
} catch (err) {
|
||||
return { success: false, error: getErrorMessage(err, 'Connection test failed.') };
|
||||
}
|
||||
}
|
||||
|
||||
public async uploadSnapshot(snapshotId: number): Promise<void> {
|
||||
const cfg = this.getResolvedConfig();
|
||||
if (!cfg) throw new Error('Cloud backup is not configured.');
|
||||
const db = DatabaseService.getInstance();
|
||||
const snapshot = db.getSnapshot(snapshotId);
|
||||
if (!snapshot) throw new Error(`Snapshot ${snapshotId} not found.`);
|
||||
const files = db.getSnapshotFiles(snapshotId);
|
||||
const objectKey = this.buildObjectKey(cfg, snapshot.id, snapshot.description, snapshot.created_at);
|
||||
|
||||
this.setStatus(snapshotId, { status: 'uploading', objectKey, updatedAt: Date.now() });
|
||||
try {
|
||||
const archive = await this.buildArchive(snapshot, files);
|
||||
const client = this.buildS3Client(cfg);
|
||||
await client.send(new PutObjectCommand({
|
||||
Bucket: cfg.bucket,
|
||||
Key: objectKey,
|
||||
Body: archive,
|
||||
ContentType: 'application/gzip',
|
||||
}));
|
||||
this.setStatus(snapshotId, { status: 'success', objectKey, updatedAt: Date.now() });
|
||||
if (isDebugEnabled()) {
|
||||
console.log(`[CloudBackup:debug] Uploaded snapshot ${snapshotId} (${archive.byteLength} bytes) to ${cfg.bucket}/${objectKey}`);
|
||||
}
|
||||
} catch (err) {
|
||||
const message = getErrorMessage(err, 'Cloud upload failed.');
|
||||
this.setStatus(snapshotId, { status: 'failed', objectKey, error: message, updatedAt: Date.now() });
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
public async downloadSnapshot(objectKey: string): Promise<Buffer> {
|
||||
const cfg = this.getResolvedConfig();
|
||||
if (!cfg) throw new Error('Cloud backup is not configured.');
|
||||
const client = this.buildS3Client(cfg);
|
||||
const result = await client.send(new GetObjectCommand({ Bucket: cfg.bucket, Key: objectKey }));
|
||||
const body = result.Body as Readable | undefined;
|
||||
if (!body) throw new Error('Empty response body.');
|
||||
return await streamToBuffer(body);
|
||||
}
|
||||
|
||||
public async listCloudSnapshots(): Promise<CloudSnapshotEntry[]> {
|
||||
const cfg = this.getResolvedConfig();
|
||||
if (!cfg) return [];
|
||||
const client = this.buildS3Client(cfg);
|
||||
const prefix = `${cfg.pathPrefix}instances/${this.getInstanceId()}/snapshots/`;
|
||||
const result = await client.send(new ListObjectsV2Command({
|
||||
Bucket: cfg.bucket,
|
||||
Prefix: prefix,
|
||||
MaxKeys: 1000,
|
||||
}));
|
||||
const objects = result.Contents || [];
|
||||
return objects
|
||||
.filter(o => !!o.Key)
|
||||
.map(o => {
|
||||
const key = o.Key as string;
|
||||
const basename = key.split('/').pop() || key;
|
||||
const idMatch = basename.match(/^(\d+)_/);
|
||||
return {
|
||||
objectKey: key,
|
||||
sizeBytes: o.Size ?? 0,
|
||||
lastModified: o.LastModified ? o.LastModified.toISOString() : null,
|
||||
snapshotId: idMatch ? parseInt(idMatch[1], 10) : null,
|
||||
};
|
||||
})
|
||||
.sort((a, b) => (b.lastModified || '').localeCompare(a.lastModified || ''));
|
||||
}
|
||||
|
||||
public async deleteCloudSnapshot(objectKey: string): Promise<void> {
|
||||
const cfg = this.getResolvedConfig();
|
||||
if (!cfg) throw new Error('Cloud backup is not configured.');
|
||||
const client = this.buildS3Client(cfg);
|
||||
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: objectKey }));
|
||||
}
|
||||
|
||||
// ─── Status tracking ───────────────────────────────────────────────────────
|
||||
|
||||
public getUploadStatus(snapshotId: number): UploadStatusEntry {
|
||||
return this.uploadStatus.get(snapshotId) || { status: 'idle', updatedAt: 0 };
|
||||
}
|
||||
|
||||
private setStatus(snapshotId: number, entry: UploadStatusEntry): void {
|
||||
this.uploadStatus.set(snapshotId, entry);
|
||||
}
|
||||
|
||||
private gcUploadStatus(): void {
|
||||
const now = Date.now();
|
||||
for (const [id, entry] of this.uploadStatus.entries()) {
|
||||
if (entry.status === 'success' && now - entry.updatedAt > SUCCESS_STATUS_TTL_MS) {
|
||||
this.uploadStatus.delete(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Internals ─────────────────────────────────────────────────────────────
|
||||
|
||||
private buildS3Client(cfg: ResolvedCloudConfig): S3Client {
|
||||
return new S3Client({
|
||||
endpoint: cfg.endpoint,
|
||||
region: cfg.region,
|
||||
credentials: { accessKeyId: cfg.accessKey, secretAccessKey: cfg.secretKey },
|
||||
forcePathStyle: true,
|
||||
});
|
||||
}
|
||||
|
||||
private getInstanceId(): string {
|
||||
return DatabaseService.getInstance().getSystemState('instance_id') || 'unknown';
|
||||
}
|
||||
|
||||
private buildObjectKey(cfg: ResolvedCloudConfig, snapshotId: number, description: string, createdAt: number): string {
|
||||
const ts = new Date(createdAt).toISOString().replace(/[:.]/g, '-');
|
||||
const slug = description.slice(0, 40).replace(/[^a-z0-9]+/gi, '-').replace(/^-+|-+$/g, '').toLowerCase() || 'snapshot';
|
||||
return `${cfg.pathPrefix}instances/${this.getInstanceId()}/snapshots/${snapshotId}_${ts}_${slug}.tar.gz`;
|
||||
}
|
||||
|
||||
private async buildArchive(
|
||||
snapshot: { id: number; description: string; created_by: string; node_count: number; stack_count: number; skipped_nodes: string; created_at: number },
|
||||
files: FleetSnapshotFile[],
|
||||
): Promise<Buffer> {
|
||||
const pack = tar.pack();
|
||||
const metadata = {
|
||||
id: snapshot.id,
|
||||
description: snapshot.description,
|
||||
created_by: snapshot.created_by,
|
||||
created_at: snapshot.created_at,
|
||||
node_count: snapshot.node_count,
|
||||
stack_count: snapshot.stack_count,
|
||||
skipped_nodes: safeParseJson(snapshot.skipped_nodes, []),
|
||||
instance_id: this.getInstanceId(),
|
||||
archive_version: 1,
|
||||
};
|
||||
pack.entry({ name: 'metadata.json' }, JSON.stringify(metadata, null, 2));
|
||||
|
||||
for (const file of files) {
|
||||
const safeNodeName = sanitizePathSegment(file.node_name);
|
||||
const safeStackName = sanitizePathSegment(file.stack_name);
|
||||
const safeFilename = sanitizePathSegment(file.filename);
|
||||
const entryPath = `nodes/${file.node_id}_${safeNodeName}/${safeStackName}/${safeFilename}`;
|
||||
pack.entry({ name: entryPath }, file.content);
|
||||
}
|
||||
pack.finalize();
|
||||
|
||||
const gzip = zlib.createGzip();
|
||||
const chunks: Buffer[] = [];
|
||||
return await new Promise<Buffer>((resolve, reject) => {
|
||||
gzip.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
gzip.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
gzip.on('error', reject);
|
||||
pack.on('error', reject);
|
||||
pack.pipe(gzip);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function streamToBuffer(stream: Readable): Promise<Buffer> {
|
||||
return new Promise<Buffer>((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
stream.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
stream.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function sanitizePathSegment(input: string): string {
|
||||
// Preserve dotfiles like `.env` while neutralising traversal (`..`) and path separators.
|
||||
const cleaned = input.replace(/\.\./g, '_').replace(/[\\/]/g, '_');
|
||||
if (/^\.+$/.test(cleaned)) return '_';
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
function safeParseJson<T>(value: string | null | undefined, fallback: T): T {
|
||||
if (!value) return fallback;
|
||||
try { return JSON.parse(value) as T; } catch { return fallback; }
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import { NotificationService } from './NotificationService';
|
||||
import TrivyService from './TrivyService';
|
||||
import type { ScanAllNodeImagesResult } from './TrivyService';
|
||||
import TrivyInstaller from './TrivyInstaller';
|
||||
import { CloudBackupService } from './CloudBackupService';
|
||||
|
||||
const TRIVY_UPDATE_CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
||||
const TRIVY_UPDATE_CHECK_STARTUP_DELAY_MS = 5 * 60 * 1000;
|
||||
@@ -501,11 +502,25 @@ export class SchedulerService {
|
||||
db.insertSnapshotFiles(snapshotId, allFiles);
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) {
|
||||
console.debug(`[SchedulerService:debug] Snapshot task ${task.id}: captured ${capturedNodes.length} node(s), ${totalStacks} stack(s), ${allFiles.length} file(s), skipped ${skippedNodes.length}`);
|
||||
let cloudUploadNote = '';
|
||||
const cloudSvc = CloudBackupService.getInstance();
|
||||
if (cloudSvc.isEnabled() && cloudSvc.isAutoUploadOn()) {
|
||||
try {
|
||||
await cloudSvc.uploadSnapshot(snapshotId);
|
||||
cloudUploadNote = ', cloud upload OK';
|
||||
} catch (err) {
|
||||
const message = getErrorMessage(err, 'Cloud upload failed');
|
||||
console.error('[SchedulerService] Cloud upload failed:', message);
|
||||
this.safeDispatch('warning', 'system', `Cloud backup failed for scheduled snapshot ${snapshotId}: ${message}`);
|
||||
cloudUploadNote = ', cloud upload FAILED';
|
||||
}
|
||||
}
|
||||
|
||||
return `Fleet snapshot created (id=${snapshotId}, ${capturedNodes.length} node(s), ${totalStacks} stack(s)${skippedNodes.length > 0 ? `, ${skippedNodes.length} skipped` : ''})`;
|
||||
if (isDebugEnabled()) {
|
||||
console.debug(`[SchedulerService:debug] Snapshot task ${task.id}: captured ${capturedNodes.length} node(s), ${totalStacks} stack(s), ${allFiles.length} file(s), skipped ${skippedNodes.length}${cloudUploadNote}`);
|
||||
}
|
||||
|
||||
return `Fleet snapshot created (id=${snapshotId}, ${capturedNodes.length} node(s), ${totalStacks} stack(s)${skippedNodes.length > 0 ? `, ${skippedNodes.length} skipped` : ''}${cloudUploadNote})`;
|
||||
}
|
||||
|
||||
private async executePrune(task: ScheduledTask): Promise<string> {
|
||||
|
||||
@@ -86,6 +86,13 @@ export const AUDIT_ROUTE_SUMMARIES: Record<string, string> = {
|
||||
'POST /fleet/nodes/*/update': 'Triggered fleet node update',
|
||||
'POST /fleet/update-all': 'Triggered fleet-wide update',
|
||||
|
||||
// Cloud backup
|
||||
'PUT /cloud-backup/config': 'Updated cloud backup config',
|
||||
'POST /cloud-backup/test': 'Tested cloud backup connection',
|
||||
'POST /cloud-backup/provision': 'Provisioned Sencho Cloud Backup',
|
||||
'POST /cloud-backup/upload': 'Uploaded snapshot to cloud',
|
||||
'DELETE /cloud-backup/object': 'Deleted cloud snapshot',
|
||||
|
||||
// SSO
|
||||
'PUT /sso/config': 'Updated SSO configuration',
|
||||
'DELETE /sso/config': 'Deleted SSO configuration',
|
||||
|
||||
Reference in New Issue
Block a user