diff --git a/backend/src/__tests__/scheduler-service.test.ts b/backend/src/__tests__/scheduler-service.test.ts index 5ac83a9c..56cf4152 100644 --- a/backend/src/__tests__/scheduler-service.test.ts +++ b/backend/src/__tests__/scheduler-service.test.ts @@ -195,6 +195,15 @@ import { SchedulerService } from '../services/SchedulerService'; beforeEach(() => { vi.clearAllMocks(); + // clearAllMocks only clears call history, not implementations, so restore the + // mocks that individual tests mutate (tier, variant, node lookup, proxy + // target) to their documented defaults. Without this a test that points + // getNode at a remote node or drops the tier leaks that state into every + // later test in the file. + mockGetTier.mockReturnValue('paid'); + mockGetVariant.mockReturnValue('admiral'); + mockGetNode.mockReturnValue({ id: 1, name: 'local', type: 'local', status: 'online' }); + mockGetProxyTarget.mockReturnValue(null); // Default: the scan-policy gate allows. Individual tests override to a block. mockEnforcePolicyPreDeploy.mockResolvedValue({ ok: true, bypassed: false, violations: [] }); (SchedulerService as any).instance = undefined; @@ -620,7 +629,12 @@ describe('SchedulerService - executeUpdate', () => { expect(mockClearStackUpdateStatus).toHaveBeenCalledWith(1, 'web-app'); }); - it('runs the update without the atomic wrapper on the community tier', async () => { + it('does not run a scheduled update on the community tier', async () => { + // Scheduled tasks are paid-only at every entry point (the tick tier check and + // the manual-run route both require paid), and executeTask guards again, so a + // community licence never runs the update. Hub-driven updates to a community + // remote worker take a different path (the /auto-update/execute route, which + // derives atomicity from the proxy tier header) and are unaffected. mockGetTier.mockReturnValue('community'); mockGetScheduledTask.mockReturnValue({ id: 82, @@ -633,21 +647,15 @@ describe('SchedulerService - executeUpdate', () => { created_by: 'admin', last_status: null, }); - mockGetContainersByStack.mockResolvedValue([ - { Id: 'c1', Image: 'nginx:latest' }, - ]); - mockCheckImage.mockResolvedValue({ hasUpdate: true }); const svc = SchedulerService.getInstance(); await svc.triggerTask(82); - // Atomic backup/rollback is a paid capability: the auto-update path derives - // the flag from the licence, so a community instance updates without it. - expect(mockUpdateStack).toHaveBeenCalledWith('web-app', undefined, false); - - // clearAllMocks does not reset return values; restore the suite default so - // later tier-agnostic tests keep the paid behavior they assume. - mockGetTier.mockReturnValue('paid'); + expect(mockUpdateStack).not.toHaveBeenCalled(); + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + 1, + expect.objectContaining({ status: 'failure' }), + ); }); it('skips when all images up to date', async () => { @@ -1611,6 +1619,184 @@ describe('SchedulerService - lifecycle actions', () => { }); }); +// ── Lifecycle remote proxy ────────────────────────────────────────────── + +describe('SchedulerService - lifecycle remote proxy', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + const remoteHeaders = expect.objectContaining({ + 'Authorization': 'Bearer tkn', + 'x-sencho-tier': 'paid', + 'x-sencho-variant': 'admiral', + }); + + function stubRemote(okBody: unknown = { success: true }) { + mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); + mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' }); + const mockFetch = vi.fn().mockResolvedValue({ ok: true, json: async () => okBody }); + vi.stubGlobal('fetch', mockFetch); + return mockFetch; + } + + it('auto_stop proxies to the remote stop endpoint instead of running locally', async () => { + const fetchMock = stubRemote(); + mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 })); + await SchedulerService.getInstance().triggerTask(300); + expect(fetchMock).toHaveBeenCalledWith( + 'http://remote:1852/api/stacks/my-stack/stop', + expect.objectContaining({ method: 'POST', headers: remoteHeaders }), + ); + expect(mockRunCommand).not.toHaveBeenCalled(); + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' })); + }); + + it('auto_down proxies to the remote down endpoint', async () => { + const fetchMock = stubRemote(); + mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_down', { node_id: 2 })); + await SchedulerService.getInstance().triggerTask(300); + expect(fetchMock).toHaveBeenCalledWith( + 'http://remote:1852/api/stacks/my-stack/down', + expect.objectContaining({ method: 'POST' }), + ); + expect(mockRunCommand).not.toHaveBeenCalled(); + }); + + it('auto_start proxies to the remote deploy endpoint and skips the hub policy gate', async () => { + const fetchMock = stubRemote(); + mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_start', { node_id: 2 })); + await SchedulerService.getInstance().triggerTask(300); + expect(fetchMock).toHaveBeenCalledWith( + 'http://remote:1852/api/stacks/my-stack/deploy', + expect.objectContaining({ method: 'POST' }), + ); + expect(mockDeployStack).not.toHaveBeenCalled(); + expect(mockEnforcePolicyPreDeploy).not.toHaveBeenCalled(); + }); + + it('auto_backup proxies to the remote backup endpoint', async () => { + const fetchMock = stubRemote(); + mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { node_id: 2 })); + await SchedulerService.getInstance().triggerTask(300); + expect(fetchMock).toHaveBeenCalledWith( + 'http://remote:1852/api/stacks/my-stack/backup', + expect.objectContaining({ method: 'POST' }), + ); + expect(mockBackupStackFiles).not.toHaveBeenCalled(); + }); + + it('restart (all services) proxies to the remote restart endpoint', async () => { + const fetchMock = stubRemote(); + mockGetScheduledTask.mockReturnValue(makeLifecycleTask('restart', { node_id: 2 })); + await SchedulerService.getInstance().triggerTask(300); + expect(fetchMock).toHaveBeenCalledWith( + 'http://remote:1852/api/stacks/my-stack/restart', + expect.objectContaining({ method: 'POST' }), + ); + expect(mockGetContainersByStack).not.toHaveBeenCalled(); + }); + + it('restart with target_services fans out to per-service restart endpoints', async () => { + const fetchMock = stubRemote(); + mockGetScheduledTask.mockReturnValue( + makeLifecycleTask('restart', { node_id: 2, target_services: JSON.stringify(['api', 'worker']) }), + ); + await SchedulerService.getInstance().triggerTask(300); + expect(fetchMock).toHaveBeenCalledWith( + 'http://remote:1852/api/stacks/my-stack/services/api/restart', + expect.objectContaining({ method: 'POST' }), + ); + expect(fetchMock).toHaveBeenCalledWith( + 'http://remote:1852/api/stacks/my-stack/services/worker/restart', + expect.objectContaining({ method: 'POST' }), + ); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it('records failure when the remote node returns an error', async () => { + mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); + mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' }); + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ + ok: false, + status: 503, + json: async () => ({ error: 'Docker daemon is unreachable' }), + })); + mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 })); + await SchedulerService.getInstance().triggerTask(300); + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + 1, + expect.objectContaining({ status: 'failure', error: expect.stringContaining('Docker daemon is unreachable') }), + ); + }); + + it('falls back to the HTTP status when the remote error body is not JSON', async () => { + mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); + mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' }); + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ + ok: false, + status: 502, + json: async () => { throw new Error('not json'); }, + })); + mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_down', { node_id: 2 })); + await SchedulerService.getInstance().triggerTask(300); + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + 1, + expect.objectContaining({ status: 'failure', error: expect.stringContaining('HTTP 502') }), + ); + }); + + it('records failure when the remote node has no proxy credentials', async () => { + mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); + // getProxyTarget defaults to null (no credentials configured). + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 })); + await SchedulerService.getInstance().triggerTask(300); + expect(fetchMock).not.toHaveBeenCalled(); + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + 1, + expect.objectContaining({ status: 'failure', error: expect.stringContaining('not configured or missing API credentials') }), + ); + }); + + it('restart fan-out fails fast and names already-restarted services', async () => { + mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); + mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' }); + const fetchMock = vi.fn() + .mockResolvedValueOnce({ ok: true, json: async () => ({ success: true }) }) + .mockResolvedValueOnce({ ok: false, status: 500, json: async () => ({ error: 'boom' }) }); + vi.stubGlobal('fetch', fetchMock); + mockGetScheduledTask.mockReturnValue( + makeLifecycleTask('restart', { node_id: 2, target_services: JSON.stringify(['api', 'worker', 'cache']) }), + ); + await SchedulerService.getInstance().triggerTask(300); + // Third service is never reached after the second fails. + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + 1, + expect.objectContaining({ status: 'failure', error: expect.stringContaining('already restarted: api') }), + ); + }); +}); + +// ── Unpaid-tier guard in executeTask ──────────────────────────────────── + +describe('SchedulerService - unpaid tier guard', () => { + it('records a failed run and does not execute when the licence is not paid', async () => { + mockGetTier.mockReturnValue('community'); + mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop')); + await SchedulerService.getInstance().triggerTask(300); + expect(mockRunCommand).not.toHaveBeenCalled(); + // The skip is visible in run history rather than silently dropped. + expect(mockCreateScheduledTaskRun).toHaveBeenCalled(); + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + 1, + expect.objectContaining({ status: 'failure', error: expect.stringContaining('paid licence') }), + ); + }); +}); + // ── delete_after_run ──────────────────────────────────────────────────── describe('SchedulerService - delete_after_run', () => { diff --git a/backend/src/__tests__/stack-backup-route.test.ts b/backend/src/__tests__/stack-backup-route.test.ts new file mode 100644 index 00000000..dfa39105 --- /dev/null +++ b/backend/src/__tests__/stack-backup-route.test.ts @@ -0,0 +1,127 @@ +/** + * Integration tests for POST /api/stacks/:stackName/backup, the on-demand + * stack-files backup trigger. Covers auth, role, paid gating, the success + * path, the missing-stack 404, name validation, and error propagation. The + * route exists so a scheduled auto_backup can run on a remote node through the + * proxy path, and so an operator can take a snapshot on demand. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import request from 'supertest'; +import bcrypt from 'bcrypt'; +import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; + +const { mockBackupStackFiles, mockHasComposeFile } = vi.hoisted(() => ({ + mockBackupStackFiles: vi.fn(), + mockHasComposeFile: vi.fn(), +})); + +vi.mock('../services/FileSystemService', () => ({ + FileSystemService: { + getInstance: () => ({ + getBaseDir: () => '/tmp/compose', + hasComposeFile: mockHasComposeFile, + backupStackFiles: mockBackupStackFiles, + getStacks: vi.fn().mockResolvedValue([]), + }), + }, +})); + +let tmpDir: string; +let app: import('express').Express; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let adminCookie: string; +let viewerCookie: string; +let tierSpy: ReturnType; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); + + const { LicenseService } = await import('../services/LicenseService'); + tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + vi.spyOn(LicenseService.getInstance(), 'getVariant').mockReturnValue('admiral'); + + ({ app } = await import('../index')); + adminCookie = await loginAsTestAdmin(app); + + const viewerHash = await bcrypt.hash('viewerpass', 1); + DatabaseService.getInstance().addUser({ username: 'backup-viewer', password_hash: viewerHash, role: 'viewer' }); + const viewerRes = await request(app).post('/api/auth/login').send({ username: 'backup-viewer', password: 'viewerpass' }); + const cookies = viewerRes.headers['set-cookie'] as string | string[]; + viewerCookie = Array.isArray(cookies) ? cookies[0] : cookies; +}); + +afterAll(() => { + vi.restoreAllMocks(); + cleanupTestDb(tmpDir); +}); + +beforeEach(() => { + mockBackupStackFiles.mockReset().mockResolvedValue(undefined); + mockHasComposeFile.mockReset().mockResolvedValue(true); + tierSpy.mockReturnValue('paid'); +}); + +describe('POST /api/stacks/:stackName/backup', () => { + it('backs up the stack files and returns success', async () => { + const res = await request(app).post('/api/stacks/web/backup').set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(mockBackupStackFiles).toHaveBeenCalledWith('web'); + }); + + it('returns 401 without an auth cookie', async () => { + const res = await request(app).post('/api/stacks/web/backup'); + expect(res.status).toBe(401); + expect(mockBackupStackFiles).not.toHaveBeenCalled(); + }); + + it('returns 403 for a viewer (no deploy permission)', async () => { + const res = await request(app).post('/api/stacks/web/backup').set('Cookie', viewerCookie); + expect(res.status).toBe(403); + expect(mockBackupStackFiles).not.toHaveBeenCalled(); + }); + + it('returns 403 on the community tier', async () => { + tierSpy.mockReturnValue('community'); + const res = await request(app).post('/api/stacks/web/backup').set('Cookie', adminCookie); + expect(res.status).toBe(403); + expect(mockBackupStackFiles).not.toHaveBeenCalled(); + }); + + it('returns 404 when the stack does not exist', async () => { + mockHasComposeFile.mockResolvedValue(false); + const res = await request(app).post('/api/stacks/ghost/backup').set('Cookie', adminCookie); + expect(res.status).toBe(404); + expect(mockBackupStackFiles).not.toHaveBeenCalled(); + }); + + it('returns 400 for an invalid stack name', async () => { + const res = await request(app).post('/api/stacks/..bad../backup').set('Cookie', adminCookie); + expect(res.status).toBe(400); + expect(mockBackupStackFiles).not.toHaveBeenCalled(); + }); + + it('returns 500 when the backup operation fails', async () => { + mockBackupStackFiles.mockRejectedValue(new Error('disk full')); + const res = await request(app).post('/api/stacks/web/backup').set('Cookie', adminCookie); + expect(res.status).toBe(500); + expect(res.body.error).toContain('disk full'); + }); + + it('returns 409 when the stack is busy with another operation', async () => { + // The backup shares the rollback slot, so it must not run while a deploy + // holds the stack-op lock for the same stack. + const { StackOpLockService } = await import('../services/StackOpLockService'); + const localNodeId = DatabaseService.getInstance().getNodes().find(n => n.type === 'local')!.id; + StackOpLockService.getInstance().tryAcquire(localNodeId, 'web', 'deploy', 'someone'); + try { + const res = await request(app).post('/api/stacks/web/backup').set('Cookie', adminCookie); + expect(res.status).toBe(409); + expect(res.body.code).toBe('stack_op_in_progress'); + expect(mockBackupStackFiles).not.toHaveBeenCalled(); + } finally { + StackOpLockService.getInstance().release(localNodeId, 'web'); + } + }); +}); diff --git a/backend/src/routes/scheduledTasks.ts b/backend/src/routes/scheduledTasks.ts index a07d52e3..6e606100 100644 --- a/backend/src/routes/scheduledTasks.ts +++ b/backend/src/routes/scheduledTasks.ts @@ -123,7 +123,10 @@ scheduledTasksRouter.get('/', (req: Request, res: Response): void => { if (!requirePaid(req, res)) return; try { let tasks = DatabaseService.getInstance().getScheduledTasks(); - // Split Auto-Update and Scheduled Operations into distinct views. + // The Scheduled Operations view manages every task type, so it lists all of + // them. `action` / `exclude_action` exist for the read-only consumers that + // want a slice: the Auto-Update readiness card and the sidebar next-run + // indicator both request `?action=update`. const actionFilter = typeof req.query.action === 'string' ? req.query.action : undefined; const excludeAction = typeof req.query.exclude_action === 'string' ? req.query.exclude_action : undefined; if (actionFilter) { diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index 76c05ef9..1988c942 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -62,6 +62,7 @@ const STACK_OP_PRESENT_PARTICIPLE: Record = { start: 'starting', update: 'updating', rollback: 'rolling back', + backup: 'backing up', }; function tryAcquireStackOpLock( @@ -1251,6 +1252,32 @@ stacksRouter.get('/:stackName/backup', async (req: Request, res: Response) => { } }); +stacksRouter.post('/:stackName/backup', async (req: Request, res: Response) => { + // Triggers a server-side backup of the stack's managed files: the same + // rollback snapshot a deploy takes. Exposed so a scheduled backup can run on + // a remote node through the proxy path, and so an operator can capture an + // on-demand snapshot. Paid-gated to match the rollback feature it feeds. + const stackName = req.params.stackName as string; + if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return; + if (!requirePaid(req, res)) return; + if (!(await requireStackExists(req.nodeId, stackName, res))) return; + // The backup slot is shared with the pre-deploy rollback snapshot, so hold the + // stack-op lock to keep a backup from interleaving with a concurrent + // deploy/update/rollback on the same stack. All early-returns stay inside the + // try so finally always releases. + if (!tryAcquireStackOpLock(req, res, stackName, 'backup')) return; + try { + await FileSystemService.getInstance(req.nodeId).backupStackFiles(stackName); + dlog(`[Stacks] Backup completed: ${sanitizeForLog(stackName)}`); + res.json({ success: true }); + } catch (error: unknown) { + console.error('[Stacks] Backup failed: %s', sanitizeForLog(stackName), error); + res.status(500).json({ error: getErrorMessage(error, 'Failed to back up stack files') }); + } finally { + releaseStackOpLock(req, stackName); + } +}); + /** * Returns the latest post-deploy scan attempt for this stack, or null if * no scan has been attempted yet. Used by the editor UI to flag stacks diff --git a/backend/src/services/FileSystemService.ts b/backend/src/services/FileSystemService.ts index b177c614..09120d64 100644 --- a/backend/src/services/FileSystemService.ts +++ b/backend/src/services/FileSystemService.ts @@ -485,7 +485,16 @@ export class FileSystemService { const debug = isDebugEnabled(); const t0 = Date.now(); const stackDir = this.resolveStackDir(stackName); - const backupDir = this.getBackupDir(stackName); + // Canonical js/path-injection barrier (mirrors restoreStackFiles): resolve the + // backup path against the backup root and confirm containment inline, so the + // mkdir/copy/write sinks below operate on a validated path. stackName is + // already validated by resolveStackDir above; this re-establishes containment + // at the backup sinks themselves so static analysis sees the barrier. + const backupRoot = path.resolve(getBackupBaseDir()); + const backupDir = path.resolve(backupRoot, String(this.nodeId), stackName); + if (!backupDir.startsWith(backupRoot + path.sep)) { + throw Object.assign(new Error('Path escapes backup directory'), { code: 'INVALID_PATH' }); + } await fsPromises.mkdir(backupDir, { recursive: true }); // Clear stale managed files from the backup slot before writing the current @@ -493,11 +502,9 @@ export class FileSystemService { // stack since the last backup (e.g. a deleted .env or a switched compose // variant) would otherwise linger here and a later restore would resurrect // it, breaking the faithful-revert guarantee. Scope is the protected set - // Sencho writes; .timestamp is rewritten below. Containment is re-checked at - // the sink, rooted at the backup base, for the same reason restoreStackFiles - // does it. A clear failure is logged but not fatal: it only risks a stale - // future rollback, so it should not block an otherwise valid deploy. - const backupRoot = path.resolve(getBackupBaseDir()); + // Sencho writes; .timestamp is rewritten below. A clear failure is logged but + // not fatal: it only risks a stale future rollback, so it should not block an + // otherwise valid deploy. for (const file of PROTECTED_STACK_FILES) { const stale = path.resolve(backupRoot, path.join(backupDir, file)); if (!stale.startsWith(backupRoot + path.sep)) continue; diff --git a/backend/src/services/SchedulerService.ts b/backend/src/services/SchedulerService.ts index 8c349d18..b9a7db9a 100644 --- a/backend/src/services/SchedulerService.ts +++ b/backend/src/services/SchedulerService.ts @@ -291,6 +291,21 @@ export class SchedulerService { triggered_by: triggeredBy, }); + // Defense in depth: every entry point that reaches here is already paid-gated + // (the route's requirePaid and the tick's tier check), but guard again so a + // task can never run on an unpaid licence regardless of the caller. Record the + // skip as a failed run so a manual trigger (which already returned 202 to the + // operator) shows in run history rather than vanishing silently. + if (LicenseService.getInstance().getTier() !== 'paid') { + console.warn(`[SchedulerService] Skipping task "${task.name}" (id=${task.id}): licence is not paid`); + db.updateScheduledTaskRun(runId, { + completed_at: Date.now(), + status: 'failure', + error: 'Scheduled tasks require a paid licence; task was not run.', + }); + return; + } + try { // Pre-check: ensure target node exists and is reachable if (task.node_id != null && task.action !== 'snapshot') { @@ -423,6 +438,9 @@ export class SchedulerService { if (!task.target_id || task.node_id == null) { throw new Error('Stack restart requires target_id and node_id'); } + if (this.isRemoteNode(task.node_id)) { + return this.executeRestartRemote(task.node_id, task.target_id, task.target_services); + } const docker = DockerController.getInstance(task.node_id); const containers = await docker.getContainersByStack(task.target_id); if (!containers || containers.length === 0) { @@ -445,32 +463,82 @@ export class SchedulerService { return `Restarted ${filtered.length} container(s) in stack "${task.target_id}"${servicesSuffix}`; } + /** + * Remote restart. The remote bulk-restart endpoint restarts every container + * in the stack, so when the task targets specific services we fan out to the + * per-service restart route to preserve the filter. + */ + private async executeRestartRemote(nodeId: number, stackName: string, targetServices: string | null): Promise { + const stackSeg = encodeURIComponent(stackName); + if (targetServices) { + const serviceNames: string[] = JSON.parse(targetServices); + // Fail fast, but name the services already restarted so a mid-loop failure + // records the partial state of the remote stack in run history. + const restarted: string[] = []; + for (const svc of serviceNames) { + try { + await this.postToRemoteStack(nodeId, `${stackSeg}/services/${encodeURIComponent(svc)}/restart`); + restarted.push(svc); + } catch (e) { + const done = restarted.length ? ` (already restarted: ${restarted.join(', ')})` : ''; + throw new Error(`Restart of service "${svc}" failed${done}: ${getErrorMessage(e, String(e))}`); + } + } + return `Restarted services [${serviceNames.join(', ')}] in stack "${stackName}" on remote node`; + } + await this.postToRemoteStack(nodeId, `${stackSeg}/restart`); + return `Restarted stack "${stackName}" on remote node`; + } + private assertStackTarget(task: ScheduledTask, label: string): asserts task is ScheduledTask & { target_id: string; node_id: number } { if (!task.target_id || task.node_id == null) { throw new Error(`${label} requires target_id and node_id`); } } + private isRemoteNode(nodeId: number): boolean { + return NodeRegistry.getInstance().getNode(nodeId)?.type === 'remote'; + } + private async executeAutoBackup(task: ScheduledTask): Promise { this.assertStackTarget(task, 'Auto-backup'); + if (this.isRemoteNode(task.node_id)) { + await this.postToRemoteStack(task.node_id, `${encodeURIComponent(task.target_id)}/backup`); + return `Backed up stack "${task.target_id}" files on remote node`; + } await FileSystemService.getInstance(task.node_id).backupStackFiles(task.target_id); return `Backed up stack "${task.target_id}" files`; } private async executeAutoStop(task: ScheduledTask): Promise { this.assertStackTarget(task, 'Auto-stop'); + if (this.isRemoteNode(task.node_id)) { + await this.postToRemoteStack(task.node_id, `${encodeURIComponent(task.target_id)}/stop`); + return `Stopped stack "${task.target_id}" (containers preserved) on remote node`; + } await ComposeService.getInstance(task.node_id).runCommand(task.target_id, 'stop'); return `Stopped stack "${task.target_id}" (containers preserved)`; } private async executeAutoDown(task: ScheduledTask): Promise { this.assertStackTarget(task, 'Auto-down'); + if (this.isRemoteNode(task.node_id)) { + await this.postToRemoteStack(task.node_id, `${encodeURIComponent(task.target_id)}/down`); + return `Took down stack "${task.target_id}" (containers removed) on remote node`; + } await ComposeService.getInstance(task.node_id).runCommand(task.target_id, 'down'); return `Took down stack "${task.target_id}" (containers removed)`; } private async executeAutoStart(task: ScheduledTask): Promise { this.assertStackTarget(task, 'Auto-start'); + // Remote auto-start proxies to the remote's own deploy route, which runs + // that node's scan-policy gate against the images it actually holds. The + // hub-side enforceSchedulerPolicyGate below is for local nodes only. + if (this.isRemoteNode(task.node_id)) { + await this.postToRemoteStack(task.node_id, `${encodeURIComponent(task.target_id)}/deploy`); + return `Started stack "${task.target_id}" on remote node`; + } await this.enforceSchedulerPolicyGate( task.target_id, task.node_id, @@ -684,6 +752,40 @@ export class SchedulerService { return body.result || 'Remote auto-update completed (no details returned).'; } + /** + * Proxy a stack lifecycle action to a remote Sencho instance. ComposeService, + * DockerController, and FileSystemService are local-only, so for a remote node + * we POST to the remote's own stack-operation endpoint with the node Bearer + * token and the licence proxy headers, exactly as executeUpdateRemote does. + * `routeSuffix` is the path under `/api/stacks/`; the caller URL-encodes each + * segment. + */ + private async postToRemoteStack(nodeId: number, routeSuffix: string): Promise { + const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId); + if (!proxyTarget) { + throw new Error('Remote node is not configured or missing API credentials'); + } + const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); + const proxyHeaders = LicenseService.getInstance().getProxyHeaders(); + if (isDebugEnabled()) { + console.log(`[SchedulerService:debug] postToRemoteStack: node=${nodeId} route=${routeSuffix}`); + } + const response = await fetch(`${baseUrl}/api/stacks/${routeSuffix}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${proxyTarget.apiToken}`, + [PROXY_TIER_HEADER]: proxyHeaders.tier, + [PROXY_VARIANT_HEADER]: proxyHeaders.variant ?? '', + }, + signal: AbortSignal.timeout(300_000), + }); + if (!response.ok) { + const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` })); + throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`); + } + } + private async executeUpdateForStack( stackName: string, nodeId: number, diff --git a/backend/src/services/StackOpLockService.ts b/backend/src/services/StackOpLockService.ts index 530800e6..e79cc326 100644 --- a/backend/src/services/StackOpLockService.ts +++ b/backend/src/services/StackOpLockService.ts @@ -1,14 +1,15 @@ /** * Tracks in-flight stack lifecycle operations (deploy, down, restart, stop, - * start, update, rollback) per (nodeId, stackName). A second request to the - * same stack while the first is still running returns 409 instead of racing - * the first. + * start, update, rollback, backup) per (nodeId, stackName). A second request to + * the same stack while the first is still running returns 409 instead of racing + * the first. Backup is included because it rewrites the shared rollback slot, so + * it must not interleave with a deploy/update/rollback on the same stack. * * State is intentionally process-local: a Sencho restart clears all locks, * which matches the lifecycle of any in-flight `docker compose` child process. */ -export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback'; +export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup'; export interface StackOpLock { action: StackOpAction; diff --git a/docs/features/scheduled-operations.mdx b/docs/features/scheduled-operations.mdx index f6315e09..7e1461e8 100644 --- a/docs/features/scheduled-operations.mdx +++ b/docs/features/scheduled-operations.mdx @@ -131,7 +131,7 @@ Stop Stack, Take Stack Down, Start Stack, and Backup Stack Files let you schedul **Backup Stack Files** keeps a single slot per stack under `/backups//`. Each run overwrites the previous backup. For a versioned, point-in-time archive across all nodes, use a [Fleet Snapshot](/features/fleet-backups) task instead. -The four lifecycle actions execute against the local Sencho instance. To schedule lifecycle operations for a remote node, switch the active node to that node first and create the schedule from its UI; the schedule then lives on the remote node and runs there. +Pick any node in your fleet from the schedule's **Node** combobox: the four lifecycle actions run against the node you select, whether it is the hub or a connected remote. The schedule itself is stored on the hub, so you manage every node's lifecycle tasks from one place. ### Delete after successful run diff --git a/frontend/src/components/__tests__/ScheduledOperationsView.test.tsx b/frontend/src/components/__tests__/ScheduledOperationsView.test.tsx new file mode 100644 index 00000000..95347906 --- /dev/null +++ b/frontend/src/components/__tests__/ScheduledOperationsView.test.tsx @@ -0,0 +1,141 @@ +/** + * Component coverage for ScheduledOperationsView. Locks the deterministic + * wiring that manual and browser testing miss: the task list renders, a prefill + * opens the create modal and is consumed once, the node filter narrows the + * table, and a create submits the correct action/target payload to the + * hub-local endpoint. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import type { ScheduledTask } from '@/types/scheduling'; + +vi.mock('@/lib/api', () => ({ apiFetch: vi.fn(), fetchForNode: vi.fn() })); +vi.mock('@/components/ui/toast-store', () => ({ + toast: { error: vi.fn(), success: vi.fn(), warning: vi.fn(), info: vi.fn() }, +})); + +import { apiFetch, fetchForNode } from '@/lib/api'; +import ScheduledOperationsView from '../ScheduledOperationsView'; + +const mockedFetch = apiFetch as unknown as ReturnType; +const mockedFetchForNode = fetchForNode as unknown as ReturnType; + +function jsonResponse(body: unknown, init: { ok?: boolean; status?: number } = {}): Response { + return { ok: init.ok ?? true, status: init.status ?? 200, json: async () => body } as unknown as Response; +} + +function makeTask(overrides: Partial = {}): ScheduledTask { + return { + id: 1, + name: 'task-1', + target_type: 'system', + target_id: null, + node_id: 1, + action: 'prune', + cron_expression: '0 3 * * *', + enabled: 1, + created_by: 'admin', + created_at: 0, + updated_at: 0, + last_run_at: null, + next_run_at: null, + last_status: null, + last_error: null, + prune_targets: null, + target_services: null, + prune_label_filter: null, + ...overrides, + }; +} + +let tasksFixture: ScheduledTask[]; +let nodesFixture: { id: number; name: string }[]; + +beforeEach(() => { + tasksFixture = []; + nodesFixture = [{ id: 1, name: 'hub' }, { id: 2, name: 'edge' }]; + mockedFetch.mockReset(); + mockedFetchForNode.mockReset(); + + mockedFetch.mockImplementation(async (url: string, opts?: { method?: string }) => { + if (url === '/scheduled-tasks' && opts?.method === 'POST') return jsonResponse({ id: 99 }, { status: 201 }); + if (url === '/scheduled-tasks') return jsonResponse(tasksFixture); + if (url === '/nodes') return jsonResponse(nodesFixture); + if (url === '/stacks') return jsonResponse([]); + return jsonResponse({}); + }); + mockedFetchForNode.mockResolvedValue(jsonResponse(['web', 'db'])); +}); + +afterEach(() => vi.clearAllMocks()); + +describe('ScheduledOperationsView', () => { + it('renders existing tasks in the table view', async () => { + tasksFixture = [makeTask({ id: 7, name: 'nightly-prune' })]; + render(); + + await userEvent.click(await screen.findByRole('button', { name: /All tasks/ })); + expect(await screen.findByText('nightly-prune')).toBeInTheDocument(); + }); + + it('opens the create modal from a prefill and consumes it once', async () => { + const onPrefillConsumed = vi.fn(); + render( + , + ); + + expect(await screen.findByText('New scheduled task')).toBeInTheDocument(); + expect(onPrefillConsumed).toHaveBeenCalledTimes(1); + // The prefilled stack drives a node-scoped stack fetch through the proxy. + await waitFor(() => expect(mockedFetchForNode).toHaveBeenCalledWith('/stacks', 1)); + }); + + it('filters the table to the selected node and clears the filter', async () => { + tasksFixture = [ + makeTask({ id: 1, name: 'hub-task', node_id: 1 }), + makeTask({ id: 2, name: 'edge-task', node_id: 2 }), + ]; + const onClearFilter = vi.fn(); + render(); + + await userEvent.click(await screen.findByRole('button', { name: /All tasks/ })); + expect(await screen.findByText('edge-task')).toBeInTheDocument(); + expect(screen.queryByText('hub-task')).not.toBeInTheDocument(); + + await userEvent.click(screen.getByRole('button', { name: /Clear filter/ })); + expect(onClearFilter).toHaveBeenCalled(); + }); + + it('submits a system-prune create with the correct target_type and payload', async () => { + render(); + + await userEvent.click(await screen.findByRole('button', { name: /New Schedule/ })); + await userEvent.type(await screen.findByPlaceholderText('e.g. Nightly stack restart'), 'cleanup'); + + // The action selector is the first combobox; switch it to System Prune. + await userEvent.click(screen.getAllByRole('combobox')[0]); + await userEvent.click(await screen.findByRole('button', { name: 'System Prune' })); + + await userEvent.click(screen.getByRole('button', { name: 'Create' })); + + await waitFor(() => { + const postCall = mockedFetch.mock.calls.find( + ([url, opts]) => url === '/scheduled-tasks' && opts?.method === 'POST', + ); + expect(postCall).toBeTruthy(); + const body = JSON.parse(postCall![1].body); + expect(body).toMatchObject({ + name: 'cleanup', + target_type: 'system', + action: 'prune', + cron_expression: '0 3 * * *', + prune_targets: ['containers', 'images', 'networks', 'volumes'], + }); + expect(postCall![1].localOnly).toBe(true); + }); + }); +});