feat(scheduler): schedule container restart, stop, and start (#1526)

* feat(scheduler): schedule container restart, stop, and start

Add container as a scheduled-task target type so operators can automate lifecycle actions against standalone containers by node and name, with matching UI pickers, validation, execution on local and remote nodes, and tests.

* fix(scheduler): stack service matching and container picker hygiene

Backfill Service on smartFallback containers so per-service stack restarts work when container_name is set. Match services by compose label and container name in stack routes and scheduled restarts. Exclude Sencho from GET /api/containers lists. Hide the Restart Stack service picker when a stack has only one service.

* test(scheduler): scope service checkbox assertion to Services block

The create dialog also has a Delete after run checkbox. Count checkboxes only inside the Services section so CI does not include unrelated form controls.

* fix(scheduler): narrow closest() result to HTMLElement in schedule test

The service-checkbox assertion passed an Element from closest() into
within(), which requires an HTMLElement, failing tsc -b in the frontend
build and Docker build stages. Use the closest<HTMLElement>() type
argument so the value type-checks without an unsafe cast.

* fix(scheduler): hide Sencho container on remote node picker lists

Remote container lists are proxied from peer Sencho instances, so id-only self filtering missed peers on older builds. Await SelfIdentity init, match ImageID, and drop official saelix/sencho images. Apply the same heuristic in the scheduled-operations UI and when the hub fetches remote containers for scheduled runs.

* test(monitor): add missing DatabaseService mocks for scan history cleanup

* test(scheduler): add missing markStaleScansAsFailed mock

SchedulerService.tick() calls db.markStaleScansAsFailed() to sweep stale
vulnerability scans. The scheduler-service test was missing this method in
its DatabaseService mock, causing TypeError failures during test initialization.

Added mockMarkStaleScansAsFailed to hoisted mocks and DatabaseService mock
object, returning safe default of 0 scans marked as failed.

* test(compose): add missing FileSystemService mocks for getStackContent/getEnvContent

* test(containers-route): mock SelfIdentityService to prevent initialize() crash

The excludeSelfContainers() helper calls SelfIdentityService.initialize(), which tries to access DockerController. Without a proper SelfIdentityService mock, the initialize() call fails silently, causing a 500 error on GET /api/containers.

Added SelfIdentityService mock with initialize(), isOwnContainer(), and isOwnImage() methods to prevent the crash.
This commit is contained in:
Anso
2026-07-02 22:31:29 -04:00
committed by GitHub
parent b65daf6845
commit 10fb93dcb1
29 changed files with 932 additions and 68 deletions
@@ -0,0 +1,36 @@
import { describe, it, expect } from 'vitest';
import {
containerBelongsToComposeService,
filterContainersByComposeService,
} from '../helpers/composeServiceMatch';
describe('composeServiceMatch', () => {
it('matches by Service field from docker compose ps', () => {
const c = { Id: 'abc', Service: 'mariadb', Names: ['/mariadb'] };
expect(containerBelongsToComposeService(c, 'mariadb')).toBe(true);
expect(containerBelongsToComposeService(c, 'phpmyadmin')).toBe(false);
});
it('matches by com.docker.compose.service label', () => {
const c = {
Id: 'abc',
Names: ['/custom-name'],
Labels: { 'com.docker.compose.service': 'mariadb' },
};
expect(containerBelongsToComposeService(c, 'mariadb')).toBe(true);
});
it('matches when container name equals service (container_name in compose)', () => {
const c = { Id: 'abc', Service: '', Names: ['/mariadb'] };
expect(containerBelongsToComposeService(c, 'mariadb')).toBe(true);
});
it('filterContainersByComposeService returns all replicas', () => {
const containers = [
{ Id: '1', Service: 'app', Names: ['/web-app-1'] },
{ Id: '2', Service: 'app', Names: ['/web-app-2'] },
{ Id: '3', Names: ['/db'] },
];
expect(filterContainersByComposeService(containers, 'app')).toHaveLength(2);
});
});
@@ -19,6 +19,7 @@ const {
mockGetComposeFilename, mockGetOverrideFilename, mockEnsureStackOverride,
mockMkdtempSync, mockWriteFileSync, mockUnlinkSync, mockRmdirSync,
mockGetGlobalSettings, mockPruneDanglingImages, mockGetBindMounts,
mockGetStackContent, mockGetEnvContent,
} = vi.hoisted(() => ({
mockSpawn: vi.fn(),
mockGetContainersByStack: vi.fn().mockResolvedValue([]),
@@ -40,6 +41,8 @@ const {
mockGetGlobalSettings: vi.fn().mockReturnValue({}),
mockPruneDanglingImages: vi.fn().mockResolvedValue({ reclaimedBytes: 0 }),
mockGetBindMounts: vi.fn().mockResolvedValue(null),
mockGetStackContent: vi.fn().mockResolvedValue(''),
mockGetEnvContent: vi.fn().mockResolvedValue(''),
}));
vi.mock('child_process', () => ({ spawn: mockSpawn, execFile: vi.fn() }));
@@ -109,6 +112,8 @@ vi.mock('../services/FileSystemService', () => ({
restoreStackFiles: mockRestoreStackFiles,
getComposeFilename: mockGetComposeFilename,
getOverrideFilename: mockGetOverrideFilename,
getStackContent: mockGetStackContent,
getEnvContent: mockGetEnvContent,
}),
},
}));
@@ -16,6 +16,7 @@ let app: import('express').Express;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let DockerController: typeof import('../services/DockerController').default;
let FileSystemService: typeof import('../services/FileSystemService').FileSystemService;
let SelfIdentityService: typeof import('../services/SelfIdentityService').default;
let ROLE_PERMISSIONS: typeof import('../middleware/permissions').ROLE_PERMISSIONS;
const VIEWER = 'container-read-viewer';
@@ -28,7 +29,7 @@ function viewerToken(): string {
}
/**
* Replace the DockerController / FileSystemService singletons with stubs so the
* Replace the DockerController / FileSystemService / SelfIdentityService singletons with stubs so the
* handlers run without a Docker daemon. The logs stub ends the response itself
* (the real streamContainerLogs flushes SSE headers and streams), so a request
* that clears the guard resolves instead of hanging. Returns the spies so a test
@@ -46,6 +47,11 @@ function stubDockerAndFs(): { docker: ReturnType<typeof vi.spyOn>; fs: ReturnTyp
const fs = vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({
getStacks: vi.fn().mockResolvedValue([]),
} as unknown as ReturnType<typeof FileSystemService.getInstance>);
vi.spyOn(SelfIdentityService, 'getInstance').mockReturnValue({
initialize: vi.fn().mockResolvedValue(undefined),
isOwnContainer: vi.fn().mockReturnValue(false),
isOwnImage: vi.fn().mockReturnValue(false),
} as unknown as ReturnType<typeof SelfIdentityService.getInstance>);
return { docker, fs };
}
@@ -54,6 +60,7 @@ beforeAll(async () => {
({ DatabaseService } = await import('../services/DatabaseService'));
({ default: DockerController } = await import('../services/DockerController'));
({ FileSystemService } = await import('../services/FileSystemService'));
({ default: SelfIdentityService } = await import('../services/SelfIdentityService'));
({ ROLE_PERMISSIONS } = await import('../middleware/permissions'));
({ app } = await import('../index'));
@@ -0,0 +1,92 @@
/**
* GET /api/containers must omit Sencho's own container from picker lists.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import bcrypt from 'bcrypt';
import { setupTestDb, cleanupTestDb, TEST_JWT_SECRET } from './helpers/setupTestDb';
let tmpDir: string;
let app: import('express').Express;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let DockerController: typeof import('../services/DockerController').default;
let SelfIdentityServiceMod: typeof import('../services/SelfIdentityService').default;
const VIEWER = 'container-self-filter-viewer';
function viewerToken(): string {
const user = DatabaseService.getInstance().getUserByUsername(VIEWER)!;
return jwt.sign({ username: VIEWER, role: 'viewer', tv: user.token_version }, TEST_JWT_SECRET, { expiresIn: '1m' });
}
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
({ default: DockerController } = await import('../services/DockerController'));
({ default: SelfIdentityServiceMod } = await import('../services/SelfIdentityService'));
({ app } = await import('../index'));
const hash = await bcrypt.hash('password123', 1);
DatabaseService.getInstance().addUser({ username: VIEWER, password_hash: hash, role: 'viewer' });
});
afterAll(() => {
cleanupTestDb(tmpDir);
});
afterEach(() => {
vi.restoreAllMocks();
});
describe('GET /api/containers self-filter', () => {
beforeEach(() => {
vi.spyOn(SelfIdentityServiceMod, 'getInstance').mockReturnValue({
initialize: vi.fn().mockResolvedValue(undefined),
isOwnContainer: (idOrName: string) => idOrName === 'sencho' || idOrName.startsWith('sencho-id'),
isOwnImage: vi.fn().mockReturnValue(false),
} as unknown as ReturnType<typeof SelfIdentityServiceMod.getInstance>);
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getAllContainers: vi.fn().mockResolvedValue([
{ Id: 'sencho-id-full', Names: ['/sencho'], State: 'running', Status: 'Up 1 day' },
{ Id: 'other-id', Names: ['/mariadb'], State: 'running', Status: 'Up 1 day' },
]),
getRunningContainers: vi.fn().mockResolvedValue([]),
} as unknown as ReturnType<typeof DockerController.getInstance>);
});
it('excludes Sencho from all=true container list', async () => {
const res = await request(app)
.get('/api/containers?all=true')
.set('Authorization', `Bearer ${viewerToken()}`);
expect(res.status).toBe(200);
expect(res.body).toHaveLength(1);
expect(res.body[0].Names).toEqual(['/mariadb']);
});
it('excludes official Sencho images when SelfIdentity does not match by id', async () => {
vi.spyOn(SelfIdentityServiceMod, 'getInstance').mockReturnValue({
initialize: vi.fn().mockResolvedValue(undefined),
isOwnContainer: vi.fn().mockReturnValue(false),
isOwnImage: vi.fn().mockReturnValue(false),
} as unknown as ReturnType<typeof SelfIdentityServiceMod.getInstance>);
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getAllContainers: vi.fn().mockResolvedValue([
{ Id: 'remote-sencho', Names: ['/sencho'], Image: 'saelix/sencho:latest', State: 'running' },
{ Id: 'other-id', Names: ['/mariadb'], State: 'running' },
]),
getRunningContainers: vi.fn().mockResolvedValue([]),
} as unknown as ReturnType<typeof DockerController.getInstance>);
const res = await request(app)
.get('/api/containers?all=true')
.set('Authorization', `Bearer ${viewerToken()}`);
expect(res.status).toBe(200);
expect(res.body).toHaveLength(1);
expect(res.body[0].Names).toEqual(['/mariadb']);
});
});
@@ -0,0 +1,11 @@
import { describe, it, expect } from 'vitest';
import { isPublishedSenchoImage } from '../helpers/excludeSelfContainers';
describe('isPublishedSenchoImage', () => {
it('matches Docker Hub and GHCR release paths', () => {
expect(isPublishedSenchoImage('saelix/sencho:latest')).toBe(true);
expect(isPublishedSenchoImage('ghcr.io/studio-saelix/sencho:0.93.1')).toBe(true);
expect(isPublishedSenchoImage('ghcr.io/studio-saelix/sencho-dev:dev')).toBe(true);
expect(isPublishedSenchoImage('lscr.io/linuxserver/mariadb:latest')).toBe(false);
});
});
@@ -10,6 +10,7 @@ import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsM
const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContainerMetric,
mockCleanupOldMetrics, mockCleanupOldNotifications, mockCleanupOldAuditLogs,
mockUpdateStackAlertLastFired, mockGetSystemState, mockSetSystemState,
mockPruneScanHistoryPerImage, mockDeleteScansByImageRef,
mockGetRunningContainers, mockGetAllContainers, mockGetContainerStatsStream,
mockGetContainerRestartCount, mockGetDiskUsage, mockGetImages, mockGetStacks,
mockDispatchAlert,
@@ -30,6 +31,8 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
mockUpdateStackAlertLastFired: vi.fn(),
mockGetSystemState: vi.fn().mockReturnValue(null),
mockSetSystemState: vi.fn(),
mockPruneScanHistoryPerImage: vi.fn().mockReturnValue(0),
mockDeleteScansByImageRef: vi.fn().mockReturnValue(0),
mockGetRunningContainers: vi.fn().mockResolvedValue([]),
mockGetAllContainers: vi.fn().mockResolvedValue([]),
mockGetContainerStatsStream: vi.fn().mockResolvedValue('{}'),
@@ -64,6 +67,8 @@ vi.mock('../services/DatabaseService', () => ({
updateStackAlertLastFired: mockUpdateStackAlertLastFired,
getSystemState: mockGetSystemState,
setSystemState: mockSetSystemState,
pruneScanHistoryPerImage: mockPruneScanHistoryPerImage,
deleteScansByImageRef: mockDeleteScansByImageRef,
}),
},
}));
@@ -19,7 +19,7 @@ const EXPECTED_ACTIONS: BackendScheduledAction[] = [
'auto_backup', 'auto_stop', 'auto_down', 'auto_start',
];
const ALL_TARGET_TYPES: TargetType[] = ['stack', 'fleet', 'system'];
const ALL_TARGET_TYPES: TargetType[] = ['stack', 'fleet', 'system', 'container'];
describe('scheduledActionRegistry', () => {
it('exposes exactly the known backend actions, in order', () => {
@@ -46,27 +46,27 @@ describe('scheduledActionRegistry', () => {
describe('validateActionTarget', () => {
const validPairs: Record<BackendScheduledAction, TargetType[]> = {
restart: ['stack'],
restart: ['stack', 'container'],
snapshot: ['fleet'],
prune: ['system'],
update: ['stack', 'fleet'],
scan: ['system'],
auto_backup: ['stack'],
auto_stop: ['stack'],
auto_stop: ['stack', 'container'],
auto_down: ['stack'],
auto_start: ['stack'],
auto_start: ['stack', 'container'],
};
const mismatchMessage: Record<BackendScheduledAction, string> = {
restart: 'Restart action requires target_type "stack".',
restart: 'Restart action requires target_type "stack" or "container".',
snapshot: 'Snapshot action requires target_type "fleet".',
prune: 'Prune action requires target_type "system".',
update: 'Update action requires target_type "stack" or "fleet".',
scan: 'Scan action requires target_type "system".',
auto_backup: 'auto_backup action requires target_type "stack".',
auto_stop: 'auto_stop action requires target_type "stack".',
auto_stop: 'auto_stop action requires target_type "stack" or "container".',
auto_down: 'auto_down action requires target_type "stack".',
auto_start: 'auto_start action requires target_type "stack".',
auto_start: 'auto_start action requires target_type "stack" or "container".',
};
for (const action of EXPECTED_ACTIONS) {
@@ -594,6 +594,64 @@ describe('POST /api/scheduled-tasks - new lifecycle actions', () => {
});
});
describe('POST /api/scheduled-tasks - container lifecycle', () => {
it('creates a container restart schedule', async () => {
const res = await request(app)
.post('/api/scheduled-tasks')
.set('Cookie', adminCookie)
.send({
name: 'daily-watchtower-restart',
target_type: 'container',
target_id: 'watchtower',
node_id: 1,
action: 'restart',
cron_expression: '0 3 * * *',
enabled: true,
});
expect(res.status).toBe(201);
expect(res.body.target_type).toBe('container');
expect(res.body.target_id).toBe('watchtower');
expect(res.body.action).toBe('restart');
});
it('rejects invalid container names', async () => {
const res = await request(app)
.post('/api/scheduled-tasks')
.set('Cookie', adminCookie)
.send({
name: 'bad-container',
target_type: 'container',
target_id: '../escape',
node_id: 1,
action: 'restart',
cron_expression: '0 3 * * *',
enabled: true,
});
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/valid container name/);
});
for (const action of ['auto_stop', 'auto_start'] as const) {
it(`creates container ${action} schedule`, async () => {
const res = await request(app)
.post('/api/scheduled-tasks')
.set('Cookie', adminCookie)
.send({
name: `ctr-${action}`,
target_type: 'container',
target_id: 'sidecar',
node_id: 1,
action,
cron_expression: '0 4 * * *',
enabled: true,
});
expect(res.status).toBe(201);
expect(res.body.target_type).toBe('container');
expect(res.body.action).toBe(action);
});
}
});
describe('POST /api/scheduled-tasks - available on the Community tier', () => {
beforeEach(() => {
tierSpy.mockReturnValue('community');
+110 -2
View File
@@ -12,9 +12,10 @@ const {
mockUpdateScheduledTask, mockCleanupOldTaskRuns, mockGetScheduledTask, mockGetNodes, mockGetNode,
mockGetGlobalSettings, mockGetStackDossier,
mockCreateSnapshot, mockInsertSnapshotFiles, mockClearStackUpdateStatus,
mockMarkStaleRunsAsFailed, mockDeleteOldScans,
mockMarkStaleRunsAsFailed, mockMarkStaleScansAsFailed, mockDeleteOldScans,
mockGetTier, mockGetProxyHeaders,
mockGetContainersByStack, mockRestartContainer, mockPruneSystem,
mockGetContainersByStack, mockRestartContainer, mockFindContainerByName,
mockStartContainer, mockStopContainer, mockPruneSystem,
mockUpdateStack,
mockGetStacks, mockGetStackContent, mockGetEnvContent,
mockCheckImage,
@@ -43,11 +44,15 @@ const {
mockInsertSnapshotFiles: vi.fn(),
mockClearStackUpdateStatus: vi.fn(),
mockMarkStaleRunsAsFailed: vi.fn().mockReturnValue(0),
mockMarkStaleScansAsFailed: vi.fn().mockReturnValue(0),
mockDeleteOldScans: vi.fn().mockReturnValue(0),
mockGetTier: vi.fn().mockReturnValue('paid'),
mockGetProxyHeaders: vi.fn().mockReturnValue({ tier: 'paid' }),
mockGetContainersByStack: vi.fn().mockResolvedValue([]),
mockRestartContainer: vi.fn().mockResolvedValue(undefined),
mockFindContainerByName: vi.fn().mockResolvedValue(null),
mockStartContainer: vi.fn().mockResolvedValue(undefined),
mockStopContainer: vi.fn().mockResolvedValue(undefined),
mockPruneSystem: vi.fn().mockResolvedValue({ success: true, reclaimedBytes: 0 }),
mockUpdateStack: vi.fn().mockResolvedValue(undefined),
mockGetStacks: vi.fn().mockResolvedValue([]),
@@ -89,6 +94,7 @@ vi.mock('../services/DatabaseService', () => ({
insertSnapshotFiles: mockInsertSnapshotFiles,
clearStackUpdateStatus: mockClearStackUpdateStatus,
markStaleRunsAsFailed: mockMarkStaleRunsAsFailed,
markStaleScansAsFailed: mockMarkStaleScansAsFailed,
deleteOldScans: mockDeleteOldScans,
deleteScheduledTask: mockDeleteScheduledTask,
getMatchingPolicy: mockGetMatchingPolicy,
@@ -117,6 +123,9 @@ vi.mock('../services/DockerController', () => ({
getInstance: () => ({
getContainersByStack: mockGetContainersByStack,
restartContainer: mockRestartContainer,
findContainerByName: mockFindContainerByName,
startContainer: mockStartContainer,
stopContainer: mockStopContainer,
pruneSystem: mockPruneSystem,
}),
},
@@ -530,6 +539,105 @@ describe('SchedulerService - executeRestart', () => {
expect.objectContaining({ status: 'failure', error: expect.stringContaining('No containers') })
);
});
it('restarts a standalone container by name', async () => {
mockGetScheduledTask.mockReturnValue({
id: 63,
name: 'restart-ctr',
action: 'restart',
target_type: 'container',
cron_expression: '0 3 * * *',
enabled: true,
target_id: 'watchtower',
node_id: 1,
created_by: 'admin',
last_status: null,
});
mockFindContainerByName.mockResolvedValue({
id: 'abc123deadbeef',
name: 'watchtower',
state: 'running',
image: 'containrrr/watchtower',
stackProject: null,
});
const svc = SchedulerService.getInstance();
await svc.triggerTask(63);
expect(mockFindContainerByName).toHaveBeenCalledWith('watchtower');
expect(mockRestartContainer).toHaveBeenCalledWith('abc123deadbeef');
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
expect.any(Number),
expect.objectContaining({
status: 'success',
output: expect.stringContaining('watchtower'),
}),
);
});
it('records failure when container name is missing', async () => {
mockGetScheduledTask.mockReturnValue({
id: 64,
name: 'restart-missing',
action: 'restart',
target_type: 'container',
cron_expression: '0 3 * * *',
enabled: true,
target_id: 'gone-container',
node_id: 1,
created_by: 'admin',
last_status: null,
});
mockFindContainerByName.mockResolvedValue(null);
const svc = SchedulerService.getInstance();
await svc.triggerTask(64);
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
expect.any(Number),
expect.objectContaining({
status: 'failure',
error: expect.stringContaining('gone-container'),
}),
);
});
it('resolves a new container ID when the name matches after recreation', async () => {
mockGetScheduledTask.mockReturnValue({
id: 65,
name: 'restart-recreated',
action: 'restart',
target_type: 'container',
cron_expression: '0 3 * * *',
enabled: true,
target_id: 'sidecar',
node_id: 1,
created_by: 'admin',
last_status: null,
});
mockFindContainerByName
.mockResolvedValueOnce({
id: 'old-id-111',
name: 'sidecar',
state: 'running',
image: 'sidecar:latest',
stackProject: null,
})
.mockResolvedValueOnce({
id: 'new-id-222',
name: 'sidecar',
state: 'running',
image: 'sidecar:latest',
stackProject: null,
});
const svc = SchedulerService.getInstance();
await svc.triggerTask(65);
await svc.triggerTask(65);
expect(mockRestartContainer).toHaveBeenNthCalledWith(1, 'old-id-111');
expect(mockRestartContainer).toHaveBeenNthCalledWith(2, 'new-id-222');
});
});
// ── executePrune ───────────────────────────────────────────────────────
@@ -140,6 +140,28 @@ describe('POST /api/stacks/:stackName/services/:serviceName/restart', () => {
expect(mockRestartContainer).toHaveBeenCalledWith('container-app-1');
expect(mockRestartContainer).not.toHaveBeenCalledWith('container-db-1');
});
it('matches smartFallback containers when Service is empty but container name equals service', async () => {
mockGetContainersByStack.mockResolvedValue([
{
Id: 'container-mariadb-1',
Service: '',
Names: ['/mariadb'],
State: 'running',
Status: 'Up 12 days',
Ports: [],
},
makeContainer('container-phpmyadmin-1', 'phpmyadmin'),
]);
const res = await request(app)
.post('/api/stacks/db-compose/services/mariadb/restart')
.set('Cookie', authCookie);
expect(res.status).toBe(200);
expect(res.body.count).toBe(1);
expect(mockRestartContainer).toHaveBeenCalledWith('container-mariadb-1');
});
});
describe('POST /api/stacks/:stackName/services/:serviceName/stop', () => {