mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-07-26 11:49:16 +00:00
fix(scheduler): harden scheduled operations with stale cleanup, cron validation, and design fixes (#549)
* fix(scheduler): clean up stale runs on startup and auto-disable invalid cron tasks
- Add markStaleRunsAsFailed() bulk DB method with status index
- Clean up orphaned 'running' records on scheduler startup
- Auto-disable tasks when cron expression becomes invalid at execution time
- Promote CRUD debug logs to standard logs for scheduled task admin actions
- Add diagnostic logging for task pre-checks, action timing, and prune fallback
* refactor(scheduling): extract shared types and fix design system violations
- Extract ScheduledTask, TaskRun, NodeOption to shared types file
- Extract getCronDescription and formatTimestamp to shared utilities
- Fix formatTimestamp falsy-zero null check
- Tighten last_status type to 'success' | 'failure' | null
- Add strokeWidth={1.5} to all action icons per design system
- Add sr-only DialogDescription for accessibility
- Wrap Sheet run history in ScrollArea
- Fix delete button styling to match design system pattern
- Change manual trigger toast from "executed" to "triggered"
* test(scheduler): add tests for snapshot, remote update, stale cleanup, and cron invalidation
- Add stale run cleanup tests (bulk markStaleRunsAsFailed, logging)
- Add cron invalidation test (auto-disable, error message)
- Add executeSnapshot tests (fleet capture, empty stacks)
- Add executeUpdateRemote tests (proxy success, remote error)
- Document stale run cleanup and cron auto-disable in troubleshooting docs
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
* Unit tests for SchedulerService — task execution, concurrent prevention,
|
||||
* license gating, cron parsing, and error handling.
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
|
||||
// ── Hoisted mocks ──────────────────────────────────────────────────────
|
||||
|
||||
@@ -10,12 +10,14 @@ const {
|
||||
mockGetDueScheduledTasks, mockCreateScheduledTaskRun, mockUpdateScheduledTaskRun,
|
||||
mockUpdateScheduledTask, mockCleanupOldTaskRuns, mockGetScheduledTask, mockGetNodes, mockGetNode,
|
||||
mockCreateSnapshot, mockInsertSnapshotFiles, mockClearStackUpdateStatus,
|
||||
mockMarkStaleRunsAsFailed,
|
||||
mockGetTier, mockGetVariant,
|
||||
mockGetContainersByStack, mockRestartContainer, mockPruneSystem,
|
||||
mockUpdateStack,
|
||||
mockGetStacks, mockGetStackContent, mockGetEnvContent,
|
||||
mockCheckImage,
|
||||
mockDispatchAlert,
|
||||
mockGetProxyTarget,
|
||||
} = vi.hoisted(() => ({
|
||||
mockGetDueScheduledTasks: vi.fn().mockReturnValue([]),
|
||||
mockCreateScheduledTaskRun: vi.fn().mockReturnValue(1),
|
||||
@@ -28,6 +30,7 @@ const {
|
||||
mockCreateSnapshot: vi.fn().mockReturnValue(1),
|
||||
mockInsertSnapshotFiles: vi.fn(),
|
||||
mockClearStackUpdateStatus: vi.fn(),
|
||||
mockMarkStaleRunsAsFailed: vi.fn().mockReturnValue(0),
|
||||
mockGetTier: vi.fn().mockReturnValue('paid'),
|
||||
mockGetVariant: vi.fn().mockReturnValue('admiral'),
|
||||
mockGetContainersByStack: vi.fn().mockResolvedValue([]),
|
||||
@@ -39,6 +42,7 @@ const {
|
||||
mockGetEnvContent: vi.fn().mockResolvedValue(''),
|
||||
mockCheckImage: vi.fn().mockResolvedValue({ hasUpdate: false }),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
|
||||
mockGetProxyTarget: vi.fn().mockReturnValue(null),
|
||||
}));
|
||||
|
||||
vi.mock('../services/DatabaseService', () => ({
|
||||
@@ -55,6 +59,7 @@ vi.mock('../services/DatabaseService', () => ({
|
||||
createSnapshot: mockCreateSnapshot,
|
||||
insertSnapshotFiles: mockInsertSnapshotFiles,
|
||||
clearStackUpdateStatus: mockClearStackUpdateStatus,
|
||||
markStaleRunsAsFailed: mockMarkStaleRunsAsFailed,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
@@ -117,6 +122,7 @@ vi.mock('../services/NodeRegistry', () => ({
|
||||
getInstance: () => ({
|
||||
getDefaultNodeId: () => 1,
|
||||
getNode: mockGetNode,
|
||||
getProxyTarget: mockGetProxyTarget,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
@@ -774,3 +780,218 @@ describe('SchedulerService - isProcessing guard', () => {
|
||||
expect((svc as any).isProcessing).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Stale run cleanup (T1) ───────────────────────────────────────────
|
||||
|
||||
describe('SchedulerService - stale run cleanup', () => {
|
||||
it('calls markStaleRunsAsFailed on start and logs when records exist', () => {
|
||||
mockMarkStaleRunsAsFailed.mockReturnValue(2);
|
||||
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
svc.start();
|
||||
|
||||
expect(mockMarkStaleRunsAsFailed).toHaveBeenCalledTimes(1);
|
||||
expect(logSpy).toHaveBeenCalledWith(expect.stringContaining('Cleaned up 2 stale run record(s)'));
|
||||
|
||||
logSpy.mockRestore();
|
||||
svc.stop();
|
||||
});
|
||||
|
||||
it('does not log when no stale runs exist', () => {
|
||||
mockMarkStaleRunsAsFailed.mockReturnValue(0);
|
||||
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
svc.start();
|
||||
|
||||
expect(mockMarkStaleRunsAsFailed).toHaveBeenCalledTimes(1);
|
||||
expect(logSpy).not.toHaveBeenCalledWith(expect.stringContaining('stale'));
|
||||
|
||||
logSpy.mockRestore();
|
||||
svc.stop();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Invalid cron at execution time (T2) ──────────────────────────────
|
||||
|
||||
describe('SchedulerService - invalid cron at execution time', () => {
|
||||
it('disables task and records error when cron becomes invalid', async () => {
|
||||
mockGetScheduledTask.mockReturnValue({
|
||||
id: 95,
|
||||
name: 'bad-cron-task',
|
||||
action: 'restart',
|
||||
cron_expression: 'INVALID CRON',
|
||||
enabled: true,
|
||||
target_id: null,
|
||||
node_id: null,
|
||||
created_by: 'admin',
|
||||
last_status: null,
|
||||
});
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(95);
|
||||
|
||||
expect(mockUpdateScheduledTask).toHaveBeenCalledWith(
|
||||
95,
|
||||
expect.objectContaining({
|
||||
enabled: 0,
|
||||
last_status: 'failure',
|
||||
last_error: expect.stringContaining('no longer valid'),
|
||||
})
|
||||
);
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
expect.stringContaining('failed'),
|
||||
undefined
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── executeSnapshot (T3) ─────────────────────────────────────────────
|
||||
|
||||
describe('SchedulerService - executeSnapshot', () => {
|
||||
it('creates a fleet snapshot capturing all local nodes', async () => {
|
||||
mockGetScheduledTask.mockReturnValue({
|
||||
id: 75,
|
||||
name: 'nightly-snapshot',
|
||||
action: 'snapshot',
|
||||
target_type: 'fleet',
|
||||
cron_expression: '0 3 * * *',
|
||||
enabled: true,
|
||||
created_by: 'admin',
|
||||
last_status: null,
|
||||
});
|
||||
mockGetNodes.mockReturnValue([
|
||||
{ id: 1, name: 'local', type: 'local' },
|
||||
]);
|
||||
mockGetStacks.mockResolvedValue(['app1']);
|
||||
mockGetStackContent.mockResolvedValue('version: "3"\nservices:\n web:\n image: nginx');
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(75);
|
||||
|
||||
expect(mockCreateSnapshot).toHaveBeenCalledWith(
|
||||
expect.stringContaining('nightly-snapshot'),
|
||||
'admin',
|
||||
1,
|
||||
1,
|
||||
expect.any(String),
|
||||
);
|
||||
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ status: 'success' })
|
||||
);
|
||||
});
|
||||
|
||||
it('handles nodes with no stacks gracefully', async () => {
|
||||
mockGetScheduledTask.mockReturnValue({
|
||||
id: 76,
|
||||
name: 'empty-snapshot',
|
||||
action: 'snapshot',
|
||||
target_type: 'fleet',
|
||||
cron_expression: '0 3 * * *',
|
||||
enabled: true,
|
||||
created_by: 'admin',
|
||||
last_status: null,
|
||||
});
|
||||
mockGetNodes.mockReturnValue([
|
||||
{ id: 1, name: 'local', type: 'local' },
|
||||
]);
|
||||
mockGetStacks.mockResolvedValue([]);
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(76);
|
||||
|
||||
expect(mockCreateSnapshot).toHaveBeenCalled();
|
||||
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ status: 'success' })
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── executeUpdateRemote (T4) ─────────────────────────────────────────
|
||||
|
||||
describe('SchedulerService - executeUpdateRemote', () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it('proxies update execution to remote node', async () => {
|
||||
mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
|
||||
mockGetProxyTarget.mockReturnValue({
|
||||
apiUrl: 'http://remote:3000',
|
||||
apiToken: 'test-token',
|
||||
});
|
||||
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ result: 'Stack "web": updated (nginx:latest).' }),
|
||||
});
|
||||
vi.stubGlobal('fetch', mockFetch);
|
||||
|
||||
mockGetScheduledTask.mockReturnValue({
|
||||
id: 88,
|
||||
name: 'remote-update',
|
||||
action: 'update',
|
||||
cron_expression: '0 4 * * *',
|
||||
enabled: true,
|
||||
target_id: 'web-app',
|
||||
node_id: 2,
|
||||
created_by: 'admin',
|
||||
last_status: null,
|
||||
});
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(88);
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
'http://remote:3000/api/auto-update/execute',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ target: 'web-app' }),
|
||||
})
|
||||
);
|
||||
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ status: 'success' })
|
||||
);
|
||||
});
|
||||
|
||||
it('records failure when remote node returns error', async () => {
|
||||
mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
|
||||
mockGetProxyTarget.mockReturnValue({
|
||||
apiUrl: 'http://remote:3000',
|
||||
apiToken: 'test-token',
|
||||
});
|
||||
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: false,
|
||||
status: 500,
|
||||
json: async () => ({ error: 'Internal error' }),
|
||||
});
|
||||
vi.stubGlobal('fetch', mockFetch);
|
||||
|
||||
mockGetScheduledTask.mockReturnValue({
|
||||
id: 89,
|
||||
name: 'remote-update-fail',
|
||||
action: 'update',
|
||||
cron_expression: '0 4 * * *',
|
||||
enabled: true,
|
||||
target_id: 'web-app',
|
||||
node_id: 2,
|
||||
created_by: 'admin',
|
||||
last_status: null,
|
||||
});
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(89);
|
||||
|
||||
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ status: 'failure', error: expect.stringContaining('Internal error') })
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5015,7 +5015,7 @@ app.post('/api/scheduled-tasks', (req: Request, res: Response): void => {
|
||||
prune_label_filter: prune_label_filter ? prune_label_filter.trim() : null,
|
||||
});
|
||||
|
||||
if (isDebugEnabled()) console.debug(`[ScheduledTasks:debug] Created task id=${id} action=${action} target=${target_id || 'none'}`);
|
||||
console.log(`[ScheduledTasks] Created task id=${id} action=${action} target=${target_id || 'none'}`);
|
||||
const task = DatabaseService.getInstance().getScheduledTask(id);
|
||||
res.status(201).json(task);
|
||||
} catch (error) {
|
||||
@@ -5131,7 +5131,7 @@ app.put('/api/scheduled-tasks/:id', (req: Request, res: Response): void => {
|
||||
}
|
||||
|
||||
db.updateScheduledTask(id, updates as Partial<Omit<ScheduledTask, 'id'>>);
|
||||
if (isDebugEnabled()) console.debug(`[ScheduledTasks:debug] Updated task id=${id}`);
|
||||
console.log(`[ScheduledTasks] Updated task id=${id}`);
|
||||
const task = db.getScheduledTask(id);
|
||||
res.json(task);
|
||||
} catch (error) {
|
||||
@@ -5153,7 +5153,7 @@ app.delete('/api/scheduled-tasks/:id', (req: Request, res: Response): void => {
|
||||
if (!requireScheduledTaskTier(existing.action, req, res)) return;
|
||||
|
||||
db.deleteScheduledTask(id);
|
||||
if (isDebugEnabled()) console.debug(`[ScheduledTasks:debug] Deleted task id=${id}`);
|
||||
console.log(`[ScheduledTasks] Deleted task id=${id}`);
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('[ScheduledTasks] Delete error:', error);
|
||||
@@ -5182,6 +5182,7 @@ app.patch('/api/scheduled-tasks/:id/toggle', (req: Request, res: Response): void
|
||||
updated_at: Date.now(),
|
||||
});
|
||||
|
||||
console.log(`[ScheduledTasks] Toggled task id=${id} enabled=${newEnabled}`);
|
||||
const task = db.getScheduledTask(id);
|
||||
res.json(task);
|
||||
} catch (error) {
|
||||
@@ -5207,6 +5208,7 @@ app.post('/api/scheduled-tasks/:id/run', (req: Request, res: Response): void =>
|
||||
res.status(409).json({ error: 'Task is already running' }); return;
|
||||
}
|
||||
|
||||
console.log(`[ScheduledTasks] Manual run requested for task id=${id}`);
|
||||
scheduler.triggerTask(id).catch((err: unknown) => {
|
||||
const msg = getErrorMessage(err, String(err));
|
||||
console.error(`[ScheduledTasks] Background run error for task ${id}:`, msg);
|
||||
@@ -5885,6 +5887,7 @@ app.post('/api/auto-update/execute', authMiddleware, async (req: Request, res: R
|
||||
if (!requireAdmin(req, res)) return;
|
||||
try {
|
||||
const { target } = req.body as { target?: string };
|
||||
console.log(`[AutoUpdate] Execute requested: target="${target || ''}"`);
|
||||
if (!target || typeof target !== 'string') {
|
||||
return res.status(400).json({ error: 'Missing "target" (stack name or "*" for all)' });
|
||||
}
|
||||
|
||||
@@ -437,6 +437,7 @@ export class DatabaseService {
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scheduled_task_runs_task ON scheduled_task_runs(task_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_scheduled_task_runs_status ON scheduled_task_runs(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_next_run ON scheduled_tasks(next_run_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS stack_labels (
|
||||
@@ -1531,6 +1532,13 @@ export class DatabaseService {
|
||||
).all(taskId) as ScheduledTaskRun[];
|
||||
}
|
||||
|
||||
public markStaleRunsAsFailed(): number {
|
||||
const result = this.db.prepare(
|
||||
'UPDATE scheduled_task_runs SET status = ?, completed_at = ?, error = ? WHERE status = ?'
|
||||
).run('failure', Date.now(), 'Server restarted during execution', 'running');
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
public cleanupOldTaskRuns(retentionDays = 30): void {
|
||||
const cutoff = Date.now() - (retentionDays * 24 * 60 * 60 * 1000);
|
||||
this.db.prepare('DELETE FROM scheduled_task_runs WHERE started_at < ?').run(cutoff);
|
||||
|
||||
@@ -29,6 +29,7 @@ export class SchedulerService {
|
||||
|
||||
public start(): void {
|
||||
if (this.intervalId) return;
|
||||
this.cleanupStaleRuns();
|
||||
this.intervalId = setInterval(() => this.tick(), 60_000);
|
||||
setTimeout(() => this.tick(), 10_000);
|
||||
console.log('[SchedulerService] Started');
|
||||
@@ -42,6 +43,17 @@ export class SchedulerService {
|
||||
console.log('[SchedulerService] Stopped');
|
||||
}
|
||||
|
||||
private cleanupStaleRuns(): void {
|
||||
try {
|
||||
const count = DatabaseService.getInstance().markStaleRunsAsFailed();
|
||||
if (count > 0) {
|
||||
console.log(`[SchedulerService] Cleaned up ${count} stale run record(s)`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[SchedulerService] Failed to clean up stale runs:', error);
|
||||
}
|
||||
}
|
||||
|
||||
public calculateNextRun(cronExpression: string): number {
|
||||
const expr = CronExpressionParser.parse(cronExpression);
|
||||
return expr.next().toDate().getTime();
|
||||
@@ -101,6 +113,7 @@ export class SchedulerService {
|
||||
const task = db.getScheduledTask(taskId);
|
||||
if (!task) throw new Error('Task not found');
|
||||
if (this.runningTasks.has(task.id)) throw new Error('Task is already running');
|
||||
console.log(`[SchedulerService] Manual trigger: task "${task.name}" (id=${task.id})`);
|
||||
this.runningTasks.add(task.id);
|
||||
try {
|
||||
await this.executeTask(task, 'manual');
|
||||
@@ -129,6 +142,8 @@ export class SchedulerService {
|
||||
if (node.status === 'offline') throw new Error(`Target node "${node.name}" is offline`);
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} pre-checks passed, executing ${task.action}`);
|
||||
const actionStart = Date.now();
|
||||
let output = '';
|
||||
switch (task.action) {
|
||||
case 'restart':
|
||||
@@ -145,6 +160,8 @@ export class SchedulerService {
|
||||
break;
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} action completed in ${Date.now() - actionStart}ms`);
|
||||
|
||||
const nextRun = this.calculateNextRun(task.cron_expression);
|
||||
db.updateScheduledTask(task.id, {
|
||||
last_run_at: Date.now(),
|
||||
@@ -169,18 +186,26 @@ export class SchedulerService {
|
||||
} catch (error: unknown) {
|
||||
const errMsg = error instanceof Error ? error.message : String(error);
|
||||
let nextRun: number | null = null;
|
||||
let cronInvalid = false;
|
||||
try {
|
||||
nextRun = this.calculateNextRun(task.cron_expression);
|
||||
} catch {
|
||||
// If cron expression is somehow invalid, disable the task
|
||||
cronInvalid = true;
|
||||
}
|
||||
db.updateScheduledTask(task.id, {
|
||||
const updates: Partial<Omit<ScheduledTask, 'id'>> = {
|
||||
last_run_at: Date.now(),
|
||||
next_run_at: nextRun,
|
||||
last_status: 'failure',
|
||||
last_error: errMsg,
|
||||
last_error: cronInvalid
|
||||
? `${errMsg}. Cron expression "${task.cron_expression}" is no longer valid; task has been disabled.`
|
||||
: errMsg,
|
||||
updated_at: Date.now(),
|
||||
});
|
||||
};
|
||||
if (cronInvalid) {
|
||||
updates.enabled = 0;
|
||||
console.warn(`[SchedulerService] Task "${task.name}" (id=${task.id}) auto-disabled: cron expression invalid`);
|
||||
}
|
||||
db.updateScheduledTask(task.id, updates);
|
||||
db.updateScheduledTaskRun(runId, {
|
||||
completed_at: Date.now(),
|
||||
status: 'failure',
|
||||
@@ -361,6 +386,9 @@ export class SchedulerService {
|
||||
|
||||
private async executePrune(task: ScheduledTask): Promise<string> {
|
||||
const nodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
|
||||
if (task.node_id == null && isDebugEnabled()) {
|
||||
console.log(`[SchedulerService:debug] Prune task ${task.id}: no node_id specified, using default node ${nodeId}`);
|
||||
}
|
||||
const docker = DockerController.getInstance(nodeId);
|
||||
const allTargets = ['containers', 'images', 'networks', 'volumes'] as const;
|
||||
type PruneTarget = typeof allTargets[number];
|
||||
|
||||
@@ -180,3 +180,15 @@ Check that the target stack exists and has at least one running container. For w
|
||||
### "Run Now" finishes instantly
|
||||
|
||||
This is expected behavior. The Run Now button triggers the update check in the background and returns immediately. The actual check runs asynchronously; check the run history panel to see the result once it completes.
|
||||
|
||||
### Policy was automatically disabled
|
||||
|
||||
If a policy's cron expression becomes invalid after creation, the scheduler disables the policy and records the reason in the status column. To fix this:
|
||||
|
||||
1. Open the policy in edit mode.
|
||||
2. Re-enter a valid cron expression (or choose a preset).
|
||||
3. Re-enable the policy using the toggle switch.
|
||||
|
||||
### Run shows "Server restarted during execution"
|
||||
|
||||
This means Sencho was restarted while this policy's update check was mid-execution. The run was marked as failed automatically on startup. The policy itself is still enabled and will run at its next scheduled time. Use **Run Now** if you want to trigger it immediately.
|
||||
|
||||
@@ -166,3 +166,19 @@ The Scheduler Service runs in the background and checks for due tasks every 60 s
|
||||
5. The next run time is recalculated from the cron expression.
|
||||
|
||||
If a task is still running from a previous execution, the scheduler skips it to prevent overlap.
|
||||
|
||||
If the server restarts while a task is mid-execution, the orphaned run record is automatically marked as failed with a "Server restarted during execution" message on the next startup. No manual cleanup is needed.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Task was automatically disabled
|
||||
|
||||
If a task's cron expression becomes invalid after creation (for example, due to a corrupt database edit or an expression that was valid in a previous version), the scheduler disables the task and records the reason in the last error column. To fix this:
|
||||
|
||||
1. Open the task in edit mode.
|
||||
2. Re-enter a valid cron expression.
|
||||
3. Re-enable the task using the toggle switch.
|
||||
|
||||
### Run shows "Server restarted during execution"
|
||||
|
||||
This means Sencho was restarted (or crashed) while this task was mid-execution. The run was marked as failed automatically on startup. The task itself is still enabled and will run at its next scheduled time. If you want to re-run it immediately, use the **Run Now** button.
|
||||
|
||||
@@ -15,41 +15,8 @@ import { RefreshCw, Plus, Pencil, Trash2, History, Play, ChevronLeft, ChevronRig
|
||||
import { toast } from '@/components/ui/toast-store';
|
||||
import { apiFetch, fetchForNode } from '@/lib/api';
|
||||
import { PaidGate } from '@/components/PaidGate';
|
||||
import cronstrue from 'cronstrue';
|
||||
|
||||
interface ScheduledTask {
|
||||
id: number;
|
||||
name: string;
|
||||
target_type: 'stack' | 'fleet' | 'system';
|
||||
target_id: string | null;
|
||||
node_id: number | null;
|
||||
action: 'restart' | 'snapshot' | 'prune' | 'update';
|
||||
cron_expression: string;
|
||||
enabled: number;
|
||||
created_by: string;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
last_run_at: number | null;
|
||||
next_run_at: number | null;
|
||||
last_status: string | null;
|
||||
last_error: string | null;
|
||||
}
|
||||
|
||||
interface TaskRun {
|
||||
id: number;
|
||||
task_id: number;
|
||||
started_at: number;
|
||||
completed_at: number | null;
|
||||
status: 'running' | 'success' | 'failure';
|
||||
output: string | null;
|
||||
error: string | null;
|
||||
triggered_by: 'scheduler' | 'manual';
|
||||
}
|
||||
|
||||
interface NodeOption {
|
||||
id: number;
|
||||
name: string;
|
||||
}
|
||||
import type { ScheduledTask, TaskRun, NodeOption } from '@/types/scheduling';
|
||||
import { getCronDescription, formatTimestamp } from '@/lib/scheduling';
|
||||
|
||||
const CRON_PRESETS = [
|
||||
{ label: 'Every 6 hours', value: '0 */6 * * *' },
|
||||
@@ -60,19 +27,6 @@ const CRON_PRESETS = [
|
||||
{ label: 'Custom', value: 'custom' },
|
||||
];
|
||||
|
||||
function getCronDescription(expression: string): string {
|
||||
try {
|
||||
return cronstrue.toString(expression);
|
||||
} catch {
|
||||
return 'Invalid expression';
|
||||
}
|
||||
}
|
||||
|
||||
function formatTimestamp(ts: number | null): string {
|
||||
if (!ts) return '-';
|
||||
return new Date(ts).toLocaleString();
|
||||
}
|
||||
|
||||
interface AutoUpdatePoliciesProps {
|
||||
filterNodeId?: number | null;
|
||||
onClearFilter?: () => void;
|
||||
|
||||
@@ -4,9 +4,10 @@ import { Button } from '@/components/ui/button';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
|
||||
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from '@/components/ui/dialog';
|
||||
import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle, DialogFooter } from '@/components/ui/dialog';
|
||||
import { AlertDialog, AlertDialogAction, AlertDialogCancel, AlertDialogContent, AlertDialogDescription, AlertDialogFooter, AlertDialogHeader, AlertDialogTitle } from '@/components/ui/alert-dialog';
|
||||
import { Sheet, SheetContent, SheetHeader, SheetTitle } from '@/components/ui/sheet';
|
||||
import { ScrollArea } from '@/components/ui/scroll-area';
|
||||
import { Switch } from '@/components/ui/switch';
|
||||
import { Label } from '@/components/ui/label';
|
||||
import { Checkbox } from '@/components/ui/checkbox';
|
||||
@@ -14,44 +15,8 @@ import { Clock, Plus, Pencil, Trash2, History, RefreshCw, Play, ChevronLeft, Che
|
||||
import { toast } from '@/components/ui/toast-store';
|
||||
import { apiFetch, fetchForNode } from '@/lib/api';
|
||||
import { Combobox } from '@/components/ui/combobox';
|
||||
import cronstrue from 'cronstrue';
|
||||
|
||||
interface ScheduledTask {
|
||||
id: number;
|
||||
name: string;
|
||||
target_type: 'stack' | 'fleet' | 'system';
|
||||
target_id: string | null;
|
||||
node_id: number | null;
|
||||
action: 'restart' | 'snapshot' | 'prune';
|
||||
cron_expression: string;
|
||||
enabled: number;
|
||||
created_by: string;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
last_run_at: number | null;
|
||||
next_run_at: number | null;
|
||||
last_status: string | null;
|
||||
last_error: string | null;
|
||||
prune_targets: string | null;
|
||||
target_services: string | null;
|
||||
prune_label_filter: string | null;
|
||||
}
|
||||
|
||||
interface TaskRun {
|
||||
id: number;
|
||||
task_id: number;
|
||||
started_at: number;
|
||||
completed_at: number | null;
|
||||
status: 'running' | 'success' | 'failure';
|
||||
output: string | null;
|
||||
error: string | null;
|
||||
triggered_by: 'scheduler' | 'manual';
|
||||
}
|
||||
|
||||
interface NodeOption {
|
||||
id: number;
|
||||
name: string;
|
||||
}
|
||||
import type { ScheduledTask, TaskRun, NodeOption } from '@/types/scheduling';
|
||||
import { getCronDescription, formatTimestamp } from '@/lib/scheduling';
|
||||
|
||||
const ACTION_OPTIONS = [
|
||||
{ value: 'restart', label: 'Restart Stack', targetType: 'stack' as const },
|
||||
@@ -59,19 +24,6 @@ const ACTION_OPTIONS = [
|
||||
{ value: 'prune', label: 'System Prune', targetType: 'system' as const },
|
||||
];
|
||||
|
||||
function getCronDescription(expression: string): string {
|
||||
try {
|
||||
return cronstrue.toString(expression);
|
||||
} catch {
|
||||
return 'Invalid expression';
|
||||
}
|
||||
}
|
||||
|
||||
function formatTimestamp(ts: number | null): string {
|
||||
if (!ts) return '-';
|
||||
return new Date(ts).toLocaleString();
|
||||
}
|
||||
|
||||
interface ScheduledOperationsViewProps {
|
||||
filterNodeId?: number | null;
|
||||
onClearFilter?: () => void;
|
||||
@@ -329,7 +281,7 @@ export default function ScheduledOperationsView({ filterNodeId, onClearFilter }:
|
||||
try {
|
||||
const res = await apiFetch(`/scheduled-tasks/${task.id}/run`, { method: 'POST', localOnly: true });
|
||||
if (res.ok) {
|
||||
toast.success(`Task "${task.name}" executed successfully`);
|
||||
toast.success(`Task "${task.name}" triggered`);
|
||||
fetchTasks();
|
||||
} else {
|
||||
const data = await res.json().catch(() => ({}));
|
||||
@@ -351,16 +303,16 @@ export default function ScheduledOperationsView({ filterNodeId, onClearFilter }:
|
||||
<CardHeader>
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2">
|
||||
<Clock className="w-5 h-5" />
|
||||
<Clock className="w-5 h-5" strokeWidth={1.5} />
|
||||
<CardTitle>Scheduled Operations</CardTitle>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Button variant="outline" size="sm" onClick={fetchTasks} disabled={loading}>
|
||||
<RefreshCw className={`w-4 h-4 mr-2 ${loading ? 'animate-spin' : ''}`} />
|
||||
<RefreshCw className={`w-4 h-4 mr-2 ${loading ? 'animate-spin' : ''}`} strokeWidth={1.5} />
|
||||
Refresh
|
||||
</Button>
|
||||
<Button size="sm" onClick={openCreate}>
|
||||
<Plus className="w-4 h-4 mr-2" />
|
||||
<Plus className="w-4 h-4 mr-2" strokeWidth={1.5} />
|
||||
New Schedule
|
||||
</Button>
|
||||
</div>
|
||||
@@ -440,16 +392,16 @@ export default function ScheduledOperationsView({ filterNodeId, onClearFilter }:
|
||||
<TableCell className="text-right">
|
||||
<div className="flex items-center justify-end gap-1">
|
||||
<Button variant="ghost" size="sm" onClick={() => handleRunNow(task)} title="Run now" disabled={runningTaskId === task.id}>
|
||||
<Play className={`w-4 h-4 ${runningTaskId === task.id ? 'animate-pulse' : ''}`} />
|
||||
<Play className={`w-4 h-4 ${runningTaskId === task.id ? 'animate-pulse' : ''}`} strokeWidth={1.5} />
|
||||
</Button>
|
||||
<Button variant="ghost" size="sm" onClick={() => openRuns(task)} title="Execution history">
|
||||
<History className="w-4 h-4" />
|
||||
<History className="w-4 h-4" strokeWidth={1.5} />
|
||||
</Button>
|
||||
<Button variant="ghost" size="sm" onClick={() => openEdit(task)} title="Edit">
|
||||
<Pencil className="w-4 h-4" />
|
||||
<Pencil className="w-4 h-4" strokeWidth={1.5} />
|
||||
</Button>
|
||||
<Button variant="ghost" size="sm" onClick={() => setDeleteTarget(task)} title="Delete" className="text-destructive hover:text-destructive">
|
||||
<Trash2 className="w-4 h-4" />
|
||||
<Button variant="ghost" size="sm" onClick={() => setDeleteTarget(task)} title="Delete" className="text-destructive/60 hover:bg-destructive hover:text-destructive-foreground">
|
||||
<Trash2 className="w-4 h-4" strokeWidth={1.5} />
|
||||
</Button>
|
||||
</div>
|
||||
</TableCell>
|
||||
@@ -466,6 +418,7 @@ export default function ScheduledOperationsView({ filterNodeId, onClearFilter }:
|
||||
<DialogContent className="sm:max-w-[480px]">
|
||||
<DialogHeader>
|
||||
<DialogTitle>{editingTask ? 'Edit Scheduled Task' : 'New Scheduled Task'}</DialogTitle>
|
||||
<DialogDescription className="sr-only">Configure a scheduled operation task.</DialogDescription>
|
||||
</DialogHeader>
|
||||
<div className="space-y-4">
|
||||
<div className="space-y-2">
|
||||
@@ -621,7 +574,8 @@ export default function ScheduledOperationsView({ filterNodeId, onClearFilter }:
|
||||
)}
|
||||
</div>
|
||||
</SheetHeader>
|
||||
<div className="mt-4">
|
||||
<ScrollArea className="mt-4 flex-1" style={{ maxHeight: 'calc(100vh - 10rem)' }}>
|
||||
<div>
|
||||
{runsLoading ? (
|
||||
<div className="text-center text-muted-foreground py-8">Loading...</div>
|
||||
) : runs.length === 0 ? (
|
||||
@@ -678,17 +632,18 @@ export default function ScheduledOperationsView({ filterNodeId, onClearFilter }:
|
||||
</p>
|
||||
<div className="flex items-center gap-2">
|
||||
<Button variant="outline" size="sm" onClick={() => openRuns(runsTask, runsPage - 1)} disabled={runsPage <= 1}>
|
||||
<ChevronLeft className="w-4 h-4" />
|
||||
<ChevronLeft className="w-4 h-4" strokeWidth={1.5} />
|
||||
</Button>
|
||||
<Button variant="outline" size="sm" onClick={() => openRuns(runsTask, runsPage + 1)} disabled={runsPage >= Math.ceil(runsTotal / runsLimit)}>
|
||||
<ChevronRight className="w-4 h-4" />
|
||||
<ChevronRight className="w-4 h-4" strokeWidth={1.5} />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</ScrollArea>
|
||||
</SheetContent>
|
||||
</Sheet>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
import cronstrue from 'cronstrue';
|
||||
|
||||
export function getCronDescription(expression: string): string {
|
||||
try {
|
||||
return cronstrue.toString(expression);
|
||||
} catch {
|
||||
return 'Invalid expression';
|
||||
}
|
||||
}
|
||||
|
||||
export function formatTimestamp(ts: number | null): string {
|
||||
if (ts == null) return '-';
|
||||
return new Date(ts).toLocaleString();
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
export interface ScheduledTask {
|
||||
id: number;
|
||||
name: string;
|
||||
target_type: 'stack' | 'fleet' | 'system';
|
||||
target_id: string | null;
|
||||
node_id: number | null;
|
||||
action: 'restart' | 'snapshot' | 'prune' | 'update';
|
||||
cron_expression: string;
|
||||
enabled: number;
|
||||
created_by: string;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
last_run_at: number | null;
|
||||
next_run_at: number | null;
|
||||
last_status: 'success' | 'failure' | null;
|
||||
last_error: string | null;
|
||||
prune_targets: string | null;
|
||||
target_services: string | null;
|
||||
prune_label_filter: string | null;
|
||||
}
|
||||
|
||||
export interface TaskRun {
|
||||
id: number;
|
||||
task_id: number;
|
||||
started_at: number;
|
||||
completed_at: number | null;
|
||||
status: 'running' | 'success' | 'failure';
|
||||
output: string | null;
|
||||
error: string | null;
|
||||
triggered_by: 'scheduler' | 'manual';
|
||||
}
|
||||
|
||||
export interface NodeOption {
|
||||
id: number;
|
||||
name: string;
|
||||
}
|
||||
Reference in New Issue
Block a user