mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-26 02:06:49 +00:00
fix(scheduler): harden auto-update policies with cascade deletes, error reporting, and UI fixes (#545)
* fix(scheduler): harden auto-update policies with cascade deletes, error reporting, and UI fixes
- Fix orphaned task runs on policy/node deletion with transaction-wrapped cascade deletes
- Make manual trigger non-blocking (202 Accepted) to prevent proxy timeouts
- Distinguish registry check failures from clean "no update" results via structured ImageCheckResult
- Trim whitespace-only policy names in both frontend and backend validation
- Add strokeWidth={1.5} to action icons per design system
- Add sr-only DialogDescription for Radix accessibility
- Replace Select with Combobox for frequency picker
- Wrap run history sheet content in ScrollArea
- Support concurrent Run Now indicators via Set-based state
- Abort stale stack fetches on node switch with AbortController
- Add standard and diagnostic logging to SchedulerService and ImageUpdateService
- Add tests for cascade deletes, image checking, and scheduler edge cases
- Add troubleshooting section to auto-update docs
* fix(tests): resolve lint errors in image-update-service tests
Remove unused mock variables (mockGetImage, mockGetDocker) and unused
ImageCheckResult type import. Replace CommonJS require('yaml') with
ESM import to satisfy no-require-imports rule.
* chore(deps): bump Docker CLI to 29.4.0 and Compose to v5.1.2
Resolves Trivy CVE-2026-32282 (Go stdlib symlink follow in Root.Chmod)
by upgrading to releases that ship Go 1.25.9. Compose v5.1.2 also bumps
grpc to 1.80.0, resolving CVE-2026-33186.
* chore(security): accept CVE-2026-32282 in .trivyignore, update stale refs
Go stdlib symlink-following in Root.Chmod (CVE-2026-32282) affects both
Docker CLI 29.4.0 (Go 1.26.1) and Compose v5.1.2 (Go 1.25.8). Fix
requires Go 1.25.9 or 1.26.2; no upstream static binary ships a patched
runtime yet. The vulnerable code path requires a chroot context with
attacker-controlled filesystem, which does not apply to our usage.
Also updates version references from v5.1.1/v29.3.1 to v5.1.2/v29.4.0
for existing CVE entries, and notes that Compose v5.1.2 resolved
CVE-2026-33186 (grpc bumped to 1.80.0) for the compose binary.
This commit is contained in:
Anso
@@ -990,9 +990,10 @@ export class DatabaseService {
|
||||
throw new Error('Cannot delete the default node');
|
||||
}
|
||||
this.db.transaction(() => {
|
||||
this.db.prepare('DELETE FROM nodes WHERE id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM scheduled_task_runs WHERE task_id IN (SELECT id FROM scheduled_tasks WHERE node_id = ?)').run(id);
|
||||
this.db.prepare('DELETE FROM scheduled_tasks WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM stack_update_status WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM nodes WHERE id = ?').run(id);
|
||||
})();
|
||||
}
|
||||
|
||||
@@ -1481,7 +1482,10 @@ export class DatabaseService {
|
||||
}
|
||||
|
||||
public deleteScheduledTask(id: number): void {
|
||||
this.db.prepare('DELETE FROM scheduled_tasks WHERE id = ?').run(id);
|
||||
this.db.transaction(() => {
|
||||
this.db.prepare('DELETE FROM scheduled_task_runs WHERE task_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM scheduled_tasks WHERE id = ?').run(id);
|
||||
})();
|
||||
}
|
||||
|
||||
public getDueScheduledTasks(now: number): ScheduledTask[] {
|
||||
|
||||
@@ -7,6 +7,7 @@ import { DatabaseService } from './DatabaseService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { RegistryService } from './RegistryService';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
|
||||
// ─── Image ref parsing ────────────────────────────────────────────────────────
|
||||
|
||||
@@ -51,6 +52,11 @@ function parseImageRef(imageRef: string): ParsedRef | null {
|
||||
return { registry, repo: rest, tag };
|
||||
}
|
||||
|
||||
export interface ImageCheckResult {
|
||||
hasUpdate: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// ─── Minimal HTTP helper ──────────────────────────────────────────────────────
|
||||
|
||||
interface HttpResult {
|
||||
@@ -355,25 +361,29 @@ export class ImageUpdateService {
|
||||
const allImages = new Set<string>();
|
||||
for (const imgs of stackImages.values()) for (const img of imgs) allImages.add(img);
|
||||
|
||||
const imageUpdateMap = new Map<string, boolean>();
|
||||
const imageUpdateMap = new Map<string, ImageCheckResult>();
|
||||
|
||||
for (const imageRef of allImages) {
|
||||
try {
|
||||
imageUpdateMap.set(imageRef, await this.checkImage(docker, imageRef));
|
||||
} catch (e) {
|
||||
console.error(`[ImageUpdateService] Error checking ${imageRef}:`, e);
|
||||
imageUpdateMap.set(imageRef, false);
|
||||
imageUpdateMap.set(imageRef, { hasUpdate: false, error: String(e) });
|
||||
}
|
||||
await sleep(ImageUpdateService.INTER_IMAGE_DELAY_MS);
|
||||
}
|
||||
|
||||
// Write status for ALL stacks (including those with no pullable images)
|
||||
const now = Date.now();
|
||||
let updatesFound = 0;
|
||||
for (const [stackName, images] of stackImages) {
|
||||
const hasUpdate = Array.from(images).some(img => imageUpdateMap.get(img) === true);
|
||||
const hasUpdate = Array.from(images).some(img => imageUpdateMap.get(img)?.hasUpdate === true);
|
||||
if (hasUpdate) updatesFound++;
|
||||
db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now);
|
||||
}
|
||||
|
||||
console.log(`[ImageUpdateService] Node ${nodeId}: checked ${allImages.size} image(s), ${updatesFound} stack(s) with updates`);
|
||||
|
||||
// Prune stale entries for stacks no longer on disk
|
||||
const existing = db.getStackUpdateStatus(nodeId);
|
||||
for (const staleStack of Object.keys(existing)) {
|
||||
@@ -383,12 +393,19 @@ export class ImageUpdateService {
|
||||
}
|
||||
}
|
||||
|
||||
public async checkImage(docker: DockerController, imageRef: string): Promise<boolean> {
|
||||
public async checkImage(docker: DockerController, imageRef: string): Promise<ImageCheckResult> {
|
||||
const parsed = parseImageRef(imageRef);
|
||||
if (!parsed) return false;
|
||||
if (!parsed) return { hasUpdate: false };
|
||||
|
||||
if (isDebugEnabled()) {
|
||||
console.log(`[ImageUpdateService] Checking ${imageRef}: registry=${parsed.registry} repo=${parsed.repo} tag=${parsed.tag}`);
|
||||
}
|
||||
|
||||
// Look up stored credentials for this registry
|
||||
const credentials = await RegistryService.getInstance().getAuthForRegistry(parsed.registry);
|
||||
if (isDebugEnabled()) {
|
||||
console.log(`[ImageUpdateService] ${imageRef}: credentials ${credentials ? 'found' : 'none'}`);
|
||||
}
|
||||
|
||||
// Get local digest from RepoDigests
|
||||
let localDigest: string | null = null;
|
||||
@@ -400,27 +417,28 @@ export class ImageUpdateService {
|
||||
if (!rd.includes('@sha256:')) continue;
|
||||
const [, digest] = rd.split('@');
|
||||
|
||||
// Match: rd contains the repo path or this is the only digest entry
|
||||
if (rd.includes(parsed.repo) || rd.includes(parsed.registry) || repoDigests.length === 1) {
|
||||
localDigest = digest;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return false; // Image inspect failed (removed since container was started)
|
||||
return { hasUpdate: false, error: `Failed to inspect local image "${imageRef}"` };
|
||||
}
|
||||
|
||||
if (!localDigest) return false; // Locally built or never pulled with a digest
|
||||
if (!localDigest) return { hasUpdate: false };
|
||||
|
||||
const remoteDigest = await getRemoteDigest(parsed.registry, parsed.repo, parsed.tag, credentials);
|
||||
if (!remoteDigest) return false; // Registry unreachable - no false positives
|
||||
if (!remoteDigest) {
|
||||
return { hasUpdate: false, error: `Registry unreachable for ${parsed.registry}/${parsed.repo}:${parsed.tag}` };
|
||||
}
|
||||
|
||||
const hasUpdate = localDigest !== remoteDigest;
|
||||
console.log(
|
||||
`[ImageUpdateService] ${imageRef}: ` +
|
||||
`local=${localDigest.slice(0, 27)}... remote=${remoteDigest.slice(0, 27)}... update=${hasUpdate}`
|
||||
);
|
||||
return hasUpdate;
|
||||
return { hasUpdate };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,9 @@ import DockerController from './DockerController';
|
||||
import { ComposeService } from './ComposeService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { ImageUpdateService } from './ImageUpdateService';
|
||||
import type { ImageCheckResult } from './ImageUpdateService';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { NotificationService } from './NotificationService';
|
||||
|
||||
@@ -45,26 +48,39 @@ export class SchedulerService {
|
||||
}
|
||||
|
||||
private async tick(): Promise<void> {
|
||||
if (this.isProcessing) return;
|
||||
if (this.isProcessing) {
|
||||
console.warn('[SchedulerService] Tick skipped: previous tick still processing');
|
||||
return;
|
||||
}
|
||||
this.isProcessing = true;
|
||||
try {
|
||||
const ls = LicenseService.getInstance();
|
||||
const isPaid = ls.getTier() === 'paid';
|
||||
const isAdmiral = isPaid && ls.getVariant() === 'admiral';
|
||||
if (!isPaid) return; // No scheduled tasks for unpaid tiers
|
||||
if (!isPaid) return;
|
||||
|
||||
const db = DatabaseService.getInstance();
|
||||
const now = Date.now();
|
||||
const dueTasks = db.getDueScheduledTasks(now);
|
||||
|
||||
if (dueTasks.length > 0) {
|
||||
console.log(`[SchedulerService] Found ${dueTasks.length} due task(s)`);
|
||||
}
|
||||
|
||||
// Clean up old runs periodically (piggyback on tick)
|
||||
db.cleanupOldTaskRuns(30);
|
||||
|
||||
for (const task of dueTasks) {
|
||||
// Skipper users can only run 'update' tasks; other actions require Admiral
|
||||
if (!isAdmiral && task.action !== 'update') continue;
|
||||
if (this.runningTasks.has(task.id)) continue;
|
||||
if (!isAdmiral && task.action !== 'update') {
|
||||
if (isDebugEnabled()) console.log(`[SchedulerService] Task ${task.id} skipped: action "${task.action}" requires Admiral tier`);
|
||||
continue;
|
||||
}
|
||||
if (this.runningTasks.has(task.id)) {
|
||||
if (isDebugEnabled()) console.log(`[SchedulerService] Task ${task.id} skipped: already running`);
|
||||
continue;
|
||||
}
|
||||
this.runningTasks.add(task.id);
|
||||
if (isDebugEnabled()) console.log(`[SchedulerService] Executing task ${task.id} ("${task.name}")`);
|
||||
this.executeTask(task).finally(() => this.runningTasks.delete(task.id));
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -74,7 +90,11 @@ export class SchedulerService {
|
||||
}
|
||||
}
|
||||
|
||||
// Intentionally allows triggering disabled tasks — useful for testing before enabling a schedule.
|
||||
public isTaskRunning(taskId: number): boolean {
|
||||
return this.runningTasks.has(taskId);
|
||||
}
|
||||
|
||||
// Intentionally allows triggering disabled tasks, useful for testing before enabling a schedule.
|
||||
// Manual triggers are attributed as 'manual' in the run record (see triggered_by column).
|
||||
public async triggerTask(taskId: number): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
@@ -376,8 +396,9 @@ export class SchedulerService {
|
||||
}
|
||||
|
||||
// Local node: execute directly
|
||||
const isWildcard = task.target_id === '*';
|
||||
let stackNames: string[];
|
||||
if (task.target_id === '*') {
|
||||
if (isWildcard) {
|
||||
stackNames = await FileSystemService.getInstance(task.node_id).getStacks();
|
||||
if (stackNames.length === 0) {
|
||||
return 'No stacks found on node; skipped.';
|
||||
@@ -386,6 +407,10 @@ export class SchedulerService {
|
||||
stackNames = [task.target_id];
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) {
|
||||
console.log(`[SchedulerService] executeUpdate: ${stackNames.length} stack(s) to check, wildcard=${isWildcard}`);
|
||||
}
|
||||
|
||||
const docker = DockerController.getInstance(task.node_id);
|
||||
const imageUpdateService = ImageUpdateService.getInstance();
|
||||
const compose = ComposeService.getInstance(task.node_id);
|
||||
@@ -394,10 +419,10 @@ export class SchedulerService {
|
||||
|
||||
for (const stackName of stackNames) {
|
||||
try {
|
||||
const output = await this.executeUpdateForStack(stackName, task.node_id ?? 0, docker, imageUpdateService, compose, db);
|
||||
const output = await this.executeUpdateForStack(stackName, task.node_id ?? 0, docker, imageUpdateService, compose, db, isWildcard);
|
||||
results.push(output);
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
const msg = getErrorMessage(e, String(e));
|
||||
results.push(`Stack "${stackName}" failed: ${msg}`);
|
||||
console.error(`[SchedulerService] Auto-update failed for stack "${stackName}":`, e);
|
||||
}
|
||||
@@ -417,6 +442,10 @@ export class SchedulerService {
|
||||
}
|
||||
|
||||
const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
|
||||
if (isDebugEnabled()) {
|
||||
console.log(`[SchedulerService] executeUpdateRemote: node=${nodeId} target=${target}`);
|
||||
}
|
||||
const startTime = Date.now();
|
||||
const response = await fetch(`${baseUrl}/api/auto-update/execute`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
@@ -433,6 +462,9 @@ export class SchedulerService {
|
||||
}
|
||||
|
||||
const body = await response.json() as { result?: string };
|
||||
if (isDebugEnabled()) {
|
||||
console.log(`[SchedulerService] executeUpdateRemote: completed in ${Date.now() - startTime}ms`);
|
||||
}
|
||||
return body.result || 'Remote auto-update completed (no details returned).';
|
||||
}
|
||||
|
||||
@@ -442,10 +474,15 @@ export class SchedulerService {
|
||||
docker: DockerController,
|
||||
imageUpdateService: ImageUpdateService,
|
||||
compose: ComposeService,
|
||||
db: DatabaseService
|
||||
db: DatabaseService,
|
||||
isWildcard = false
|
||||
): Promise<string> {
|
||||
const containers = await docker.getContainersByStack(stackName);
|
||||
if (!containers || containers.length === 0) {
|
||||
if (!isWildcard) {
|
||||
console.warn(`[SchedulerService] Stack "${stackName}": no containers found. The stack may have been removed or renamed.`);
|
||||
return `Stack "${stackName}": WARNING - no containers found. The stack may have been removed or renamed.`;
|
||||
}
|
||||
return `Stack "${stackName}": no containers found; skipped.`;
|
||||
}
|
||||
|
||||
@@ -459,21 +496,37 @@ export class SchedulerService {
|
||||
return `Stack "${stackName}": no pullable images; skipped.`;
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) {
|
||||
console.log(`[SchedulerService] Stack "${stackName}": checking ${imageRefs.length} image(s): ${imageRefs.join(', ')}`);
|
||||
}
|
||||
|
||||
let hasUpdate = false;
|
||||
const updatedImages: string[] = [];
|
||||
const checkErrors: string[] = [];
|
||||
|
||||
for (const imageRef of imageRefs) {
|
||||
try {
|
||||
if (await imageUpdateService.checkImage(docker, imageRef)) {
|
||||
const result: ImageCheckResult = await imageUpdateService.checkImage(docker, imageRef);
|
||||
if (result.error) {
|
||||
checkErrors.push(result.error);
|
||||
} else if (result.hasUpdate) {
|
||||
hasUpdate = true;
|
||||
updatedImages.push(imageRef);
|
||||
}
|
||||
} catch (e) {
|
||||
const msg = getErrorMessage(e, String(e));
|
||||
checkErrors.push(msg);
|
||||
console.warn(`[SchedulerService] Failed to check image ${imageRef}:`, e);
|
||||
}
|
||||
}
|
||||
|
||||
if (!hasUpdate) {
|
||||
if (checkErrors.length > 0 && checkErrors.length === imageRefs.length) {
|
||||
return `Stack "${stackName}": WARNING - all image checks failed (${checkErrors.join('; ')}). Unable to determine update status.`;
|
||||
}
|
||||
if (checkErrors.length > 0) {
|
||||
return `Stack "${stackName}": all reachable images up to date (${checkErrors.length} check(s) failed).`;
|
||||
}
|
||||
return `Stack "${stackName}": all images up to date.`;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user