mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-08 09:54:26 +00:00
feat(schedules): auto-update stacks by Stack Label (#1717)
* feat(schedules): auto-update stacks by Stack Label Add a reusable selector_type/selector_value on scheduled tasks so admins can schedule image updates against live Stack Label membership across the fleet or one node, reusing fleet label resolution and the existing auto-update orchestrator. * fix(image-updates): sanitize auto-update execute failure logs Use a static format string and sanitizeForLog so CodeQL no longer flags user-controlled stack names and error text in the execute catch. * fix(ui): space Scope label from fleet/node segmented control Match the Schedule row layout so the inline SegmentedControl no longer sits flush against the Scope label. * fix(ui): remove redundant wrapper around Scope segmented control
This commit is contained in:
Anso
@@ -714,6 +714,10 @@ export interface ScheduledTask {
|
||||
prune_targets: string | null;
|
||||
target_services: string | null;
|
||||
prune_label_filter: string | null;
|
||||
/** Optional dynamic target selector; currently only 'stack-label'. */
|
||||
selector_type: string | null;
|
||||
/** Selector payload (e.g. Stack Label name when selector_type is stack-label). */
|
||||
selector_value: string | null;
|
||||
delete_after_run?: number;
|
||||
// Absolute epoch-ms fire time for a one-time ('once') schedule. A 5-field
|
||||
// cron has no year field, so the chosen instant (including year) is persisted
|
||||
@@ -1936,6 +1940,8 @@ export class DatabaseService {
|
||||
maybeAddCol('scheduled_tasks', 'prune_targets', 'TEXT DEFAULT NULL');
|
||||
maybeAddCol('scheduled_tasks', 'target_services', 'TEXT DEFAULT NULL');
|
||||
maybeAddCol('scheduled_tasks', 'prune_label_filter', 'TEXT DEFAULT NULL');
|
||||
maybeAddCol('scheduled_tasks', 'selector_type', 'TEXT DEFAULT NULL');
|
||||
maybeAddCol('scheduled_tasks', 'selector_value', 'TEXT DEFAULT NULL');
|
||||
maybeAddCol('scheduled_tasks', 'delete_after_run', 'INTEGER DEFAULT 0');
|
||||
maybeAddCol('scheduled_tasks', 'run_at', 'INTEGER DEFAULT NULL');
|
||||
|
||||
@@ -6041,13 +6047,14 @@ export class DatabaseService {
|
||||
|
||||
public createScheduledTask(task: Omit<ScheduledTask, 'id'>): number {
|
||||
const result = this.db.prepare(
|
||||
'INSERT INTO scheduled_tasks (name, target_type, target_id, node_id, action, cron_expression, enabled, created_by, created_at, updated_at, last_run_at, next_run_at, last_status, last_error, prune_targets, target_services, prune_label_filter, delete_after_run, run_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
|
||||
'INSERT INTO scheduled_tasks (name, target_type, target_id, node_id, action, cron_expression, enabled, created_by, created_at, updated_at, last_run_at, next_run_at, last_status, last_error, prune_targets, target_services, prune_label_filter, selector_type, selector_value, delete_after_run, run_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
|
||||
).run(
|
||||
task.name, task.target_type, task.target_id, task.node_id,
|
||||
task.action, task.cron_expression, task.enabled, task.created_by,
|
||||
task.created_at, task.updated_at, task.last_run_at, task.next_run_at,
|
||||
task.last_status, task.last_error, task.prune_targets, task.target_services,
|
||||
task.prune_label_filter, task.delete_after_run ?? 0, task.run_at ?? null
|
||||
task.prune_label_filter, task.selector_type ?? null, task.selector_value ?? null,
|
||||
task.delete_after_run ?? 0, task.run_at ?? null
|
||||
);
|
||||
return result.lastInsertRowid as number;
|
||||
}
|
||||
@@ -6064,6 +6071,8 @@ export class DatabaseService {
|
||||
last_status: updates.last_status, last_error: updates.last_error,
|
||||
prune_targets: updates.prune_targets, target_services: updates.target_services,
|
||||
prune_label_filter: updates.prune_label_filter,
|
||||
selector_type: updates.selector_type,
|
||||
selector_value: updates.selector_value,
|
||||
delete_after_run: updates.delete_after_run,
|
||||
run_at: updates.run_at,
|
||||
};
|
||||
|
||||
@@ -739,6 +739,10 @@ export class SchedulerService {
|
||||
}
|
||||
|
||||
private async executeUpdate(task: ScheduledTask): Promise<string> {
|
||||
if (task.selector_type === 'stack-label') {
|
||||
return this.executeUpdateByStackLabel(task);
|
||||
}
|
||||
|
||||
if (task.node_id == null) {
|
||||
throw new Error('Auto-update requires node_id');
|
||||
}
|
||||
@@ -791,6 +795,140 @@ export class SchedulerService {
|
||||
return results.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve live stack-label membership (fleet-wide or one node) and run the
|
||||
* existing per-stack auto-update path. Remotes receive an explicit stack
|
||||
* list; they do not evaluate the selector themselves.
|
||||
*/
|
||||
private async executeUpdateByStackLabel(task: ScheduledTask): Promise<string> {
|
||||
const labelName = (task.selector_value ?? '').trim();
|
||||
if (!labelName) {
|
||||
throw new Error('Label-targeted auto-update requires selector_value');
|
||||
}
|
||||
|
||||
const { collectFleetLabelSummaries } = await import('../helpers/fleetLabelSummary');
|
||||
let summaries = await collectFleetLabelSummaries();
|
||||
if (task.node_id != null) {
|
||||
summaries = summaries.filter(s => s.nodeId === task.node_id);
|
||||
if (summaries.length === 0) {
|
||||
throw new Error(`Target node (id=${task.node_id}) no longer exists`);
|
||||
}
|
||||
}
|
||||
|
||||
type NodePlan = {
|
||||
nodeId: number;
|
||||
nodeName: string;
|
||||
reachable: boolean;
|
||||
stacks: string[];
|
||||
error?: string;
|
||||
};
|
||||
|
||||
const plans: NodePlan[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const summary of summaries) {
|
||||
if (!summary.reachable) {
|
||||
plans.push({
|
||||
nodeId: summary.nodeId,
|
||||
nodeName: summary.nodeName,
|
||||
reachable: false,
|
||||
stacks: [],
|
||||
error: summary.error ?? 'unreachable',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const match = summary.labels.find(l => l.name === labelName);
|
||||
const stacks: string[] = [];
|
||||
if (match) {
|
||||
for (const stackName of match.stackNames) {
|
||||
const key = `${summary.nodeId}\0${stackName}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
stacks.push(stackName);
|
||||
}
|
||||
}
|
||||
plans.push({
|
||||
nodeId: summary.nodeId,
|
||||
nodeName: summary.nodeName,
|
||||
reachable: true,
|
||||
stacks,
|
||||
});
|
||||
}
|
||||
|
||||
const lines: string[] = [
|
||||
`Selector: stack-label="${labelName}" · scope=${task.node_id == null ? 'entire fleet' : `node ${task.node_id}`}`,
|
||||
];
|
||||
for (const plan of plans) {
|
||||
if (!plan.reachable) {
|
||||
lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}): unreachable (${plan.error})`);
|
||||
} else if (plan.stacks.length === 0) {
|
||||
lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}): no stacks with label "${labelName}"`);
|
||||
} else {
|
||||
lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}): ${plan.stacks.length} stack(s) → ${plan.stacks.join(', ')}`);
|
||||
}
|
||||
}
|
||||
|
||||
const matchedStacks = plans.reduce((n, p) => n + p.stacks.length, 0);
|
||||
const unreachableCount = plans.filter(p => !p.reachable).length;
|
||||
if (matchedStacks === 0 && unreachableCount === 0) {
|
||||
lines.push(`No stacks currently match label "${labelName}"; skipped.`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
let materialFailure = unreachableCount > 0;
|
||||
const work = plans.filter(p => p.reachable && p.stacks.length > 0);
|
||||
const NODE_CONCURRENCY = 3;
|
||||
// Label-targeted runs fail closed on material stack failures or
|
||||
// unreachable scoped nodes (unlike plain node fleet update, which
|
||||
// historically returns failure lines as a successful run output).
|
||||
|
||||
const looksLikeStackFailure = (text: string): boolean =>
|
||||
/^Stack ".+" failed:/m.test(text);
|
||||
|
||||
const runNode = async (plan: NodePlan): Promise<void> => {
|
||||
const node = NodeRegistry.getInstance().getNode(plan.nodeId);
|
||||
try {
|
||||
if (node?.type === 'remote') {
|
||||
const remoteOut = await this.executeUpdateRemoteTargets(plan.nodeId, plan.stacks);
|
||||
lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}) results:\n${remoteOut}`);
|
||||
if (looksLikeStackFailure(remoteOut)) materialFailure = true;
|
||||
} else {
|
||||
const docker = DockerController.getInstance(plan.nodeId);
|
||||
const imageUpdateService = ImageUpdateService.getInstance();
|
||||
const stackLines: string[] = [];
|
||||
for (const stackName of plan.stacks) {
|
||||
try {
|
||||
const out = await this.executeUpdateForStack(
|
||||
stackName, plan.nodeId, docker, imageUpdateService, true,
|
||||
);
|
||||
stackLines.push(out);
|
||||
} catch (e) {
|
||||
materialFailure = true;
|
||||
const msg = getErrorMessage(e, String(e));
|
||||
stackLines.push(`Stack "${stackName}" failed: ${msg}`);
|
||||
console.error(`[SchedulerService] Label auto-update failed for stack "${stackName}" on node ${plan.nodeId}:`, e);
|
||||
}
|
||||
}
|
||||
lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}) results:\n${stackLines.join('\n')}`);
|
||||
}
|
||||
} catch (e) {
|
||||
materialFailure = true;
|
||||
const msg = getErrorMessage(e, String(e));
|
||||
lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}): failed (${msg})`);
|
||||
console.error(`[SchedulerService] Label auto-update failed for node ${plan.nodeId}:`, e);
|
||||
}
|
||||
};
|
||||
|
||||
for (let i = 0; i < work.length; i += NODE_CONCURRENCY) {
|
||||
const batch = work.slice(i, i + NODE_CONCURRENCY);
|
||||
await Promise.all(batch.map(runNode));
|
||||
}
|
||||
|
||||
if (materialFailure) {
|
||||
throw new Error(lines.join('\n'));
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Proxy auto-update execution to a remote Sencho instance.
|
||||
* The remote node runs the image checks and compose update locally.
|
||||
@@ -829,6 +967,55 @@ export class SchedulerService {
|
||||
}
|
||||
}
|
||||
|
||||
/** Proxy auto-update for an explicit stack list on a remote node (label selector). */
|
||||
private async executeUpdateRemoteTargets(nodeId: number, targets: string[]): Promise<string> {
|
||||
const proxyTarget = this.requireRemoteProxyTarget(nodeId);
|
||||
const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
|
||||
const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
|
||||
if (isDebugEnabled()) {
|
||||
console.log(`[SchedulerService] executeUpdateRemoteTargets: node=${nodeId} count=${targets.length}`);
|
||||
}
|
||||
const startTime = Date.now();
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/api/auto-update/execute`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${proxyTarget.apiToken}`,
|
||||
[PROXY_TIER_HEADER]: proxyHeaders.tier,
|
||||
},
|
||||
body: JSON.stringify({ targets }),
|
||||
signal: AbortSignal.timeout(300_000),
|
||||
});
|
||||
|
||||
// Older remotes only accept { target }. Fall back to one call per
|
||||
// stack so mixed-version fleets still complete the label schedule.
|
||||
if (response.status === 400) {
|
||||
const detail = await this.remoteResponseDetail(response);
|
||||
if (/target/i.test(detail)) {
|
||||
const parts: string[] = [];
|
||||
for (const stackName of targets) {
|
||||
parts.push(await this.executeUpdateRemote(nodeId, stackName));
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
throw new Error(this.remoteProxyFailureMessage(nodeId, detail));
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
|
||||
}
|
||||
|
||||
const body = await response.json() as { result?: string };
|
||||
if (isDebugEnabled()) {
|
||||
console.log(`[SchedulerService] executeUpdateRemoteTargets: completed in ${Date.now() - startTime}ms`);
|
||||
}
|
||||
return body.result || 'Remote auto-update completed (no details returned).';
|
||||
} catch (err) {
|
||||
this.rethrowRemoteProxyError(nodeId, err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Proxy a stack lifecycle action to a remote Sencho instance. ComposeService,
|
||||
* DockerController, and FileSystemService are local-only, so for a remote node
|
||||
|
||||
Reference in New Issue
Block a user