mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-07 09:24:09 +00:00
3f1f15a6f4
* fix: keep running containers until stack pull/build succeeds Acquire images before reconcile, capture a recovery generation for compensation, and only remove classified orphans after handoff. * fix: address recovery audit blockers for safe stack updates Retire abandoned and expired recovery artifacts, probe compensated runtimes before reporting rollback success, preserve local Docker when deleting a node, validate the exact Compose invocation before capture, and repair updateStack return-contract fixtures. * fix: resolve ESLint errors blocking CI on this branch Unused-import and unused-variable errors left over from the stack deletion refactor: MeshService in stacks.ts (its opt-out cascade moved into DeployedStackDeletionService), a redundant pruneVolumes destructure in deleteDeployedStack (the real one is re-derived from the same input object inside runDeletionBody), and an unused beforeAll import in a Docker-integration test stub. Also scopes the webhook pull-action case body in a block to satisfy no-case-declarations; purely syntactic, no behavior change. * fix: harden recovery probe, cleanup retry, and failed-pull Docker test Reject absent or unhealthy expected replicas before reporting rollback success, keep cleanup records until artifacts are actually removed, fail closed when a mesh override cannot be generated, and assert a real failed pull leaves the original container running. * fix: verify recovery probe image identity and stack-scoped override paths Reject recovered runtimes that use the wrong image or leave scale-zero services running, and confine tombstone override deletion to the intent stack directory so forged cross-stack paths cannot be swept. * test: batch notification cap fixtures in a SQLite transaction Unbatched 1200-row inserts were timing out at the default 30s under CI load even though the same assertions pass in under 2s when green.
135 lines
4.5 KiB
TypeScript
135 lines
4.5 KiB
TypeScript
import { DatabaseService } from './DatabaseService';
|
|
/**
|
|
* Tracks in-flight stack lifecycle operations (deploy, down, restart, stop,
|
|
* start, update, rollback, backup) per (nodeId, stackName). A second request to
|
|
* the same stack while the first is still running returns 409 instead of racing
|
|
* the first. Backup is included because it rewrites the shared rollback slot, so
|
|
* it must not interleave with a deploy/update/rollback on the same stack.
|
|
*
|
|
* State is intentionally process-local: a Sencho restart clears all locks,
|
|
* which matches the lifecycle of any in-flight `docker compose` child process.
|
|
*/
|
|
|
|
export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup' | 'delete';
|
|
|
|
/**
|
|
* Note returned by a background path that skipped its operation because a manual
|
|
* or concurrent operation already held the stack's lock.
|
|
*/
|
|
export function stackOpSkipMessage(stackName: string, existingAction: StackOpAction): string {
|
|
return `Skipped "${stackName}": another operation (${existingAction}) is already in progress.`;
|
|
}
|
|
|
|
export interface StackOpLock {
|
|
action: StackOpAction;
|
|
startedAt: number;
|
|
user: string;
|
|
}
|
|
|
|
interface AcquireSuccess {
|
|
acquired: true;
|
|
}
|
|
|
|
interface AcquireConflict {
|
|
acquired: false;
|
|
existing: StackOpLock;
|
|
}
|
|
|
|
export type AcquireResult = AcquireSuccess | AcquireConflict;
|
|
|
|
export class StackOpLockService {
|
|
private static instance: StackOpLockService;
|
|
private readonly locks = new Map<string, StackOpLock>();
|
|
|
|
public static getInstance(): StackOpLockService {
|
|
if (!this.instance) this.instance = new StackOpLockService();
|
|
return this.instance;
|
|
}
|
|
|
|
public static resetForTests(): void {
|
|
this.instance = new StackOpLockService();
|
|
}
|
|
|
|
private key(nodeId: number, stackName: string): string {
|
|
return `${nodeId}:${stackName}`;
|
|
}
|
|
|
|
public tryAcquire(
|
|
nodeId: number,
|
|
stackName: string,
|
|
action: StackOpAction,
|
|
user: string,
|
|
): AcquireResult {
|
|
const k = this.key(nodeId, stackName);
|
|
const existing = this.locks.get(k);
|
|
if (existing) return { acquired: false, existing };
|
|
this.locks.set(k, { action, startedAt: Date.now(), user });
|
|
return { acquired: true };
|
|
}
|
|
|
|
public release(nodeId: number, stackName: string): void {
|
|
this.locks.delete(this.key(nodeId, stackName));
|
|
}
|
|
|
|
/**
|
|
* Matching-intent continuation for crash-resume of a prepared deletion.
|
|
* Succeeds only when the persisted intent matches and no other in-memory lock holds.
|
|
*/
|
|
public tryAcquireDeletionContinuation(args: {
|
|
intentId: string;
|
|
nodeId: number;
|
|
stackName: string;
|
|
}): AcquireResult {
|
|
const intent = DatabaseService.getInstance().getDeletionIntentById(args.intentId);
|
|
if (
|
|
!intent
|
|
|| intent.status !== 'prepared'
|
|
|| intent.node_id !== args.nodeId
|
|
|| intent.stack_name !== args.stackName
|
|
) {
|
|
const existing = this.locks.get(this.key(args.nodeId, args.stackName));
|
|
return {
|
|
acquired: false,
|
|
existing: existing ?? { action: 'delete', startedAt: Date.now(), user: 'system' },
|
|
};
|
|
}
|
|
return this.tryAcquire(args.nodeId, args.stackName, 'delete', 'system:deletion-continuation');
|
|
}
|
|
|
|
/**
|
|
* Acquire the per-(nodeId, stackName) lock for the duration of `fn`, then
|
|
* release it. Returns `{ ran: true, result }` when the lock was free, or
|
|
* `{ ran: false, existing }` when another operation already holds it, so the
|
|
* caller can skip rather than race. Background/system paths (scheduler,
|
|
* webhook, Git source, image auto-update, label bulk actions, fleet snapshot
|
|
* redeploy, mesh redeploy) run their lifecycle calls through this so they
|
|
* cannot interleave with a manual deploy/update/rollback/backup on the same
|
|
* stack and node. An error thrown by `fn` still releases the lock, then
|
|
* propagates to the caller.
|
|
*/
|
|
public async runExclusive<T>(
|
|
nodeId: number,
|
|
stackName: string,
|
|
action: StackOpAction,
|
|
user: string,
|
|
fn: () => Promise<T>,
|
|
): Promise<{ ran: true; result: T } | { ran: false; existing: StackOpLock }> {
|
|
const acquired = this.tryAcquire(nodeId, stackName, action, user);
|
|
if (!acquired.acquired) return { ran: false, existing: acquired.existing };
|
|
try {
|
|
const result = await fn();
|
|
return { ran: true, result };
|
|
} finally {
|
|
this.release(nodeId, stackName);
|
|
}
|
|
}
|
|
|
|
public get(nodeId: number, stackName: string): StackOpLock | undefined {
|
|
return this.locks.get(this.key(nodeId, stackName));
|
|
}
|
|
|
|
public size(): number {
|
|
return this.locks.size;
|
|
}
|
|
}
|