Files
sencho/backend/src/services/StackOpLockService.ts
T
Anso 3f1f15a6f4 fix: keep running containers until stack pull/build succeeds (#1657)
* fix: keep running containers until stack pull/build succeeds

Acquire images before reconcile, capture a recovery generation for
compensation, and only remove classified orphans after handoff.

* fix: address recovery audit blockers for safe stack updates

Retire abandoned and expired recovery artifacts, probe compensated
runtimes before reporting rollback success, preserve local Docker when
deleting a node, validate the exact Compose invocation before capture,
and repair updateStack return-contract fixtures.

* fix: resolve ESLint errors blocking CI on this branch

Unused-import and unused-variable errors left over from the stack
deletion refactor: MeshService in stacks.ts (its opt-out cascade moved
into DeployedStackDeletionService), a redundant pruneVolumes
destructure in deleteDeployedStack (the real one is re-derived from
the same input object inside runDeletionBody), and an unused beforeAll
import in a Docker-integration test stub. Also scopes the webhook
pull-action case body in a block to satisfy no-case-declarations;
purely syntactic, no behavior change.

* fix: harden recovery probe, cleanup retry, and failed-pull Docker test

Reject absent or unhealthy expected replicas before reporting rollback
success, keep cleanup records until artifacts are actually removed, fail
closed when a mesh override cannot be generated, and assert a real
failed pull leaves the original container running.

* fix: verify recovery probe image identity and stack-scoped override paths

Reject recovered runtimes that use the wrong image or leave scale-zero
services running, and confine tombstone override deletion to the intent
stack directory so forged cross-stack paths cannot be swept.

* test: batch notification cap fixtures in a SQLite transaction

Unbatched 1200-row inserts were timing out at the default 30s under
CI load even though the same assertions pass in under 2s when green.
2026-07-21 12:18:01 -04:00

135 lines
4.5 KiB
TypeScript

import { DatabaseService } from './DatabaseService';
/**
* Tracks in-flight stack lifecycle operations (deploy, down, restart, stop,
* start, update, rollback, backup) per (nodeId, stackName). A second request to
* the same stack while the first is still running returns 409 instead of racing
* the first. Backup is included because it rewrites the shared rollback slot, so
* it must not interleave with a deploy/update/rollback on the same stack.
*
* State is intentionally process-local: a Sencho restart clears all locks,
* which matches the lifecycle of any in-flight `docker compose` child process.
*/
export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup' | 'delete';
/**
* Note returned by a background path that skipped its operation because a manual
* or concurrent operation already held the stack's lock.
*/
export function stackOpSkipMessage(stackName: string, existingAction: StackOpAction): string {
return `Skipped "${stackName}": another operation (${existingAction}) is already in progress.`;
}
export interface StackOpLock {
action: StackOpAction;
startedAt: number;
user: string;
}
interface AcquireSuccess {
acquired: true;
}
interface AcquireConflict {
acquired: false;
existing: StackOpLock;
}
export type AcquireResult = AcquireSuccess | AcquireConflict;
export class StackOpLockService {
private static instance: StackOpLockService;
private readonly locks = new Map<string, StackOpLock>();
public static getInstance(): StackOpLockService {
if (!this.instance) this.instance = new StackOpLockService();
return this.instance;
}
public static resetForTests(): void {
this.instance = new StackOpLockService();
}
private key(nodeId: number, stackName: string): string {
return `${nodeId}:${stackName}`;
}
public tryAcquire(
nodeId: number,
stackName: string,
action: StackOpAction,
user: string,
): AcquireResult {
const k = this.key(nodeId, stackName);
const existing = this.locks.get(k);
if (existing) return { acquired: false, existing };
this.locks.set(k, { action, startedAt: Date.now(), user });
return { acquired: true };
}
public release(nodeId: number, stackName: string): void {
this.locks.delete(this.key(nodeId, stackName));
}
/**
* Matching-intent continuation for crash-resume of a prepared deletion.
* Succeeds only when the persisted intent matches and no other in-memory lock holds.
*/
public tryAcquireDeletionContinuation(args: {
intentId: string;
nodeId: number;
stackName: string;
}): AcquireResult {
const intent = DatabaseService.getInstance().getDeletionIntentById(args.intentId);
if (
!intent
|| intent.status !== 'prepared'
|| intent.node_id !== args.nodeId
|| intent.stack_name !== args.stackName
) {
const existing = this.locks.get(this.key(args.nodeId, args.stackName));
return {
acquired: false,
existing: existing ?? { action: 'delete', startedAt: Date.now(), user: 'system' },
};
}
return this.tryAcquire(args.nodeId, args.stackName, 'delete', 'system:deletion-continuation');
}
/**
* Acquire the per-(nodeId, stackName) lock for the duration of `fn`, then
* release it. Returns `{ ran: true, result }` when the lock was free, or
* `{ ran: false, existing }` when another operation already holds it, so the
* caller can skip rather than race. Background/system paths (scheduler,
* webhook, Git source, image auto-update, label bulk actions, fleet snapshot
* redeploy, mesh redeploy) run their lifecycle calls through this so they
* cannot interleave with a manual deploy/update/rollback/backup on the same
* stack and node. An error thrown by `fn` still releases the lock, then
* propagates to the caller.
*/
public async runExclusive<T>(
nodeId: number,
stackName: string,
action: StackOpAction,
user: string,
fn: () => Promise<T>,
): Promise<{ ran: true; result: T } | { ran: false; existing: StackOpLock }> {
const acquired = this.tryAcquire(nodeId, stackName, action, user);
if (!acquired.acquired) return { ran: false, existing: acquired.existing };
try {
const result = await fn();
return { ran: true, result };
} finally {
this.release(nodeId, stackName);
}
}
public get(nodeId: number, stackName: string): StackOpLock | undefined {
return this.locks.get(this.key(nodeId, stackName));
}
public size(): number {
return this.locks.size;
}
}