mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-20 23:32:19 +00:00
fix: keep running containers until stack pull/build succeeds (#1657)
* fix: keep running containers until stack pull/build succeeds Acquire images before reconcile, capture a recovery generation for compensation, and only remove classified orphans after handoff. * fix: address recovery audit blockers for safe stack updates Retire abandoned and expired recovery artifacts, probe compensated runtimes before reporting rollback success, preserve local Docker when deleting a node, validate the exact Compose invocation before capture, and repair updateStack return-contract fixtures. * fix: resolve ESLint errors blocking CI on this branch Unused-import and unused-variable errors left over from the stack deletion refactor: MeshService in stacks.ts (its opt-out cascade moved into DeployedStackDeletionService), a redundant pruneVolumes destructure in deleteDeployedStack (the real one is re-derived from the same input object inside runDeletionBody), and an unused beforeAll import in a Docker-integration test stub. Also scopes the webhook pull-action case body in a block to satisfy no-case-declarations; purely syntactic, no behavior change. * fix: harden recovery probe, cleanup retry, and failed-pull Docker test Reject absent or unhealthy expected replicas before reporting rollback success, keep cleanup records until artifacts are actually removed, fail closed when a mesh override cannot be generated, and assert a real failed pull leaves the original container running. * fix: verify recovery probe image identity and stack-scoped override paths Reject recovered runtimes that use the wrong image or leave scale-zero services running, and confine tombstone override deletion to the intent stack directory so forged cross-stack paths cannot be swept. * test: batch notification cap fixtures in a SQLite transaction Unbatched 1200-row inserts were timing out at the default 30s under CI load even though the same assertions pass in under 2s when green.
This commit is contained in:
Anso
@@ -10,6 +10,7 @@ import {
|
||||
} from './DatabaseService';
|
||||
import { ComposeService } from './ComposeService';
|
||||
import { StackOpLockService, stackOpSkipMessage, type StackOpAction } from './StackOpLockService';
|
||||
import { DeployedStackDeletionService } from './DeployedStackDeletionService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { PROXY_TIER_HEADER, deployProvenanceHeaders } from './license-headers';
|
||||
@@ -463,32 +464,17 @@ export class BlueprintService {
|
||||
}
|
||||
|
||||
private async withdrawLocal(blueprint: Blueprint, node: Node): Promise<void> {
|
||||
// Hold the per-stack lock across both the compose down and the directory
|
||||
// delete so a withdraw cannot race a manual operation, nor tear the
|
||||
// files out from under one that starts mid-withdraw.
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
node.id, blueprint.name, 'down', 'system',
|
||||
async () => {
|
||||
try {
|
||||
await ComposeService.getInstance(node.id).downStack(blueprint.name);
|
||||
} catch (err) {
|
||||
// best-effort: continue to delete the directory even if down fails
|
||||
console.warn(`[BlueprintService] downStack failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(err)}`);
|
||||
}
|
||||
if (await this.stackDirExists(node.id, blueprint.name)) {
|
||||
await FileSystemService.getInstance(node.id).deleteStack(blueprint.name);
|
||||
}
|
||||
// Remove the exposure descriptor so a withdrawn blueprint
|
||||
// stack does not leave a stale row that escalates posture.
|
||||
try {
|
||||
DatabaseService.getInstance().deleteStackExposure(node.id, blueprint.name);
|
||||
} catch (e) {
|
||||
console.warn(`[BlueprintService] deleteStackExposure failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(e)}`);
|
||||
}
|
||||
},
|
||||
);
|
||||
if (!lock.ran) {
|
||||
throw new Error(stackOpSkipMessage(blueprint.name, lock.existing.action));
|
||||
const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({
|
||||
nodeId: node.id,
|
||||
stackName: blueprint.name,
|
||||
pruneVolumes: false,
|
||||
actor: 'system:blueprint',
|
||||
});
|
||||
if (!result.ok) {
|
||||
if (result.code === 'lock_conflict') {
|
||||
throw new Error(result.error);
|
||||
}
|
||||
throw new Error(result.error);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -143,6 +143,33 @@ export class ComposeService {
|
||||
* same `authoredComposeFileArgs` prefix directly but intentionally omit the mesh
|
||||
* override, rendering the user's authored model without mesh injection.
|
||||
*/
|
||||
/** Public wrapper for dual-arg assembly and recovery Compose invocations. */
|
||||
public async buildAuthoredComposeArgs(stackName: string, action: string[]): Promise<string[]> {
|
||||
return this.authoredComposeArgs(stackName, action);
|
||||
}
|
||||
|
||||
public async validateStackForMutation(stackName: string): Promise<void> {
|
||||
await this.assertRequiredEnvPresent(stackName);
|
||||
await this.assertSafePilotBindMapping(stackName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Render/validate the exact Compose invocation used by mutating operations
|
||||
* (authored files, env pins, and generated Mesh override) before capture.
|
||||
*/
|
||||
public async validateExactComposeInvocation(stackName: string): Promise<void> {
|
||||
if (!isValidStackName(stackName)) {
|
||||
throw new Error('Invalid stack path');
|
||||
}
|
||||
const baseResolved = path.resolve(this.baseDir);
|
||||
const stackDir = path.resolve(baseResolved, stackName);
|
||||
if (!stackDir.startsWith(baseResolved + path.sep)) {
|
||||
throw new Error('Invalid stack path');
|
||||
}
|
||||
const args = await this.authoredComposeArgs(stackName, ['config', '--quiet']);
|
||||
await this.execute('docker', args, stackDir, undefined, true);
|
||||
}
|
||||
|
||||
private async authoredComposeArgs(stackName: string, action: string[]): Promise<string[]> {
|
||||
const args: string[] = ['compose'];
|
||||
const filePrefix = authoredComposeFileArgs(stackName, this.nodeId);
|
||||
@@ -151,12 +178,23 @@ export class ComposeService {
|
||||
// directory, so deploy/update resolve the same effective config the validator did.
|
||||
args.push(...await authoredComposeEnvFileArgs(stackName, this.nodeId));
|
||||
|
||||
const meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(this.nodeId, stackName);
|
||||
let overridePath: string | null = null;
|
||||
try {
|
||||
overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName);
|
||||
} catch (err) {
|
||||
if (meshEnabled) {
|
||||
throw err instanceof Error
|
||||
? err
|
||||
: new Error(`Mesh override generation failed: ${String(err)}`);
|
||||
}
|
||||
console.warn('[ComposeService] mesh override skipped:', sanitizeForLog((err as Error).message));
|
||||
}
|
||||
if (meshEnabled && !overridePath) {
|
||||
throw new Error(
|
||||
`Mesh override is required for stack "${stackName}" but could not be generated`,
|
||||
);
|
||||
}
|
||||
if (overridePath) {
|
||||
if (filePrefix.length === 0) {
|
||||
// Single-file stack: passing any -f disables compose's auto-discovery, so name
|
||||
@@ -799,7 +837,50 @@ export class ComposeService {
|
||||
startStream();
|
||||
}
|
||||
|
||||
async updateStack(stackName: string, ws?: WebSocket, atomic?: boolean): Promise<void> {
|
||||
|
||||
/**
|
||||
* Authored (+ mesh) compose args with an optional recovery override layered LAST.
|
||||
* Used for pinned lifecycle / compensation ups (`--pull never --no-build`).
|
||||
*/
|
||||
public async buildComposeArgsWithRecoveryOverride(
|
||||
stackName: string,
|
||||
action: string[],
|
||||
recoveryOverridePath: string | null,
|
||||
): Promise<string[]> {
|
||||
const withSentinel = await this.authoredComposeArgs(stackName, ['__SENCHO_ACTION_SENTINEL__']);
|
||||
const idx = withSentinel.indexOf('__SENCHO_ACTION_SENTINEL__');
|
||||
const prefix = idx >= 0 ? withSentinel.slice(0, idx) : withSentinel;
|
||||
const out = [...prefix];
|
||||
if (recoveryOverridePath) {
|
||||
if (!out.includes('-f')) {
|
||||
const fsSvc = FileSystemService.getInstance(this.nodeId);
|
||||
const baseFilename = await fsSvc.getComposeFilename(stackName);
|
||||
out.push('-f', baseFilename);
|
||||
try {
|
||||
const userOverride = await fsSvc.getOverrideFilename(stackName);
|
||||
if (userOverride) out.push('-f', userOverride);
|
||||
} catch (err) {
|
||||
const code = (err as { code?: string }).code;
|
||||
if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') {
|
||||
throw err;
|
||||
}
|
||||
console.warn(
|
||||
'[ComposeService] could not resolve user compose override for recovery args:',
|
||||
sanitizeForLog((err as Error).message),
|
||||
);
|
||||
}
|
||||
}
|
||||
out.push('-f', recoveryOverridePath);
|
||||
}
|
||||
out.push(...action);
|
||||
return out;
|
||||
}
|
||||
|
||||
async updateStack(
|
||||
stackName: string,
|
||||
ws?: WebSocket,
|
||||
atomic?: boolean,
|
||||
): Promise<{ recoveryId: string | null }> {
|
||||
await this.assertRequiredEnvPresent(stackName);
|
||||
await this.assertSafePilotBindMapping(stackName);
|
||||
const stackDir = path.join(this.baseDir, stackName);
|
||||
@@ -810,48 +891,106 @@ export class ComposeService {
|
||||
if (ws && ws.readyState === WebSocket.OPEN) ws.send(data);
|
||||
};
|
||||
|
||||
if (atomic) {
|
||||
await this.createAtomicBackup(stackName, 'update', sendOutput);
|
||||
}
|
||||
// Dynamic import avoids a static cycle (recovery imports getComposeCommandTimeoutMs).
|
||||
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
||||
const recoverySvc = StackUpdateRecoveryService.getInstance();
|
||||
let recoveryId: string | null = null;
|
||||
let handedOff = false;
|
||||
|
||||
try {
|
||||
try {
|
||||
const dockerController = DockerController.getInstance(this.nodeId);
|
||||
const legacyContainers = await dockerController.getContainersByStack(stackName);
|
||||
if (legacyContainers && legacyContainers.length > 0) {
|
||||
sendOutput(`=== Cleaning up existing containers for clean update ===\n`);
|
||||
await dockerController.removeContainers(legacyContainers.map((c: any) => c.Id));
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('Failed to clean up legacy containers for %s:', sanitizeForLog(stackName), e);
|
||||
}
|
||||
sendOutput('=== Validating stack for update ===\n');
|
||||
sendOutput('=== Capturing rollback generation ===\n');
|
||||
const candidate = await recoverySvc.captureCandidate({
|
||||
nodeId: this.nodeId,
|
||||
stackName,
|
||||
createdBy: null,
|
||||
});
|
||||
recoveryId = candidate.id;
|
||||
|
||||
const buildServices = await loadStackBuildServices(this.nodeId, stackName);
|
||||
const buildAware = buildServices.length > 0;
|
||||
|
||||
await this.withRegistryAuth(async (env) => {
|
||||
if (buildAware) {
|
||||
sendOutput('=== Building images ===\n');
|
||||
await this.execute('docker', await this.authoredComposeArgs(stackName, ['build', '--pull']), stackDir, ws, true, env, getComposeStallTimeoutMs());
|
||||
try {
|
||||
await this.withRegistryAuth(async (env) => {
|
||||
if (buildAware) {
|
||||
sendOutput('=== Building images ===\n');
|
||||
await this.execute(
|
||||
'docker',
|
||||
await this.authoredComposeArgs(stackName, ['build', '--pull']),
|
||||
stackDir, ws, true, env, getComposeStallTimeoutMs(),
|
||||
);
|
||||
sendOutput('=== Pulling registry images ===\n');
|
||||
await this.execute(
|
||||
'docker',
|
||||
await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']),
|
||||
stackDir, ws, true, env, getComposeStallTimeoutMs(),
|
||||
);
|
||||
} else {
|
||||
sendOutput('=== Pulling latest images ===\n');
|
||||
await this.execute(
|
||||
'docker',
|
||||
await this.authoredComposeArgs(stackName, ['pull']),
|
||||
stackDir, ws, true, env, getComposeStallTimeoutMs(),
|
||||
);
|
||||
}
|
||||
}, sendOutput);
|
||||
} catch (acquireError) {
|
||||
// Acquisition failure: abandon candidate; leave runtime untouched.
|
||||
await recoverySvc.abandon(candidate.id);
|
||||
recoveryId = null;
|
||||
throw acquireError;
|
||||
}
|
||||
|
||||
sendOutput('=== Pulling registry images ===\n');
|
||||
await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']), stackDir, ws, true, env, getComposeStallTimeoutMs());
|
||||
} else {
|
||||
sendOutput('=== Pulling latest images ===\n');
|
||||
await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull']), stackDir, ws, true, env, getComposeStallTimeoutMs());
|
||||
}
|
||||
|
||||
sendOutput('=== Recreating containers ===\n');
|
||||
await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs());
|
||||
}, sendOutput);
|
||||
|
||||
// Post-Update Health Probe
|
||||
await new Promise(resolve => setTimeout(resolve, 3000));
|
||||
if (!recoverySvc.markAcquired(candidate.id)) {
|
||||
await recoverySvc.abandon(candidate.id);
|
||||
throw new Error('Failed to mark recovery generation as acquired');
|
||||
}
|
||||
|
||||
const dockerController = DockerController.getInstance(this.nodeId);
|
||||
sendOutput('=== Classifying legacy orphans ===\n');
|
||||
const classified = await dockerController.classifyLegacyOrphansForUpdate(stackName);
|
||||
if (classified.status === 'classification_failed') {
|
||||
await recoverySvc.abandon(candidate.id);
|
||||
recoveryId = null;
|
||||
throw new Error(`Legacy orphan classification failed: ${classified.error}`);
|
||||
}
|
||||
|
||||
if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) {
|
||||
await recoverySvc.abandon(candidate.id);
|
||||
recoveryId = null;
|
||||
throw new Error('Failed to hand off recovery generation');
|
||||
}
|
||||
handedOff = true;
|
||||
if (!recoverySvc.markReconciling(candidate.id)) {
|
||||
throw new Error('Failed to mark recovery generation as reconciling after handoff');
|
||||
}
|
||||
|
||||
if (classified.status === 'orphans') {
|
||||
sendOutput(`=== Removing ${classified.ids.length} legacy orphan container(s) ===\n`);
|
||||
const results = await dockerController.removeContainers(classified.ids);
|
||||
const failed = results.filter((r) => !r.success);
|
||||
if (failed.length > 0) {
|
||||
throw new Error(
|
||||
`Failed to remove ${failed.length} legacy orphan container(s) after handoff`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await this.withRegistryAuth(async (env) => {
|
||||
sendOutput('=== Recreating containers ===\n');
|
||||
await this.execute(
|
||||
'docker',
|
||||
await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']),
|
||||
stackDir, ws, true, env, getComposeStallTimeoutMs(),
|
||||
);
|
||||
}, sendOutput);
|
||||
|
||||
// Immediate verification probe
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
|
||||
const containers = await dockerController.getDocker().listContainers({
|
||||
all: true,
|
||||
filters: { label: [`com.docker.compose.project=${stackName}`] }
|
||||
filters: { label: [`com.docker.compose.project=${stackName}`] },
|
||||
});
|
||||
|
||||
for (const containerInfo of containers) {
|
||||
@@ -859,56 +998,90 @@ export class ComposeService {
|
||||
const container = dockerController.getDocker().getContainer(containerInfo.Id);
|
||||
const inspectData = await container.inspect();
|
||||
const exitCode = inspectData.State.ExitCode;
|
||||
|
||||
if (exitCode !== 0) {
|
||||
throw this.createContainerCrashError(exitCode);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!recoverySvc.markImmediateVerified(candidate.id)) {
|
||||
console.warn(
|
||||
'[ComposeService] Could not CAS immediate_verified for recovery %s',
|
||||
sanitizeForLog(candidate.id),
|
||||
);
|
||||
}
|
||||
|
||||
sendOutput('=== Stack updated successfully ===\n');
|
||||
// Opt-out (default ON): after a clean update, prune the node's dangling
|
||||
// (untagged) image layers, including the one this pull just orphaned. Read
|
||||
// fresh each run so a remote node honors its own setting. Wrapped so a
|
||||
// prune failure can never reach the atomic-rollback catch below.
|
||||
|
||||
// Defer prune until gate retention / gate link: only prune when no active holds
|
||||
// would be violated. Still honor prune_on_update, but use unified holds so
|
||||
// candidate/current rollback images are retained.
|
||||
try {
|
||||
const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1';
|
||||
if (pruneOnUpdate) {
|
||||
// Dynamic import: ServiceUpdateRecoveryService imports getComposeCommandTimeoutMs
|
||||
// from this module, so a static import here would be circular.
|
||||
const { ServiceUpdateRecoveryService } = await import('./ServiceUpdateRecoveryService');
|
||||
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(this.nodeId);
|
||||
const isImageHeld = recoverySvc.buildUnifiedHeldImagePredicate(this.nodeId);
|
||||
const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld);
|
||||
// The Docker prune API does not report SpaceReclaimed on the containerd
|
||||
// image store, so only show the figure when the daemon actually returns one.
|
||||
const reclaimed = result.reclaimedBytes > 0
|
||||
? ` · reclaimed ${(result.reclaimedBytes / (1024 * 1024)).toFixed(1)} MB`
|
||||
: '';
|
||||
sendOutput(`=== Pruned dangling images${reclaimed} ===\n`);
|
||||
}
|
||||
} catch (pruneError) {
|
||||
console.warn('Failed to prune dangling images after update for %s:', sanitizeForLog(stackName), pruneError);
|
||||
console.warn(
|
||||
'Failed to prune dangling images after update for %s:',
|
||||
sanitizeForLog(stackName),
|
||||
pruneError,
|
||||
);
|
||||
}
|
||||
|
||||
if (debug) {
|
||||
console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName });
|
||||
}
|
||||
if (debug) console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName });
|
||||
} catch (updateError) {
|
||||
if (atomic) {
|
||||
sendOutput('\n=== Update failed - restoring previous compose and env files ===\n');
|
||||
const rolledBack = await this.restoreAtomicBackup(stackName, stackDir, ws, sendOutput);
|
||||
if (!handedOff && recoveryId) {
|
||||
await recoverySvc.abandon(recoveryId);
|
||||
recoveryId = null;
|
||||
}
|
||||
if (handedOff && recoveryId) {
|
||||
sendOutput('\n=== Update failed - restoring previous runtime from recovery generation ===\n');
|
||||
const rolledBack = await recoverySvc.compensateWithCandidate(
|
||||
recoveryId,
|
||||
async (overridePath) => {
|
||||
await this.withRegistryAuth(async (env) => {
|
||||
await this.execute(
|
||||
'docker',
|
||||
await this.buildComposeArgsWithRecoveryOverride(
|
||||
stackName,
|
||||
['up', '-d', '--remove-orphans', '--pull', 'never', '--no-build'],
|
||||
overridePath,
|
||||
),
|
||||
stackDir,
|
||||
ws,
|
||||
true,
|
||||
env,
|
||||
getComposeStallTimeoutMs(),
|
||||
);
|
||||
}, sendOutput);
|
||||
},
|
||||
);
|
||||
throw new ComposeRollbackError(updateError, true, rolledBack);
|
||||
}
|
||||
// Pre-handoff failure: abandon already handled on acquire/classify; runtime untouched.
|
||||
throw updateError;
|
||||
}
|
||||
// Reached only on a successful update; re-baseline so temporal drift compares
|
||||
// against what is now deployed (see deployStack for why this lives here), then
|
||||
// reconcile the ledger against the updated runtime.
|
||||
|
||||
await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName);
|
||||
await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName);
|
||||
try {
|
||||
await this.refreshExposureCache(stackName);
|
||||
} catch (err) {
|
||||
console.warn('[ComposeService] Exposure refresh failed after update for %s:',
|
||||
sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown')));
|
||||
console.warn(
|
||||
'[ComposeService] Exposure refresh failed after update for %s:',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(err, 'unknown')),
|
||||
);
|
||||
}
|
||||
return { recoveryId };
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -213,6 +213,42 @@ export interface HealthGateRunRow {
|
||||
}
|
||||
|
||||
/** Pre-update image snapshot enabling a manual per-service restore after a service-scoped update. */
|
||||
|
||||
/** Full-stack update recovery generation. Separate from service_update_recovery. */
|
||||
export interface StackUpdateRecoveryGenerationRow {
|
||||
id: string;
|
||||
node_id: number;
|
||||
stack_name: string;
|
||||
status: 'candidate' | 'active' | 'restored_current' | 'superseded' | 'abandoned' | 'recovery_required';
|
||||
phase: 'captured' | 'acquired' | 'handoff_committed' | 'reconciling' | 'immediate_verified';
|
||||
is_current: number;
|
||||
backup_slot_id: string | null;
|
||||
override_path: string | null;
|
||||
services_json: string;
|
||||
health_gate_id: string | null;
|
||||
gate_retain_until: number | null;
|
||||
artifact_expires_at: number | null;
|
||||
operation_lease_expires_at: number | null;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
created_by: string | null;
|
||||
artifacts_retired: number;
|
||||
}
|
||||
|
||||
/** Durable cleanup tombstone for stack/node deletion artifact sweep. */
|
||||
export interface StackUpdateCleanupPendingRow {
|
||||
id: string;
|
||||
node_id: number | null;
|
||||
stack_name: string | null;
|
||||
status: 'prepared' | 'ready' | 'cancelled';
|
||||
target_kind: 'local_socket';
|
||||
rollback_tags_json: string;
|
||||
override_paths_json: string;
|
||||
prune_volumes_requested: number;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
export interface ServiceUpdateRecoveryRow {
|
||||
id: string;
|
||||
node_id: number;
|
||||
@@ -1647,6 +1683,50 @@ export class DatabaseService {
|
||||
CREATE INDEX IF NOT EXISTS idx_service_update_recovery_status_claim_expires
|
||||
ON service_update_recovery(status, claim_expires_at);
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS stack_update_recovery_generations (
|
||||
id TEXT PRIMARY KEY,
|
||||
node_id INTEGER NOT NULL,
|
||||
stack_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL CHECK (status IN ('candidate','active','restored_current','superseded','abandoned','recovery_required')),
|
||||
phase TEXT NOT NULL CHECK (phase IN ('captured','acquired','handoff_committed','reconciling','immediate_verified')),
|
||||
is_current INTEGER NOT NULL DEFAULT 0,
|
||||
backup_slot_id TEXT,
|
||||
override_path TEXT,
|
||||
services_json TEXT NOT NULL,
|
||||
health_gate_id TEXT,
|
||||
gate_retain_until INTEGER,
|
||||
artifact_expires_at INTEGER,
|
||||
operation_lease_expires_at INTEGER,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL,
|
||||
created_by TEXT,
|
||||
artifacts_retired INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_node_stack_current
|
||||
ON stack_update_recovery_generations(node_id, stack_name, is_current);
|
||||
CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_status_expires
|
||||
ON stack_update_recovery_generations(status, artifact_expires_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_phase_lease
|
||||
ON stack_update_recovery_generations(phase, operation_lease_expires_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS stack_update_cleanup_pending (
|
||||
id TEXT PRIMARY KEY,
|
||||
node_id INTEGER,
|
||||
stack_name TEXT,
|
||||
status TEXT NOT NULL CHECK (status IN ('prepared','ready','cancelled')),
|
||||
target_kind TEXT NOT NULL CHECK (target_kind IN ('local_socket')),
|
||||
rollback_tags_json TEXT NOT NULL,
|
||||
override_paths_json TEXT NOT NULL,
|
||||
prune_volumes_requested INTEGER NOT NULL DEFAULT 0,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_stack_update_cleanup_status
|
||||
ON stack_update_cleanup_pending(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_stack_update_cleanup_node_stack
|
||||
ON stack_update_cleanup_pending(node_id, stack_name);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS secrets (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
@@ -1699,6 +1779,8 @@ export class DatabaseService {
|
||||
try { this.db.prepare(`ALTER TABLE ${table} ADD COLUMN ${col} ${def}`).run(); } catch (e) { /* ignore */ }
|
||||
};
|
||||
|
||||
maybeAddCol('stack_update_recovery_generations', 'artifacts_retired', 'INTEGER NOT NULL DEFAULT 0');
|
||||
|
||||
// Distributed API model columns
|
||||
maybeAddCol('nodes', 'api_url', "TEXT DEFAULT ''");
|
||||
maybeAddCol('nodes', 'api_token', "TEXT DEFAULT ''");
|
||||
@@ -3510,6 +3592,268 @@ export class DatabaseService {
|
||||
this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
|
||||
}
|
||||
|
||||
public insertStackUpdateRecoveryGeneration(row: StackUpdateRecoveryGenerationRow): void {
|
||||
this.db.prepare(
|
||||
`INSERT INTO stack_update_recovery_generations (
|
||||
id, node_id, stack_name, status, phase, is_current, backup_slot_id, override_path,
|
||||
services_json, health_gate_id, gate_retain_until, artifact_expires_at,
|
||||
operation_lease_expires_at, created_at, updated_at, created_by, artifacts_retired
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
).run(
|
||||
row.id, row.node_id, row.stack_name, row.status, row.phase, row.is_current,
|
||||
row.backup_slot_id, row.override_path, row.services_json, row.health_gate_id,
|
||||
row.gate_retain_until, row.artifact_expires_at, row.operation_lease_expires_at,
|
||||
row.created_at, row.updated_at, row.created_by, row.artifacts_retired ?? 0,
|
||||
);
|
||||
}
|
||||
|
||||
public getStackUpdateRecoveryGeneration(id: string): StackUpdateRecoveryGenerationRow | undefined {
|
||||
return this.db.prepare('SELECT * FROM stack_update_recovery_generations WHERE id = ?').get(id) as StackUpdateRecoveryGenerationRow | undefined;
|
||||
}
|
||||
|
||||
public getCurrentStackUpdateRecovery(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow | undefined {
|
||||
return this.db.prepare(
|
||||
'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ? AND is_current = 1 LIMIT 1'
|
||||
).get(nodeId, stackName) as StackUpdateRecoveryGenerationRow | undefined;
|
||||
}
|
||||
|
||||
public listStackUpdateRecoveryForStack(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow[] {
|
||||
return this.db.prepare(
|
||||
'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ? ORDER BY created_at DESC'
|
||||
).all(nodeId, stackName) as StackUpdateRecoveryGenerationRow[];
|
||||
}
|
||||
|
||||
public listStackUpdateRecoveryGenerationsForNode(nodeId: number): StackUpdateRecoveryGenerationRow[] {
|
||||
return this.db.prepare(
|
||||
'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? ORDER BY created_at DESC'
|
||||
).all(nodeId) as StackUpdateRecoveryGenerationRow[];
|
||||
}
|
||||
|
||||
public updateStackUpdateRecoveryGeneration(
|
||||
id: string,
|
||||
patch: Partial<Pick<StackUpdateRecoveryGenerationRow,
|
||||
'status' | 'phase' | 'is_current' | 'override_path' | 'health_gate_id' |
|
||||
'gate_retain_until' | 'artifact_expires_at' | 'operation_lease_expires_at' | 'services_json'>>,
|
||||
): void {
|
||||
const keys = Object.keys(patch) as Array<keyof typeof patch>;
|
||||
if (keys.length === 0) return;
|
||||
const sets = keys.map((k) => `${String(k)} = ?`).join(', ');
|
||||
const values = keys.map((k) => patch[k]);
|
||||
this.db.prepare(
|
||||
`UPDATE stack_update_recovery_generations SET ${sets}, updated_at = ? WHERE id = ?`
|
||||
).run(...values, Date.now(), id);
|
||||
}
|
||||
|
||||
public casStackUpdateRecoveryPhase(
|
||||
id: string,
|
||||
expectedPhase: StackUpdateRecoveryGenerationRow['phase'],
|
||||
nextPhase: StackUpdateRecoveryGenerationRow['phase'],
|
||||
): boolean {
|
||||
const result = this.db.prepare(
|
||||
'UPDATE stack_update_recovery_generations SET phase = ?, updated_at = ? WHERE id = ? AND phase = ?'
|
||||
).run(nextPhase, Date.now(), id, expectedPhase);
|
||||
return result.changes === 1;
|
||||
}
|
||||
|
||||
public casHandoffGeneration(candidateId: string, nodeId: number, stackName: string): boolean {
|
||||
const handoff = this.db.transaction(() => {
|
||||
const candidate = this.getStackUpdateRecoveryGeneration(candidateId);
|
||||
if (!candidate || candidate.node_id !== nodeId || candidate.stack_name !== stackName) return false;
|
||||
if (candidate.status !== 'candidate' || candidate.phase !== 'acquired') return false;
|
||||
this.db.prepare(
|
||||
`UPDATE stack_update_recovery_generations
|
||||
SET status = 'superseded', is_current = 0, artifact_expires_at = ?, updated_at = ?
|
||||
WHERE node_id = ? AND stack_name = ? AND is_current = 1 AND id != ?`
|
||||
).run(Date.now() + 7 * 24 * 60 * 60 * 1000, Date.now(), nodeId, stackName, candidateId);
|
||||
const result = this.db.prepare(
|
||||
`UPDATE stack_update_recovery_generations
|
||||
SET status = 'active', is_current = 1, phase = 'handoff_committed', updated_at = ?
|
||||
WHERE id = ? AND status = 'candidate' AND phase = 'acquired'`
|
||||
).run(Date.now(), candidateId);
|
||||
return result.changes === 1;
|
||||
});
|
||||
return handoff();
|
||||
}
|
||||
|
||||
|
||||
/** Generations whose Docker/FS artifacts can be retired (not actively held). */
|
||||
public listStackUpdateRecoveryGenerationsForArtifactRetirement(now: number): StackUpdateRecoveryGenerationRow[] {
|
||||
return this.db.prepare(
|
||||
`SELECT * FROM stack_update_recovery_generations
|
||||
WHERE artifacts_retired = 0
|
||||
AND is_current = 0
|
||||
AND status IN ('abandoned', 'superseded')
|
||||
AND (artifact_expires_at IS NULL OR artifact_expires_at <= ?)
|
||||
AND (gate_retain_until IS NULL OR gate_retain_until <= ?)`
|
||||
).all(now, now) as StackUpdateRecoveryGenerationRow[];
|
||||
}
|
||||
|
||||
public markStackUpdateRecoveryArtifactsRetired(id: string): boolean {
|
||||
const result = this.db.prepare(
|
||||
`UPDATE stack_update_recovery_generations
|
||||
SET artifacts_retired = 1, override_path = NULL, updated_at = ?
|
||||
WHERE id = ? AND artifacts_retired = 0`
|
||||
).run(Date.now(), id);
|
||||
return result.changes === 1;
|
||||
}
|
||||
|
||||
public abandonStackUpdateRecoveryGeneration(id: string): boolean {
|
||||
const now = Date.now();
|
||||
const result = this.db.prepare(
|
||||
`UPDATE stack_update_recovery_generations
|
||||
SET status = 'abandoned', is_current = 0, artifact_expires_at = ?,
|
||||
operation_lease_expires_at = NULL, updated_at = ?
|
||||
WHERE id = ? AND status = 'candidate'`
|
||||
).run(now, now, id);
|
||||
return result.changes === 1;
|
||||
}
|
||||
|
||||
/** Pre-handoff candidates whose operation lease has expired. */
|
||||
public listStaleStackUpdateRecoveryCandidates(now: number): StackUpdateRecoveryGenerationRow[] {
|
||||
return this.db.prepare(
|
||||
`SELECT * FROM stack_update_recovery_generations
|
||||
WHERE status = 'candidate'
|
||||
AND operation_lease_expires_at IS NOT NULL
|
||||
AND operation_lease_expires_at <= ?`
|
||||
).all(now) as StackUpdateRecoveryGenerationRow[];
|
||||
}
|
||||
|
||||
/** Post-handoff generations stuck before immediate_verified with an expired lease. */
|
||||
public listStuckStackUpdateRecoveryGenerations(now: number): StackUpdateRecoveryGenerationRow[] {
|
||||
return this.db.prepare(
|
||||
`SELECT * FROM stack_update_recovery_generations
|
||||
WHERE status IN ('active', 'restored_current')
|
||||
AND phase IN ('handoff_committed', 'reconciling')
|
||||
AND operation_lease_expires_at IS NOT NULL
|
||||
AND operation_lease_expires_at <= ?`
|
||||
).all(now) as StackUpdateRecoveryGenerationRow[];
|
||||
}
|
||||
|
||||
public linkStackUpdateRecoveryHealthGate(id: string, healthGateId: string): void {
|
||||
this.db.prepare(
|
||||
'UPDATE stack_update_recovery_generations SET health_gate_id = ?, updated_at = ? WHERE id = ?'
|
||||
).run(healthGateId, Date.now(), id);
|
||||
}
|
||||
|
||||
public setStackUpdateRecoveryGateRetainUntil(id: string, until: number): void {
|
||||
this.db.prepare(
|
||||
'UPDATE stack_update_recovery_generations SET gate_retain_until = ?, updated_at = ? WHERE id = ?'
|
||||
).run(until, Date.now(), id);
|
||||
}
|
||||
|
||||
public listHeldStackUpdateRecoveryImageIds(nodeId: number, now: number): string[] {
|
||||
const rows = this.db.prepare(
|
||||
`SELECT services_json FROM stack_update_recovery_generations
|
||||
WHERE node_id = ?
|
||||
AND status IN ('candidate','active','restored_current','recovery_required')
|
||||
AND (artifact_expires_at IS NULL OR artifact_expires_at > ? OR gate_retain_until > ? OR is_current = 1)`
|
||||
).all(nodeId, now, now) as Array<{ services_json: string }>;
|
||||
const ids = new Set<string>();
|
||||
for (const row of rows) {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(row.services_json);
|
||||
if (!Array.isArray(parsed)) continue;
|
||||
for (const item of parsed) {
|
||||
if (!item || typeof item !== 'object') continue;
|
||||
const replicas = (item as { replicas?: unknown }).replicas;
|
||||
if (Array.isArray(replicas)) {
|
||||
for (const replica of replicas) {
|
||||
if (replica && typeof replica === 'object'
|
||||
&& typeof (replica as { imageId?: unknown }).imageId === 'string'
|
||||
&& (replica as { imageId: string }).imageId.trim()) {
|
||||
ids.add((replica as { imageId: string }).imageId);
|
||||
}
|
||||
}
|
||||
} else if (typeof (item as { imageId?: unknown }).imageId === 'string') {
|
||||
ids.add((item as { imageId: string }).imageId);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Corrupt JSON: skip.
|
||||
}
|
||||
}
|
||||
return [...ids];
|
||||
}
|
||||
|
||||
public deleteStackUpdateRecoveryGenerations(nodeId: number, stackName: string): void {
|
||||
this.db.prepare(
|
||||
'DELETE FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ?'
|
||||
).run(nodeId, stackName);
|
||||
}
|
||||
|
||||
public insertCleanupPending(row: StackUpdateCleanupPendingRow): void {
|
||||
this.db.prepare(
|
||||
`INSERT INTO stack_update_cleanup_pending (
|
||||
id, node_id, stack_name, status, target_kind, rollback_tags_json,
|
||||
override_paths_json, prune_volumes_requested, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
).run(
|
||||
row.id, row.node_id, row.stack_name, row.status, row.target_kind,
|
||||
row.rollback_tags_json, row.override_paths_json, row.prune_volumes_requested,
|
||||
row.created_at, row.updated_at,
|
||||
);
|
||||
}
|
||||
|
||||
public getCleanupPending(id: string): StackUpdateCleanupPendingRow | undefined {
|
||||
return this.db.prepare('SELECT * FROM stack_update_cleanup_pending WHERE id = ?').get(id) as StackUpdateCleanupPendingRow | undefined;
|
||||
}
|
||||
|
||||
public listReadyCleanupPending(): StackUpdateCleanupPendingRow[] {
|
||||
return this.db.prepare(
|
||||
"SELECT * FROM stack_update_cleanup_pending WHERE status = 'ready' ORDER BY created_at ASC"
|
||||
).all() as StackUpdateCleanupPendingRow[];
|
||||
}
|
||||
|
||||
public listPreparedCleanupPending(): StackUpdateCleanupPendingRow[] {
|
||||
return this.db.prepare(
|
||||
"SELECT * FROM stack_update_cleanup_pending WHERE status = 'prepared' ORDER BY created_at ASC"
|
||||
).all() as StackUpdateCleanupPendingRow[];
|
||||
}
|
||||
|
||||
public updateCleanupPendingStatus(id: string, status: StackUpdateCleanupPendingRow['status']): void {
|
||||
this.db.prepare(
|
||||
'UPDATE stack_update_cleanup_pending SET status = ?, updated_at = ? WHERE id = ?'
|
||||
).run(status, Date.now(), id);
|
||||
}
|
||||
|
||||
public deleteCleanupPending(id: string): void {
|
||||
this.db.prepare('DELETE FROM stack_update_cleanup_pending WHERE id = ?').run(id);
|
||||
}
|
||||
|
||||
public hasBlockingDeletionIntent(nodeId: number, stackName: string): boolean {
|
||||
const row = this.db.prepare(
|
||||
`SELECT id FROM stack_update_cleanup_pending
|
||||
WHERE node_id = ? AND stack_name = ? AND status IN ('prepared','ready') LIMIT 1`
|
||||
).get(nodeId, stackName);
|
||||
return !!row;
|
||||
}
|
||||
|
||||
public getPreparedDeletionIntent(nodeId: number, stackName: string): StackUpdateCleanupPendingRow | undefined {
|
||||
return this.db.prepare(
|
||||
`SELECT * FROM stack_update_cleanup_pending
|
||||
WHERE node_id = ? AND stack_name = ? AND status = 'prepared' LIMIT 1`
|
||||
).get(nodeId, stackName) as StackUpdateCleanupPendingRow | undefined;
|
||||
}
|
||||
|
||||
public getDeletionIntentById(id: string): StackUpdateCleanupPendingRow | undefined {
|
||||
return this.getCleanupPending(id);
|
||||
}
|
||||
|
||||
public commitStackDeletionReadyTransaction(intentId: string, nodeId: number, stackName: string): boolean {
|
||||
const run = this.db.transaction(() => {
|
||||
const intent = this.getCleanupPending(intentId);
|
||||
if (!intent || intent.status !== 'prepared') return false;
|
||||
if (intent.node_id !== nodeId || intent.stack_name !== stackName) return false;
|
||||
this.db.prepare(
|
||||
"UPDATE stack_update_cleanup_pending SET status = 'ready', updated_at = ? WHERE id = ? AND status = 'prepared'"
|
||||
).run(Date.now(), intentId);
|
||||
this.deleteStackUpdateRecoveryGenerations(nodeId, stackName);
|
||||
this.deleteServiceUpdateRecoveries(nodeId, stackName);
|
||||
return true;
|
||||
});
|
||||
return run();
|
||||
}
|
||||
|
||||
// --- Notification History ---
|
||||
|
||||
private mapNotificationRow(row: any): NotificationHistory {
|
||||
@@ -3878,7 +4222,16 @@ export class DatabaseService {
|
||||
this.db.prepare(`UPDATE nodes SET ${fields.join(', ')} WHERE id = ?`).run(...values);
|
||||
}
|
||||
|
||||
public deleteNode(id: number): void {
|
||||
/**
|
||||
* Delete a node row and related per-node state.
|
||||
* For a non-default local-socket node, pass localCleanup so a ready tombstone
|
||||
* is inserted in the same transaction (caller enumerates tags/paths first).
|
||||
* Remote hub node records must omit localCleanup (no Docker tombstone).
|
||||
*/
|
||||
public deleteNode(
|
||||
id: number,
|
||||
localCleanup?: { tombstoneId: string; tags: string[]; overridePaths: string[] },
|
||||
): void {
|
||||
const node = this.getNode(id);
|
||||
// Protect the last local node regardless of is_default flag.
|
||||
if (node && node.type === 'local' && this.getLocalNodeCount() <= 1) {
|
||||
@@ -3887,7 +4240,25 @@ export class DatabaseService {
|
||||
if (node?.is_default) {
|
||||
throw new Error('Cannot delete the default node');
|
||||
}
|
||||
if (localCleanup && node?.type !== 'local') {
|
||||
throw new Error('Local cleanup tombstones are only valid for local-socket nodes');
|
||||
}
|
||||
this.db.transaction(() => {
|
||||
if (localCleanup && node?.type === 'local') {
|
||||
const now = Date.now();
|
||||
this.insertCleanupPending({
|
||||
id: localCleanup.tombstoneId,
|
||||
node_id: id,
|
||||
stack_name: null,
|
||||
status: 'ready',
|
||||
target_kind: 'local_socket',
|
||||
rollback_tags_json: JSON.stringify(localCleanup.tags),
|
||||
override_paths_json: JSON.stringify(localCleanup.overridePaths),
|
||||
prune_volumes_requested: 0,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
});
|
||||
}
|
||||
this.db.prepare('DELETE FROM scheduled_task_runs WHERE task_id IN (SELECT id FROM scheduled_tasks WHERE node_id = ?)').run(id);
|
||||
this.db.prepare('DELETE FROM scheduled_tasks WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM stack_update_status WHERE node_id = ?').run(id);
|
||||
@@ -3901,6 +4272,7 @@ export class DatabaseService {
|
||||
this.db.prepare('DELETE FROM preflight_acknowledgements WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM stack_exposure WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM health_gate_runs WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM stack_update_recovery_generations WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ?').run(id);
|
||||
this.db.prepare('UPDATE blueprints SET pinned_node_id = NULL WHERE pinned_node_id = ?').run(id);
|
||||
this.deleteRoleAssignmentsByResource('node', String(id));
|
||||
|
||||
@@ -0,0 +1,521 @@
|
||||
/**
|
||||
* Shared deployed-stack deletion lifecycle (manual DELETE + Blueprint withdrawLocal).
|
||||
*
|
||||
* prepared -> (down, optional volume prune, FS delete) -> ready transaction that
|
||||
* retires full-stack recovery generations and service_update_recovery rows, then
|
||||
* sweeper removes ready tombstone tags/overrides.
|
||||
*/
|
||||
import { randomUUID } from 'crypto';
|
||||
import Docker from 'dockerode';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import {
|
||||
DatabaseService,
|
||||
type StackUpdateCleanupPendingRow,
|
||||
} from './DatabaseService';
|
||||
import { ComposeService } from './ComposeService';
|
||||
import DockerController from './DockerController';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { MeshService } from './MeshService';
|
||||
import { StackOpLockService, stackOpSkipMessage } from './StackOpLockService';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { isPathWithinBase, isValidStackName } from '../utils/validation';
|
||||
|
||||
/**
|
||||
* Directory that may contain recovery override files for a tombstone sweep.
|
||||
* Stack-scoped intents are limited to `<composeDir>/<stackName>/`;
|
||||
* node-wide intents (null stack) use the whole compose root.
|
||||
*/
|
||||
export function overrideDeletionContainmentBase(
|
||||
composeDir: string,
|
||||
stackName: string | null,
|
||||
): string | null {
|
||||
const resolvedCompose = path.resolve(composeDir);
|
||||
if (!stackName) return resolvedCompose;
|
||||
if (!isValidStackName(stackName)) return null;
|
||||
const stackDir = path.resolve(resolvedCompose, stackName);
|
||||
if (!isPathWithinBase(stackDir, resolvedCompose)) return null;
|
||||
return stackDir;
|
||||
}
|
||||
|
||||
export interface DeleteDeployedStackInput {
|
||||
nodeId: number;
|
||||
stackName: string;
|
||||
pruneVolumes: boolean;
|
||||
actor: string;
|
||||
/** When true, skip acquiring a new lock (caller already holds delete via continuation). */
|
||||
continuationIntentId?: string;
|
||||
}
|
||||
|
||||
export type DeleteDeployedStackResult =
|
||||
| { ok: true }
|
||||
| { ok: false; code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed'; error: string; existingAction?: string };
|
||||
|
||||
function collectArtifactsFromGenerations(
|
||||
generations: Array<{ override_path: string | null; services_json: string }>,
|
||||
): { tags: string[]; overridePaths: string[] } {
|
||||
const tags = new Set<string>();
|
||||
const overridePaths = new Set<string>();
|
||||
|
||||
for (const gen of generations) {
|
||||
if (gen.override_path) overridePaths.add(gen.override_path);
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(gen.services_json);
|
||||
if (!Array.isArray(parsed)) continue;
|
||||
for (const svc of parsed) {
|
||||
if (!svc || typeof svc !== 'object') continue;
|
||||
const replicas = (svc as { replicas?: unknown }).replicas;
|
||||
if (!Array.isArray(replicas)) continue;
|
||||
for (const replica of replicas) {
|
||||
const tag = replica && typeof replica === 'object'
|
||||
? (replica as { rollbackTag?: unknown }).rollbackTag
|
||||
: null;
|
||||
if (typeof tag === 'string' && tag.trim()) tags.add(tag);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Corrupt capture JSON: skip tags for this generation.
|
||||
}
|
||||
}
|
||||
|
||||
return { tags: [...tags], overridePaths: [...overridePaths] };
|
||||
}
|
||||
|
||||
function collectArtifacts(nodeId: number, stackName: string): { tags: string[]; overridePaths: string[] } {
|
||||
return collectArtifactsFromGenerations(
|
||||
DatabaseService.getInstance().listStackUpdateRecoveryForStack(nodeId, stackName),
|
||||
);
|
||||
}
|
||||
|
||||
function parseJsonStringArray(raw: string): string[] {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed.filter((item): item is string => typeof item === 'string');
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export class DeployedStackDeletionService {
|
||||
private static instance: DeployedStackDeletionService;
|
||||
|
||||
public static getInstance(): DeployedStackDeletionService {
|
||||
if (!DeployedStackDeletionService.instance) {
|
||||
DeployedStackDeletionService.instance = new DeployedStackDeletionService();
|
||||
}
|
||||
return DeployedStackDeletionService.instance;
|
||||
}
|
||||
|
||||
public assertNoBlockingDeletionIntent(nodeId: number, stackName: string): void {
|
||||
if (DatabaseService.getInstance().hasBlockingDeletionIntent(nodeId, stackName)) {
|
||||
throw new Error(
|
||||
`Stack "${stackName}" has a deletion in progress and cannot be created or mutated until cleanup finishes.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public async deleteDeployedStack(input: DeleteDeployedStackInput): Promise<DeleteDeployedStackResult> {
|
||||
const { nodeId, stackName, actor } = input;
|
||||
const locks = StackOpLockService.getInstance();
|
||||
|
||||
if (input.continuationIntentId) {
|
||||
const cont = locks.tryAcquireDeletionContinuation({
|
||||
intentId: input.continuationIntentId,
|
||||
nodeId,
|
||||
stackName,
|
||||
});
|
||||
if (!cont.acquired) {
|
||||
return {
|
||||
ok: false,
|
||||
code: 'lock_conflict',
|
||||
error: stackOpSkipMessage(stackName, cont.existing.action),
|
||||
existingAction: cont.existing.action,
|
||||
};
|
||||
}
|
||||
try {
|
||||
return await this.runDeletionBody(input, input.continuationIntentId);
|
||||
} finally {
|
||||
locks.release(nodeId, stackName);
|
||||
}
|
||||
}
|
||||
|
||||
const exclusive = await locks.runExclusive(nodeId, stackName, 'delete', actor, async () => {
|
||||
return this.runDeletionBody(input);
|
||||
});
|
||||
if (!exclusive.ran) {
|
||||
return {
|
||||
ok: false,
|
||||
code: 'lock_conflict',
|
||||
error: stackOpSkipMessage(stackName, exclusive.existing.action),
|
||||
existingAction: exclusive.existing.action,
|
||||
};
|
||||
}
|
||||
return exclusive.result;
|
||||
}
|
||||
|
||||
private async runDeletionBody(
|
||||
input: DeleteDeployedStackInput,
|
||||
existingIntentId?: string,
|
||||
): Promise<DeleteDeployedStackResult> {
|
||||
const { nodeId, stackName, pruneVolumes } = input;
|
||||
const db = DatabaseService.getInstance();
|
||||
|
||||
let intentId = existingIntentId;
|
||||
if (!intentId) {
|
||||
const { tags, overridePaths } = collectArtifacts(nodeId, stackName);
|
||||
const now = Date.now();
|
||||
const row: StackUpdateCleanupPendingRow = {
|
||||
id: randomUUID(),
|
||||
node_id: nodeId,
|
||||
stack_name: stackName,
|
||||
status: 'prepared',
|
||||
target_kind: 'local_socket',
|
||||
rollback_tags_json: JSON.stringify(tags),
|
||||
override_paths_json: JSON.stringify(overridePaths),
|
||||
prune_volumes_requested: pruneVolumes ? 1 : 0,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
};
|
||||
try {
|
||||
db.insertCleanupPending(row);
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
code: 'tombstone_failed',
|
||||
error: getErrorMessage(error, 'Failed to prepare stack deletion'),
|
||||
};
|
||||
}
|
||||
intentId = row.id;
|
||||
}
|
||||
|
||||
const intent = db.getDeletionIntentById(intentId);
|
||||
if (!intent || intent.status !== 'prepared') {
|
||||
return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' };
|
||||
}
|
||||
|
||||
try {
|
||||
await ComposeService.getInstance(nodeId).downStack(stackName);
|
||||
} catch (downErr) {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Compose down failed or no-op for %s:',
|
||||
sanitizeForLog(stackName),
|
||||
downErr,
|
||||
);
|
||||
}
|
||||
|
||||
if (intent.prune_volumes_requested === 1) {
|
||||
try {
|
||||
await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]);
|
||||
} catch (pruneErr) {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Volume prune failed for %s, continuing delete:',
|
||||
sanitizeForLog(stackName),
|
||||
pruneErr,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await FileSystemService.getInstance(nodeId).deleteStack(stackName);
|
||||
} catch (fsErr) {
|
||||
db.updateCleanupPendingStatus(intentId, 'cancelled');
|
||||
return {
|
||||
ok: false,
|
||||
code: 'fs_failed',
|
||||
error: getErrorMessage(fsErr, 'Failed to remove stack files'),
|
||||
};
|
||||
}
|
||||
|
||||
if (!db.commitStackDeletionReadyTransaction(intentId, nodeId, stackName)) {
|
||||
return {
|
||||
ok: false,
|
||||
code: 'db_failed',
|
||||
error: 'Failed to commit stack deletion ready transaction',
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
db.clearStackUpdateStatus(nodeId, stackName);
|
||||
db.clearStackScanAttempts(nodeId, stackName);
|
||||
db.deleteRoleAssignmentsByResource('stack', stackName);
|
||||
db.deleteGitSource(stackName);
|
||||
db.deleteStackDossier(nodeId, stackName);
|
||||
db.deleteStackDriftFindings(nodeId, stackName);
|
||||
db.deleteStackExposureIntents(nodeId, stackName);
|
||||
db.deleteStackExposure(nodeId, stackName);
|
||||
db.deleteStackProjectEnvFiles(nodeId, stackName);
|
||||
db.deleteStackScans(nodeId, stackName);
|
||||
} catch (dbErr) {
|
||||
console.error(
|
||||
'[DeployedStackDeletion] Secondary DB cleanup failed for %s; recovery rows already retired:',
|
||||
sanitizeForLog(stackName),
|
||||
dbErr,
|
||||
);
|
||||
return {
|
||||
ok: false,
|
||||
code: 'db_failed',
|
||||
error: getErrorMessage(dbErr, 'Failed to clear stack database state'),
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
await MeshService.getInstance().optOutStack(nodeId, stackName, input.actor);
|
||||
} catch (meshErr) {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Mesh opt-out failed for %s:',
|
||||
sanitizeForLog(stackName),
|
||||
meshErr,
|
||||
);
|
||||
}
|
||||
|
||||
await this.sweepReadyIntent(intentId);
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove rollback tags + override paths for a ready tombstone, then drop the row.
|
||||
* When the node row is already gone (local-node delete), pass a preserved local
|
||||
* Docker handle and composeDir so we never fall back to a remote default node.
|
||||
*/
|
||||
public async sweepReadyIntent(
|
||||
intentId: string,
|
||||
opts?: { docker?: Docker; composeDir?: string },
|
||||
): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const intent = db.getCleanupPending(intentId);
|
||||
if (!intent || intent.status !== 'ready') return;
|
||||
if (intent.node_id == null) {
|
||||
db.deleteCleanupPending(intentId);
|
||||
return;
|
||||
}
|
||||
|
||||
const tags = parseJsonStringArray(intent.rollback_tags_json);
|
||||
const overridePaths = parseJsonStringArray(intent.override_paths_json);
|
||||
|
||||
let docker: Docker;
|
||||
if (opts?.docker) {
|
||||
docker = opts.docker;
|
||||
} else if (db.getNode(intent.node_id)) {
|
||||
docker = DockerController.getInstance(intent.node_id).getDocker();
|
||||
} else if (intent.target_kind === 'local_socket') {
|
||||
// Deleted local node: local_socket tombstones always target the host Docker socket.
|
||||
docker = new Docker();
|
||||
} else {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Cannot sweep tombstone %s: node gone and target is not local_socket',
|
||||
sanitizeForLog(intentId),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let incomplete = false;
|
||||
|
||||
for (const tag of tags) {
|
||||
try {
|
||||
await docker.getImage(tag).remove({ force: true });
|
||||
} catch (error) {
|
||||
const status = (error as { statusCode?: number }).statusCode;
|
||||
const message = getErrorMessage(error, 'unknown').toLowerCase();
|
||||
if (status === 404 || message.includes('no such image') || message.includes('not found')) {
|
||||
continue;
|
||||
}
|
||||
incomplete = true;
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Failed to remove rollback tag %s: %s',
|
||||
sanitizeForLog(tag),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const baseDir = opts?.composeDir
|
||||
?? (db.getNode(intent.node_id)
|
||||
? FileSystemService.getInstance(intent.node_id).getBaseDir()
|
||||
: null);
|
||||
|
||||
const containmentBase = baseDir
|
||||
? overrideDeletionContainmentBase(baseDir, intent.stack_name)
|
||||
: null;
|
||||
if (baseDir && !containmentBase) {
|
||||
incomplete = true;
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Refusing override sweep: invalid stack containment for tombstone %s',
|
||||
sanitizeForLog(intentId),
|
||||
);
|
||||
}
|
||||
|
||||
for (const overridePath of overridePaths) {
|
||||
try {
|
||||
const resolved = path.resolve(overridePath);
|
||||
const basename = path.basename(resolved);
|
||||
if (!/^\.sencho-recovery-[a-f0-9]+\.yml$/i.test(basename)) {
|
||||
incomplete = true;
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Refusing to delete non-recovery override: %s',
|
||||
sanitizeForLog(overridePath),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (containmentBase && !isPathWithinBase(resolved, containmentBase)) {
|
||||
incomplete = true;
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Refusing to delete override outside stack containment: %s',
|
||||
sanitizeForLog(overridePath),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (!baseDir && !path.isAbsolute(resolved)) {
|
||||
incomplete = true;
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Refusing relative override without compose dir: %s',
|
||||
sanitizeForLog(overridePath),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (!containmentBase && baseDir) {
|
||||
incomplete = true;
|
||||
continue;
|
||||
}
|
||||
await fs.unlink(resolved);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
incomplete = true;
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Failed to delete override %s: %s',
|
||||
sanitizeForLog(overridePath),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the ready tombstone when cleanup is incomplete so startup can retry.
|
||||
if (!incomplete) {
|
||||
db.deleteCleanupPending(intentId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Enumerate rollback tags and override paths for every stack recovery on a node. */
|
||||
public collectNodeArtifacts(nodeId: number): { tags: string[]; overridePaths: string[] } {
|
||||
return collectArtifactsFromGenerations(
|
||||
DatabaseService.getInstance().listStackUpdateRecoveryGenerationsForNode(nodeId),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a local-socket node with an atomic ready tombstone, then sweep.
|
||||
* Remote node records call DatabaseService.deleteNode without cleanup.
|
||||
*/
|
||||
public async deleteLocalNode(nodeId: number): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const node = db.getNode(nodeId);
|
||||
if (!node) throw new Error('Node not found');
|
||||
if (node.type !== 'local') {
|
||||
db.deleteNode(nodeId);
|
||||
return;
|
||||
}
|
||||
// Preserve Docker + compose dir before the row disappears so sweep never
|
||||
// targets a remote default node.
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
const composeDir = FileSystemService.getInstance(nodeId).getBaseDir();
|
||||
const { tags, overridePaths } = this.collectNodeArtifacts(nodeId);
|
||||
const tombstoneId = randomUUID();
|
||||
db.deleteNode(nodeId, { tombstoneId, tags, overridePaths });
|
||||
NodeRegistry.getInstance().evictConnection(nodeId);
|
||||
try {
|
||||
await this.sweepReadyIntent(tombstoneId, { docker, composeDir });
|
||||
} catch (error) {
|
||||
// Node row is already gone; leave the ready tombstone for startup resume.
|
||||
console.error(
|
||||
'[DeployedStackDeletion] Sweep after local-node delete deferred for tombstone %s: %s',
|
||||
sanitizeForLog(tombstoneId),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Startup reconciliation: resume prepared intents, complete ready transitions
|
||||
* when the directory is already gone, and sweep ready tombstones.
|
||||
*/
|
||||
public async reconcileAtStartup(): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
for (const intent of db.listPreparedCleanupPending()) {
|
||||
if (intent.node_id == null || !intent.stack_name) continue;
|
||||
const nodeId = intent.node_id;
|
||||
const stackName = intent.stack_name;
|
||||
const stackDir = path.join(FileSystemService.getInstance(nodeId).getBaseDir(), stackName);
|
||||
let dirExists = true;
|
||||
try {
|
||||
await fs.access(stackDir);
|
||||
} catch (accessError) {
|
||||
const code = (accessError as NodeJS.ErrnoException).code;
|
||||
if (code === 'ENOENT') {
|
||||
dirExists = false;
|
||||
} else {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Startup access error for %s (not treating as absent): %s',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(accessError, 'unknown')),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (!dirExists) {
|
||||
if (!db.commitStackDeletionReadyTransaction(intent.id, nodeId, stackName)) {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Startup ready commit failed for %s/%s',
|
||||
nodeId,
|
||||
sanitizeForLog(stackName),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await MeshService.getInstance().optOutStack(nodeId, stackName, 'system:startup');
|
||||
} catch (meshErr) {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Startup mesh opt-out failed for %s: %s',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(meshErr, 'unknown')),
|
||||
);
|
||||
}
|
||||
await this.sweepReadyIntent(intent.id);
|
||||
continue;
|
||||
}
|
||||
|
||||
const result = await this.deleteDeployedStack({
|
||||
nodeId,
|
||||
stackName,
|
||||
pruneVolumes: intent.prune_volumes_requested === 1,
|
||||
actor: 'system:startup',
|
||||
continuationIntentId: intent.id,
|
||||
});
|
||||
if (!result.ok) {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Startup resume failed for %s: %s',
|
||||
sanitizeForLog(stackName),
|
||||
result.error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const intent of db.listReadyCleanupPending()) {
|
||||
if (intent.node_id != null && intent.stack_name) {
|
||||
try {
|
||||
await MeshService.getInstance().optOutStack(intent.node_id, intent.stack_name, 'system:startup');
|
||||
} catch (meshErr) {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Ready-resume mesh opt-out failed for %s: %s',
|
||||
sanitizeForLog(intent.stack_name),
|
||||
sanitizeForLog(getErrorMessage(meshErr, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
await this.sweepReadyIntent(intent.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
import type { NetworkingNetworkBase } from './network/networkingTypes';
|
||||
import { isPathWithinBase } from '../utils/validation';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { describeSpawnError } from '../utils/spawnErrors';
|
||||
import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs';
|
||||
@@ -2258,6 +2259,57 @@ class DockerController {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Strict Result API for full-stack updates. Never converts Compose-ps + fallback
|
||||
* failure into empty success. Compose-managed containers are never orphan IDs.
|
||||
*/
|
||||
public async classifyLegacyOrphansForUpdate(
|
||||
stackName: string,
|
||||
): Promise<
|
||||
| { status: 'none' }
|
||||
| { status: 'orphans'; ids: string[] }
|
||||
| { status: 'classification_failed'; error: string }
|
||||
> {
|
||||
const stackDir = path.join(NodeRegistry.getInstance().getComposeDir(this.nodeId), stackName);
|
||||
const toIds = (list: Array<{ Id?: string }>) =>
|
||||
list.filter((c): c is { Id: string } => typeof c.Id === 'string' && c.Id.length > 0)
|
||||
.map((c) => c.Id);
|
||||
|
||||
const fallbackOrphans = async (): Promise<
|
||||
| { status: 'none' }
|
||||
| { status: 'orphans'; ids: string[] }
|
||||
| { status: 'classification_failed'; error: string }
|
||||
> => {
|
||||
try {
|
||||
const ids = toIds(await this.smartFallback(stackName, stackDir));
|
||||
return ids.length === 0 ? { status: 'none' } : { status: 'orphans', ids };
|
||||
} catch (fallbackError) {
|
||||
return {
|
||||
status: 'classification_failed',
|
||||
error: getErrorMessage(fallbackError, 'Legacy orphan classification failed'),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
const composeContainers = await this.fetchComposePsContainers(stackName, stackDir);
|
||||
// Compose already manages this stack: no legacy orphan cleanup (same as deploy).
|
||||
if (composeContainers.length > 0) return { status: 'none' };
|
||||
return await fallbackOrphans();
|
||||
} catch (error) {
|
||||
const execError = error as NodeJS.ErrnoException & { stderr?: string };
|
||||
const mapped = describeSpawnError(execError, { command: 'docker compose ps' });
|
||||
const detail = execError.stderr || mapped.message || getErrorMessage(error, 'docker compose ps failed');
|
||||
console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(detail));
|
||||
// Unlike getLegacyOrphanContainersByStack, never convert dual failure into empty success.
|
||||
const fallback = await fallbackOrphans();
|
||||
if (fallback.status === 'classification_failed') {
|
||||
return { status: 'classification_failed', error: String(detail) };
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
public async getContainersByStack(stackName: string) {
|
||||
// Resolve the compose dir and the authored prefix for THIS controller's node,
|
||||
// not the process default, so a non-default local node sees its own stack dir
|
||||
|
||||
@@ -545,6 +545,11 @@ export class FileSystemService {
|
||||
}
|
||||
|
||||
async createStack(stackName: string): Promise<void> {
|
||||
if (DatabaseService.getInstance().hasBlockingDeletionIntent(this.nodeId, stackName)) {
|
||||
throw new Error(
|
||||
`Stack "${stackName}" has a deletion in progress and cannot be created until cleanup finishes.`,
|
||||
);
|
||||
}
|
||||
const stackDir = this.resolveStackDir(stackName);
|
||||
await this.assertRealWithinBase(stackDir);
|
||||
|
||||
|
||||
@@ -1104,7 +1104,13 @@ export class SchedulerService {
|
||||
);
|
||||
if (!lock.ran) return skipMessage(stackName, lock.existing.action);
|
||||
db.clearStackUpdateStatus(nodeId, stackName);
|
||||
HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler');
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler');
|
||||
const orchResult = lock.result;
|
||||
const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
|
||||
if (recoveryId) {
|
||||
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
||||
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
|
||||
}
|
||||
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
|
||||
@@ -237,7 +237,17 @@ export class ServiceUpdateRecoveryService {
|
||||
return (imageId: string) => {
|
||||
const held = this.getHeldImageIds(nodeId);
|
||||
if (held === null) return true;
|
||||
return held.has(imageId);
|
||||
if (held.has(imageId)) return true;
|
||||
try {
|
||||
// Dynamic import avoids a static cycle with StackUpdateRecoveryService.
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const { StackUpdateRecoveryService } = require('./StackUpdateRecoveryService') as typeof import('./StackUpdateRecoveryService');
|
||||
const stackHeld = StackUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
|
||||
if (stackHeld === null) return true;
|
||||
return stackHeld.has(imageId);
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
/**
|
||||
* Tracks in-flight stack lifecycle operations (deploy, down, restart, stop,
|
||||
* start, update, rollback, backup) per (nodeId, stackName). A second request to
|
||||
@@ -9,7 +10,7 @@
|
||||
* which matches the lifecycle of any in-flight `docker compose` child process.
|
||||
*/
|
||||
|
||||
export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup';
|
||||
export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup' | 'delete';
|
||||
|
||||
/**
|
||||
* Note returned by a background path that skipped its operation because a manual
|
||||
@@ -70,6 +71,31 @@ export class StackOpLockService {
|
||||
this.locks.delete(this.key(nodeId, stackName));
|
||||
}
|
||||
|
||||
/**
|
||||
* Matching-intent continuation for crash-resume of a prepared deletion.
|
||||
* Succeeds only when the persisted intent matches and no other in-memory lock holds.
|
||||
*/
|
||||
public tryAcquireDeletionContinuation(args: {
|
||||
intentId: string;
|
||||
nodeId: number;
|
||||
stackName: string;
|
||||
}): AcquireResult {
|
||||
const intent = DatabaseService.getInstance().getDeletionIntentById(args.intentId);
|
||||
if (
|
||||
!intent
|
||||
|| intent.status !== 'prepared'
|
||||
|| intent.node_id !== args.nodeId
|
||||
|| intent.stack_name !== args.stackName
|
||||
) {
|
||||
const existing = this.locks.get(this.key(args.nodeId, args.stackName));
|
||||
return {
|
||||
acquired: false,
|
||||
existing: existing ?? { action: 'delete', startedAt: Date.now(), user: 'system' },
|
||||
};
|
||||
}
|
||||
return this.tryAcquire(args.nodeId, args.stackName, 'delete', 'system:deletion-continuation');
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire the per-(nodeId, stackName) lock for the duration of `fn`, then
|
||||
* release it. Returns `{ ran: true, result }` when the lock was free, or
|
||||
|
||||
@@ -47,7 +47,7 @@ export interface ServiceUpdateOptions {
|
||||
}
|
||||
|
||||
export type OrchestratorResult =
|
||||
| { kind: 'stack_compose_done' }
|
||||
| { kind: 'stack_compose_done'; recoveryId: string | null }
|
||||
| {
|
||||
kind: 'service_done';
|
||||
serviceName: string;
|
||||
@@ -182,10 +182,11 @@ export class StackUpdateOrchestrator {
|
||||
ctx: UpdateOperationContext,
|
||||
options: StackComposeOptions,
|
||||
): Promise<OrchestratorResult> {
|
||||
await ComposeService.getInstance(ctx.nodeId).updateStack(
|
||||
const updateResult = await ComposeService.getInstance(ctx.nodeId).updateStack(
|
||||
ctx.stackName, options.terminalWs ?? undefined, options.atomic,
|
||||
);
|
||||
return { kind: 'stack_compose_done' };
|
||||
const recoveryId = updateResult?.recoveryId ?? null;
|
||||
return { kind: 'stack_compose_done', recoveryId };
|
||||
}
|
||||
|
||||
private async executeServiceUpdate(
|
||||
@@ -195,6 +196,16 @@ export class StackUpdateOrchestrator {
|
||||
): Promise<OrchestratorResult> {
|
||||
const { nodeId, stackName } = ctx;
|
||||
|
||||
{
|
||||
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
||||
if (StackUpdateRecoveryService.getInstance().isRestoredCurrentPinActive(nodeId, stackName)) {
|
||||
return serviceFailed(
|
||||
'stack_recovery_pin_active',
|
||||
'This stack is pinned to a restored recovery generation. Run a full-stack Update to continue.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const loaded = await this.loadServiceSpec(nodeId, stackName, serviceName);
|
||||
if (!loaded.ok) return loaded.result;
|
||||
const { spec, services } = loaded;
|
||||
@@ -308,6 +319,16 @@ export class StackUpdateOrchestrator {
|
||||
const { nodeId, stackName } = ctx;
|
||||
const recoveryId = options.recoveryId as string;
|
||||
|
||||
{
|
||||
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
||||
if (StackUpdateRecoveryService.getInstance().isRestoredCurrentPinActive(nodeId, stackName)) {
|
||||
return serviceFailed(
|
||||
'stack_recovery_pin_active',
|
||||
'This stack is pinned to a restored recovery generation. Run a full-stack Update to continue.',
|
||||
{ recoveryId },
|
||||
);
|
||||
}
|
||||
}
|
||||
const recovery = ServiceUpdateRecoveryService.getInstance().get(recoveryId);
|
||||
if (
|
||||
!recovery ||
|
||||
|
||||
@@ -0,0 +1,767 @@
|
||||
/**
|
||||
* Full-stack update recovery generations: capture, opaque rollback tags,
|
||||
* stack-local recovery override, handoff, compensation, and prune holds.
|
||||
*
|
||||
* Separate from ServiceUpdateRecoveryService (service-scoped snapshots).
|
||||
* Does not run Compose; ComposeService / orchestrator own Docker mutations.
|
||||
*
|
||||
* Recovery fidelity contract (supported):
|
||||
* - Prior images are restored via opaque hold tags (`--pull never --no-build`).
|
||||
* - Observed running replica count is restored via compose `scale`.
|
||||
* - Services that were fully stopped at capture are kept at `scale: 0`.
|
||||
* - Authored replica counts are not used when they diverge from observed state.
|
||||
*/
|
||||
import { randomUUID } from 'crypto';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import {
|
||||
DatabaseService,
|
||||
type StackUpdateRecoveryGenerationRow,
|
||||
} from './DatabaseService';
|
||||
import DockerController from './DockerController';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { buildEffectiveServiceModel } from './effectiveServiceModel';
|
||||
import {
|
||||
classifyReferenceKind,
|
||||
type ImageReferenceKind,
|
||||
resolveComposeProjectContext,
|
||||
} from './composeProjectContext';
|
||||
import { getComposeCommandTimeoutMs } from './ComposeService';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { isValidStackName } from '../utils/validation';
|
||||
|
||||
const SWEEP_INTERVAL_MS = 5 * 60_000;
|
||||
const INITIAL_SWEEP_DELAY_MS = 30_000;
|
||||
const MIN_RECOVERY_WINDOW_SECONDS = 90;
|
||||
const RECOVERY_TTL_BUFFER_MS = 30 * 60_000;
|
||||
const GATE_RETAIN_DEFAULT_MS = 2 * 60 * 60_000;
|
||||
const RECOVERY_PROBE_DELAY_MS = 3_000;
|
||||
|
||||
export interface StackRecoveryReplicaCapture {
|
||||
containerId: string | null;
|
||||
imageId: string | null;
|
||||
repoDigest: string | null;
|
||||
state: 'running' | 'stopped' | 'none';
|
||||
rollbackTag: string | null;
|
||||
}
|
||||
|
||||
export interface StackRecoveryServiceCapture {
|
||||
serviceName: string;
|
||||
/** Observed running replica count at capture (supported restore scale). */
|
||||
scale: number;
|
||||
hasBuild: boolean;
|
||||
declaredImageRef: string | null;
|
||||
referenceKind: ImageReferenceKind;
|
||||
replicas: StackRecoveryReplicaCapture[];
|
||||
}
|
||||
|
||||
export interface CaptureStackUpdateInput {
|
||||
nodeId: number;
|
||||
stackName: string;
|
||||
createdBy: string | null;
|
||||
}
|
||||
|
||||
function yamlQuote(value: string): string {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function sanitizeServiceSlug(name: string): string {
|
||||
return name.replace(/[^a-zA-Z0-9._-]/g, '-').toLowerCase() || 'svc';
|
||||
}
|
||||
|
||||
function opaqueRollbackTag(generationId: string, serviceName: string): string {
|
||||
const short = generationId.replace(/-/g, '').slice(0, 12);
|
||||
return `sencho-rb/${short}/${sanitizeServiceSlug(serviceName)}:hold`;
|
||||
}
|
||||
|
||||
function parseServicesJson(raw: string): StackRecoveryServiceCapture[] {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed as StackRecoveryServiceCapture[];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function collectImageIdsFromServicesJson(servicesJson: string): string[] {
|
||||
const ids = new Set<string>();
|
||||
for (const svc of parseServicesJson(servicesJson)) {
|
||||
for (const replica of svc.replicas ?? []) {
|
||||
if (replica.imageId && replica.imageId.trim()) ids.add(replica.imageId);
|
||||
}
|
||||
}
|
||||
return [...ids];
|
||||
}
|
||||
|
||||
function collectRollbackTags(services: StackRecoveryServiceCapture[]): string[] {
|
||||
const tags = new Set<string>();
|
||||
for (const svc of services) {
|
||||
for (const replica of svc.replicas ?? []) {
|
||||
if (replica.rollbackTag) tags.add(replica.rollbackTag);
|
||||
}
|
||||
}
|
||||
return [...tags];
|
||||
}
|
||||
|
||||
export class StackUpdateRecoveryService {
|
||||
private static instance: StackUpdateRecoveryService;
|
||||
private started = false;
|
||||
private intervalId: NodeJS.Timeout | null = null;
|
||||
private initialTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
private constructor() {}
|
||||
|
||||
public static getInstance(): StackUpdateRecoveryService {
|
||||
if (!StackUpdateRecoveryService.instance) {
|
||||
StackUpdateRecoveryService.instance = new StackUpdateRecoveryService();
|
||||
}
|
||||
return StackUpdateRecoveryService.instance;
|
||||
}
|
||||
|
||||
public static resetForTests(): void {
|
||||
if (StackUpdateRecoveryService.instance) {
|
||||
StackUpdateRecoveryService.instance.stop();
|
||||
}
|
||||
StackUpdateRecoveryService.instance = new StackUpdateRecoveryService();
|
||||
}
|
||||
|
||||
public start(): void {
|
||||
this.started = true;
|
||||
if (this.initialTimer || this.intervalId) return;
|
||||
this.initialTimer = setTimeout(() => {
|
||||
void this.reconcileIncomplete();
|
||||
this.intervalId = setInterval(() => {
|
||||
void this.reconcileIncomplete();
|
||||
}, SWEEP_INTERVAL_MS);
|
||||
}, INITIAL_SWEEP_DELAY_MS);
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
this.started = false;
|
||||
if (this.initialTimer) {
|
||||
clearTimeout(this.initialTimer);
|
||||
this.initialTimer = null;
|
||||
}
|
||||
if (this.intervalId) {
|
||||
clearInterval(this.intervalId);
|
||||
this.intervalId = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate exact Compose invocation, backup files, snapshot runtime, create
|
||||
* opaque tags + recovery override, insert candidate generation.
|
||||
*/
|
||||
public async captureCandidate(input: CaptureStackUpdateInput): Promise<StackUpdateRecoveryGenerationRow> {
|
||||
const { nodeId, stackName, createdBy } = input;
|
||||
if (!isValidStackName(stackName)) {
|
||||
throw new Error('Invalid stack name');
|
||||
}
|
||||
|
||||
const context = await resolveComposeProjectContext(nodeId, stackName);
|
||||
await context.validateForMutation();
|
||||
// Exact mutating invocation (authored files + env + generated Mesh override)
|
||||
// must validate before any backup, tag, or override write.
|
||||
const { ComposeService } = await import('./ComposeService');
|
||||
await ComposeService.getInstance(nodeId).validateExactComposeInvocation(stackName);
|
||||
|
||||
const backupSlotId = await context.backupFromContext('update');
|
||||
|
||||
const model = await buildEffectiveServiceModel(nodeId, stackName);
|
||||
if (!model.renderable) {
|
||||
throw new Error(model.error || 'Effective Compose model failed to render');
|
||||
}
|
||||
|
||||
const generationId = randomUUID();
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
const services: StackRecoveryServiceCapture[] = [];
|
||||
const createdTags: string[] = [];
|
||||
let overridePath: string | null = null;
|
||||
|
||||
try {
|
||||
for (const spec of model.services) {
|
||||
const declaredImageRef = spec.declaredImage;
|
||||
const referenceKind = classifyReferenceKind(declaredImageRef);
|
||||
const listed = await docker.listContainers({
|
||||
all: true,
|
||||
filters: {
|
||||
label: [
|
||||
`com.docker.compose.project=${stackName}`,
|
||||
`com.docker.compose.service=${spec.name}`,
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const replicas: StackRecoveryReplicaCapture[] = [];
|
||||
for (const info of listed) {
|
||||
try {
|
||||
const inspect = await docker.getContainer(info.Id).inspect();
|
||||
const status = inspect.State?.Status;
|
||||
const state: StackRecoveryReplicaCapture['state'] =
|
||||
status === 'running' ? 'running' : status ? 'stopped' : 'none';
|
||||
const imageId = typeof inspect.Image === 'string' && inspect.Image.length > 0
|
||||
? inspect.Image
|
||||
: null;
|
||||
if ((state === 'running' || state === 'stopped') && !imageId) {
|
||||
throw new Error(
|
||||
`Service "${spec.name}" replica ${info.Id.slice(0, 12)} has no protectable image id`,
|
||||
);
|
||||
}
|
||||
let repoDigest: string | null = null;
|
||||
if (imageId) {
|
||||
try {
|
||||
const image = await docker.getImage(imageId).inspect();
|
||||
const digests = (image.RepoDigests ?? []) as string[];
|
||||
repoDigest = digests.length > 0 ? digests[0] : null;
|
||||
} catch {
|
||||
repoDigest = null;
|
||||
}
|
||||
}
|
||||
replicas.push({
|
||||
containerId: info.Id,
|
||||
imageId,
|
||||
repoDigest,
|
||||
state,
|
||||
rollbackTag: null,
|
||||
});
|
||||
} catch (error) {
|
||||
if ((error as { statusCode?: number })?.statusCode === 404) continue;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const runningCount = replicas.filter((r) => r.state === 'running').length;
|
||||
services.push({
|
||||
serviceName: spec.name,
|
||||
scale: runningCount,
|
||||
hasBuild: spec.hasBuild,
|
||||
declaredImageRef,
|
||||
referenceKind,
|
||||
replicas,
|
||||
});
|
||||
}
|
||||
|
||||
const taggedIds = new Set<string>();
|
||||
for (const svc of services) {
|
||||
const primary = svc.replicas.find((r) => r.imageId) ?? null;
|
||||
if (!primary?.imageId) continue;
|
||||
|
||||
const tag = opaqueRollbackTag(generationId, svc.serviceName);
|
||||
const tagKey = `${primary.imageId}|${tag}`;
|
||||
if (!taggedIds.has(tagKey)) {
|
||||
const { repo, tagName } = splitOpaqueTag(tag);
|
||||
await docker.getImage(primary.imageId).tag({ repo, tag: tagName });
|
||||
taggedIds.add(tagKey);
|
||||
createdTags.push(tag);
|
||||
}
|
||||
for (const replica of svc.replicas) {
|
||||
if (replica.imageId === primary.imageId) {
|
||||
replica.rollbackTag = tag;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
overridePath = await this.writeRecoveryOverride(nodeId, stackName, generationId, services);
|
||||
|
||||
const now = Date.now();
|
||||
const row: StackUpdateRecoveryGenerationRow = {
|
||||
id: generationId,
|
||||
node_id: nodeId,
|
||||
stack_name: stackName,
|
||||
status: 'candidate',
|
||||
phase: 'captured',
|
||||
is_current: 0,
|
||||
backup_slot_id: backupSlotId,
|
||||
override_path: overridePath,
|
||||
services_json: JSON.stringify(services),
|
||||
health_gate_id: null,
|
||||
gate_retain_until: null,
|
||||
artifact_expires_at: null,
|
||||
operation_lease_expires_at: now + getComposeCommandTimeoutMs() + RECOVERY_TTL_BUFFER_MS,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
created_by: createdBy,
|
||||
artifacts_retired: 0,
|
||||
};
|
||||
DatabaseService.getInstance().insertStackUpdateRecoveryGeneration(row);
|
||||
return row;
|
||||
} catch (error) {
|
||||
await this.bestEffortRemoveTags(nodeId, createdTags);
|
||||
if (overridePath) {
|
||||
try {
|
||||
await fs.unlink(overridePath);
|
||||
} catch {
|
||||
// Best-effort mid-capture cleanup.
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async writeRecoveryOverride(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
generationId: string,
|
||||
services: StackRecoveryServiceCapture[],
|
||||
): Promise<string> {
|
||||
if (!isValidStackName(stackName)) {
|
||||
throw new Error('Invalid stack name');
|
||||
}
|
||||
// Canonical inline path barrier (same pattern as ComposeService.renderConfig).
|
||||
const baseResolved = path.resolve(FileSystemService.getInstance(nodeId).getBaseDir());
|
||||
const stackDir = path.resolve(baseResolved, stackName);
|
||||
if (!stackDir.startsWith(baseResolved + path.sep)) {
|
||||
throw new Error('Invalid stack path');
|
||||
}
|
||||
let stackDirReal: string;
|
||||
let baseReal: string;
|
||||
try {
|
||||
[stackDirReal, baseReal] = await Promise.all([
|
||||
fs.realpath(stackDir),
|
||||
fs.realpath(baseResolved),
|
||||
]);
|
||||
} catch {
|
||||
throw new Error('Stack directory not found');
|
||||
}
|
||||
if (stackDirReal !== baseReal && !stackDirReal.startsWith(baseReal + path.sep)) {
|
||||
throw new Error('Stack directory escapes compose base');
|
||||
}
|
||||
|
||||
const short = generationId.replace(/-/g, '').slice(0, 12);
|
||||
if (!/^[a-f0-9]{12}$/i.test(short)) {
|
||||
throw new Error('Invalid recovery generation id');
|
||||
}
|
||||
const filename = `.sencho-recovery-${short}.yml`;
|
||||
const abs = path.resolve(stackDirReal, filename);
|
||||
if (!abs.startsWith(stackDirReal + path.sep)) {
|
||||
throw new Error('Recovery override path escapes stack directory');
|
||||
}
|
||||
|
||||
const lines: string[] = ['services:'];
|
||||
let wroteAny = false;
|
||||
for (const svc of services) {
|
||||
const tag = svc.replicas.find((r) => r.rollbackTag)?.rollbackTag ?? null;
|
||||
const stoppedOnly =
|
||||
svc.replicas.length > 0
|
||||
&& svc.replicas.every((r) => r.state === 'stopped' || r.state === 'none');
|
||||
if (!tag && svc.scale !== 0 && !stoppedOnly) continue;
|
||||
const key = /^[a-zA-Z0-9._-]+$/.test(svc.serviceName) ? svc.serviceName : yamlQuote(svc.serviceName);
|
||||
lines.push(` ${key}:`);
|
||||
if (tag) {
|
||||
lines.push(` image: ${yamlQuote(tag)}`);
|
||||
}
|
||||
// Observed running count; fully stopped services stay at scale 0.
|
||||
if (svc.scale === 0 || stoppedOnly) {
|
||||
lines.push(' scale: 0');
|
||||
} else if (svc.scale > 0) {
|
||||
lines.push(` scale: ${svc.scale}`);
|
||||
}
|
||||
wroteAny = true;
|
||||
}
|
||||
const body = wroteAny ? `${lines.join('\n')}\n` : 'services: {}\n';
|
||||
await fs.writeFile(abs, body, 'utf8');
|
||||
return abs;
|
||||
}
|
||||
|
||||
public markAcquired(id: string): boolean {
|
||||
return DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'captured', 'acquired');
|
||||
}
|
||||
|
||||
public handoff(candidateId: string, nodeId: number, stackName: string): boolean {
|
||||
return DatabaseService.getInstance().casHandoffGeneration(candidateId, nodeId, stackName);
|
||||
}
|
||||
|
||||
public markReconciling(id: string): boolean {
|
||||
return DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'handoff_committed', 'reconciling');
|
||||
}
|
||||
|
||||
public markImmediateVerified(id: string): boolean {
|
||||
const ok = DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'reconciling', 'immediate_verified');
|
||||
if (ok) {
|
||||
DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(id, {
|
||||
artifact_expires_at: Date.now() + this.activeRecoveryTtlMs() + RECOVERY_TTL_BUFFER_MS,
|
||||
operation_lease_expires_at: null,
|
||||
});
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
/** Mark candidate abandoned in DB and retire Docker/FS artifacts. */
|
||||
public async abandon(id: string): Promise<boolean> {
|
||||
const row = this.get(id);
|
||||
const ok = DatabaseService.getInstance().abandonStackUpdateRecoveryGeneration(id);
|
||||
if (ok && row) {
|
||||
await this.retireGenerationArtifacts({ ...row, status: 'abandoned', artifacts_retired: 0 });
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
public linkHealthGate(id: string, healthGateId: string): void {
|
||||
DatabaseService.getInstance().linkStackUpdateRecoveryHealthGate(id, healthGateId);
|
||||
}
|
||||
|
||||
public setGateRetainUntil(id: string, until: number = Date.now() + GATE_RETAIN_DEFAULT_MS): void {
|
||||
DatabaseService.getInstance().setStackUpdateRecoveryGateRetainUntil(id, until);
|
||||
}
|
||||
|
||||
/** Link a health gate when present; otherwise set bounded gate_retain_until. */
|
||||
public linkGateOrRetain(id: string, healthGateId: string | null): void {
|
||||
try {
|
||||
if (healthGateId) {
|
||||
this.linkHealthGate(id, healthGateId);
|
||||
} else {
|
||||
this.setGateRetainUntil(id);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] linkGateOrRetain failed for %s; setting retain window:',
|
||||
sanitizeForLog(id),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
try {
|
||||
this.setGateRetainUntil(id);
|
||||
} catch (retainError) {
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] setGateRetainUntil failed for %s:',
|
||||
sanitizeForLog(id),
|
||||
sanitizeForLog(getErrorMessage(retainError, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public get(id: string): StackUpdateRecoveryGenerationRow | undefined {
|
||||
return DatabaseService.getInstance().getStackUpdateRecoveryGeneration(id);
|
||||
}
|
||||
|
||||
public getCurrent(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow | undefined {
|
||||
return DatabaseService.getInstance().getCurrentStackUpdateRecovery(nodeId, stackName);
|
||||
}
|
||||
|
||||
public isRestoredCurrentPinActive(nodeId: number, stackName: string): boolean {
|
||||
const current = this.getCurrent(nodeId, stackName);
|
||||
return !!current && current.status === 'restored_current';
|
||||
}
|
||||
|
||||
public getHeldImageIds(nodeId: number): Set<string> | null {
|
||||
try {
|
||||
const now = Date.now();
|
||||
const fromStack = DatabaseService.getInstance().listHeldStackUpdateRecoveryImageIds(nodeId, now);
|
||||
return new Set(fromStack);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] Failed to compute held image ids for node %d:',
|
||||
nodeId,
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unified held-image predicate: service-scoped + full-stack holds.
|
||||
* Fail closed (skip prune) when either lookup fails.
|
||||
*/
|
||||
public buildUnifiedHeldImagePredicate(nodeId: number): (imageId: string) => boolean {
|
||||
// Dynamic require avoids a static cycle with ServiceUpdateRecoveryService.
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const { ServiceUpdateRecoveryService } = require('./ServiceUpdateRecoveryService') as typeof import('./ServiceUpdateRecoveryService');
|
||||
const serviceHeld = ServiceUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
|
||||
const stackHeld = this.getHeldImageIds(nodeId);
|
||||
if (serviceHeld === null || stackHeld === null) {
|
||||
return () => true;
|
||||
}
|
||||
return (imageId: string) => serviceHeld.has(imageId) || stackHeld.has(imageId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Post-handoff compensation: restore files + pinned up, then probe before
|
||||
* reporting restored_current / immediate_verified.
|
||||
*/
|
||||
public async compensateWithCandidate(
|
||||
generationId: string,
|
||||
composeUp: (overridePath: string) => Promise<void>,
|
||||
): Promise<boolean> {
|
||||
const row = this.get(generationId);
|
||||
if (!row) return false;
|
||||
try {
|
||||
const context = await resolveComposeProjectContext(row.node_id, row.stack_name);
|
||||
await context.restoreFromContext();
|
||||
if (!row.override_path) {
|
||||
throw new Error('Recovery generation has no override path');
|
||||
}
|
||||
await composeUp(row.override_path);
|
||||
const probeOk = await this.probeRecoveredStack(
|
||||
row.node_id,
|
||||
row.stack_name,
|
||||
row.services_json,
|
||||
);
|
||||
if (!probeOk) {
|
||||
DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, {
|
||||
status: 'recovery_required',
|
||||
});
|
||||
return false;
|
||||
}
|
||||
DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, {
|
||||
status: 'restored_current',
|
||||
phase: 'immediate_verified',
|
||||
is_current: 1,
|
||||
artifact_expires_at: null,
|
||||
});
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'[StackUpdateRecovery] Compensation failed for %s: %s',
|
||||
sanitizeForLog(generationId),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, {
|
||||
status: 'recovery_required',
|
||||
});
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify recovered runtime against the captured generation.
|
||||
* Rejects absent, restarting, dead, exited, or unhealthy expected replicas,
|
||||
* image-id mismatches vs capture, and any running replica of a scale-0 service.
|
||||
*/
|
||||
public async probeRecoveredStack(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
servicesJson: string,
|
||||
): Promise<boolean> {
|
||||
await new Promise((resolve) => setTimeout(resolve, RECOVERY_PROBE_DELAY_MS));
|
||||
try {
|
||||
const expected = parseServicesJson(servicesJson);
|
||||
const expectedRunning = new Map<string, number>();
|
||||
const expectedImageIds = new Map<string, Set<string>>();
|
||||
const scaleZeroServices = new Set<string>();
|
||||
|
||||
for (const svc of expected) {
|
||||
const imageIds = new Set<string>();
|
||||
for (const replica of svc.replicas ?? []) {
|
||||
if (replica.imageId?.trim()) imageIds.add(replica.imageId);
|
||||
}
|
||||
if (svc.scale > 0) {
|
||||
// Fail closed when we cannot verify image identity for expected runners.
|
||||
if (imageIds.size === 0) return false;
|
||||
expectedRunning.set(svc.serviceName, svc.scale);
|
||||
expectedImageIds.set(svc.serviceName, imageIds);
|
||||
} else {
|
||||
scaleZeroServices.add(svc.serviceName);
|
||||
}
|
||||
}
|
||||
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
const containers = await docker.listContainers({
|
||||
all: true,
|
||||
filters: { label: [`com.docker.compose.project=${stackName}`] },
|
||||
});
|
||||
|
||||
if (expectedRunning.size > 0 && containers.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const runningByService = new Map<string, number>();
|
||||
for (const containerInfo of containers) {
|
||||
const labels = (containerInfo.Labels ?? {}) as Record<string, string>;
|
||||
const serviceName = labels['com.docker.compose.service'];
|
||||
const state = (containerInfo.State || '').toLowerCase();
|
||||
|
||||
if (state === 'restarting' || state === 'dead') {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (state === 'exited' || state === 'created' || state === 'removing') {
|
||||
if (serviceName && expectedRunning.has(serviceName)) {
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (state !== 'running') {
|
||||
if (serviceName && expectedRunning.has(serviceName)) {
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Captured at scale 0 must stay stopped; any running replica fails the probe.
|
||||
if (serviceName && scaleZeroServices.has(serviceName)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const inspectData = await docker.getContainer(containerInfo.Id).inspect();
|
||||
const health = inspectData.State?.Health?.Status;
|
||||
if (health === 'unhealthy') {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (serviceName && expectedImageIds.has(serviceName)) {
|
||||
const allowedIds = expectedImageIds.get(serviceName)!;
|
||||
const actualImageId = typeof inspectData.Image === 'string' ? inspectData.Image : '';
|
||||
if (!actualImageId || !allowedIds.has(actualImageId)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (serviceName) {
|
||||
runningByService.set(serviceName, (runningByService.get(serviceName) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [serviceName, need] of expectedRunning) {
|
||||
if ((runningByService.get(serviceName) ?? 0) < need) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] Recovery probe failed for %s: %s',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotent removal of opaque tags and override files for a generation.
|
||||
* Marks artifacts_retired only when every artifact is removed or confirmed absent,
|
||||
* so transient failures remain retryable via reconcileIncomplete.
|
||||
*/
|
||||
public async retireGenerationArtifacts(row: StackUpdateRecoveryGenerationRow): Promise<boolean> {
|
||||
if (row.artifacts_retired === 1) return true;
|
||||
const services = parseServicesJson(row.services_json);
|
||||
const tagsOk = await this.removeRollbackTags(row.node_id, collectRollbackTags(services));
|
||||
let overrideOk = true;
|
||||
if (row.override_path) {
|
||||
try {
|
||||
await fs.unlink(row.override_path);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
overrideOk = false;
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] Failed to delete override %s: %s',
|
||||
sanitizeForLog(row.override_path),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!tagsOk || !overrideOk) return false;
|
||||
DatabaseService.getInstance().markStackUpdateRecoveryArtifactsRetired(row.id);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Abandon lease-expired candidates, flag stuck post-handoff generations,
|
||||
* and retire expired abandoned/superseded artifacts.
|
||||
*/
|
||||
public async reconcileIncomplete(): Promise<void> {
|
||||
if (!this.started) return;
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const now = Date.now();
|
||||
let abandoned = 0;
|
||||
for (const row of db.listStaleStackUpdateRecoveryCandidates(now)) {
|
||||
if (await this.abandon(row.id)) abandoned += 1;
|
||||
}
|
||||
let flagged = 0;
|
||||
for (const row of db.listStuckStackUpdateRecoveryGenerations(now)) {
|
||||
db.updateStackUpdateRecoveryGeneration(row.id, {
|
||||
status: 'recovery_required',
|
||||
operation_lease_expires_at: null,
|
||||
});
|
||||
flagged += 1;
|
||||
}
|
||||
let retired = 0;
|
||||
for (const row of db.listStackUpdateRecoveryGenerationsForArtifactRetirement(now)) {
|
||||
// Never retire an active/current or recovery_required hold target.
|
||||
if (row.is_current === 1 || row.status === 'recovery_required') continue;
|
||||
if (await this.retireGenerationArtifacts(row)) retired += 1;
|
||||
}
|
||||
if (abandoned > 0 || flagged > 0 || retired > 0) {
|
||||
console.log(
|
||||
`[StackUpdateRecovery] Reconciled ${abandoned} stale candidate(s), `
|
||||
+ `${flagged} stuck generation(s), retired ${retired} artifact set(s)`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'[StackUpdateRecovery] Reconcile failed: %s',
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns true when every tag is removed or already absent. */
|
||||
private async removeRollbackTags(nodeId: number, tags: string[]): Promise<boolean> {
|
||||
if (tags.length === 0) return true;
|
||||
try {
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
let allOk = true;
|
||||
for (const tag of tags) {
|
||||
try {
|
||||
await docker.getImage(tag).remove({ force: true });
|
||||
} catch (error) {
|
||||
const status = (error as { statusCode?: number }).statusCode;
|
||||
const message = getErrorMessage(error, 'unknown').toLowerCase();
|
||||
if (status === 404 || message.includes('no such image') || message.includes('not found')) {
|
||||
continue;
|
||||
}
|
||||
allOk = false;
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] Failed to remove rollback tag %s: %s',
|
||||
sanitizeForLog(tag),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
return allOk;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] Docker unavailable while removing tags: %s',
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async bestEffortRemoveTags(nodeId: number, tags: string[]): Promise<void> {
|
||||
await this.removeRollbackTags(nodeId, tags);
|
||||
}
|
||||
|
||||
private activeRecoveryTtlMs(): number {
|
||||
return Math.max(getComposeCommandTimeoutMs(), this.readRecoveryWindowSeconds() * 1000);
|
||||
}
|
||||
|
||||
private readRecoveryWindowSeconds(): number {
|
||||
try {
|
||||
const raw = parseInt(
|
||||
DatabaseService.getInstance().getGlobalSettings()['health_gate_window_seconds'] ?? '',
|
||||
10,
|
||||
);
|
||||
return Number.isFinite(raw) ? Math.max(raw, MIN_RECOVERY_WINDOW_SECONDS) : MIN_RECOVERY_WINDOW_SECONDS;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] Settings read failed; using default window: %s',
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return MIN_RECOVERY_WINDOW_SECONDS;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function splitOpaqueTag(tag: string): { repo: string; tagName: string } {
|
||||
const lastColon = tag.lastIndexOf(':');
|
||||
if (lastColon > 0) {
|
||||
return { repo: tag.slice(0, lastColon), tagName: tag.slice(lastColon + 1) };
|
||||
}
|
||||
return { repo: tag, tagName: 'hold' };
|
||||
}
|
||||
@@ -174,18 +174,24 @@ export class WebhookService {
|
||||
case 'start':
|
||||
await compose.runCommand(stackName, 'start');
|
||||
break;
|
||||
case 'pull':
|
||||
case 'pull': {
|
||||
await assertPolicyGateAllows(
|
||||
stackName,
|
||||
nodeId,
|
||||
buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }),
|
||||
);
|
||||
await StackUpdateOrchestrator.getInstance().execute(
|
||||
const orchResult = await StackUpdateOrchestrator.getInstance().execute(
|
||||
{ nodeId, stackName, target: { scope: 'stack' }, trigger: 'webhook', actor: 'system:webhook' },
|
||||
{ atomic: atomic ?? false, terminalWs: null },
|
||||
);
|
||||
HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook');
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook');
|
||||
const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
|
||||
if (recoveryId) {
|
||||
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
||||
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* Thin Compose project context for safe full-stack updates.
|
||||
*
|
||||
* Wraps the current authored compose argument path and atomic file backup/restore.
|
||||
* When a richer shared Compose project context lands, migrate callers to that type;
|
||||
* this module must not become a competing full-manifest resolver.
|
||||
*/
|
||||
import path from 'path';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { ComposeService } from './ComposeService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { buildEffectiveServiceModel } from './effectiveServiceModel';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
|
||||
export type ImageReferenceKind = 'moving_tag' | 'digest_pinned' | 'none';
|
||||
|
||||
const DIGEST_PIN_PATTERN = /@sha256:[a-f0-9]{64}$/i;
|
||||
|
||||
export function classifyReferenceKind(declaredImageRef: string | null): ImageReferenceKind {
|
||||
if (!declaredImageRef) return 'none';
|
||||
if (DIGEST_PIN_PATTERN.test(declaredImageRef)) return 'digest_pinned';
|
||||
return 'moving_tag';
|
||||
}
|
||||
|
||||
async function requireRenderableModel(nodeId: number, stackName: string) {
|
||||
const model = await buildEffectiveServiceModel(nodeId, stackName);
|
||||
if (!model.renderable) {
|
||||
throw new Error(model.error || 'Effective Compose model failed to render');
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
export interface ComposeProjectContext {
|
||||
readonly nodeId: number;
|
||||
readonly stackName: string;
|
||||
readonly stackDir: string;
|
||||
backupSlotId: string | null;
|
||||
toComposeArgs(action: string[]): Promise<string[]>;
|
||||
validateForMutation(): Promise<void>;
|
||||
backupFromContext(operation: 'update' | 'deployment'): Promise<string>;
|
||||
restoreFromContext(): Promise<void>;
|
||||
resolveServiceImageMap(): Promise<Map<string, string | null>>;
|
||||
}
|
||||
|
||||
class AuthoredComposeProjectContext implements ComposeProjectContext {
|
||||
backupSlotId: string | null = null;
|
||||
|
||||
constructor(
|
||||
readonly nodeId: number,
|
||||
readonly stackName: string,
|
||||
readonly stackDir: string,
|
||||
) {}
|
||||
|
||||
async toComposeArgs(action: string[]): Promise<string[]> {
|
||||
return ComposeService.getInstance(this.nodeId).buildAuthoredComposeArgs(this.stackName, action);
|
||||
}
|
||||
|
||||
async validateForMutation(): Promise<void> {
|
||||
await ComposeService.getInstance(this.nodeId).validateStackForMutation(this.stackName);
|
||||
await requireRenderableModel(this.nodeId, this.stackName);
|
||||
}
|
||||
|
||||
async backupFromContext(_operation: 'update' | 'deployment'): Promise<string> {
|
||||
await FileSystemService.getInstance(this.nodeId).backupStackFiles(this.stackName);
|
||||
const slotId = randomUUID();
|
||||
this.backupSlotId = slotId;
|
||||
return slotId;
|
||||
}
|
||||
|
||||
async restoreFromContext(): Promise<void> {
|
||||
await FileSystemService.getInstance(this.nodeId).restoreStackFiles(this.stackName);
|
||||
}
|
||||
|
||||
async resolveServiceImageMap(): Promise<Map<string, string | null>> {
|
||||
const model = await requireRenderableModel(this.nodeId, this.stackName);
|
||||
const map = new Map<string, string | null>();
|
||||
for (const svc of model.services) {
|
||||
map.set(svc.name, svc.declaredImage);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveComposeProjectContext(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
): Promise<ComposeProjectContext> {
|
||||
const stackDir = path.join(FileSystemService.getInstance(nodeId).getBaseDir(), stackName);
|
||||
return new AuthoredComposeProjectContext(nodeId, stackName, stackDir);
|
||||
}
|
||||
|
||||
export function describeContextError(error: unknown): string {
|
||||
return getErrorMessage(error, 'Compose project context failed');
|
||||
}
|
||||
@@ -304,11 +304,11 @@ export function buildRollbackItems(inputs: RollbackInputs, now: number): Rollbac
|
||||
if (inputs.rollbackTarget === 'error') {
|
||||
items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The update preview is unavailable.' });
|
||||
} else if (inputs.rollbackTarget.target && inputs.rollbackTarget.moving) {
|
||||
items.push({ id: 'previous_images', state: 'not_covered', label: 'Previous image tag', detail: `Rollback target ${inputs.rollbackTarget.target}. This stack uses a moving image tag, so restoring the compose and env files does not revert the image: the local tag still resolves to the newer digest. Pin every image to an immutable version tag for a true image rollback.` });
|
||||
items.push({ id: 'previous_images', state: 'ready', label: 'Previous image tag', detail: `Rollback target ${inputs.rollbackTarget.target}. This stack uses a moving image tag. Full-stack updates capture the running image ID before pull/build and retain it through the recovery window so an immediate restore can retarget the exact prior image, not only the tag string.` });
|
||||
} else if (inputs.rollbackTarget.target) {
|
||||
items.push({ id: 'previous_images', state: 'ready', label: 'Previous image tag', detail: `Known rollback target: ${inputs.rollbackTarget.target}. The compose file pins an immutable tag, so restoring files also restores the image.` });
|
||||
} else {
|
||||
items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The previous image tag could not be determined. A rollback restores compose and env files; a moving tag may keep the newer image.' });
|
||||
items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The previous image tag could not be determined. When no prior image is recorded, a restore may fall back to compose and env files alone; a moving tag can still resolve to a newer digest.' });
|
||||
}
|
||||
|
||||
if (inputs.lastDeployAt === 'error') {
|
||||
|
||||
Reference in New Issue
Block a user