mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-07-26 11:49:16 +00:00
fix: keep running containers until stack pull/build succeeds (#1657)
* fix: keep running containers until stack pull/build succeeds Acquire images before reconcile, capture a recovery generation for compensation, and only remove classified orphans after handoff. * fix: address recovery audit blockers for safe stack updates Retire abandoned and expired recovery artifacts, probe compensated runtimes before reporting rollback success, preserve local Docker when deleting a node, validate the exact Compose invocation before capture, and repair updateStack return-contract fixtures. * fix: resolve ESLint errors blocking CI on this branch Unused-import and unused-variable errors left over from the stack deletion refactor: MeshService in stacks.ts (its opt-out cascade moved into DeployedStackDeletionService), a redundant pruneVolumes destructure in deleteDeployedStack (the real one is re-derived from the same input object inside runDeletionBody), and an unused beforeAll import in a Docker-integration test stub. Also scopes the webhook pull-action case body in a block to satisfy no-case-declarations; purely syntactic, no behavior change. * fix: harden recovery probe, cleanup retry, and failed-pull Docker test Reject absent or unhealthy expected replicas before reporting rollback success, keep cleanup records until artifacts are actually removed, fail closed when a mesh override cannot be generated, and assert a real failed pull leaves the original container running. * fix: verify recovery probe image identity and stack-scoped override paths Reject recovered runtimes that use the wrong image or leave scale-zero services running, and confine tombstone override deletion to the intent stack directory so forged cross-stack paths cannot be swept. * test: batch notification cap fixtures in a SQLite transaction Unbatched 1200-row inserts were timing out at the default 30s under CI load even though the same assertions pass in under 2s when green.
This commit is contained in:
Anso
@@ -49,6 +49,10 @@ jobs:
|
||||
working-directory: ./backend
|
||||
run: npm test
|
||||
|
||||
- name: Docker integration tests (dedicated Vitest config)
|
||||
working-directory: ./backend
|
||||
run: npm run test:docker-integration
|
||||
|
||||
- name: Lint (ESLint)
|
||||
working-directory: ./backend
|
||||
run: npm run lint
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
"start": "node dist/index.js",
|
||||
"predev": "node scripts/generate-version.js",
|
||||
"dev": "nodemon --watch src --ext ts,json --exec ts-node src/index.ts",
|
||||
"test:docker-integration": "vitest run --config vitest.docker-integration.config.ts",
|
||||
"test": "vitest run",
|
||||
"lint": "eslint src",
|
||||
"reset-mfa": "node dist/cli/resetMfa.js",
|
||||
|
||||
@@ -24,6 +24,17 @@ const {
|
||||
mockResolveMissingExternalNetworks,
|
||||
mockCreateNetwork,
|
||||
mockAddNotificationHistory,
|
||||
mockIsMeshStackEnabled,
|
||||
mockClassifyLegacyOrphansForUpdate,
|
||||
mockCaptureCandidate,
|
||||
mockMarkAcquired,
|
||||
mockHandoff,
|
||||
mockMarkReconciling,
|
||||
mockMarkImmediateVerified,
|
||||
mockAbandon,
|
||||
mockCompensateWithCandidate,
|
||||
mockBuildUnifiedHeldImagePredicate,
|
||||
mockGetRecovery,
|
||||
} = vi.hoisted(() => ({
|
||||
mockSpawn: vi.fn(),
|
||||
mockGetContainersByStack: vi.fn().mockResolvedValue([]),
|
||||
@@ -58,6 +69,24 @@ const {
|
||||
}),
|
||||
mockCreateNetwork: vi.fn().mockResolvedValue({ id: 'net-1' }),
|
||||
mockAddNotificationHistory: vi.fn(),
|
||||
mockIsMeshStackEnabled: vi.fn().mockReturnValue(false),
|
||||
mockClassifyLegacyOrphansForUpdate: vi.fn().mockResolvedValue({ status: 'none' }),
|
||||
mockCaptureCandidate: vi.fn().mockResolvedValue({
|
||||
id: 'recovery-1',
|
||||
node_id: 1,
|
||||
stack_name: 'my-stack',
|
||||
status: 'candidate',
|
||||
phase: 'captured',
|
||||
override_path: '/test/compose/my-stack/.sencho-recovery-abc.yml',
|
||||
}),
|
||||
mockMarkAcquired: vi.fn().mockReturnValue(true),
|
||||
mockHandoff: vi.fn().mockReturnValue(true),
|
||||
mockMarkReconciling: vi.fn().mockReturnValue(true),
|
||||
mockMarkImmediateVerified: vi.fn().mockReturnValue(true),
|
||||
mockAbandon: vi.fn().mockResolvedValue(true),
|
||||
mockCompensateWithCandidate: vi.fn().mockResolvedValue(true),
|
||||
mockBuildUnifiedHeldImagePredicate: vi.fn().mockReturnValue(() => false),
|
||||
mockGetRecovery: vi.fn().mockReturnValue(undefined),
|
||||
}));
|
||||
|
||||
vi.mock('child_process', () => ({ spawn: mockSpawn, execFile: vi.fn() }));
|
||||
@@ -89,6 +118,7 @@ vi.mock('../services/DockerController', () => ({
|
||||
getInstance: () => ({
|
||||
getContainersByStack: mockGetContainersByStack,
|
||||
getLegacyOrphanContainersByStack: mockGetLegacyOrphanContainersByStack,
|
||||
classifyLegacyOrphansForUpdate: mockClassifyLegacyOrphansForUpdate,
|
||||
removeContainers: mockRemoveContainers,
|
||||
pruneDanglingImages: mockPruneDanglingImages,
|
||||
createNetwork: mockCreateNetwork,
|
||||
@@ -111,6 +141,7 @@ vi.mock('../services/DatabaseService', () => ({
|
||||
getGitSource: () => undefined,
|
||||
getStackProjectEnvFiles: () => [],
|
||||
addNotificationHistory: mockAddNotificationHistory,
|
||||
isMeshStackEnabled: (...args: unknown[]) => mockIsMeshStackEnabled(...args),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
@@ -150,6 +181,24 @@ vi.mock('../services/MeshService', () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/StackUpdateRecoveryService', () => ({
|
||||
StackUpdateRecoveryService: {
|
||||
getInstance: () => ({
|
||||
captureCandidate: mockCaptureCandidate,
|
||||
markAcquired: mockMarkAcquired,
|
||||
handoff: mockHandoff,
|
||||
markReconciling: mockMarkReconciling,
|
||||
markImmediateVerified: mockMarkImmediateVerified,
|
||||
abandon: mockAbandon,
|
||||
compensateWithCandidate: mockCompensateWithCandidate,
|
||||
buildUnifiedHeldImagePredicate: mockBuildUnifiedHeldImagePredicate,
|
||||
get: mockGetRecovery,
|
||||
linkGateOrRetain: vi.fn(),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
|
||||
vi.mock('../services/SelfIdentityService', () => ({
|
||||
default: {
|
||||
getInstance: () => ({ getBindMounts: mockGetBindMounts }),
|
||||
@@ -247,6 +296,23 @@ beforeEach(() => {
|
||||
declaredExternalCount: 0,
|
||||
});
|
||||
mockCreateNetwork.mockResolvedValue({ id: 'net-1' });
|
||||
mockClassifyLegacyOrphansForUpdate.mockResolvedValue({ status: 'none' });
|
||||
mockCaptureCandidate.mockResolvedValue({
|
||||
id: 'recovery-1',
|
||||
node_id: 1,
|
||||
stack_name: 'my-stack',
|
||||
status: 'candidate',
|
||||
phase: 'captured',
|
||||
override_path: '/test/compose/my-stack/.sencho-recovery-abc.yml',
|
||||
});
|
||||
mockMarkAcquired.mockReturnValue(true);
|
||||
mockHandoff.mockReturnValue(true);
|
||||
mockMarkReconciling.mockReturnValue(true);
|
||||
mockMarkImmediateVerified.mockReturnValue(true);
|
||||
mockAbandon.mockResolvedValue(true);
|
||||
mockIsMeshStackEnabled.mockReturnValue(false);
|
||||
mockCompensateWithCandidate.mockResolvedValue(true);
|
||||
mockBuildUnifiedHeldImagePredicate.mockReturnValue(() => false);
|
||||
delete process.env.SENCHO_MODE;
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||
});
|
||||
@@ -586,6 +652,16 @@ describe('ComposeService - authoredComposeArgs mesh override', () => {
|
||||
|
||||
// ── deployStack ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
it('fails closed when a mesh-enabled stack cannot generate its override', async () => {
|
||||
mockIsMeshStackEnabled.mockReturnValue(true);
|
||||
mockEnsureStackOverride.mockRejectedValue(new Error('mesh override write failed'));
|
||||
|
||||
const svc = ComposeService.getInstance(1);
|
||||
await expect(svc.validateExactComposeInvocation('my-stack')).rejects.toThrow(/mesh override write failed/i);
|
||||
expect(mockSpawn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe('ComposeService - deployStack', () => {
|
||||
it('blocks a pilot deploy with relative binds before replacing containers when path mapping differs', async () => {
|
||||
process.env.SENCHO_MODE = 'pilot';
|
||||
@@ -987,7 +1063,7 @@ describe('ComposeService - updateStack build-aware', () => {
|
||||
expect(spawnArgs.some(args => args.includes('pull') && !args.includes('--ignore-buildable'))).toBe(true);
|
||||
});
|
||||
|
||||
it('rolls back compose files when a build step fails during atomic update', async () => {
|
||||
it('abandons recovery and leaves runtime untouched when acquire fails before handoff', async () => {
|
||||
mockLoadStackBuildServices.mockResolvedValueOnce(['app']);
|
||||
let spawnCount = 0;
|
||||
mockSpawn.mockImplementation(() => {
|
||||
@@ -1004,8 +1080,12 @@ describe('ComposeService - updateStack build-aware', () => {
|
||||
const error = await result;
|
||||
|
||||
expect(error).not.toBeNull();
|
||||
expect(mockRestoreStackFiles).toHaveBeenCalled();
|
||||
expect(getComposeRollbackInfo(error)?.attempted).toBe(true);
|
||||
expect(mockAbandon).toHaveBeenCalledWith('recovery-1');
|
||||
expect(mockHandoff).not.toHaveBeenCalled();
|
||||
expect(mockRemoveContainers).not.toHaveBeenCalled();
|
||||
expect(mockCompensateWithCandidate).not.toHaveBeenCalled();
|
||||
const spawnArgs = mockSpawn.mock.calls.map(c => c[1] as string[]);
|
||||
expect(spawnArgs.some(args => args.includes('up'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1106,7 +1186,7 @@ describe('ComposeService - updateStack prune-on-update', () => {
|
||||
|
||||
// The update already succeeded before the prune ran, so a prune failure
|
||||
// must neither reject nor trigger the atomic restore.
|
||||
await expect(promise).resolves.toBeUndefined();
|
||||
await expect(promise).resolves.toEqual({ recoveryId: 'recovery-1' });
|
||||
expect(mockRestoreStackFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -1120,7 +1200,7 @@ describe('ComposeService - updateStack prune-on-update', () => {
|
||||
const promise = svc.updateStack('my-stack');
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
|
||||
await expect(promise).resolves.toBeUndefined();
|
||||
await expect(promise).resolves.toEqual({ recoveryId: 'recovery-1' });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1428,19 +1508,10 @@ describe('ComposeService - idle-output stall backstop', () => {
|
||||
await promise;
|
||||
});
|
||||
|
||||
it('preserves STACK_STALLED_OUTPUT through the atomic rollback wrapper', async () => {
|
||||
it('surfaces STACK_STALLED_OUTPUT on acquire stall without runtime compensation', async () => {
|
||||
process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS = '1000';
|
||||
const pullProc = createMockProcess();
|
||||
let call = 0;
|
||||
// First spawn is the stalling pull; later spawns (the rollback restore's
|
||||
// `up`) close cleanly, proving the restore is not idle-timeout armed.
|
||||
mockSpawn.mockImplementation(() => {
|
||||
call += 1;
|
||||
if (call === 1) return pullProc;
|
||||
const p = createMockProcess();
|
||||
Promise.resolve().then(() => p.emit('close', 0));
|
||||
return p;
|
||||
});
|
||||
mockSpawn.mockImplementation(() => pullProc);
|
||||
|
||||
const svc = ComposeService.getInstance(1);
|
||||
const result = svc.updateStack('my-stack', undefined, true).then(() => null, (e: Error) => e);
|
||||
@@ -1451,7 +1522,10 @@ describe('ComposeService - idle-output stall backstop', () => {
|
||||
const error = await result;
|
||||
expect(error).not.toBeNull();
|
||||
expect(error!.message).toContain('STACK_STALLED_OUTPUT');
|
||||
expect(getComposeRollbackInfo(error)).toMatchObject({ attempted: true });
|
||||
// Acquire failure is pre-handoff: abandon candidate, do not wrap as ComposeRollbackError.
|
||||
expect(getComposeRollbackInfo(error)).toBeNull();
|
||||
expect(mockAbandon).toHaveBeenCalledWith('recovery-1');
|
||||
expect(mockCompensateWithCandidate).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1580,3 +1654,41 @@ describe('ComposeService - streamLogs', () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe('ComposeService - updateStack safe ordering', () => {
|
||||
it('never broad-removes Compose-managed containers before up', async () => {
|
||||
setupAutoCloseSpawn();
|
||||
mockListContainers.mockResolvedValue([]);
|
||||
mockGetContainersByStack.mockResolvedValue([{ Id: 'should-not-remove' }]);
|
||||
|
||||
const svc = ComposeService.getInstance(1);
|
||||
const promise = svc.updateStack('my-stack');
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await promise;
|
||||
|
||||
expect(mockGetContainersByStack).not.toHaveBeenCalled();
|
||||
expect(mockRemoveContainers).not.toHaveBeenCalled();
|
||||
expect(mockCaptureCandidate).toHaveBeenCalled();
|
||||
expect(mockClassifyLegacyOrphansForUpdate).toHaveBeenCalledWith('my-stack');
|
||||
expect(mockHandoff).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('removes only classified orphans after handoff', async () => {
|
||||
setupAutoCloseSpawn();
|
||||
mockListContainers.mockResolvedValue([]);
|
||||
mockClassifyLegacyOrphansForUpdate.mockResolvedValueOnce({
|
||||
status: 'orphans',
|
||||
ids: ['orphan-1'],
|
||||
});
|
||||
mockRemoveContainers.mockResolvedValueOnce([{ id: 'orphan-1', success: true }]);
|
||||
|
||||
const svc = ComposeService.getInstance(1);
|
||||
const promise = svc.updateStack('my-stack');
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await promise;
|
||||
|
||||
expect(mockHandoff).toHaveBeenCalled();
|
||||
expect(mockRemoveContainers).toHaveBeenCalledWith(['orphan-1']);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -225,23 +225,25 @@ describe('DatabaseService - notification history cap (periodic)', () => {
|
||||
|
||||
// A chatty stack writes 600 events.
|
||||
const base = Date.now();
|
||||
for (let i = 0; i < 600; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `chatty-${i}`,
|
||||
timestamp: base + i,
|
||||
stack_name: 'chatty',
|
||||
});
|
||||
}
|
||||
// A quiet stack writes 3 events long before the chatty burst.
|
||||
for (let i = 0; i < 3; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `quiet-${i}`,
|
||||
timestamp: base - 10_000 + i,
|
||||
stack_name: 'quiet',
|
||||
});
|
||||
}
|
||||
db.transaction(() => {
|
||||
for (let i = 0; i < 600; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `chatty-${i}`,
|
||||
timestamp: base + i,
|
||||
stack_name: 'chatty',
|
||||
});
|
||||
}
|
||||
// A quiet stack writes 3 events long before the chatty burst.
|
||||
for (let i = 0; i < 3; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `quiet-${i}`,
|
||||
timestamp: base - 10_000 + i,
|
||||
stack_name: 'quiet',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// No per-insert prune: every row is present.
|
||||
const beforeCleanup = db.getNotificationHistory(0, 2000);
|
||||
@@ -261,13 +263,15 @@ describe('DatabaseService - notification history cap (periodic)', () => {
|
||||
db.deleteAllNotifications(0);
|
||||
|
||||
const base = Date.now();
|
||||
for (let i = 0; i < 1200; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `system-${i}`,
|
||||
timestamp: base + i,
|
||||
});
|
||||
}
|
||||
db.transaction(() => {
|
||||
for (let i = 0; i < 1200; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `system-${i}`,
|
||||
timestamp: base + i,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
db.cleanupOldNotifications(30, { perStackCap: 500, perNodeUnattachedCap: 1000 });
|
||||
|
||||
@@ -279,14 +283,16 @@ describe('DatabaseService - notification history cap (periodic)', () => {
|
||||
it('keeps the newest entries per (node, stack) after periodic cap', () => {
|
||||
db.deleteAllNotifications(0);
|
||||
const base = Date.now();
|
||||
for (let i = 0; i < 600; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `ordered-${i}`,
|
||||
timestamp: base + i * 10,
|
||||
stack_name: 'ordered',
|
||||
});
|
||||
}
|
||||
db.transaction(() => {
|
||||
for (let i = 0; i < 600; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `ordered-${i}`,
|
||||
timestamp: base + i * 10,
|
||||
stack_name: 'ordered',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
db.cleanupOldNotifications(30, { perStackCap: 500, perNodeUnattachedCap: 1000 });
|
||||
|
||||
@@ -301,9 +307,11 @@ describe('DatabaseService - notification history cap (periodic)', () => {
|
||||
it('uses safe defaults when called with only the retention argument', () => {
|
||||
db.deleteAllNotifications(0);
|
||||
const base = Date.now();
|
||||
for (let i = 0; i < 600; i++) {
|
||||
db.addNotificationHistory(0, { level: 'info', message: `d-${i}`, timestamp: base + i, stack_name: 'default' });
|
||||
}
|
||||
db.transaction(() => {
|
||||
for (let i = 0; i < 600; i++) {
|
||||
db.addNotificationHistory(0, { level: 'info', message: `d-${i}`, timestamp: base + i, stack_name: 'default' });
|
||||
}
|
||||
});
|
||||
// Production caller (MonitorService) only passes daysToKeep; the cap defaults must enforce the per-stack 500 limit.
|
||||
const summary = db.cleanupOldNotifications(30);
|
||||
const after = db.getNotificationHistory(0, 2000).filter((n: any) => n.stack_name === 'default');
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
/**
|
||||
* Deployed-stack deletion: ready transaction retires both recovery models;
|
||||
* blocking intents gate same-name create.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
|
||||
import { randomUUID } from 'crypto';
|
||||
import path from 'path';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
import type {
|
||||
StackUpdateRecoveryGenerationRow,
|
||||
ServiceUpdateRecoveryRow,
|
||||
StackUpdateCleanupPendingRow,
|
||||
} from '../services/DatabaseService';
|
||||
|
||||
let tmpDir: string;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
let DeployedStackDeletionService: typeof import('../services/DeployedStackDeletionService').DeployedStackDeletionService;
|
||||
let overrideDeletionContainmentBase: typeof import('../services/DeployedStackDeletionService').overrideDeletionContainmentBase;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
({ DeployedStackDeletionService, overrideDeletionContainmentBase } = await import('../services/DeployedStackDeletionService'));
|
||||
});
|
||||
|
||||
afterAll(() => cleanupTestDb(tmpDir));
|
||||
|
||||
function db() {
|
||||
return DatabaseService.getInstance();
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
const raw = (db() as unknown as { db: { prepare: (s: string) => { run: () => void } } }).db;
|
||||
raw.prepare('DELETE FROM stack_update_recovery_generations').run();
|
||||
raw.prepare('DELETE FROM service_update_recovery').run();
|
||||
raw.prepare('DELETE FROM stack_update_cleanup_pending').run();
|
||||
});
|
||||
|
||||
const NODE = 1;
|
||||
|
||||
describe('DeployedStackDeletionService ready transaction', () => {
|
||||
it('commitStackDeletionReadyTransaction deletes full-stack and service recovery rows', () => {
|
||||
const now = Date.now();
|
||||
const stackName = 'del-stack';
|
||||
const gen: StackUpdateRecoveryGenerationRow = {
|
||||
id: randomUUID(),
|
||||
node_id: NODE,
|
||||
stack_name: stackName,
|
||||
status: 'active',
|
||||
phase: 'immediate_verified',
|
||||
is_current: 1,
|
||||
backup_slot_id: null,
|
||||
override_path: null,
|
||||
services_json: '[]',
|
||||
health_gate_id: null,
|
||||
gate_retain_until: null,
|
||||
artifact_expires_at: null,
|
||||
operation_lease_expires_at: null,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
};
|
||||
db().insertStackUpdateRecoveryGeneration(gen);
|
||||
const svc: ServiceUpdateRecoveryRow = {
|
||||
id: randomUUID(),
|
||||
node_id: NODE,
|
||||
stack_name: stackName,
|
||||
service_name: 'web',
|
||||
replicas_json: '[]',
|
||||
majority_image_id: 'sha256:abc',
|
||||
declared_image_ref: 'nginx:latest',
|
||||
weak_floating_tag: 0,
|
||||
health_gate_id: null,
|
||||
status: 'active',
|
||||
expires_at: now + 60_000,
|
||||
claim_expires_at: null,
|
||||
created_at: now,
|
||||
created_by: null,
|
||||
};
|
||||
db().insertServiceUpdateRecovery(svc);
|
||||
const intentId = randomUUID();
|
||||
const intent: StackUpdateCleanupPendingRow = {
|
||||
id: intentId,
|
||||
node_id: NODE,
|
||||
stack_name: stackName,
|
||||
status: 'prepared',
|
||||
target_kind: 'local_socket',
|
||||
rollback_tags_json: '[]',
|
||||
override_paths_json: '[]',
|
||||
prune_volumes_requested: 0,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
};
|
||||
db().insertCleanupPending(intent);
|
||||
|
||||
expect(db().commitStackDeletionReadyTransaction(intentId, NODE, stackName)).toBe(true);
|
||||
expect(db().listStackUpdateRecoveryForStack(NODE, stackName)).toHaveLength(0);
|
||||
expect(db().listActiveServiceUpdateRecoveries(NODE, stackName, 'web')).toHaveLength(0);
|
||||
expect(db().getCleanupPending(intentId)?.status).toBe('ready');
|
||||
});
|
||||
|
||||
it('hasBlockingDeletionIntent is true for prepared intents', () => {
|
||||
const now = Date.now();
|
||||
db().insertCleanupPending({
|
||||
id: randomUUID(),
|
||||
node_id: NODE,
|
||||
stack_name: 'blocked',
|
||||
status: 'prepared',
|
||||
target_kind: 'local_socket',
|
||||
rollback_tags_json: '[]',
|
||||
override_paths_json: '[]',
|
||||
prune_volumes_requested: 0,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
});
|
||||
expect(db().hasBlockingDeletionIntent(NODE, 'blocked')).toBe(true);
|
||||
expect(db().hasBlockingDeletionIntent(NODE, 'other')).toBe(false);
|
||||
});
|
||||
|
||||
it('assertNoBlockingDeletionIntent throws for prepared stacks', () => {
|
||||
const now = Date.now();
|
||||
db().insertCleanupPending({
|
||||
id: randomUUID(),
|
||||
node_id: NODE,
|
||||
stack_name: 'prep',
|
||||
status: 'prepared',
|
||||
target_kind: 'local_socket',
|
||||
rollback_tags_json: '[]',
|
||||
override_paths_json: '[]',
|
||||
prune_volumes_requested: 0,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
});
|
||||
expect(() => {
|
||||
DeployedStackDeletionService.getInstance().assertNoBlockingDeletionIntent(NODE, 'prep');
|
||||
}).toThrow(/deletion in progress/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe('overrideDeletionContainmentBase', () => {
|
||||
it('confines stack-scoped intents to the stack directory', () => {
|
||||
expect(overrideDeletionContainmentBase('/app/compose', 'my-stack')).toBe(
|
||||
path.resolve('/app/compose', 'my-stack'),
|
||||
);
|
||||
expect(overrideDeletionContainmentBase('/app/compose', null)).toBe(
|
||||
path.resolve('/app/compose'),
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects invalid stack names that could traverse', () => {
|
||||
expect(overrideDeletionContainmentBase('/app/compose', '../other')).toBeNull();
|
||||
expect(overrideDeletionContainmentBase('/app/compose', 'bad/name')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1714,3 +1714,23 @@ describe('DockerController - getLegacyOrphanContainersByStack', () => {
|
||||
fallbackSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('DockerController - classifyLegacyOrphansForUpdate', () => {
|
||||
it('returns none when compose ps already manages containers', async () => {
|
||||
// Reuse the same mocks as getLegacyOrphanContainersByStack tests in this file.
|
||||
const { default: DockerController } = await import('../services/DockerController');
|
||||
const dc = DockerController.getInstance(1);
|
||||
vi.spyOn(dc as any, 'fetchComposePsContainers').mockResolvedValue([{ ID: 'c1' }]);
|
||||
await expect(dc.classifyLegacyOrphansForUpdate('my-stack')).resolves.toEqual({ status: 'none' });
|
||||
});
|
||||
|
||||
it('returns classification_failed when compose ps and fallback both fail', async () => {
|
||||
const { default: DockerController } = await import('../services/DockerController');
|
||||
const dc = DockerController.getInstance(1);
|
||||
vi.spyOn(dc as any, 'fetchComposePsContainers').mockRejectedValue(new Error('compose boom'));
|
||||
vi.spyOn(dc as any, 'smartFallback').mockRejectedValue(new Error('fallback boom'));
|
||||
const result = await dc.classifyLegacyOrphansForUpdate('my-stack');
|
||||
expect(result.status).toBe('classification_failed');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* Docker-backed integration: a failed pull must leave the running stack untouched.
|
||||
* Skipped automatically when Docker is unavailable.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { execFileSync } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { setupTestDb, cleanupTestDb } from '../helpers/setupTestDb';
|
||||
|
||||
function dockerAvailable(): boolean {
|
||||
try {
|
||||
execFileSync('docker', ['info'], { stdio: 'ignore' });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const hasDocker = dockerAvailable();
|
||||
const STACK = 'fpkeep';
|
||||
|
||||
function compose(args: string[], cwd: string): string {
|
||||
return execFileSync('docker', ['compose', '-p', STACK, ...args], {
|
||||
cwd,
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
}
|
||||
|
||||
describe.skipIf(!hasDocker)('failed pull keeps stack running', () => {
|
||||
let tmpDir: string;
|
||||
let composeDir: string;
|
||||
let stackDir: string;
|
||||
let nodeId: number;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
composeDir = process.env.COMPOSE_DIR!;
|
||||
stackDir = path.join(composeDir, STACK);
|
||||
fs.mkdirSync(stackDir, { recursive: true });
|
||||
|
||||
const { DatabaseService } = await import('../../services/DatabaseService');
|
||||
const db = DatabaseService.getInstance();
|
||||
const local = db.getDefaultNode();
|
||||
if (!local?.id) throw new Error('Test DB has no default local node');
|
||||
nodeId = local.id;
|
||||
db.updateGlobalSetting('prune_on_update', '0');
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(stackDir, 'compose.yaml'),
|
||||
[
|
||||
'services:',
|
||||
' web:',
|
||||
' image: busybox:1.36.1',
|
||||
' command: ["sleep", "3600"]',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
compose(['up', '-d', '--pull', 'always'], stackDir);
|
||||
}, 180_000);
|
||||
|
||||
afterAll(async () => {
|
||||
try {
|
||||
if (stackDir && fs.existsSync(stackDir)) {
|
||||
compose(['down', '--remove-orphans'], stackDir);
|
||||
}
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
if (tmpDir) cleanupTestDb(tmpDir);
|
||||
}, 120_000);
|
||||
|
||||
it('leaves the original container running when ComposeService updateStack pull fails', async () => {
|
||||
const beforeId = compose(['ps', '-q'], stackDir).trim();
|
||||
expect(beforeId.length).toBeGreaterThan(0);
|
||||
|
||||
const beforeInspect = JSON.parse(
|
||||
execFileSync('docker', ['inspect', beforeId], { encoding: 'utf8' }),
|
||||
) as Array<{ State: { Running: boolean; Status: string } }>;
|
||||
expect(beforeInspect[0].State.Running).toBe(true);
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(stackDir, 'compose.yaml'),
|
||||
[
|
||||
'services:',
|
||||
' web:',
|
||||
' image: busybox:sencho-does-not-exist-fpkeep-xyz',
|
||||
' command: ["sleep", "3600"]',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
const { StackUpdateRecoveryService } = await import('../../services/StackUpdateRecoveryService');
|
||||
StackUpdateRecoveryService.resetForTests();
|
||||
const { ComposeService } = await import('../../services/ComposeService');
|
||||
await expect(
|
||||
ComposeService.getInstance(nodeId).updateStack(STACK, undefined, true),
|
||||
).rejects.toThrow();
|
||||
|
||||
const afterId = compose(['ps', '-q'], stackDir).trim();
|
||||
expect(afterId).toBe(beforeId);
|
||||
|
||||
const afterInspect = JSON.parse(
|
||||
execFileSync('docker', ['inspect', beforeId], { encoding: 'utf8' }),
|
||||
) as Array<{ State: { Running: boolean } }>;
|
||||
expect(afterInspect[0].State.Running).toBe(true);
|
||||
}, 180_000);
|
||||
});
|
||||
@@ -375,7 +375,7 @@ describe('POST /api/auto-update/execute', () => {
|
||||
.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never);
|
||||
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
|
||||
.mockResolvedValue({ hasUpdate: true } as never);
|
||||
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue();
|
||||
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
|
||||
const gateSpy = vi.spyOn(PolicyEnforcement, 'enforcePolicyPreDeploy').mockResolvedValue({
|
||||
ok: false,
|
||||
bypassed: false,
|
||||
@@ -421,7 +421,7 @@ describe('POST /api/auto-update/execute', () => {
|
||||
.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never);
|
||||
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
|
||||
.mockResolvedValue({ hasUpdate: true } as never);
|
||||
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue();
|
||||
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-au');
|
||||
try {
|
||||
const res = await request(app)
|
||||
|
||||
@@ -54,7 +54,7 @@ const {
|
||||
mockStartContainer: vi.fn().mockResolvedValue(undefined),
|
||||
mockStopContainer: vi.fn().mockResolvedValue(undefined),
|
||||
mockPruneSystem: vi.fn().mockResolvedValue({ success: true, reclaimedBytes: 0 }),
|
||||
mockUpdateStack: vi.fn().mockResolvedValue(undefined),
|
||||
mockUpdateStack: vi.fn().mockResolvedValue({ recoveryId: null }),
|
||||
mockGetStacks: vi.fn().mockResolvedValue([]),
|
||||
mockGetStackContent: vi.fn().mockResolvedValue(''),
|
||||
mockGetEnvContent: vi.fn().mockResolvedValue(''),
|
||||
|
||||
@@ -260,7 +260,7 @@ describe('POST /api/stacks/bulk execution', () => {
|
||||
});
|
||||
|
||||
it('handles update action (paid tier) by calling ComposeService.updateStack', async () => {
|
||||
mockUpdateStack.mockResolvedValue(undefined);
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
const { LicenseService } = await import('../services/LicenseService');
|
||||
const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
try {
|
||||
@@ -287,7 +287,7 @@ describe('POST /api/stacks/bulk execution', () => {
|
||||
policy: { id: 1, name: 'block-criticals', node_id: null, node_identity: '', stack_pattern: null, max_severity: 'HIGH', block_on_deploy: 1, block_on_severity: 1, block_on_kev: 0, block_on_fixable: 0, enabled: 1, replicated_from_control: 0, created_at: Date.now(), updated_at: Date.now() },
|
||||
violations: [{ imageRef: 'nginx:latest', severity: 'CRITICAL', criticalCount: 3, highCount: 0, kevCount: 0, fixableCount: 0, reasons: ['severity'], scanId: 1 }],
|
||||
});
|
||||
mockUpdateStack.mockResolvedValue(undefined);
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
try {
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/bulk')
|
||||
|
||||
@@ -124,7 +124,9 @@ describe('DELETE /api/stacks/:stackName mesh opt-out cascade (F-1 / F-14)', () =
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const sawCascadeWarning = warnSpy.mock.calls.some(args =>
|
||||
typeof args[0] === 'string' && args[0].includes('Mesh opt-out cascade failed')
|
||||
typeof args[0] === 'string'
|
||||
&& (args[0].includes('Mesh opt-out cascade failed')
|
||||
|| args[0].includes('Mesh opt-out failed'))
|
||||
);
|
||||
expect(sawCascadeWarning).toBe(true);
|
||||
});
|
||||
|
||||
@@ -164,7 +164,7 @@ describe('self stack lifecycle refusal', () => {
|
||||
});
|
||||
|
||||
it('allows update on a non-self stack', async () => {
|
||||
mockUpdateStack.mockResolvedValue(undefined);
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/web/update')
|
||||
.set('Cookie', authCookie);
|
||||
@@ -177,7 +177,7 @@ describe('self stack lifecycle refusal', () => {
|
||||
describe('POST /api/stacks/bulk self stack skip', () => {
|
||||
beforeEach(() => {
|
||||
stubSelfProject('sencho');
|
||||
mockUpdateStack.mockResolvedValue(undefined);
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1' }]);
|
||||
});
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ function spec(name: string): EffectiveServiceSpec {
|
||||
|
||||
beforeEach(() => {
|
||||
state.updateStack.mockReset();
|
||||
state.updateStack.mockResolvedValue(undefined);
|
||||
state.updateStack.mockResolvedValue({ recoveryId: null });
|
||||
state.model = null;
|
||||
});
|
||||
|
||||
@@ -85,8 +85,9 @@ describe('StackUpdateOrchestrator stack branch', () => {
|
||||
{ nodeId: 0, stackName: 'web', target: { scope: 'stack' }, trigger: 'manual', actor: 'tester' },
|
||||
{ atomic: true, terminalWs: null },
|
||||
);
|
||||
expect(result).toEqual({ kind: 'stack_compose_done' });
|
||||
expect(result).toEqual({ kind: 'stack_compose_done', recoveryId: null });
|
||||
expect(state.updateStack).toHaveBeenCalledWith('web', undefined, true);
|
||||
// recoveryId is forwarded from ComposeService.updateStack
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
/**
|
||||
* Direct tests for StackUpdateRecoveryService artifact lifecycle and compensation probe.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const mockTag = vi.fn().mockResolvedValue(undefined);
|
||||
const mockRemove = vi.fn().mockResolvedValue(undefined);
|
||||
const mockListContainers = vi.fn().mockResolvedValue([]);
|
||||
const mockInspectContainer = vi.fn();
|
||||
const mockGetContainer = vi.fn(() => ({ inspect: mockInspectContainer }));
|
||||
const mockGetImage = vi.fn((ref: string) => ({
|
||||
tag: mockTag,
|
||||
remove: mockRemove,
|
||||
inspect: vi.fn().mockResolvedValue({ RepoDigests: [] }),
|
||||
_ref: ref,
|
||||
}));
|
||||
|
||||
vi.mock('../services/DockerController', () => ({
|
||||
default: {
|
||||
getInstance: () => ({
|
||||
getDocker: () => ({
|
||||
listContainers: mockListContainers,
|
||||
getContainer: mockGetContainer,
|
||||
getImage: mockGetImage,
|
||||
}),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/FileSystemService', () => ({
|
||||
FileSystemService: {
|
||||
getInstance: () => ({
|
||||
getBaseDir: () => '/test/compose',
|
||||
backupStackFiles: vi.fn().mockResolvedValue(undefined),
|
||||
restoreStackFiles: vi.fn().mockResolvedValue(undefined),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/composeProjectContext', () => ({
|
||||
classifyReferenceKind: () => 'moving_tag',
|
||||
resolveComposeProjectContext: vi.fn().mockResolvedValue({
|
||||
validateForMutation: vi.fn().mockResolvedValue(undefined),
|
||||
backupFromContext: vi.fn().mockResolvedValue('backup-1'),
|
||||
restoreFromContext: vi.fn().mockResolvedValue(undefined),
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock('../services/effectiveServiceModel', () => ({
|
||||
buildEffectiveServiceModel: vi.fn().mockResolvedValue({
|
||||
renderable: true,
|
||||
services: [{ name: 'web', declaredImage: 'nginx:latest', hasBuild: false, expectedReplicas: 1 }],
|
||||
}),
|
||||
}));
|
||||
|
||||
const mockValidateExact = vi.fn().mockResolvedValue(undefined);
|
||||
vi.mock('../services/ComposeService', async () => {
|
||||
const actual = await vi.importActual<typeof import('../services/ComposeService')>('../services/ComposeService');
|
||||
return {
|
||||
...actual,
|
||||
ComposeService: {
|
||||
getInstance: () => ({
|
||||
validateExactComposeInvocation: mockValidateExact,
|
||||
buildAuthoredComposeArgs: vi.fn(),
|
||||
}),
|
||||
},
|
||||
getComposeCommandTimeoutMs: () => 60_000,
|
||||
};
|
||||
});
|
||||
|
||||
const mockUnlink = vi.fn().mockResolvedValue(undefined);
|
||||
const mockWriteFile = vi.fn().mockResolvedValue(undefined);
|
||||
const mockRealpath = vi.fn(async (p: string) => p);
|
||||
vi.mock('fs/promises', () => ({
|
||||
default: {
|
||||
unlink: (p: string) => mockUnlink(p),
|
||||
writeFile: (p: string, data: string, enc?: string) => mockWriteFile(p, data, enc),
|
||||
realpath: (p: string) => mockRealpath(p),
|
||||
},
|
||||
unlink: (p: string) => mockUnlink(p),
|
||||
writeFile: (p: string, data: string, enc?: string) => mockWriteFile(p, data, enc),
|
||||
realpath: (p: string) => mockRealpath(p),
|
||||
}));
|
||||
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService';
|
||||
|
||||
describe('StackUpdateRecoveryService', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
StackUpdateRecoveryService.resetForTests();
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'c1', State: 'running', Labels: {} },
|
||||
]);
|
||||
mockInspectContainer.mockResolvedValue({
|
||||
State: { Status: 'running', ExitCode: 0 },
|
||||
Image: 'sha256:abc',
|
||||
});
|
||||
mockValidateExact.mockResolvedValue(undefined);
|
||||
mockUnlink.mockResolvedValue(undefined);
|
||||
mockWriteFile.mockResolvedValue(undefined);
|
||||
mockRealpath.mockImplementation(async (p: string) => p);
|
||||
mockRemove.mockResolvedValue(undefined);
|
||||
mockTag.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
it('validates exact invocation before tagging images', async () => {
|
||||
const order: string[] = [];
|
||||
mockValidateExact.mockImplementation(async () => { order.push('validate'); });
|
||||
mockTag.mockImplementation(async () => { order.push('tag'); });
|
||||
mockWriteFile.mockImplementation(async () => { order.push('write'); });
|
||||
|
||||
const spyInsert = vi.spyOn(DatabaseService.prototype, 'insertStackUpdateRecoveryGeneration')
|
||||
.mockImplementation(() => { order.push('insert'); });
|
||||
vi.spyOn(DatabaseService.prototype, 'getGlobalSettings').mockReturnValue({});
|
||||
|
||||
await StackUpdateRecoveryService.getInstance().captureCandidate({
|
||||
nodeId: 1,
|
||||
stackName: 'my-stack',
|
||||
createdBy: 'test',
|
||||
});
|
||||
|
||||
expect(order.indexOf('validate')).toBeLessThan(order.indexOf('tag'));
|
||||
expect(order.indexOf('tag')).toBeLessThan(order.indexOf('write'));
|
||||
expect(order.indexOf('write')).toBeLessThan(order.indexOf('insert'));
|
||||
spyInsert.mockRestore();
|
||||
});
|
||||
|
||||
it('retires opaque tags and override on abandon', async () => {
|
||||
const row = {
|
||||
id: 'gen-1',
|
||||
node_id: 1,
|
||||
stack_name: 'my-stack',
|
||||
status: 'candidate' as const,
|
||||
phase: 'captured' as const,
|
||||
is_current: 0,
|
||||
backup_slot_id: null,
|
||||
override_path: '/test/compose/my-stack/.sencho-recovery-aaaaaaaaaaaa.yml',
|
||||
services_json: JSON.stringify([{
|
||||
serviceName: 'web',
|
||||
scale: 1,
|
||||
hasBuild: false,
|
||||
declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag',
|
||||
replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold' }],
|
||||
}]),
|
||||
health_gate_id: null,
|
||||
gate_retain_until: null,
|
||||
artifact_expires_at: null,
|
||||
operation_lease_expires_at: null,
|
||||
created_at: Date.now(),
|
||||
updated_at: Date.now(),
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
};
|
||||
|
||||
vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row);
|
||||
vi.spyOn(DatabaseService.prototype, 'abandonStackUpdateRecoveryGeneration').mockReturnValue(true);
|
||||
const markRetired = vi.spyOn(DatabaseService.prototype, 'markStackUpdateRecoveryArtifactsRetired')
|
||||
.mockReturnValue(true);
|
||||
|
||||
const ok = await StackUpdateRecoveryService.getInstance().abandon('gen-1');
|
||||
expect(ok).toBe(true);
|
||||
expect(mockRemove).toHaveBeenCalled();
|
||||
expect(mockUnlink).toHaveBeenCalledWith(row.override_path);
|
||||
expect(markRetired).toHaveBeenCalledWith('gen-1');
|
||||
});
|
||||
|
||||
it('does not mark restored_current when recovery probe finds crashed containers', async () => {
|
||||
const servicesJson = JSON.stringify([{
|
||||
serviceName: 'web',
|
||||
scale: 1,
|
||||
hasBuild: false,
|
||||
declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag',
|
||||
replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/x/web:hold' }],
|
||||
}]);
|
||||
const row = {
|
||||
id: 'gen-2',
|
||||
node_id: 1,
|
||||
stack_name: 'my-stack',
|
||||
status: 'active' as const,
|
||||
phase: 'reconciling' as const,
|
||||
is_current: 1,
|
||||
backup_slot_id: 'b1',
|
||||
override_path: '/test/compose/my-stack/.sencho-recovery-bbbbbbbbbbbb.yml',
|
||||
services_json: servicesJson,
|
||||
health_gate_id: null,
|
||||
gate_retain_until: null,
|
||||
artifact_expires_at: null,
|
||||
operation_lease_expires_at: null,
|
||||
created_at: Date.now(),
|
||||
updated_at: Date.now(),
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
};
|
||||
vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row);
|
||||
const update = vi.spyOn(DatabaseService.prototype, 'updateStackUpdateRecoveryGeneration')
|
||||
.mockImplementation(() => undefined);
|
||||
|
||||
mockListContainers.mockResolvedValue([{ Id: 'c1', State: 'exited', Labels: { 'com.docker.compose.service': 'web' } }]);
|
||||
mockInspectContainer.mockResolvedValue({ State: { ExitCode: 0 } });
|
||||
|
||||
vi.useFakeTimers();
|
||||
const promise = StackUpdateRecoveryService.getInstance().compensateWithCandidate(
|
||||
'gen-2',
|
||||
async () => undefined,
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
const ok = await promise;
|
||||
vi.useRealTimers();
|
||||
|
||||
expect(ok).toBe(false);
|
||||
expect(update).toHaveBeenCalledWith('gen-2', expect.objectContaining({ status: 'recovery_required' }));
|
||||
expect(update).not.toHaveBeenCalledWith(
|
||||
'gen-2',
|
||||
expect.objectContaining({ status: 'restored_current' }),
|
||||
);
|
||||
});
|
||||
|
||||
const capturedWebReplica = {
|
||||
containerId: 'c1',
|
||||
imageId: 'sha256:oldimg',
|
||||
repoDigest: null,
|
||||
state: 'running' as const,
|
||||
rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold',
|
||||
};
|
||||
|
||||
it('probeRecoveredStack rejects empty runtime when expected replicas were running', async () => {
|
||||
const servicesJson = JSON.stringify([{
|
||||
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag', replicas: [capturedWebReplica],
|
||||
}]);
|
||||
mockListContainers.mockResolvedValue([]);
|
||||
vi.useFakeTimers();
|
||||
const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await expect(promise).resolves.toBe(false);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('probeRecoveredStack rejects restarting and unhealthy expected containers', async () => {
|
||||
const servicesJson = JSON.stringify([{
|
||||
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag', replicas: [capturedWebReplica],
|
||||
}]);
|
||||
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'c1', State: 'restarting', Labels: { 'com.docker.compose.service': 'web' } },
|
||||
]);
|
||||
vi.useFakeTimers();
|
||||
let promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await expect(promise).resolves.toBe(false);
|
||||
vi.useRealTimers();
|
||||
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } },
|
||||
]);
|
||||
mockInspectContainer.mockResolvedValue({
|
||||
Image: 'sha256:oldimg',
|
||||
State: { Status: 'running', Health: { Status: 'unhealthy' } },
|
||||
});
|
||||
vi.useFakeTimers();
|
||||
promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await expect(promise).resolves.toBe(false);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('probeRecoveredStack rejects healthy containers running a mismatched image', async () => {
|
||||
const servicesJson = JSON.stringify([{
|
||||
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag', replicas: [capturedWebReplica],
|
||||
}]);
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } },
|
||||
]);
|
||||
mockInspectContainer.mockResolvedValue({
|
||||
Image: 'sha256:newfailed',
|
||||
Config: { Image: 'nginx:alpine' },
|
||||
State: { Status: 'running' },
|
||||
});
|
||||
vi.useFakeTimers();
|
||||
const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await expect(promise).resolves.toBe(false);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('probeRecoveredStack rejects a running replica for a service captured at scale zero', async () => {
|
||||
const servicesJson = JSON.stringify([{
|
||||
serviceName: 'worker', scale: 0, hasBuild: false, declaredImageRef: 'busybox:latest',
|
||||
referenceKind: 'moving_tag',
|
||||
replicas: [{
|
||||
containerId: 'c0', imageId: 'sha256:worker', repoDigest: null, state: 'stopped',
|
||||
rollbackTag: 'sencho-rb/aaaaaaaaaaaa/worker:hold',
|
||||
}],
|
||||
}]);
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'c0', State: 'running', Labels: { 'com.docker.compose.service': 'worker' } },
|
||||
]);
|
||||
mockInspectContainer.mockResolvedValue({
|
||||
Image: 'sha256:worker',
|
||||
State: { Status: 'running' },
|
||||
});
|
||||
vi.useFakeTimers();
|
||||
const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await expect(promise).resolves.toBe(false);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('probeRecoveredStack accepts healthy running replicas matching capture scale and image', async () => {
|
||||
const servicesJson = JSON.stringify([{
|
||||
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag', replicas: [capturedWebReplica],
|
||||
}]);
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } },
|
||||
]);
|
||||
mockInspectContainer.mockResolvedValue({
|
||||
Image: 'sha256:oldimg',
|
||||
Config: { Image: 'sencho-rb/aaaaaaaaaaaa/web:hold' },
|
||||
State: { Status: 'running' },
|
||||
});
|
||||
vi.useFakeTimers();
|
||||
const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await expect(promise).resolves.toBe(true);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('does not mark artifacts retired when tag removal fails', async () => {
|
||||
const row = {
|
||||
id: 'gen-3',
|
||||
node_id: 1,
|
||||
stack_name: 'my-stack',
|
||||
status: 'abandoned' as const,
|
||||
phase: 'captured' as const,
|
||||
is_current: 0,
|
||||
backup_slot_id: null,
|
||||
override_path: null,
|
||||
services_json: JSON.stringify([{
|
||||
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag',
|
||||
replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/cccccccccccc/web:hold' }],
|
||||
}]),
|
||||
health_gate_id: null,
|
||||
gate_retain_until: null,
|
||||
artifact_expires_at: Date.now() - 1,
|
||||
operation_lease_expires_at: null,
|
||||
created_at: Date.now(),
|
||||
updated_at: Date.now(),
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
};
|
||||
mockRemove.mockRejectedValueOnce(Object.assign(new Error('docker busy'), { statusCode: 500 }));
|
||||
const markRetired = vi.spyOn(DatabaseService.prototype, 'markStackUpdateRecoveryArtifactsRetired')
|
||||
.mockReturnValue(true);
|
||||
const ok = await StackUpdateRecoveryService.getInstance().retireGenerationArtifacts(row);
|
||||
expect(ok).toBe(false);
|
||||
expect(markRetired).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -270,7 +270,7 @@ describe('health gate begin call sites', () => {
|
||||
});
|
||||
|
||||
it('begins a gate after a manual update and returns its id', async () => {
|
||||
mockUpdateStack.mockResolvedValue(undefined);
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/myapp/update')
|
||||
.set('Cookie', authCookie)
|
||||
@@ -281,7 +281,7 @@ describe('health gate begin call sites', () => {
|
||||
});
|
||||
|
||||
it('begins a gate per stack in a bulk update and carries ids in the results', async () => {
|
||||
mockUpdateStack.mockResolvedValue(undefined);
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/bulk')
|
||||
.set('Cookie', authCookie)
|
||||
@@ -490,7 +490,7 @@ describe('deploy_failure notification on /update error', () => {
|
||||
});
|
||||
|
||||
it('uses trusted proxy tier headers for remote atomic updates', async () => {
|
||||
mockUpdateStack.mockResolvedValue(undefined);
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
const token = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' });
|
||||
|
||||
const res = await request(app)
|
||||
|
||||
@@ -96,11 +96,12 @@ describe('buildRollbackItems', () => {
|
||||
expect(known.detail).toContain('nginx:1.27.1');
|
||||
});
|
||||
|
||||
it('downgrades the previous image to not_covered for a moving tag', () => {
|
||||
it('treats a moving tag as ready because full-stack updates retain the prior image ID', () => {
|
||||
const item = itemById(baseInputs({ rollbackTarget: { target: 'nginx:latest', moving: true } }), 'previous_images');
|
||||
expect(item.state).toBe('not_covered');
|
||||
expect(item.state).toBe('ready');
|
||||
expect(item.detail).toContain('moving image tag');
|
||||
expect(item.detail).toContain('nginx:latest');
|
||||
expect(item.detail).toContain('recovery window');
|
||||
});
|
||||
|
||||
it('does not mistake an image literally named error for a failed preview', () => {
|
||||
@@ -139,9 +140,9 @@ describe('aggregateRollbackOverall', () => {
|
||||
expect(aggregateRollbackOverall(items)).toBe('partial');
|
||||
});
|
||||
|
||||
it('is partial when the rollback target is a moving tag', () => {
|
||||
it('is ready when the rollback target is a moving tag (image ID capture covers it)', () => {
|
||||
const items = buildRollbackItems(baseInputs({ rollbackTarget: { target: 'nginx:latest', moving: true } }), NOW);
|
||||
expect(aggregateRollbackOverall(items)).toBe('partial');
|
||||
expect(aggregateRollbackOverall(items)).toBe('ready');
|
||||
});
|
||||
|
||||
it('is partial when env coverage is missing', () => {
|
||||
|
||||
@@ -295,12 +295,11 @@ describe('UpdateGuardService.computeRollbackReadiness moving-tag wiring', () =>
|
||||
mockListContainers.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
it('marks previous_images not_covered (overall partial) when any image uses a moving tag', async () => {
|
||||
// Primary pinned, sidecar on a moving tag: a file rollback cannot revert it.
|
||||
it('marks previous_images ready when any image uses a moving tag (full-stack image ID capture)', async () => {
|
||||
mockGetPreview.mockResolvedValue(preview([{ current_tag: '1.2.3' }, { current_tag: 'latest' }]));
|
||||
const report = await UpdateGuardService.getInstance().computeRollbackReadiness(0, 'app');
|
||||
expect(report.items.find(i => i.id === 'previous_images')?.state).toBe('not_covered');
|
||||
expect(report.overall).toBe('partial');
|
||||
expect(report.items.find(i => i.id === 'previous_images')?.state).toBe('ready');
|
||||
expect(report.overall).toBe('ready');
|
||||
});
|
||||
|
||||
it('marks previous_images ready (overall ready) when every image is pinned', async () => {
|
||||
|
||||
@@ -481,7 +481,7 @@ describe('WebhookService.execute: health gate begin call sites', () => {
|
||||
const { HealthGateService } = await import('../services/HealthGateService');
|
||||
vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined);
|
||||
vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]);
|
||||
vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue(undefined);
|
||||
vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook');
|
||||
|
||||
const result = await WebhookService.getInstance().execute(webhook, 'pull', 'test', true);
|
||||
|
||||
@@ -12,6 +12,8 @@ import { MonitorService } from '../services/MonitorService';
|
||||
import { AutoHealService } from '../services/AutoHealService';
|
||||
import { HealthGateService } from '../services/HealthGateService';
|
||||
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
|
||||
import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService';
|
||||
import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService';
|
||||
import { FleetSyncRetryService } from '../services/FleetSyncRetryService';
|
||||
import { DockerEventManager } from '../services/DockerEventManager';
|
||||
import TrivyService, { sweepStaleTrivyTempDirs } from '../services/TrivyService';
|
||||
@@ -134,6 +136,16 @@ export async function startServer(server: Server): Promise<void> {
|
||||
// Initialize the license service before any tier-gated code can run.
|
||||
LicenseService.getInstance().initialize();
|
||||
|
||||
// Deletion-intent reconciliation must finish before mutation-capable
|
||||
// background services or HTTP accept traffic that could recreate a stack
|
||||
// name still covered by a prepared/ready tombstone.
|
||||
try {
|
||||
await DeployedStackDeletionService.getInstance().reconcileAtStartup();
|
||||
} catch (err) {
|
||||
console.error('[Startup] Deployed stack deletion reconcile failed:', (err as Error).message);
|
||||
}
|
||||
StackUpdateRecoveryService.getInstance().start();
|
||||
|
||||
// Synchronous starts: schedule background timers and continue. None of
|
||||
// these fire their first tick for at least a few seconds, so they
|
||||
// safely run alongside the async initializers below.
|
||||
|
||||
@@ -426,7 +426,13 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
|
||||
continue;
|
||||
}
|
||||
db.clearStackUpdateStatus(req.nodeId, stackName);
|
||||
HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`);
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`);
|
||||
const orchResult = lock.result;
|
||||
const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
|
||||
if (recoveryId) {
|
||||
const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService');
|
||||
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
|
||||
}
|
||||
|
||||
NotificationService.getInstance().broadcastEvent({
|
||||
type: 'state-invalidate',
|
||||
|
||||
@@ -7,6 +7,7 @@ import { requirePermission } from '../middleware/permissions';
|
||||
import { rejectApiTokenScope } from '../middleware/apiTokenScope';
|
||||
import { requireAdmin, requirePaid } from '../middleware/tierGates';
|
||||
import { enrollmentLimiter } from '../middleware/rateLimiters';
|
||||
import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import { CacheService } from '../services/CacheService';
|
||||
@@ -401,7 +402,14 @@ nodesRouter.delete('/:id', async (req: Request, res: Response) => {
|
||||
// local node, or one with no active connection). Mirrors the re-enroll path.
|
||||
MeshProxyTunnelDialer.getInstance().closeBridge(id, 'node deleted');
|
||||
PilotTunnelManager.getInstance().closeTunnel(id, PilotCloseCode.NormalClosure, 'node deleted');
|
||||
DatabaseService.getInstance().deleteNode(id);
|
||||
// Local-socket nodes: ready tombstone + recovery-row retirement in the same
|
||||
// transaction as the node delete, then sweep tags/paths. Remote hub records
|
||||
// create no Docker cleanup tombstone.
|
||||
if (existing.type === 'local') {
|
||||
await DeployedStackDeletionService.getInstance().deleteLocalNode(id);
|
||||
} else {
|
||||
DatabaseService.getInstance().deleteNode(id);
|
||||
}
|
||||
NodeRegistry.getInstance().evictConnection(id);
|
||||
NodeRegistry.getInstance().notifyNodeRemoved(id);
|
||||
CacheService.getInstance().invalidate(`${REMOTE_META_NAMESPACE}:${id}`);
|
||||
|
||||
@@ -16,7 +16,6 @@ import { ComposeService, getComposeRollbackInfo } from '../services/ComposeServi
|
||||
import { StackUpdateOrchestrator, shortImageId, type OrchestratorResult } from '../services/StackUpdateOrchestrator';
|
||||
import DockerController, { type BulkStackInfo } from '../services/DockerController';
|
||||
import { DatabaseService, type StackDossierFields } from '../services/DatabaseService';
|
||||
import { MeshService } from '../services/MeshService';
|
||||
import { CacheService, type CacheFetchOutcome } from '../services/CacheService';
|
||||
import { UpdatePreviewService } from '../services/UpdatePreviewService';
|
||||
import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService';
|
||||
@@ -38,6 +37,8 @@ import { labelInventoryOptionsFromRequest, requireRevealAdmin } from '../helpers
|
||||
import { EXPOSURE_INTENTS, type ExposureIntent } from '../services/network/types';
|
||||
import { UpdateGuardService, SingleServiceUpdateReadinessError } from '../services/UpdateGuardService';
|
||||
import { HealthGateService } from '../services/HealthGateService';
|
||||
import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService';
|
||||
import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService';
|
||||
import { classifyFailure } from '../services/updateGuard/failureClassifier';
|
||||
import { requirePermission, checkPermission } from '../middleware/permissions';
|
||||
import { NotificationService, type NotificationCategory } from '../services/NotificationService';
|
||||
@@ -99,8 +100,14 @@ const STACK_OP_PRESENT_PARTICIPLE: Record<StackOpAction, string> = {
|
||||
update: 'updating',
|
||||
rollback: 'rolling back',
|
||||
backup: 'backing up',
|
||||
delete: 'deleting',
|
||||
};
|
||||
|
||||
function linkStackUpdateRecoveryGate(recoveryId: string | null | undefined, healthGateId: string | null): void {
|
||||
if (!recoveryId) return;
|
||||
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
|
||||
}
|
||||
|
||||
function tryAcquireStackOpLock(
|
||||
req: Request,
|
||||
res: Response,
|
||||
@@ -474,7 +481,7 @@ async function runStackBulkOp(
|
||||
};
|
||||
}
|
||||
const atomic = true;
|
||||
await StackUpdateOrchestrator.getInstance().execute(
|
||||
const orchResult = await StackUpdateOrchestrator.getInstance().execute(
|
||||
{ nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'bulk', actor: user },
|
||||
{ atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) },
|
||||
);
|
||||
@@ -492,6 +499,8 @@ async function runStackBulkOp(
|
||||
console.error('[Security] Post-deploy scan failed for %s:', sanitizeForLog(stackName), err),
|
||||
);
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
|
||||
const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
|
||||
linkStackUpdateRecoveryGate(recoveryId, healthGateId);
|
||||
return { stackName, ok: true, healthGateId };
|
||||
} else {
|
||||
const outcome = await containerActionForStack(req.nodeId, stackName, action);
|
||||
@@ -929,6 +938,12 @@ stacksRouter.post('/', async (req: Request, res: Response) => {
|
||||
if (!isValidStackName(stackName)) {
|
||||
return res.status(400).json({ error: 'Stack name can only contain alphanumeric characters, hyphens, and underscores' });
|
||||
}
|
||||
try {
|
||||
DeployedStackDeletionService.getInstance().assertNoBlockingDeletionIntent(req.nodeId, stackName);
|
||||
} catch (guardError) {
|
||||
res.status(409).json({ error: getErrorMessage(guardError, 'Stack deletion in progress'), code: 'stack_deletion_in_progress' });
|
||||
return;
|
||||
}
|
||||
await FileSystemService.getInstance(req.nodeId).createStack(stackName);
|
||||
invalidateNodeCaches(req.nodeId);
|
||||
dlog(`[Stacks] Stack created: ${sanitizeForLog(stackName)}`);
|
||||
@@ -1109,82 +1124,32 @@ stacksRouter.delete('/:stackName', async (req: Request, res: Response) => {
|
||||
const sanitizedName = sanitizeForLog(stackName);
|
||||
if (debug) console.debug(`[Stacks:debug] Delete starting`, { stackName: sanitizedName, pruneVolumes, nodeId: req.nodeId });
|
||||
|
||||
// Step 1: compose down. Best-effort: a stack with corrupt compose files or
|
||||
// a temporarily-unreachable daemon should still be removable. Logged so the
|
||||
// operator can investigate orphaned containers separately.
|
||||
try {
|
||||
await ComposeService.getInstance(req.nodeId).downStack(stackName);
|
||||
if (debug) console.debug(`[Stacks:debug] Delete: down OK`, { stackName: sanitizedName });
|
||||
} catch (downErr) {
|
||||
console.warn('[Stacks] Compose down failed or no-op for %s:', sanitizeForLog(stackName), downErr);
|
||||
}
|
||||
const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({
|
||||
nodeId: req.nodeId,
|
||||
stackName,
|
||||
pruneVolumes,
|
||||
actor: req.user?.username ?? 'system',
|
||||
});
|
||||
|
||||
// Step 2: volume prune (only if requested). Best-effort.
|
||||
if (pruneVolumes) {
|
||||
try {
|
||||
const result = await DockerController.getInstance().pruneManagedOnly('volumes', [stackName]);
|
||||
dlog(`[Stacks] Pruned volumes for ${sanitizeForLog(stackName)}: ${result.reclaimedBytes} bytes reclaimed`);
|
||||
} catch (pruneErr) {
|
||||
console.warn('[Stacks] Volume prune failed for %s, continuing delete:', sanitizeForLog(stackName), pruneErr);
|
||||
if (!result.ok) {
|
||||
if (result.code === 'lock_conflict') {
|
||||
res.status(409).json({
|
||||
error: result.error,
|
||||
code: 'stack_op_in_progress',
|
||||
action: result.existingAction ?? 'delete',
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: filesystem delete. If this fails the on-disk compose files are
|
||||
// still present, so we abort BEFORE touching the database — keeping DB and
|
||||
// FS in sync so the operator can retry. Otherwise a half-deleted stack
|
||||
// (rows gone, files remain) becomes invisible to the UI.
|
||||
try {
|
||||
await FileSystemService.getInstance(req.nodeId).deleteStack(stackName);
|
||||
if (debug) console.debug(`[Stacks:debug] Delete: fs OK`, { stackName: sanitizedName });
|
||||
} catch (fsErr) {
|
||||
console.error('[Stacks] File deletion failed for %s; database state untouched:', sanitizeForLog(stackName), fsErr);
|
||||
const message = getErrorMessage(fsErr, 'Failed to remove stack files');
|
||||
res.status(500).json({
|
||||
error: `${message}. Stack containers may have been stopped but on-disk files remain. Retry the delete or clean the files manually.`,
|
||||
});
|
||||
if (result.code === 'fs_failed') {
|
||||
res.status(500).json({
|
||||
error: `${result.error}. Stack containers may have been stopped but on-disk files remain. Retry the delete or clean the files manually.`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
res.status(500).json({ error: result.error });
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 4: database cleanup. Per-call idempotent; safe to run sequentially.
|
||||
try {
|
||||
DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName);
|
||||
DatabaseService.getInstance().clearStackScanAttempts(req.nodeId, stackName);
|
||||
DatabaseService.getInstance().deleteServiceUpdateRecoveries(req.nodeId, stackName);
|
||||
DatabaseService.getInstance().deleteRoleAssignmentsByResource('stack', stackName);
|
||||
DatabaseService.getInstance().deleteGitSource(stackName);
|
||||
DatabaseService.getInstance().deleteStackDossier(req.nodeId, stackName);
|
||||
DatabaseService.getInstance().deleteStackDriftFindings(req.nodeId, stackName);
|
||||
DatabaseService.getInstance().deleteStackExposureIntents(req.nodeId, stackName);
|
||||
DatabaseService.getInstance().deleteStackExposure(req.nodeId, stackName);
|
||||
DatabaseService.getInstance().deleteStackProjectEnvFiles(req.nodeId, stackName);
|
||||
DatabaseService.getInstance().deleteStackScans(req.nodeId, stackName);
|
||||
if (debug) console.debug(`[Stacks:debug] Delete: db OK`, { stackName: sanitizedName });
|
||||
} catch (dbErr) {
|
||||
console.error('[Stacks] Database cleanup failed for %s; files already removed:', sanitizeForLog(stackName), dbErr);
|
||||
const message = getErrorMessage(dbErr, 'Failed to clear stack database state');
|
||||
res.status(500).json({
|
||||
error: `${message}. Stack files have been removed; some database rows for this stack may remain. Recreating the stack with the same name will reuse those rows.`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 5: mesh opt-out cascade. Idempotent; best-effort cleanup of derived
|
||||
// aliases / override files. A mesh failure here must not flip the delete
|
||||
// result, since the stack itself is already gone.
|
||||
try {
|
||||
await MeshService.getInstance().optOutStack(
|
||||
req.nodeId,
|
||||
stackName,
|
||||
req.user?.username ?? 'system',
|
||||
);
|
||||
} catch (meshErr) {
|
||||
console.warn(
|
||||
'[Stacks] Mesh opt-out cascade failed for %s, continuing delete:',
|
||||
sanitizeForLog(stackName),
|
||||
meshErr,
|
||||
);
|
||||
}
|
||||
|
||||
invalidateNodeCaches(req.nodeId);
|
||||
dlog(`[Stacks] Stack deleted: ${sanitizeForLog(stackName)}`);
|
||||
res.json({ success: true });
|
||||
@@ -2267,7 +2232,7 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
|
||||
const debug = isDebugEnabled();
|
||||
const atomic = true;
|
||||
if (debug) console.debug('[Stacks:debug] Update starting', { stackName, atomic, nodeId: req.nodeId });
|
||||
await StackUpdateOrchestrator.getInstance().execute(
|
||||
const orchResult = await StackUpdateOrchestrator.getInstance().execute(
|
||||
{ nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'manual', actor: req.user?.username ?? null },
|
||||
{ atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) },
|
||||
);
|
||||
@@ -2285,6 +2250,8 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
|
||||
if (debug) console.debug(`[Stacks:debug] Update finished in ${Date.now() - t0}ms`);
|
||||
ok = true;
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
|
||||
const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
|
||||
linkStackUpdateRecoveryGate(recoveryId, healthGateId);
|
||||
res.json({ status: 'Update completed', healthGateId });
|
||||
notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, req.user?.username ?? 'system');
|
||||
if (!skipScan) {
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
} from './DatabaseService';
|
||||
import { ComposeService } from './ComposeService';
|
||||
import { StackOpLockService, stackOpSkipMessage, type StackOpAction } from './StackOpLockService';
|
||||
import { DeployedStackDeletionService } from './DeployedStackDeletionService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { PROXY_TIER_HEADER, deployProvenanceHeaders } from './license-headers';
|
||||
@@ -463,32 +464,17 @@ export class BlueprintService {
|
||||
}
|
||||
|
||||
private async withdrawLocal(blueprint: Blueprint, node: Node): Promise<void> {
|
||||
// Hold the per-stack lock across both the compose down and the directory
|
||||
// delete so a withdraw cannot race a manual operation, nor tear the
|
||||
// files out from under one that starts mid-withdraw.
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
node.id, blueprint.name, 'down', 'system',
|
||||
async () => {
|
||||
try {
|
||||
await ComposeService.getInstance(node.id).downStack(blueprint.name);
|
||||
} catch (err) {
|
||||
// best-effort: continue to delete the directory even if down fails
|
||||
console.warn(`[BlueprintService] downStack failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(err)}`);
|
||||
}
|
||||
if (await this.stackDirExists(node.id, blueprint.name)) {
|
||||
await FileSystemService.getInstance(node.id).deleteStack(blueprint.name);
|
||||
}
|
||||
// Remove the exposure descriptor so a withdrawn blueprint
|
||||
// stack does not leave a stale row that escalates posture.
|
||||
try {
|
||||
DatabaseService.getInstance().deleteStackExposure(node.id, blueprint.name);
|
||||
} catch (e) {
|
||||
console.warn(`[BlueprintService] deleteStackExposure failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(e)}`);
|
||||
}
|
||||
},
|
||||
);
|
||||
if (!lock.ran) {
|
||||
throw new Error(stackOpSkipMessage(blueprint.name, lock.existing.action));
|
||||
const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({
|
||||
nodeId: node.id,
|
||||
stackName: blueprint.name,
|
||||
pruneVolumes: false,
|
||||
actor: 'system:blueprint',
|
||||
});
|
||||
if (!result.ok) {
|
||||
if (result.code === 'lock_conflict') {
|
||||
throw new Error(result.error);
|
||||
}
|
||||
throw new Error(result.error);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -143,6 +143,33 @@ export class ComposeService {
|
||||
* same `authoredComposeFileArgs` prefix directly but intentionally omit the mesh
|
||||
* override, rendering the user's authored model without mesh injection.
|
||||
*/
|
||||
/** Public wrapper for dual-arg assembly and recovery Compose invocations. */
|
||||
public async buildAuthoredComposeArgs(stackName: string, action: string[]): Promise<string[]> {
|
||||
return this.authoredComposeArgs(stackName, action);
|
||||
}
|
||||
|
||||
public async validateStackForMutation(stackName: string): Promise<void> {
|
||||
await this.assertRequiredEnvPresent(stackName);
|
||||
await this.assertSafePilotBindMapping(stackName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Render/validate the exact Compose invocation used by mutating operations
|
||||
* (authored files, env pins, and generated Mesh override) before capture.
|
||||
*/
|
||||
public async validateExactComposeInvocation(stackName: string): Promise<void> {
|
||||
if (!isValidStackName(stackName)) {
|
||||
throw new Error('Invalid stack path');
|
||||
}
|
||||
const baseResolved = path.resolve(this.baseDir);
|
||||
const stackDir = path.resolve(baseResolved, stackName);
|
||||
if (!stackDir.startsWith(baseResolved + path.sep)) {
|
||||
throw new Error('Invalid stack path');
|
||||
}
|
||||
const args = await this.authoredComposeArgs(stackName, ['config', '--quiet']);
|
||||
await this.execute('docker', args, stackDir, undefined, true);
|
||||
}
|
||||
|
||||
private async authoredComposeArgs(stackName: string, action: string[]): Promise<string[]> {
|
||||
const args: string[] = ['compose'];
|
||||
const filePrefix = authoredComposeFileArgs(stackName, this.nodeId);
|
||||
@@ -151,12 +178,23 @@ export class ComposeService {
|
||||
// directory, so deploy/update resolve the same effective config the validator did.
|
||||
args.push(...await authoredComposeEnvFileArgs(stackName, this.nodeId));
|
||||
|
||||
const meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(this.nodeId, stackName);
|
||||
let overridePath: string | null = null;
|
||||
try {
|
||||
overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName);
|
||||
} catch (err) {
|
||||
if (meshEnabled) {
|
||||
throw err instanceof Error
|
||||
? err
|
||||
: new Error(`Mesh override generation failed: ${String(err)}`);
|
||||
}
|
||||
console.warn('[ComposeService] mesh override skipped:', sanitizeForLog((err as Error).message));
|
||||
}
|
||||
if (meshEnabled && !overridePath) {
|
||||
throw new Error(
|
||||
`Mesh override is required for stack "${stackName}" but could not be generated`,
|
||||
);
|
||||
}
|
||||
if (overridePath) {
|
||||
if (filePrefix.length === 0) {
|
||||
// Single-file stack: passing any -f disables compose's auto-discovery, so name
|
||||
@@ -799,7 +837,50 @@ export class ComposeService {
|
||||
startStream();
|
||||
}
|
||||
|
||||
async updateStack(stackName: string, ws?: WebSocket, atomic?: boolean): Promise<void> {
|
||||
|
||||
/**
|
||||
* Authored (+ mesh) compose args with an optional recovery override layered LAST.
|
||||
* Used for pinned lifecycle / compensation ups (`--pull never --no-build`).
|
||||
*/
|
||||
public async buildComposeArgsWithRecoveryOverride(
|
||||
stackName: string,
|
||||
action: string[],
|
||||
recoveryOverridePath: string | null,
|
||||
): Promise<string[]> {
|
||||
const withSentinel = await this.authoredComposeArgs(stackName, ['__SENCHO_ACTION_SENTINEL__']);
|
||||
const idx = withSentinel.indexOf('__SENCHO_ACTION_SENTINEL__');
|
||||
const prefix = idx >= 0 ? withSentinel.slice(0, idx) : withSentinel;
|
||||
const out = [...prefix];
|
||||
if (recoveryOverridePath) {
|
||||
if (!out.includes('-f')) {
|
||||
const fsSvc = FileSystemService.getInstance(this.nodeId);
|
||||
const baseFilename = await fsSvc.getComposeFilename(stackName);
|
||||
out.push('-f', baseFilename);
|
||||
try {
|
||||
const userOverride = await fsSvc.getOverrideFilename(stackName);
|
||||
if (userOverride) out.push('-f', userOverride);
|
||||
} catch (err) {
|
||||
const code = (err as { code?: string }).code;
|
||||
if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') {
|
||||
throw err;
|
||||
}
|
||||
console.warn(
|
||||
'[ComposeService] could not resolve user compose override for recovery args:',
|
||||
sanitizeForLog((err as Error).message),
|
||||
);
|
||||
}
|
||||
}
|
||||
out.push('-f', recoveryOverridePath);
|
||||
}
|
||||
out.push(...action);
|
||||
return out;
|
||||
}
|
||||
|
||||
async updateStack(
|
||||
stackName: string,
|
||||
ws?: WebSocket,
|
||||
atomic?: boolean,
|
||||
): Promise<{ recoveryId: string | null }> {
|
||||
await this.assertRequiredEnvPresent(stackName);
|
||||
await this.assertSafePilotBindMapping(stackName);
|
||||
const stackDir = path.join(this.baseDir, stackName);
|
||||
@@ -810,48 +891,106 @@ export class ComposeService {
|
||||
if (ws && ws.readyState === WebSocket.OPEN) ws.send(data);
|
||||
};
|
||||
|
||||
if (atomic) {
|
||||
await this.createAtomicBackup(stackName, 'update', sendOutput);
|
||||
}
|
||||
// Dynamic import avoids a static cycle (recovery imports getComposeCommandTimeoutMs).
|
||||
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
||||
const recoverySvc = StackUpdateRecoveryService.getInstance();
|
||||
let recoveryId: string | null = null;
|
||||
let handedOff = false;
|
||||
|
||||
try {
|
||||
try {
|
||||
const dockerController = DockerController.getInstance(this.nodeId);
|
||||
const legacyContainers = await dockerController.getContainersByStack(stackName);
|
||||
if (legacyContainers && legacyContainers.length > 0) {
|
||||
sendOutput(`=== Cleaning up existing containers for clean update ===\n`);
|
||||
await dockerController.removeContainers(legacyContainers.map((c: any) => c.Id));
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('Failed to clean up legacy containers for %s:', sanitizeForLog(stackName), e);
|
||||
}
|
||||
sendOutput('=== Validating stack for update ===\n');
|
||||
sendOutput('=== Capturing rollback generation ===\n');
|
||||
const candidate = await recoverySvc.captureCandidate({
|
||||
nodeId: this.nodeId,
|
||||
stackName,
|
||||
createdBy: null,
|
||||
});
|
||||
recoveryId = candidate.id;
|
||||
|
||||
const buildServices = await loadStackBuildServices(this.nodeId, stackName);
|
||||
const buildAware = buildServices.length > 0;
|
||||
|
||||
await this.withRegistryAuth(async (env) => {
|
||||
if (buildAware) {
|
||||
sendOutput('=== Building images ===\n');
|
||||
await this.execute('docker', await this.authoredComposeArgs(stackName, ['build', '--pull']), stackDir, ws, true, env, getComposeStallTimeoutMs());
|
||||
try {
|
||||
await this.withRegistryAuth(async (env) => {
|
||||
if (buildAware) {
|
||||
sendOutput('=== Building images ===\n');
|
||||
await this.execute(
|
||||
'docker',
|
||||
await this.authoredComposeArgs(stackName, ['build', '--pull']),
|
||||
stackDir, ws, true, env, getComposeStallTimeoutMs(),
|
||||
);
|
||||
sendOutput('=== Pulling registry images ===\n');
|
||||
await this.execute(
|
||||
'docker',
|
||||
await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']),
|
||||
stackDir, ws, true, env, getComposeStallTimeoutMs(),
|
||||
);
|
||||
} else {
|
||||
sendOutput('=== Pulling latest images ===\n');
|
||||
await this.execute(
|
||||
'docker',
|
||||
await this.authoredComposeArgs(stackName, ['pull']),
|
||||
stackDir, ws, true, env, getComposeStallTimeoutMs(),
|
||||
);
|
||||
}
|
||||
}, sendOutput);
|
||||
} catch (acquireError) {
|
||||
// Acquisition failure: abandon candidate; leave runtime untouched.
|
||||
await recoverySvc.abandon(candidate.id);
|
||||
recoveryId = null;
|
||||
throw acquireError;
|
||||
}
|
||||
|
||||
sendOutput('=== Pulling registry images ===\n');
|
||||
await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']), stackDir, ws, true, env, getComposeStallTimeoutMs());
|
||||
} else {
|
||||
sendOutput('=== Pulling latest images ===\n');
|
||||
await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull']), stackDir, ws, true, env, getComposeStallTimeoutMs());
|
||||
}
|
||||
|
||||
sendOutput('=== Recreating containers ===\n');
|
||||
await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs());
|
||||
}, sendOutput);
|
||||
|
||||
// Post-Update Health Probe
|
||||
await new Promise(resolve => setTimeout(resolve, 3000));
|
||||
if (!recoverySvc.markAcquired(candidate.id)) {
|
||||
await recoverySvc.abandon(candidate.id);
|
||||
throw new Error('Failed to mark recovery generation as acquired');
|
||||
}
|
||||
|
||||
const dockerController = DockerController.getInstance(this.nodeId);
|
||||
sendOutput('=== Classifying legacy orphans ===\n');
|
||||
const classified = await dockerController.classifyLegacyOrphansForUpdate(stackName);
|
||||
if (classified.status === 'classification_failed') {
|
||||
await recoverySvc.abandon(candidate.id);
|
||||
recoveryId = null;
|
||||
throw new Error(`Legacy orphan classification failed: ${classified.error}`);
|
||||
}
|
||||
|
||||
if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) {
|
||||
await recoverySvc.abandon(candidate.id);
|
||||
recoveryId = null;
|
||||
throw new Error('Failed to hand off recovery generation');
|
||||
}
|
||||
handedOff = true;
|
||||
if (!recoverySvc.markReconciling(candidate.id)) {
|
||||
throw new Error('Failed to mark recovery generation as reconciling after handoff');
|
||||
}
|
||||
|
||||
if (classified.status === 'orphans') {
|
||||
sendOutput(`=== Removing ${classified.ids.length} legacy orphan container(s) ===\n`);
|
||||
const results = await dockerController.removeContainers(classified.ids);
|
||||
const failed = results.filter((r) => !r.success);
|
||||
if (failed.length > 0) {
|
||||
throw new Error(
|
||||
`Failed to remove ${failed.length} legacy orphan container(s) after handoff`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await this.withRegistryAuth(async (env) => {
|
||||
sendOutput('=== Recreating containers ===\n');
|
||||
await this.execute(
|
||||
'docker',
|
||||
await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']),
|
||||
stackDir, ws, true, env, getComposeStallTimeoutMs(),
|
||||
);
|
||||
}, sendOutput);
|
||||
|
||||
// Immediate verification probe
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
|
||||
const containers = await dockerController.getDocker().listContainers({
|
||||
all: true,
|
||||
filters: { label: [`com.docker.compose.project=${stackName}`] }
|
||||
filters: { label: [`com.docker.compose.project=${stackName}`] },
|
||||
});
|
||||
|
||||
for (const containerInfo of containers) {
|
||||
@@ -859,56 +998,90 @@ export class ComposeService {
|
||||
const container = dockerController.getDocker().getContainer(containerInfo.Id);
|
||||
const inspectData = await container.inspect();
|
||||
const exitCode = inspectData.State.ExitCode;
|
||||
|
||||
if (exitCode !== 0) {
|
||||
throw this.createContainerCrashError(exitCode);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!recoverySvc.markImmediateVerified(candidate.id)) {
|
||||
console.warn(
|
||||
'[ComposeService] Could not CAS immediate_verified for recovery %s',
|
||||
sanitizeForLog(candidate.id),
|
||||
);
|
||||
}
|
||||
|
||||
sendOutput('=== Stack updated successfully ===\n');
|
||||
// Opt-out (default ON): after a clean update, prune the node's dangling
|
||||
// (untagged) image layers, including the one this pull just orphaned. Read
|
||||
// fresh each run so a remote node honors its own setting. Wrapped so a
|
||||
// prune failure can never reach the atomic-rollback catch below.
|
||||
|
||||
// Defer prune until gate retention / gate link: only prune when no active holds
|
||||
// would be violated. Still honor prune_on_update, but use unified holds so
|
||||
// candidate/current rollback images are retained.
|
||||
try {
|
||||
const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1';
|
||||
if (pruneOnUpdate) {
|
||||
// Dynamic import: ServiceUpdateRecoveryService imports getComposeCommandTimeoutMs
|
||||
// from this module, so a static import here would be circular.
|
||||
const { ServiceUpdateRecoveryService } = await import('./ServiceUpdateRecoveryService');
|
||||
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(this.nodeId);
|
||||
const isImageHeld = recoverySvc.buildUnifiedHeldImagePredicate(this.nodeId);
|
||||
const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld);
|
||||
// The Docker prune API does not report SpaceReclaimed on the containerd
|
||||
// image store, so only show the figure when the daemon actually returns one.
|
||||
const reclaimed = result.reclaimedBytes > 0
|
||||
? ` · reclaimed ${(result.reclaimedBytes / (1024 * 1024)).toFixed(1)} MB`
|
||||
: '';
|
||||
sendOutput(`=== Pruned dangling images${reclaimed} ===\n`);
|
||||
}
|
||||
} catch (pruneError) {
|
||||
console.warn('Failed to prune dangling images after update for %s:', sanitizeForLog(stackName), pruneError);
|
||||
console.warn(
|
||||
'Failed to prune dangling images after update for %s:',
|
||||
sanitizeForLog(stackName),
|
||||
pruneError,
|
||||
);
|
||||
}
|
||||
|
||||
if (debug) {
|
||||
console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName });
|
||||
}
|
||||
if (debug) console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName });
|
||||
} catch (updateError) {
|
||||
if (atomic) {
|
||||
sendOutput('\n=== Update failed - restoring previous compose and env files ===\n');
|
||||
const rolledBack = await this.restoreAtomicBackup(stackName, stackDir, ws, sendOutput);
|
||||
if (!handedOff && recoveryId) {
|
||||
await recoverySvc.abandon(recoveryId);
|
||||
recoveryId = null;
|
||||
}
|
||||
if (handedOff && recoveryId) {
|
||||
sendOutput('\n=== Update failed - restoring previous runtime from recovery generation ===\n');
|
||||
const rolledBack = await recoverySvc.compensateWithCandidate(
|
||||
recoveryId,
|
||||
async (overridePath) => {
|
||||
await this.withRegistryAuth(async (env) => {
|
||||
await this.execute(
|
||||
'docker',
|
||||
await this.buildComposeArgsWithRecoveryOverride(
|
||||
stackName,
|
||||
['up', '-d', '--remove-orphans', '--pull', 'never', '--no-build'],
|
||||
overridePath,
|
||||
),
|
||||
stackDir,
|
||||
ws,
|
||||
true,
|
||||
env,
|
||||
getComposeStallTimeoutMs(),
|
||||
);
|
||||
}, sendOutput);
|
||||
},
|
||||
);
|
||||
throw new ComposeRollbackError(updateError, true, rolledBack);
|
||||
}
|
||||
// Pre-handoff failure: abandon already handled on acquire/classify; runtime untouched.
|
||||
throw updateError;
|
||||
}
|
||||
// Reached only on a successful update; re-baseline so temporal drift compares
|
||||
// against what is now deployed (see deployStack for why this lives here), then
|
||||
// reconcile the ledger against the updated runtime.
|
||||
|
||||
await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName);
|
||||
await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName);
|
||||
try {
|
||||
await this.refreshExposureCache(stackName);
|
||||
} catch (err) {
|
||||
console.warn('[ComposeService] Exposure refresh failed after update for %s:',
|
||||
sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown')));
|
||||
console.warn(
|
||||
'[ComposeService] Exposure refresh failed after update for %s:',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(err, 'unknown')),
|
||||
);
|
||||
}
|
||||
return { recoveryId };
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -213,6 +213,42 @@ export interface HealthGateRunRow {
|
||||
}
|
||||
|
||||
/** Pre-update image snapshot enabling a manual per-service restore after a service-scoped update. */
|
||||
|
||||
/** Full-stack update recovery generation. Separate from service_update_recovery. */
|
||||
export interface StackUpdateRecoveryGenerationRow {
|
||||
id: string;
|
||||
node_id: number;
|
||||
stack_name: string;
|
||||
status: 'candidate' | 'active' | 'restored_current' | 'superseded' | 'abandoned' | 'recovery_required';
|
||||
phase: 'captured' | 'acquired' | 'handoff_committed' | 'reconciling' | 'immediate_verified';
|
||||
is_current: number;
|
||||
backup_slot_id: string | null;
|
||||
override_path: string | null;
|
||||
services_json: string;
|
||||
health_gate_id: string | null;
|
||||
gate_retain_until: number | null;
|
||||
artifact_expires_at: number | null;
|
||||
operation_lease_expires_at: number | null;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
created_by: string | null;
|
||||
artifacts_retired: number;
|
||||
}
|
||||
|
||||
/** Durable cleanup tombstone for stack/node deletion artifact sweep. */
|
||||
export interface StackUpdateCleanupPendingRow {
|
||||
id: string;
|
||||
node_id: number | null;
|
||||
stack_name: string | null;
|
||||
status: 'prepared' | 'ready' | 'cancelled';
|
||||
target_kind: 'local_socket';
|
||||
rollback_tags_json: string;
|
||||
override_paths_json: string;
|
||||
prune_volumes_requested: number;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
export interface ServiceUpdateRecoveryRow {
|
||||
id: string;
|
||||
node_id: number;
|
||||
@@ -1647,6 +1683,50 @@ export class DatabaseService {
|
||||
CREATE INDEX IF NOT EXISTS idx_service_update_recovery_status_claim_expires
|
||||
ON service_update_recovery(status, claim_expires_at);
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS stack_update_recovery_generations (
|
||||
id TEXT PRIMARY KEY,
|
||||
node_id INTEGER NOT NULL,
|
||||
stack_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL CHECK (status IN ('candidate','active','restored_current','superseded','abandoned','recovery_required')),
|
||||
phase TEXT NOT NULL CHECK (phase IN ('captured','acquired','handoff_committed','reconciling','immediate_verified')),
|
||||
is_current INTEGER NOT NULL DEFAULT 0,
|
||||
backup_slot_id TEXT,
|
||||
override_path TEXT,
|
||||
services_json TEXT NOT NULL,
|
||||
health_gate_id TEXT,
|
||||
gate_retain_until INTEGER,
|
||||
artifact_expires_at INTEGER,
|
||||
operation_lease_expires_at INTEGER,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL,
|
||||
created_by TEXT,
|
||||
artifacts_retired INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_node_stack_current
|
||||
ON stack_update_recovery_generations(node_id, stack_name, is_current);
|
||||
CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_status_expires
|
||||
ON stack_update_recovery_generations(status, artifact_expires_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_phase_lease
|
||||
ON stack_update_recovery_generations(phase, operation_lease_expires_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS stack_update_cleanup_pending (
|
||||
id TEXT PRIMARY KEY,
|
||||
node_id INTEGER,
|
||||
stack_name TEXT,
|
||||
status TEXT NOT NULL CHECK (status IN ('prepared','ready','cancelled')),
|
||||
target_kind TEXT NOT NULL CHECK (target_kind IN ('local_socket')),
|
||||
rollback_tags_json TEXT NOT NULL,
|
||||
override_paths_json TEXT NOT NULL,
|
||||
prune_volumes_requested INTEGER NOT NULL DEFAULT 0,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_stack_update_cleanup_status
|
||||
ON stack_update_cleanup_pending(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_stack_update_cleanup_node_stack
|
||||
ON stack_update_cleanup_pending(node_id, stack_name);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS secrets (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
@@ -1699,6 +1779,8 @@ export class DatabaseService {
|
||||
try { this.db.prepare(`ALTER TABLE ${table} ADD COLUMN ${col} ${def}`).run(); } catch (e) { /* ignore */ }
|
||||
};
|
||||
|
||||
maybeAddCol('stack_update_recovery_generations', 'artifacts_retired', 'INTEGER NOT NULL DEFAULT 0');
|
||||
|
||||
// Distributed API model columns
|
||||
maybeAddCol('nodes', 'api_url', "TEXT DEFAULT ''");
|
||||
maybeAddCol('nodes', 'api_token', "TEXT DEFAULT ''");
|
||||
@@ -3510,6 +3592,268 @@ export class DatabaseService {
|
||||
this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
|
||||
}
|
||||
|
||||
public insertStackUpdateRecoveryGeneration(row: StackUpdateRecoveryGenerationRow): void {
|
||||
this.db.prepare(
|
||||
`INSERT INTO stack_update_recovery_generations (
|
||||
id, node_id, stack_name, status, phase, is_current, backup_slot_id, override_path,
|
||||
services_json, health_gate_id, gate_retain_until, artifact_expires_at,
|
||||
operation_lease_expires_at, created_at, updated_at, created_by, artifacts_retired
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
).run(
|
||||
row.id, row.node_id, row.stack_name, row.status, row.phase, row.is_current,
|
||||
row.backup_slot_id, row.override_path, row.services_json, row.health_gate_id,
|
||||
row.gate_retain_until, row.artifact_expires_at, row.operation_lease_expires_at,
|
||||
row.created_at, row.updated_at, row.created_by, row.artifacts_retired ?? 0,
|
||||
);
|
||||
}
|
||||
|
||||
public getStackUpdateRecoveryGeneration(id: string): StackUpdateRecoveryGenerationRow | undefined {
|
||||
return this.db.prepare('SELECT * FROM stack_update_recovery_generations WHERE id = ?').get(id) as StackUpdateRecoveryGenerationRow | undefined;
|
||||
}
|
||||
|
||||
public getCurrentStackUpdateRecovery(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow | undefined {
|
||||
return this.db.prepare(
|
||||
'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ? AND is_current = 1 LIMIT 1'
|
||||
).get(nodeId, stackName) as StackUpdateRecoveryGenerationRow | undefined;
|
||||
}
|
||||
|
||||
public listStackUpdateRecoveryForStack(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow[] {
|
||||
return this.db.prepare(
|
||||
'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ? ORDER BY created_at DESC'
|
||||
).all(nodeId, stackName) as StackUpdateRecoveryGenerationRow[];
|
||||
}
|
||||
|
||||
public listStackUpdateRecoveryGenerationsForNode(nodeId: number): StackUpdateRecoveryGenerationRow[] {
|
||||
return this.db.prepare(
|
||||
'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? ORDER BY created_at DESC'
|
||||
).all(nodeId) as StackUpdateRecoveryGenerationRow[];
|
||||
}
|
||||
|
||||
public updateStackUpdateRecoveryGeneration(
|
||||
id: string,
|
||||
patch: Partial<Pick<StackUpdateRecoveryGenerationRow,
|
||||
'status' | 'phase' | 'is_current' | 'override_path' | 'health_gate_id' |
|
||||
'gate_retain_until' | 'artifact_expires_at' | 'operation_lease_expires_at' | 'services_json'>>,
|
||||
): void {
|
||||
const keys = Object.keys(patch) as Array<keyof typeof patch>;
|
||||
if (keys.length === 0) return;
|
||||
const sets = keys.map((k) => `${String(k)} = ?`).join(', ');
|
||||
const values = keys.map((k) => patch[k]);
|
||||
this.db.prepare(
|
||||
`UPDATE stack_update_recovery_generations SET ${sets}, updated_at = ? WHERE id = ?`
|
||||
).run(...values, Date.now(), id);
|
||||
}
|
||||
|
||||
public casStackUpdateRecoveryPhase(
|
||||
id: string,
|
||||
expectedPhase: StackUpdateRecoveryGenerationRow['phase'],
|
||||
nextPhase: StackUpdateRecoveryGenerationRow['phase'],
|
||||
): boolean {
|
||||
const result = this.db.prepare(
|
||||
'UPDATE stack_update_recovery_generations SET phase = ?, updated_at = ? WHERE id = ? AND phase = ?'
|
||||
).run(nextPhase, Date.now(), id, expectedPhase);
|
||||
return result.changes === 1;
|
||||
}
|
||||
|
||||
public casHandoffGeneration(candidateId: string, nodeId: number, stackName: string): boolean {
|
||||
const handoff = this.db.transaction(() => {
|
||||
const candidate = this.getStackUpdateRecoveryGeneration(candidateId);
|
||||
if (!candidate || candidate.node_id !== nodeId || candidate.stack_name !== stackName) return false;
|
||||
if (candidate.status !== 'candidate' || candidate.phase !== 'acquired') return false;
|
||||
this.db.prepare(
|
||||
`UPDATE stack_update_recovery_generations
|
||||
SET status = 'superseded', is_current = 0, artifact_expires_at = ?, updated_at = ?
|
||||
WHERE node_id = ? AND stack_name = ? AND is_current = 1 AND id != ?`
|
||||
).run(Date.now() + 7 * 24 * 60 * 60 * 1000, Date.now(), nodeId, stackName, candidateId);
|
||||
const result = this.db.prepare(
|
||||
`UPDATE stack_update_recovery_generations
|
||||
SET status = 'active', is_current = 1, phase = 'handoff_committed', updated_at = ?
|
||||
WHERE id = ? AND status = 'candidate' AND phase = 'acquired'`
|
||||
).run(Date.now(), candidateId);
|
||||
return result.changes === 1;
|
||||
});
|
||||
return handoff();
|
||||
}
|
||||
|
||||
|
||||
/** Generations whose Docker/FS artifacts can be retired (not actively held). */
|
||||
public listStackUpdateRecoveryGenerationsForArtifactRetirement(now: number): StackUpdateRecoveryGenerationRow[] {
|
||||
return this.db.prepare(
|
||||
`SELECT * FROM stack_update_recovery_generations
|
||||
WHERE artifacts_retired = 0
|
||||
AND is_current = 0
|
||||
AND status IN ('abandoned', 'superseded')
|
||||
AND (artifact_expires_at IS NULL OR artifact_expires_at <= ?)
|
||||
AND (gate_retain_until IS NULL OR gate_retain_until <= ?)`
|
||||
).all(now, now) as StackUpdateRecoveryGenerationRow[];
|
||||
}
|
||||
|
||||
public markStackUpdateRecoveryArtifactsRetired(id: string): boolean {
|
||||
const result = this.db.prepare(
|
||||
`UPDATE stack_update_recovery_generations
|
||||
SET artifacts_retired = 1, override_path = NULL, updated_at = ?
|
||||
WHERE id = ? AND artifacts_retired = 0`
|
||||
).run(Date.now(), id);
|
||||
return result.changes === 1;
|
||||
}
|
||||
|
||||
public abandonStackUpdateRecoveryGeneration(id: string): boolean {
|
||||
const now = Date.now();
|
||||
const result = this.db.prepare(
|
||||
`UPDATE stack_update_recovery_generations
|
||||
SET status = 'abandoned', is_current = 0, artifact_expires_at = ?,
|
||||
operation_lease_expires_at = NULL, updated_at = ?
|
||||
WHERE id = ? AND status = 'candidate'`
|
||||
).run(now, now, id);
|
||||
return result.changes === 1;
|
||||
}
|
||||
|
||||
/** Pre-handoff candidates whose operation lease has expired. */
|
||||
public listStaleStackUpdateRecoveryCandidates(now: number): StackUpdateRecoveryGenerationRow[] {
|
||||
return this.db.prepare(
|
||||
`SELECT * FROM stack_update_recovery_generations
|
||||
WHERE status = 'candidate'
|
||||
AND operation_lease_expires_at IS NOT NULL
|
||||
AND operation_lease_expires_at <= ?`
|
||||
).all(now) as StackUpdateRecoveryGenerationRow[];
|
||||
}
|
||||
|
||||
/** Post-handoff generations stuck before immediate_verified with an expired lease. */
|
||||
public listStuckStackUpdateRecoveryGenerations(now: number): StackUpdateRecoveryGenerationRow[] {
|
||||
return this.db.prepare(
|
||||
`SELECT * FROM stack_update_recovery_generations
|
||||
WHERE status IN ('active', 'restored_current')
|
||||
AND phase IN ('handoff_committed', 'reconciling')
|
||||
AND operation_lease_expires_at IS NOT NULL
|
||||
AND operation_lease_expires_at <= ?`
|
||||
).all(now) as StackUpdateRecoveryGenerationRow[];
|
||||
}
|
||||
|
||||
public linkStackUpdateRecoveryHealthGate(id: string, healthGateId: string): void {
|
||||
this.db.prepare(
|
||||
'UPDATE stack_update_recovery_generations SET health_gate_id = ?, updated_at = ? WHERE id = ?'
|
||||
).run(healthGateId, Date.now(), id);
|
||||
}
|
||||
|
||||
public setStackUpdateRecoveryGateRetainUntil(id: string, until: number): void {
|
||||
this.db.prepare(
|
||||
'UPDATE stack_update_recovery_generations SET gate_retain_until = ?, updated_at = ? WHERE id = ?'
|
||||
).run(until, Date.now(), id);
|
||||
}
|
||||
|
||||
public listHeldStackUpdateRecoveryImageIds(nodeId: number, now: number): string[] {
|
||||
const rows = this.db.prepare(
|
||||
`SELECT services_json FROM stack_update_recovery_generations
|
||||
WHERE node_id = ?
|
||||
AND status IN ('candidate','active','restored_current','recovery_required')
|
||||
AND (artifact_expires_at IS NULL OR artifact_expires_at > ? OR gate_retain_until > ? OR is_current = 1)`
|
||||
).all(nodeId, now, now) as Array<{ services_json: string }>;
|
||||
const ids = new Set<string>();
|
||||
for (const row of rows) {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(row.services_json);
|
||||
if (!Array.isArray(parsed)) continue;
|
||||
for (const item of parsed) {
|
||||
if (!item || typeof item !== 'object') continue;
|
||||
const replicas = (item as { replicas?: unknown }).replicas;
|
||||
if (Array.isArray(replicas)) {
|
||||
for (const replica of replicas) {
|
||||
if (replica && typeof replica === 'object'
|
||||
&& typeof (replica as { imageId?: unknown }).imageId === 'string'
|
||||
&& (replica as { imageId: string }).imageId.trim()) {
|
||||
ids.add((replica as { imageId: string }).imageId);
|
||||
}
|
||||
}
|
||||
} else if (typeof (item as { imageId?: unknown }).imageId === 'string') {
|
||||
ids.add((item as { imageId: string }).imageId);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Corrupt JSON: skip.
|
||||
}
|
||||
}
|
||||
return [...ids];
|
||||
}
|
||||
|
||||
public deleteStackUpdateRecoveryGenerations(nodeId: number, stackName: string): void {
|
||||
this.db.prepare(
|
||||
'DELETE FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ?'
|
||||
).run(nodeId, stackName);
|
||||
}
|
||||
|
||||
public insertCleanupPending(row: StackUpdateCleanupPendingRow): void {
|
||||
this.db.prepare(
|
||||
`INSERT INTO stack_update_cleanup_pending (
|
||||
id, node_id, stack_name, status, target_kind, rollback_tags_json,
|
||||
override_paths_json, prune_volumes_requested, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
).run(
|
||||
row.id, row.node_id, row.stack_name, row.status, row.target_kind,
|
||||
row.rollback_tags_json, row.override_paths_json, row.prune_volumes_requested,
|
||||
row.created_at, row.updated_at,
|
||||
);
|
||||
}
|
||||
|
||||
public getCleanupPending(id: string): StackUpdateCleanupPendingRow | undefined {
|
||||
return this.db.prepare('SELECT * FROM stack_update_cleanup_pending WHERE id = ?').get(id) as StackUpdateCleanupPendingRow | undefined;
|
||||
}
|
||||
|
||||
public listReadyCleanupPending(): StackUpdateCleanupPendingRow[] {
|
||||
return this.db.prepare(
|
||||
"SELECT * FROM stack_update_cleanup_pending WHERE status = 'ready' ORDER BY created_at ASC"
|
||||
).all() as StackUpdateCleanupPendingRow[];
|
||||
}
|
||||
|
||||
public listPreparedCleanupPending(): StackUpdateCleanupPendingRow[] {
|
||||
return this.db.prepare(
|
||||
"SELECT * FROM stack_update_cleanup_pending WHERE status = 'prepared' ORDER BY created_at ASC"
|
||||
).all() as StackUpdateCleanupPendingRow[];
|
||||
}
|
||||
|
||||
public updateCleanupPendingStatus(id: string, status: StackUpdateCleanupPendingRow['status']): void {
|
||||
this.db.prepare(
|
||||
'UPDATE stack_update_cleanup_pending SET status = ?, updated_at = ? WHERE id = ?'
|
||||
).run(status, Date.now(), id);
|
||||
}
|
||||
|
||||
public deleteCleanupPending(id: string): void {
|
||||
this.db.prepare('DELETE FROM stack_update_cleanup_pending WHERE id = ?').run(id);
|
||||
}
|
||||
|
||||
public hasBlockingDeletionIntent(nodeId: number, stackName: string): boolean {
|
||||
const row = this.db.prepare(
|
||||
`SELECT id FROM stack_update_cleanup_pending
|
||||
WHERE node_id = ? AND stack_name = ? AND status IN ('prepared','ready') LIMIT 1`
|
||||
).get(nodeId, stackName);
|
||||
return !!row;
|
||||
}
|
||||
|
||||
public getPreparedDeletionIntent(nodeId: number, stackName: string): StackUpdateCleanupPendingRow | undefined {
|
||||
return this.db.prepare(
|
||||
`SELECT * FROM stack_update_cleanup_pending
|
||||
WHERE node_id = ? AND stack_name = ? AND status = 'prepared' LIMIT 1`
|
||||
).get(nodeId, stackName) as StackUpdateCleanupPendingRow | undefined;
|
||||
}
|
||||
|
||||
public getDeletionIntentById(id: string): StackUpdateCleanupPendingRow | undefined {
|
||||
return this.getCleanupPending(id);
|
||||
}
|
||||
|
||||
public commitStackDeletionReadyTransaction(intentId: string, nodeId: number, stackName: string): boolean {
|
||||
const run = this.db.transaction(() => {
|
||||
const intent = this.getCleanupPending(intentId);
|
||||
if (!intent || intent.status !== 'prepared') return false;
|
||||
if (intent.node_id !== nodeId || intent.stack_name !== stackName) return false;
|
||||
this.db.prepare(
|
||||
"UPDATE stack_update_cleanup_pending SET status = 'ready', updated_at = ? WHERE id = ? AND status = 'prepared'"
|
||||
).run(Date.now(), intentId);
|
||||
this.deleteStackUpdateRecoveryGenerations(nodeId, stackName);
|
||||
this.deleteServiceUpdateRecoveries(nodeId, stackName);
|
||||
return true;
|
||||
});
|
||||
return run();
|
||||
}
|
||||
|
||||
// --- Notification History ---
|
||||
|
||||
private mapNotificationRow(row: any): NotificationHistory {
|
||||
@@ -3878,7 +4222,16 @@ export class DatabaseService {
|
||||
this.db.prepare(`UPDATE nodes SET ${fields.join(', ')} WHERE id = ?`).run(...values);
|
||||
}
|
||||
|
||||
public deleteNode(id: number): void {
|
||||
/**
|
||||
* Delete a node row and related per-node state.
|
||||
* For a non-default local-socket node, pass localCleanup so a ready tombstone
|
||||
* is inserted in the same transaction (caller enumerates tags/paths first).
|
||||
* Remote hub node records must omit localCleanup (no Docker tombstone).
|
||||
*/
|
||||
public deleteNode(
|
||||
id: number,
|
||||
localCleanup?: { tombstoneId: string; tags: string[]; overridePaths: string[] },
|
||||
): void {
|
||||
const node = this.getNode(id);
|
||||
// Protect the last local node regardless of is_default flag.
|
||||
if (node && node.type === 'local' && this.getLocalNodeCount() <= 1) {
|
||||
@@ -3887,7 +4240,25 @@ export class DatabaseService {
|
||||
if (node?.is_default) {
|
||||
throw new Error('Cannot delete the default node');
|
||||
}
|
||||
if (localCleanup && node?.type !== 'local') {
|
||||
throw new Error('Local cleanup tombstones are only valid for local-socket nodes');
|
||||
}
|
||||
this.db.transaction(() => {
|
||||
if (localCleanup && node?.type === 'local') {
|
||||
const now = Date.now();
|
||||
this.insertCleanupPending({
|
||||
id: localCleanup.tombstoneId,
|
||||
node_id: id,
|
||||
stack_name: null,
|
||||
status: 'ready',
|
||||
target_kind: 'local_socket',
|
||||
rollback_tags_json: JSON.stringify(localCleanup.tags),
|
||||
override_paths_json: JSON.stringify(localCleanup.overridePaths),
|
||||
prune_volumes_requested: 0,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
});
|
||||
}
|
||||
this.db.prepare('DELETE FROM scheduled_task_runs WHERE task_id IN (SELECT id FROM scheduled_tasks WHERE node_id = ?)').run(id);
|
||||
this.db.prepare('DELETE FROM scheduled_tasks WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM stack_update_status WHERE node_id = ?').run(id);
|
||||
@@ -3901,6 +4272,7 @@ export class DatabaseService {
|
||||
this.db.prepare('DELETE FROM preflight_acknowledgements WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM stack_exposure WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM health_gate_runs WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM stack_update_recovery_generations WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ?').run(id);
|
||||
this.db.prepare('UPDATE blueprints SET pinned_node_id = NULL WHERE pinned_node_id = ?').run(id);
|
||||
this.deleteRoleAssignmentsByResource('node', String(id));
|
||||
|
||||
@@ -0,0 +1,521 @@
|
||||
/**
|
||||
* Shared deployed-stack deletion lifecycle (manual DELETE + Blueprint withdrawLocal).
|
||||
*
|
||||
* prepared -> (down, optional volume prune, FS delete) -> ready transaction that
|
||||
* retires full-stack recovery generations and service_update_recovery rows, then
|
||||
* sweeper removes ready tombstone tags/overrides.
|
||||
*/
|
||||
import { randomUUID } from 'crypto';
|
||||
import Docker from 'dockerode';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import {
|
||||
DatabaseService,
|
||||
type StackUpdateCleanupPendingRow,
|
||||
} from './DatabaseService';
|
||||
import { ComposeService } from './ComposeService';
|
||||
import DockerController from './DockerController';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { MeshService } from './MeshService';
|
||||
import { StackOpLockService, stackOpSkipMessage } from './StackOpLockService';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { isPathWithinBase, isValidStackName } from '../utils/validation';
|
||||
|
||||
/**
|
||||
* Directory that may contain recovery override files for a tombstone sweep.
|
||||
* Stack-scoped intents are limited to `<composeDir>/<stackName>/`;
|
||||
* node-wide intents (null stack) use the whole compose root.
|
||||
*/
|
||||
export function overrideDeletionContainmentBase(
|
||||
composeDir: string,
|
||||
stackName: string | null,
|
||||
): string | null {
|
||||
const resolvedCompose = path.resolve(composeDir);
|
||||
if (!stackName) return resolvedCompose;
|
||||
if (!isValidStackName(stackName)) return null;
|
||||
const stackDir = path.resolve(resolvedCompose, stackName);
|
||||
if (!isPathWithinBase(stackDir, resolvedCompose)) return null;
|
||||
return stackDir;
|
||||
}
|
||||
|
||||
export interface DeleteDeployedStackInput {
|
||||
nodeId: number;
|
||||
stackName: string;
|
||||
pruneVolumes: boolean;
|
||||
actor: string;
|
||||
/** When true, skip acquiring a new lock (caller already holds delete via continuation). */
|
||||
continuationIntentId?: string;
|
||||
}
|
||||
|
||||
export type DeleteDeployedStackResult =
|
||||
| { ok: true }
|
||||
| { ok: false; code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed'; error: string; existingAction?: string };
|
||||
|
||||
function collectArtifactsFromGenerations(
|
||||
generations: Array<{ override_path: string | null; services_json: string }>,
|
||||
): { tags: string[]; overridePaths: string[] } {
|
||||
const tags = new Set<string>();
|
||||
const overridePaths = new Set<string>();
|
||||
|
||||
for (const gen of generations) {
|
||||
if (gen.override_path) overridePaths.add(gen.override_path);
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(gen.services_json);
|
||||
if (!Array.isArray(parsed)) continue;
|
||||
for (const svc of parsed) {
|
||||
if (!svc || typeof svc !== 'object') continue;
|
||||
const replicas = (svc as { replicas?: unknown }).replicas;
|
||||
if (!Array.isArray(replicas)) continue;
|
||||
for (const replica of replicas) {
|
||||
const tag = replica && typeof replica === 'object'
|
||||
? (replica as { rollbackTag?: unknown }).rollbackTag
|
||||
: null;
|
||||
if (typeof tag === 'string' && tag.trim()) tags.add(tag);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Corrupt capture JSON: skip tags for this generation.
|
||||
}
|
||||
}
|
||||
|
||||
return { tags: [...tags], overridePaths: [...overridePaths] };
|
||||
}
|
||||
|
||||
function collectArtifacts(nodeId: number, stackName: string): { tags: string[]; overridePaths: string[] } {
|
||||
return collectArtifactsFromGenerations(
|
||||
DatabaseService.getInstance().listStackUpdateRecoveryForStack(nodeId, stackName),
|
||||
);
|
||||
}
|
||||
|
||||
function parseJsonStringArray(raw: string): string[] {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed.filter((item): item is string => typeof item === 'string');
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export class DeployedStackDeletionService {
|
||||
private static instance: DeployedStackDeletionService;
|
||||
|
||||
public static getInstance(): DeployedStackDeletionService {
|
||||
if (!DeployedStackDeletionService.instance) {
|
||||
DeployedStackDeletionService.instance = new DeployedStackDeletionService();
|
||||
}
|
||||
return DeployedStackDeletionService.instance;
|
||||
}
|
||||
|
||||
public assertNoBlockingDeletionIntent(nodeId: number, stackName: string): void {
|
||||
if (DatabaseService.getInstance().hasBlockingDeletionIntent(nodeId, stackName)) {
|
||||
throw new Error(
|
||||
`Stack "${stackName}" has a deletion in progress and cannot be created or mutated until cleanup finishes.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public async deleteDeployedStack(input: DeleteDeployedStackInput): Promise<DeleteDeployedStackResult> {
|
||||
const { nodeId, stackName, actor } = input;
|
||||
const locks = StackOpLockService.getInstance();
|
||||
|
||||
if (input.continuationIntentId) {
|
||||
const cont = locks.tryAcquireDeletionContinuation({
|
||||
intentId: input.continuationIntentId,
|
||||
nodeId,
|
||||
stackName,
|
||||
});
|
||||
if (!cont.acquired) {
|
||||
return {
|
||||
ok: false,
|
||||
code: 'lock_conflict',
|
||||
error: stackOpSkipMessage(stackName, cont.existing.action),
|
||||
existingAction: cont.existing.action,
|
||||
};
|
||||
}
|
||||
try {
|
||||
return await this.runDeletionBody(input, input.continuationIntentId);
|
||||
} finally {
|
||||
locks.release(nodeId, stackName);
|
||||
}
|
||||
}
|
||||
|
||||
const exclusive = await locks.runExclusive(nodeId, stackName, 'delete', actor, async () => {
|
||||
return this.runDeletionBody(input);
|
||||
});
|
||||
if (!exclusive.ran) {
|
||||
return {
|
||||
ok: false,
|
||||
code: 'lock_conflict',
|
||||
error: stackOpSkipMessage(stackName, exclusive.existing.action),
|
||||
existingAction: exclusive.existing.action,
|
||||
};
|
||||
}
|
||||
return exclusive.result;
|
||||
}
|
||||
|
||||
private async runDeletionBody(
|
||||
input: DeleteDeployedStackInput,
|
||||
existingIntentId?: string,
|
||||
): Promise<DeleteDeployedStackResult> {
|
||||
const { nodeId, stackName, pruneVolumes } = input;
|
||||
const db = DatabaseService.getInstance();
|
||||
|
||||
let intentId = existingIntentId;
|
||||
if (!intentId) {
|
||||
const { tags, overridePaths } = collectArtifacts(nodeId, stackName);
|
||||
const now = Date.now();
|
||||
const row: StackUpdateCleanupPendingRow = {
|
||||
id: randomUUID(),
|
||||
node_id: nodeId,
|
||||
stack_name: stackName,
|
||||
status: 'prepared',
|
||||
target_kind: 'local_socket',
|
||||
rollback_tags_json: JSON.stringify(tags),
|
||||
override_paths_json: JSON.stringify(overridePaths),
|
||||
prune_volumes_requested: pruneVolumes ? 1 : 0,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
};
|
||||
try {
|
||||
db.insertCleanupPending(row);
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
code: 'tombstone_failed',
|
||||
error: getErrorMessage(error, 'Failed to prepare stack deletion'),
|
||||
};
|
||||
}
|
||||
intentId = row.id;
|
||||
}
|
||||
|
||||
const intent = db.getDeletionIntentById(intentId);
|
||||
if (!intent || intent.status !== 'prepared') {
|
||||
return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' };
|
||||
}
|
||||
|
||||
try {
|
||||
await ComposeService.getInstance(nodeId).downStack(stackName);
|
||||
} catch (downErr) {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Compose down failed or no-op for %s:',
|
||||
sanitizeForLog(stackName),
|
||||
downErr,
|
||||
);
|
||||
}
|
||||
|
||||
if (intent.prune_volumes_requested === 1) {
|
||||
try {
|
||||
await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]);
|
||||
} catch (pruneErr) {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Volume prune failed for %s, continuing delete:',
|
||||
sanitizeForLog(stackName),
|
||||
pruneErr,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await FileSystemService.getInstance(nodeId).deleteStack(stackName);
|
||||
} catch (fsErr) {
|
||||
db.updateCleanupPendingStatus(intentId, 'cancelled');
|
||||
return {
|
||||
ok: false,
|
||||
code: 'fs_failed',
|
||||
error: getErrorMessage(fsErr, 'Failed to remove stack files'),
|
||||
};
|
||||
}
|
||||
|
||||
if (!db.commitStackDeletionReadyTransaction(intentId, nodeId, stackName)) {
|
||||
return {
|
||||
ok: false,
|
||||
code: 'db_failed',
|
||||
error: 'Failed to commit stack deletion ready transaction',
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
db.clearStackUpdateStatus(nodeId, stackName);
|
||||
db.clearStackScanAttempts(nodeId, stackName);
|
||||
db.deleteRoleAssignmentsByResource('stack', stackName);
|
||||
db.deleteGitSource(stackName);
|
||||
db.deleteStackDossier(nodeId, stackName);
|
||||
db.deleteStackDriftFindings(nodeId, stackName);
|
||||
db.deleteStackExposureIntents(nodeId, stackName);
|
||||
db.deleteStackExposure(nodeId, stackName);
|
||||
db.deleteStackProjectEnvFiles(nodeId, stackName);
|
||||
db.deleteStackScans(nodeId, stackName);
|
||||
} catch (dbErr) {
|
||||
console.error(
|
||||
'[DeployedStackDeletion] Secondary DB cleanup failed for %s; recovery rows already retired:',
|
||||
sanitizeForLog(stackName),
|
||||
dbErr,
|
||||
);
|
||||
return {
|
||||
ok: false,
|
||||
code: 'db_failed',
|
||||
error: getErrorMessage(dbErr, 'Failed to clear stack database state'),
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
await MeshService.getInstance().optOutStack(nodeId, stackName, input.actor);
|
||||
} catch (meshErr) {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Mesh opt-out failed for %s:',
|
||||
sanitizeForLog(stackName),
|
||||
meshErr,
|
||||
);
|
||||
}
|
||||
|
||||
await this.sweepReadyIntent(intentId);
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove rollback tags + override paths for a ready tombstone, then drop the row.
|
||||
* When the node row is already gone (local-node delete), pass a preserved local
|
||||
* Docker handle and composeDir so we never fall back to a remote default node.
|
||||
*/
|
||||
public async sweepReadyIntent(
|
||||
intentId: string,
|
||||
opts?: { docker?: Docker; composeDir?: string },
|
||||
): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const intent = db.getCleanupPending(intentId);
|
||||
if (!intent || intent.status !== 'ready') return;
|
||||
if (intent.node_id == null) {
|
||||
db.deleteCleanupPending(intentId);
|
||||
return;
|
||||
}
|
||||
|
||||
const tags = parseJsonStringArray(intent.rollback_tags_json);
|
||||
const overridePaths = parseJsonStringArray(intent.override_paths_json);
|
||||
|
||||
let docker: Docker;
|
||||
if (opts?.docker) {
|
||||
docker = opts.docker;
|
||||
} else if (db.getNode(intent.node_id)) {
|
||||
docker = DockerController.getInstance(intent.node_id).getDocker();
|
||||
} else if (intent.target_kind === 'local_socket') {
|
||||
// Deleted local node: local_socket tombstones always target the host Docker socket.
|
||||
docker = new Docker();
|
||||
} else {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Cannot sweep tombstone %s: node gone and target is not local_socket',
|
||||
sanitizeForLog(intentId),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let incomplete = false;
|
||||
|
||||
for (const tag of tags) {
|
||||
try {
|
||||
await docker.getImage(tag).remove({ force: true });
|
||||
} catch (error) {
|
||||
const status = (error as { statusCode?: number }).statusCode;
|
||||
const message = getErrorMessage(error, 'unknown').toLowerCase();
|
||||
if (status === 404 || message.includes('no such image') || message.includes('not found')) {
|
||||
continue;
|
||||
}
|
||||
incomplete = true;
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Failed to remove rollback tag %s: %s',
|
||||
sanitizeForLog(tag),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const baseDir = opts?.composeDir
|
||||
?? (db.getNode(intent.node_id)
|
||||
? FileSystemService.getInstance(intent.node_id).getBaseDir()
|
||||
: null);
|
||||
|
||||
const containmentBase = baseDir
|
||||
? overrideDeletionContainmentBase(baseDir, intent.stack_name)
|
||||
: null;
|
||||
if (baseDir && !containmentBase) {
|
||||
incomplete = true;
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Refusing override sweep: invalid stack containment for tombstone %s',
|
||||
sanitizeForLog(intentId),
|
||||
);
|
||||
}
|
||||
|
||||
for (const overridePath of overridePaths) {
|
||||
try {
|
||||
const resolved = path.resolve(overridePath);
|
||||
const basename = path.basename(resolved);
|
||||
if (!/^\.sencho-recovery-[a-f0-9]+\.yml$/i.test(basename)) {
|
||||
incomplete = true;
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Refusing to delete non-recovery override: %s',
|
||||
sanitizeForLog(overridePath),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (containmentBase && !isPathWithinBase(resolved, containmentBase)) {
|
||||
incomplete = true;
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Refusing to delete override outside stack containment: %s',
|
||||
sanitizeForLog(overridePath),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (!baseDir && !path.isAbsolute(resolved)) {
|
||||
incomplete = true;
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Refusing relative override without compose dir: %s',
|
||||
sanitizeForLog(overridePath),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (!containmentBase && baseDir) {
|
||||
incomplete = true;
|
||||
continue;
|
||||
}
|
||||
await fs.unlink(resolved);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
incomplete = true;
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Failed to delete override %s: %s',
|
||||
sanitizeForLog(overridePath),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the ready tombstone when cleanup is incomplete so startup can retry.
|
||||
if (!incomplete) {
|
||||
db.deleteCleanupPending(intentId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Enumerate rollback tags and override paths for every stack recovery on a node. */
|
||||
public collectNodeArtifacts(nodeId: number): { tags: string[]; overridePaths: string[] } {
|
||||
return collectArtifactsFromGenerations(
|
||||
DatabaseService.getInstance().listStackUpdateRecoveryGenerationsForNode(nodeId),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a local-socket node with an atomic ready tombstone, then sweep.
|
||||
* Remote node records call DatabaseService.deleteNode without cleanup.
|
||||
*/
|
||||
public async deleteLocalNode(nodeId: number): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const node = db.getNode(nodeId);
|
||||
if (!node) throw new Error('Node not found');
|
||||
if (node.type !== 'local') {
|
||||
db.deleteNode(nodeId);
|
||||
return;
|
||||
}
|
||||
// Preserve Docker + compose dir before the row disappears so sweep never
|
||||
// targets a remote default node.
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
const composeDir = FileSystemService.getInstance(nodeId).getBaseDir();
|
||||
const { tags, overridePaths } = this.collectNodeArtifacts(nodeId);
|
||||
const tombstoneId = randomUUID();
|
||||
db.deleteNode(nodeId, { tombstoneId, tags, overridePaths });
|
||||
NodeRegistry.getInstance().evictConnection(nodeId);
|
||||
try {
|
||||
await this.sweepReadyIntent(tombstoneId, { docker, composeDir });
|
||||
} catch (error) {
|
||||
// Node row is already gone; leave the ready tombstone for startup resume.
|
||||
console.error(
|
||||
'[DeployedStackDeletion] Sweep after local-node delete deferred for tombstone %s: %s',
|
||||
sanitizeForLog(tombstoneId),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Startup reconciliation: resume prepared intents, complete ready transitions
|
||||
* when the directory is already gone, and sweep ready tombstones.
|
||||
*/
|
||||
public async reconcileAtStartup(): Promise<void> {
|
||||
const db = DatabaseService.getInstance();
|
||||
for (const intent of db.listPreparedCleanupPending()) {
|
||||
if (intent.node_id == null || !intent.stack_name) continue;
|
||||
const nodeId = intent.node_id;
|
||||
const stackName = intent.stack_name;
|
||||
const stackDir = path.join(FileSystemService.getInstance(nodeId).getBaseDir(), stackName);
|
||||
let dirExists = true;
|
||||
try {
|
||||
await fs.access(stackDir);
|
||||
} catch (accessError) {
|
||||
const code = (accessError as NodeJS.ErrnoException).code;
|
||||
if (code === 'ENOENT') {
|
||||
dirExists = false;
|
||||
} else {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Startup access error for %s (not treating as absent): %s',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(accessError, 'unknown')),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (!dirExists) {
|
||||
if (!db.commitStackDeletionReadyTransaction(intent.id, nodeId, stackName)) {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Startup ready commit failed for %s/%s',
|
||||
nodeId,
|
||||
sanitizeForLog(stackName),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await MeshService.getInstance().optOutStack(nodeId, stackName, 'system:startup');
|
||||
} catch (meshErr) {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Startup mesh opt-out failed for %s: %s',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(meshErr, 'unknown')),
|
||||
);
|
||||
}
|
||||
await this.sweepReadyIntent(intent.id);
|
||||
continue;
|
||||
}
|
||||
|
||||
const result = await this.deleteDeployedStack({
|
||||
nodeId,
|
||||
stackName,
|
||||
pruneVolumes: intent.prune_volumes_requested === 1,
|
||||
actor: 'system:startup',
|
||||
continuationIntentId: intent.id,
|
||||
});
|
||||
if (!result.ok) {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Startup resume failed for %s: %s',
|
||||
sanitizeForLog(stackName),
|
||||
result.error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const intent of db.listReadyCleanupPending()) {
|
||||
if (intent.node_id != null && intent.stack_name) {
|
||||
try {
|
||||
await MeshService.getInstance().optOutStack(intent.node_id, intent.stack_name, 'system:startup');
|
||||
} catch (meshErr) {
|
||||
console.warn(
|
||||
'[DeployedStackDeletion] Ready-resume mesh opt-out failed for %s: %s',
|
||||
sanitizeForLog(intent.stack_name),
|
||||
sanitizeForLog(getErrorMessage(meshErr, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
await this.sweepReadyIntent(intent.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
import type { NetworkingNetworkBase } from './network/networkingTypes';
|
||||
import { isPathWithinBase } from '../utils/validation';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { describeSpawnError } from '../utils/spawnErrors';
|
||||
import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs';
|
||||
@@ -2258,6 +2259,57 @@ class DockerController {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Strict Result API for full-stack updates. Never converts Compose-ps + fallback
|
||||
* failure into empty success. Compose-managed containers are never orphan IDs.
|
||||
*/
|
||||
public async classifyLegacyOrphansForUpdate(
|
||||
stackName: string,
|
||||
): Promise<
|
||||
| { status: 'none' }
|
||||
| { status: 'orphans'; ids: string[] }
|
||||
| { status: 'classification_failed'; error: string }
|
||||
> {
|
||||
const stackDir = path.join(NodeRegistry.getInstance().getComposeDir(this.nodeId), stackName);
|
||||
const toIds = (list: Array<{ Id?: string }>) =>
|
||||
list.filter((c): c is { Id: string } => typeof c.Id === 'string' && c.Id.length > 0)
|
||||
.map((c) => c.Id);
|
||||
|
||||
const fallbackOrphans = async (): Promise<
|
||||
| { status: 'none' }
|
||||
| { status: 'orphans'; ids: string[] }
|
||||
| { status: 'classification_failed'; error: string }
|
||||
> => {
|
||||
try {
|
||||
const ids = toIds(await this.smartFallback(stackName, stackDir));
|
||||
return ids.length === 0 ? { status: 'none' } : { status: 'orphans', ids };
|
||||
} catch (fallbackError) {
|
||||
return {
|
||||
status: 'classification_failed',
|
||||
error: getErrorMessage(fallbackError, 'Legacy orphan classification failed'),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
const composeContainers = await this.fetchComposePsContainers(stackName, stackDir);
|
||||
// Compose already manages this stack: no legacy orphan cleanup (same as deploy).
|
||||
if (composeContainers.length > 0) return { status: 'none' };
|
||||
return await fallbackOrphans();
|
||||
} catch (error) {
|
||||
const execError = error as NodeJS.ErrnoException & { stderr?: string };
|
||||
const mapped = describeSpawnError(execError, { command: 'docker compose ps' });
|
||||
const detail = execError.stderr || mapped.message || getErrorMessage(error, 'docker compose ps failed');
|
||||
console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(detail));
|
||||
// Unlike getLegacyOrphanContainersByStack, never convert dual failure into empty success.
|
||||
const fallback = await fallbackOrphans();
|
||||
if (fallback.status === 'classification_failed') {
|
||||
return { status: 'classification_failed', error: String(detail) };
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
public async getContainersByStack(stackName: string) {
|
||||
// Resolve the compose dir and the authored prefix for THIS controller's node,
|
||||
// not the process default, so a non-default local node sees its own stack dir
|
||||
|
||||
@@ -545,6 +545,11 @@ export class FileSystemService {
|
||||
}
|
||||
|
||||
async createStack(stackName: string): Promise<void> {
|
||||
if (DatabaseService.getInstance().hasBlockingDeletionIntent(this.nodeId, stackName)) {
|
||||
throw new Error(
|
||||
`Stack "${stackName}" has a deletion in progress and cannot be created until cleanup finishes.`,
|
||||
);
|
||||
}
|
||||
const stackDir = this.resolveStackDir(stackName);
|
||||
await this.assertRealWithinBase(stackDir);
|
||||
|
||||
|
||||
@@ -1104,7 +1104,13 @@ export class SchedulerService {
|
||||
);
|
||||
if (!lock.ran) return skipMessage(stackName, lock.existing.action);
|
||||
db.clearStackUpdateStatus(nodeId, stackName);
|
||||
HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler');
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler');
|
||||
const orchResult = lock.result;
|
||||
const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
|
||||
if (recoveryId) {
|
||||
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
||||
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
|
||||
}
|
||||
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
|
||||
@@ -237,7 +237,17 @@ export class ServiceUpdateRecoveryService {
|
||||
return (imageId: string) => {
|
||||
const held = this.getHeldImageIds(nodeId);
|
||||
if (held === null) return true;
|
||||
return held.has(imageId);
|
||||
if (held.has(imageId)) return true;
|
||||
try {
|
||||
// Dynamic import avoids a static cycle with StackUpdateRecoveryService.
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const { StackUpdateRecoveryService } = require('./StackUpdateRecoveryService') as typeof import('./StackUpdateRecoveryService');
|
||||
const stackHeld = StackUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
|
||||
if (stackHeld === null) return true;
|
||||
return stackHeld.has(imageId);
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
/**
|
||||
* Tracks in-flight stack lifecycle operations (deploy, down, restart, stop,
|
||||
* start, update, rollback, backup) per (nodeId, stackName). A second request to
|
||||
@@ -9,7 +10,7 @@
|
||||
* which matches the lifecycle of any in-flight `docker compose` child process.
|
||||
*/
|
||||
|
||||
export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup';
|
||||
export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup' | 'delete';
|
||||
|
||||
/**
|
||||
* Note returned by a background path that skipped its operation because a manual
|
||||
@@ -70,6 +71,31 @@ export class StackOpLockService {
|
||||
this.locks.delete(this.key(nodeId, stackName));
|
||||
}
|
||||
|
||||
/**
|
||||
* Matching-intent continuation for crash-resume of a prepared deletion.
|
||||
* Succeeds only when the persisted intent matches and no other in-memory lock holds.
|
||||
*/
|
||||
public tryAcquireDeletionContinuation(args: {
|
||||
intentId: string;
|
||||
nodeId: number;
|
||||
stackName: string;
|
||||
}): AcquireResult {
|
||||
const intent = DatabaseService.getInstance().getDeletionIntentById(args.intentId);
|
||||
if (
|
||||
!intent
|
||||
|| intent.status !== 'prepared'
|
||||
|| intent.node_id !== args.nodeId
|
||||
|| intent.stack_name !== args.stackName
|
||||
) {
|
||||
const existing = this.locks.get(this.key(args.nodeId, args.stackName));
|
||||
return {
|
||||
acquired: false,
|
||||
existing: existing ?? { action: 'delete', startedAt: Date.now(), user: 'system' },
|
||||
};
|
||||
}
|
||||
return this.tryAcquire(args.nodeId, args.stackName, 'delete', 'system:deletion-continuation');
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire the per-(nodeId, stackName) lock for the duration of `fn`, then
|
||||
* release it. Returns `{ ran: true, result }` when the lock was free, or
|
||||
|
||||
@@ -47,7 +47,7 @@ export interface ServiceUpdateOptions {
|
||||
}
|
||||
|
||||
export type OrchestratorResult =
|
||||
| { kind: 'stack_compose_done' }
|
||||
| { kind: 'stack_compose_done'; recoveryId: string | null }
|
||||
| {
|
||||
kind: 'service_done';
|
||||
serviceName: string;
|
||||
@@ -182,10 +182,11 @@ export class StackUpdateOrchestrator {
|
||||
ctx: UpdateOperationContext,
|
||||
options: StackComposeOptions,
|
||||
): Promise<OrchestratorResult> {
|
||||
await ComposeService.getInstance(ctx.nodeId).updateStack(
|
||||
const updateResult = await ComposeService.getInstance(ctx.nodeId).updateStack(
|
||||
ctx.stackName, options.terminalWs ?? undefined, options.atomic,
|
||||
);
|
||||
return { kind: 'stack_compose_done' };
|
||||
const recoveryId = updateResult?.recoveryId ?? null;
|
||||
return { kind: 'stack_compose_done', recoveryId };
|
||||
}
|
||||
|
||||
private async executeServiceUpdate(
|
||||
@@ -195,6 +196,16 @@ export class StackUpdateOrchestrator {
|
||||
): Promise<OrchestratorResult> {
|
||||
const { nodeId, stackName } = ctx;
|
||||
|
||||
{
|
||||
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
||||
if (StackUpdateRecoveryService.getInstance().isRestoredCurrentPinActive(nodeId, stackName)) {
|
||||
return serviceFailed(
|
||||
'stack_recovery_pin_active',
|
||||
'This stack is pinned to a restored recovery generation. Run a full-stack Update to continue.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const loaded = await this.loadServiceSpec(nodeId, stackName, serviceName);
|
||||
if (!loaded.ok) return loaded.result;
|
||||
const { spec, services } = loaded;
|
||||
@@ -308,6 +319,16 @@ export class StackUpdateOrchestrator {
|
||||
const { nodeId, stackName } = ctx;
|
||||
const recoveryId = options.recoveryId as string;
|
||||
|
||||
{
|
||||
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
||||
if (StackUpdateRecoveryService.getInstance().isRestoredCurrentPinActive(nodeId, stackName)) {
|
||||
return serviceFailed(
|
||||
'stack_recovery_pin_active',
|
||||
'This stack is pinned to a restored recovery generation. Run a full-stack Update to continue.',
|
||||
{ recoveryId },
|
||||
);
|
||||
}
|
||||
}
|
||||
const recovery = ServiceUpdateRecoveryService.getInstance().get(recoveryId);
|
||||
if (
|
||||
!recovery ||
|
||||
|
||||
@@ -0,0 +1,767 @@
|
||||
/**
|
||||
* Full-stack update recovery generations: capture, opaque rollback tags,
|
||||
* stack-local recovery override, handoff, compensation, and prune holds.
|
||||
*
|
||||
* Separate from ServiceUpdateRecoveryService (service-scoped snapshots).
|
||||
* Does not run Compose; ComposeService / orchestrator own Docker mutations.
|
||||
*
|
||||
* Recovery fidelity contract (supported):
|
||||
* - Prior images are restored via opaque hold tags (`--pull never --no-build`).
|
||||
* - Observed running replica count is restored via compose `scale`.
|
||||
* - Services that were fully stopped at capture are kept at `scale: 0`.
|
||||
* - Authored replica counts are not used when they diverge from observed state.
|
||||
*/
|
||||
import { randomUUID } from 'crypto';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import {
|
||||
DatabaseService,
|
||||
type StackUpdateRecoveryGenerationRow,
|
||||
} from './DatabaseService';
|
||||
import DockerController from './DockerController';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { buildEffectiveServiceModel } from './effectiveServiceModel';
|
||||
import {
|
||||
classifyReferenceKind,
|
||||
type ImageReferenceKind,
|
||||
resolveComposeProjectContext,
|
||||
} from './composeProjectContext';
|
||||
import { getComposeCommandTimeoutMs } from './ComposeService';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { isValidStackName } from '../utils/validation';
|
||||
|
||||
const SWEEP_INTERVAL_MS = 5 * 60_000;
|
||||
const INITIAL_SWEEP_DELAY_MS = 30_000;
|
||||
const MIN_RECOVERY_WINDOW_SECONDS = 90;
|
||||
const RECOVERY_TTL_BUFFER_MS = 30 * 60_000;
|
||||
const GATE_RETAIN_DEFAULT_MS = 2 * 60 * 60_000;
|
||||
const RECOVERY_PROBE_DELAY_MS = 3_000;
|
||||
|
||||
export interface StackRecoveryReplicaCapture {
|
||||
containerId: string | null;
|
||||
imageId: string | null;
|
||||
repoDigest: string | null;
|
||||
state: 'running' | 'stopped' | 'none';
|
||||
rollbackTag: string | null;
|
||||
}
|
||||
|
||||
export interface StackRecoveryServiceCapture {
|
||||
serviceName: string;
|
||||
/** Observed running replica count at capture (supported restore scale). */
|
||||
scale: number;
|
||||
hasBuild: boolean;
|
||||
declaredImageRef: string | null;
|
||||
referenceKind: ImageReferenceKind;
|
||||
replicas: StackRecoveryReplicaCapture[];
|
||||
}
|
||||
|
||||
export interface CaptureStackUpdateInput {
|
||||
nodeId: number;
|
||||
stackName: string;
|
||||
createdBy: string | null;
|
||||
}
|
||||
|
||||
function yamlQuote(value: string): string {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function sanitizeServiceSlug(name: string): string {
|
||||
return name.replace(/[^a-zA-Z0-9._-]/g, '-').toLowerCase() || 'svc';
|
||||
}
|
||||
|
||||
function opaqueRollbackTag(generationId: string, serviceName: string): string {
|
||||
const short = generationId.replace(/-/g, '').slice(0, 12);
|
||||
return `sencho-rb/${short}/${sanitizeServiceSlug(serviceName)}:hold`;
|
||||
}
|
||||
|
||||
function parseServicesJson(raw: string): StackRecoveryServiceCapture[] {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed as StackRecoveryServiceCapture[];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function collectImageIdsFromServicesJson(servicesJson: string): string[] {
|
||||
const ids = new Set<string>();
|
||||
for (const svc of parseServicesJson(servicesJson)) {
|
||||
for (const replica of svc.replicas ?? []) {
|
||||
if (replica.imageId && replica.imageId.trim()) ids.add(replica.imageId);
|
||||
}
|
||||
}
|
||||
return [...ids];
|
||||
}
|
||||
|
||||
function collectRollbackTags(services: StackRecoveryServiceCapture[]): string[] {
|
||||
const tags = new Set<string>();
|
||||
for (const svc of services) {
|
||||
for (const replica of svc.replicas ?? []) {
|
||||
if (replica.rollbackTag) tags.add(replica.rollbackTag);
|
||||
}
|
||||
}
|
||||
return [...tags];
|
||||
}
|
||||
|
||||
export class StackUpdateRecoveryService {
|
||||
private static instance: StackUpdateRecoveryService;
|
||||
private started = false;
|
||||
private intervalId: NodeJS.Timeout | null = null;
|
||||
private initialTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
private constructor() {}
|
||||
|
||||
public static getInstance(): StackUpdateRecoveryService {
|
||||
if (!StackUpdateRecoveryService.instance) {
|
||||
StackUpdateRecoveryService.instance = new StackUpdateRecoveryService();
|
||||
}
|
||||
return StackUpdateRecoveryService.instance;
|
||||
}
|
||||
|
||||
public static resetForTests(): void {
|
||||
if (StackUpdateRecoveryService.instance) {
|
||||
StackUpdateRecoveryService.instance.stop();
|
||||
}
|
||||
StackUpdateRecoveryService.instance = new StackUpdateRecoveryService();
|
||||
}
|
||||
|
||||
public start(): void {
|
||||
this.started = true;
|
||||
if (this.initialTimer || this.intervalId) return;
|
||||
this.initialTimer = setTimeout(() => {
|
||||
void this.reconcileIncomplete();
|
||||
this.intervalId = setInterval(() => {
|
||||
void this.reconcileIncomplete();
|
||||
}, SWEEP_INTERVAL_MS);
|
||||
}, INITIAL_SWEEP_DELAY_MS);
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
this.started = false;
|
||||
if (this.initialTimer) {
|
||||
clearTimeout(this.initialTimer);
|
||||
this.initialTimer = null;
|
||||
}
|
||||
if (this.intervalId) {
|
||||
clearInterval(this.intervalId);
|
||||
this.intervalId = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate exact Compose invocation, backup files, snapshot runtime, create
|
||||
* opaque tags + recovery override, insert candidate generation.
|
||||
*/
|
||||
public async captureCandidate(input: CaptureStackUpdateInput): Promise<StackUpdateRecoveryGenerationRow> {
|
||||
const { nodeId, stackName, createdBy } = input;
|
||||
if (!isValidStackName(stackName)) {
|
||||
throw new Error('Invalid stack name');
|
||||
}
|
||||
|
||||
const context = await resolveComposeProjectContext(nodeId, stackName);
|
||||
await context.validateForMutation();
|
||||
// Exact mutating invocation (authored files + env + generated Mesh override)
|
||||
// must validate before any backup, tag, or override write.
|
||||
const { ComposeService } = await import('./ComposeService');
|
||||
await ComposeService.getInstance(nodeId).validateExactComposeInvocation(stackName);
|
||||
|
||||
const backupSlotId = await context.backupFromContext('update');
|
||||
|
||||
const model = await buildEffectiveServiceModel(nodeId, stackName);
|
||||
if (!model.renderable) {
|
||||
throw new Error(model.error || 'Effective Compose model failed to render');
|
||||
}
|
||||
|
||||
const generationId = randomUUID();
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
const services: StackRecoveryServiceCapture[] = [];
|
||||
const createdTags: string[] = [];
|
||||
let overridePath: string | null = null;
|
||||
|
||||
try {
|
||||
for (const spec of model.services) {
|
||||
const declaredImageRef = spec.declaredImage;
|
||||
const referenceKind = classifyReferenceKind(declaredImageRef);
|
||||
const listed = await docker.listContainers({
|
||||
all: true,
|
||||
filters: {
|
||||
label: [
|
||||
`com.docker.compose.project=${stackName}`,
|
||||
`com.docker.compose.service=${spec.name}`,
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const replicas: StackRecoveryReplicaCapture[] = [];
|
||||
for (const info of listed) {
|
||||
try {
|
||||
const inspect = await docker.getContainer(info.Id).inspect();
|
||||
const status = inspect.State?.Status;
|
||||
const state: StackRecoveryReplicaCapture['state'] =
|
||||
status === 'running' ? 'running' : status ? 'stopped' : 'none';
|
||||
const imageId = typeof inspect.Image === 'string' && inspect.Image.length > 0
|
||||
? inspect.Image
|
||||
: null;
|
||||
if ((state === 'running' || state === 'stopped') && !imageId) {
|
||||
throw new Error(
|
||||
`Service "${spec.name}" replica ${info.Id.slice(0, 12)} has no protectable image id`,
|
||||
);
|
||||
}
|
||||
let repoDigest: string | null = null;
|
||||
if (imageId) {
|
||||
try {
|
||||
const image = await docker.getImage(imageId).inspect();
|
||||
const digests = (image.RepoDigests ?? []) as string[];
|
||||
repoDigest = digests.length > 0 ? digests[0] : null;
|
||||
} catch {
|
||||
repoDigest = null;
|
||||
}
|
||||
}
|
||||
replicas.push({
|
||||
containerId: info.Id,
|
||||
imageId,
|
||||
repoDigest,
|
||||
state,
|
||||
rollbackTag: null,
|
||||
});
|
||||
} catch (error) {
|
||||
if ((error as { statusCode?: number })?.statusCode === 404) continue;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const runningCount = replicas.filter((r) => r.state === 'running').length;
|
||||
services.push({
|
||||
serviceName: spec.name,
|
||||
scale: runningCount,
|
||||
hasBuild: spec.hasBuild,
|
||||
declaredImageRef,
|
||||
referenceKind,
|
||||
replicas,
|
||||
});
|
||||
}
|
||||
|
||||
const taggedIds = new Set<string>();
|
||||
for (const svc of services) {
|
||||
const primary = svc.replicas.find((r) => r.imageId) ?? null;
|
||||
if (!primary?.imageId) continue;
|
||||
|
||||
const tag = opaqueRollbackTag(generationId, svc.serviceName);
|
||||
const tagKey = `${primary.imageId}|${tag}`;
|
||||
if (!taggedIds.has(tagKey)) {
|
||||
const { repo, tagName } = splitOpaqueTag(tag);
|
||||
await docker.getImage(primary.imageId).tag({ repo, tag: tagName });
|
||||
taggedIds.add(tagKey);
|
||||
createdTags.push(tag);
|
||||
}
|
||||
for (const replica of svc.replicas) {
|
||||
if (replica.imageId === primary.imageId) {
|
||||
replica.rollbackTag = tag;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
overridePath = await this.writeRecoveryOverride(nodeId, stackName, generationId, services);
|
||||
|
||||
const now = Date.now();
|
||||
const row: StackUpdateRecoveryGenerationRow = {
|
||||
id: generationId,
|
||||
node_id: nodeId,
|
||||
stack_name: stackName,
|
||||
status: 'candidate',
|
||||
phase: 'captured',
|
||||
is_current: 0,
|
||||
backup_slot_id: backupSlotId,
|
||||
override_path: overridePath,
|
||||
services_json: JSON.stringify(services),
|
||||
health_gate_id: null,
|
||||
gate_retain_until: null,
|
||||
artifact_expires_at: null,
|
||||
operation_lease_expires_at: now + getComposeCommandTimeoutMs() + RECOVERY_TTL_BUFFER_MS,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
created_by: createdBy,
|
||||
artifacts_retired: 0,
|
||||
};
|
||||
DatabaseService.getInstance().insertStackUpdateRecoveryGeneration(row);
|
||||
return row;
|
||||
} catch (error) {
|
||||
await this.bestEffortRemoveTags(nodeId, createdTags);
|
||||
if (overridePath) {
|
||||
try {
|
||||
await fs.unlink(overridePath);
|
||||
} catch {
|
||||
// Best-effort mid-capture cleanup.
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async writeRecoveryOverride(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
generationId: string,
|
||||
services: StackRecoveryServiceCapture[],
|
||||
): Promise<string> {
|
||||
if (!isValidStackName(stackName)) {
|
||||
throw new Error('Invalid stack name');
|
||||
}
|
||||
// Canonical inline path barrier (same pattern as ComposeService.renderConfig).
|
||||
const baseResolved = path.resolve(FileSystemService.getInstance(nodeId).getBaseDir());
|
||||
const stackDir = path.resolve(baseResolved, stackName);
|
||||
if (!stackDir.startsWith(baseResolved + path.sep)) {
|
||||
throw new Error('Invalid stack path');
|
||||
}
|
||||
let stackDirReal: string;
|
||||
let baseReal: string;
|
||||
try {
|
||||
[stackDirReal, baseReal] = await Promise.all([
|
||||
fs.realpath(stackDir),
|
||||
fs.realpath(baseResolved),
|
||||
]);
|
||||
} catch {
|
||||
throw new Error('Stack directory not found');
|
||||
}
|
||||
if (stackDirReal !== baseReal && !stackDirReal.startsWith(baseReal + path.sep)) {
|
||||
throw new Error('Stack directory escapes compose base');
|
||||
}
|
||||
|
||||
const short = generationId.replace(/-/g, '').slice(0, 12);
|
||||
if (!/^[a-f0-9]{12}$/i.test(short)) {
|
||||
throw new Error('Invalid recovery generation id');
|
||||
}
|
||||
const filename = `.sencho-recovery-${short}.yml`;
|
||||
const abs = path.resolve(stackDirReal, filename);
|
||||
if (!abs.startsWith(stackDirReal + path.sep)) {
|
||||
throw new Error('Recovery override path escapes stack directory');
|
||||
}
|
||||
|
||||
const lines: string[] = ['services:'];
|
||||
let wroteAny = false;
|
||||
for (const svc of services) {
|
||||
const tag = svc.replicas.find((r) => r.rollbackTag)?.rollbackTag ?? null;
|
||||
const stoppedOnly =
|
||||
svc.replicas.length > 0
|
||||
&& svc.replicas.every((r) => r.state === 'stopped' || r.state === 'none');
|
||||
if (!tag && svc.scale !== 0 && !stoppedOnly) continue;
|
||||
const key = /^[a-zA-Z0-9._-]+$/.test(svc.serviceName) ? svc.serviceName : yamlQuote(svc.serviceName);
|
||||
lines.push(` ${key}:`);
|
||||
if (tag) {
|
||||
lines.push(` image: ${yamlQuote(tag)}`);
|
||||
}
|
||||
// Observed running count; fully stopped services stay at scale 0.
|
||||
if (svc.scale === 0 || stoppedOnly) {
|
||||
lines.push(' scale: 0');
|
||||
} else if (svc.scale > 0) {
|
||||
lines.push(` scale: ${svc.scale}`);
|
||||
}
|
||||
wroteAny = true;
|
||||
}
|
||||
const body = wroteAny ? `${lines.join('\n')}\n` : 'services: {}\n';
|
||||
await fs.writeFile(abs, body, 'utf8');
|
||||
return abs;
|
||||
}
|
||||
|
||||
public markAcquired(id: string): boolean {
|
||||
return DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'captured', 'acquired');
|
||||
}
|
||||
|
||||
public handoff(candidateId: string, nodeId: number, stackName: string): boolean {
|
||||
return DatabaseService.getInstance().casHandoffGeneration(candidateId, nodeId, stackName);
|
||||
}
|
||||
|
||||
public markReconciling(id: string): boolean {
|
||||
return DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'handoff_committed', 'reconciling');
|
||||
}
|
||||
|
||||
public markImmediateVerified(id: string): boolean {
|
||||
const ok = DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'reconciling', 'immediate_verified');
|
||||
if (ok) {
|
||||
DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(id, {
|
||||
artifact_expires_at: Date.now() + this.activeRecoveryTtlMs() + RECOVERY_TTL_BUFFER_MS,
|
||||
operation_lease_expires_at: null,
|
||||
});
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
/** Mark candidate abandoned in DB and retire Docker/FS artifacts. */
|
||||
public async abandon(id: string): Promise<boolean> {
|
||||
const row = this.get(id);
|
||||
const ok = DatabaseService.getInstance().abandonStackUpdateRecoveryGeneration(id);
|
||||
if (ok && row) {
|
||||
await this.retireGenerationArtifacts({ ...row, status: 'abandoned', artifacts_retired: 0 });
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
public linkHealthGate(id: string, healthGateId: string): void {
|
||||
DatabaseService.getInstance().linkStackUpdateRecoveryHealthGate(id, healthGateId);
|
||||
}
|
||||
|
||||
public setGateRetainUntil(id: string, until: number = Date.now() + GATE_RETAIN_DEFAULT_MS): void {
|
||||
DatabaseService.getInstance().setStackUpdateRecoveryGateRetainUntil(id, until);
|
||||
}
|
||||
|
||||
/** Link a health gate when present; otherwise set bounded gate_retain_until. */
|
||||
public linkGateOrRetain(id: string, healthGateId: string | null): void {
|
||||
try {
|
||||
if (healthGateId) {
|
||||
this.linkHealthGate(id, healthGateId);
|
||||
} else {
|
||||
this.setGateRetainUntil(id);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] linkGateOrRetain failed for %s; setting retain window:',
|
||||
sanitizeForLog(id),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
try {
|
||||
this.setGateRetainUntil(id);
|
||||
} catch (retainError) {
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] setGateRetainUntil failed for %s:',
|
||||
sanitizeForLog(id),
|
||||
sanitizeForLog(getErrorMessage(retainError, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public get(id: string): StackUpdateRecoveryGenerationRow | undefined {
|
||||
return DatabaseService.getInstance().getStackUpdateRecoveryGeneration(id);
|
||||
}
|
||||
|
||||
public getCurrent(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow | undefined {
|
||||
return DatabaseService.getInstance().getCurrentStackUpdateRecovery(nodeId, stackName);
|
||||
}
|
||||
|
||||
public isRestoredCurrentPinActive(nodeId: number, stackName: string): boolean {
|
||||
const current = this.getCurrent(nodeId, stackName);
|
||||
return !!current && current.status === 'restored_current';
|
||||
}
|
||||
|
||||
public getHeldImageIds(nodeId: number): Set<string> | null {
|
||||
try {
|
||||
const now = Date.now();
|
||||
const fromStack = DatabaseService.getInstance().listHeldStackUpdateRecoveryImageIds(nodeId, now);
|
||||
return new Set(fromStack);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] Failed to compute held image ids for node %d:',
|
||||
nodeId,
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unified held-image predicate: service-scoped + full-stack holds.
|
||||
* Fail closed (skip prune) when either lookup fails.
|
||||
*/
|
||||
public buildUnifiedHeldImagePredicate(nodeId: number): (imageId: string) => boolean {
|
||||
// Dynamic require avoids a static cycle with ServiceUpdateRecoveryService.
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const { ServiceUpdateRecoveryService } = require('./ServiceUpdateRecoveryService') as typeof import('./ServiceUpdateRecoveryService');
|
||||
const serviceHeld = ServiceUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
|
||||
const stackHeld = this.getHeldImageIds(nodeId);
|
||||
if (serviceHeld === null || stackHeld === null) {
|
||||
return () => true;
|
||||
}
|
||||
return (imageId: string) => serviceHeld.has(imageId) || stackHeld.has(imageId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Post-handoff compensation: restore files + pinned up, then probe before
|
||||
* reporting restored_current / immediate_verified.
|
||||
*/
|
||||
public async compensateWithCandidate(
|
||||
generationId: string,
|
||||
composeUp: (overridePath: string) => Promise<void>,
|
||||
): Promise<boolean> {
|
||||
const row = this.get(generationId);
|
||||
if (!row) return false;
|
||||
try {
|
||||
const context = await resolveComposeProjectContext(row.node_id, row.stack_name);
|
||||
await context.restoreFromContext();
|
||||
if (!row.override_path) {
|
||||
throw new Error('Recovery generation has no override path');
|
||||
}
|
||||
await composeUp(row.override_path);
|
||||
const probeOk = await this.probeRecoveredStack(
|
||||
row.node_id,
|
||||
row.stack_name,
|
||||
row.services_json,
|
||||
);
|
||||
if (!probeOk) {
|
||||
DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, {
|
||||
status: 'recovery_required',
|
||||
});
|
||||
return false;
|
||||
}
|
||||
DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, {
|
||||
status: 'restored_current',
|
||||
phase: 'immediate_verified',
|
||||
is_current: 1,
|
||||
artifact_expires_at: null,
|
||||
});
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'[StackUpdateRecovery] Compensation failed for %s: %s',
|
||||
sanitizeForLog(generationId),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, {
|
||||
status: 'recovery_required',
|
||||
});
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify recovered runtime against the captured generation.
|
||||
* Rejects absent, restarting, dead, exited, or unhealthy expected replicas,
|
||||
* image-id mismatches vs capture, and any running replica of a scale-0 service.
|
||||
*/
|
||||
public async probeRecoveredStack(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
servicesJson: string,
|
||||
): Promise<boolean> {
|
||||
await new Promise((resolve) => setTimeout(resolve, RECOVERY_PROBE_DELAY_MS));
|
||||
try {
|
||||
const expected = parseServicesJson(servicesJson);
|
||||
const expectedRunning = new Map<string, number>();
|
||||
const expectedImageIds = new Map<string, Set<string>>();
|
||||
const scaleZeroServices = new Set<string>();
|
||||
|
||||
for (const svc of expected) {
|
||||
const imageIds = new Set<string>();
|
||||
for (const replica of svc.replicas ?? []) {
|
||||
if (replica.imageId?.trim()) imageIds.add(replica.imageId);
|
||||
}
|
||||
if (svc.scale > 0) {
|
||||
// Fail closed when we cannot verify image identity for expected runners.
|
||||
if (imageIds.size === 0) return false;
|
||||
expectedRunning.set(svc.serviceName, svc.scale);
|
||||
expectedImageIds.set(svc.serviceName, imageIds);
|
||||
} else {
|
||||
scaleZeroServices.add(svc.serviceName);
|
||||
}
|
||||
}
|
||||
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
const containers = await docker.listContainers({
|
||||
all: true,
|
||||
filters: { label: [`com.docker.compose.project=${stackName}`] },
|
||||
});
|
||||
|
||||
if (expectedRunning.size > 0 && containers.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const runningByService = new Map<string, number>();
|
||||
for (const containerInfo of containers) {
|
||||
const labels = (containerInfo.Labels ?? {}) as Record<string, string>;
|
||||
const serviceName = labels['com.docker.compose.service'];
|
||||
const state = (containerInfo.State || '').toLowerCase();
|
||||
|
||||
if (state === 'restarting' || state === 'dead') {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (state === 'exited' || state === 'created' || state === 'removing') {
|
||||
if (serviceName && expectedRunning.has(serviceName)) {
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (state !== 'running') {
|
||||
if (serviceName && expectedRunning.has(serviceName)) {
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Captured at scale 0 must stay stopped; any running replica fails the probe.
|
||||
if (serviceName && scaleZeroServices.has(serviceName)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const inspectData = await docker.getContainer(containerInfo.Id).inspect();
|
||||
const health = inspectData.State?.Health?.Status;
|
||||
if (health === 'unhealthy') {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (serviceName && expectedImageIds.has(serviceName)) {
|
||||
const allowedIds = expectedImageIds.get(serviceName)!;
|
||||
const actualImageId = typeof inspectData.Image === 'string' ? inspectData.Image : '';
|
||||
if (!actualImageId || !allowedIds.has(actualImageId)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (serviceName) {
|
||||
runningByService.set(serviceName, (runningByService.get(serviceName) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [serviceName, need] of expectedRunning) {
|
||||
if ((runningByService.get(serviceName) ?? 0) < need) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] Recovery probe failed for %s: %s',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotent removal of opaque tags and override files for a generation.
|
||||
* Marks artifacts_retired only when every artifact is removed or confirmed absent,
|
||||
* so transient failures remain retryable via reconcileIncomplete.
|
||||
*/
|
||||
public async retireGenerationArtifacts(row: StackUpdateRecoveryGenerationRow): Promise<boolean> {
|
||||
if (row.artifacts_retired === 1) return true;
|
||||
const services = parseServicesJson(row.services_json);
|
||||
const tagsOk = await this.removeRollbackTags(row.node_id, collectRollbackTags(services));
|
||||
let overrideOk = true;
|
||||
if (row.override_path) {
|
||||
try {
|
||||
await fs.unlink(row.override_path);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
overrideOk = false;
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] Failed to delete override %s: %s',
|
||||
sanitizeForLog(row.override_path),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!tagsOk || !overrideOk) return false;
|
||||
DatabaseService.getInstance().markStackUpdateRecoveryArtifactsRetired(row.id);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Abandon lease-expired candidates, flag stuck post-handoff generations,
|
||||
* and retire expired abandoned/superseded artifacts.
|
||||
*/
|
||||
public async reconcileIncomplete(): Promise<void> {
|
||||
if (!this.started) return;
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const now = Date.now();
|
||||
let abandoned = 0;
|
||||
for (const row of db.listStaleStackUpdateRecoveryCandidates(now)) {
|
||||
if (await this.abandon(row.id)) abandoned += 1;
|
||||
}
|
||||
let flagged = 0;
|
||||
for (const row of db.listStuckStackUpdateRecoveryGenerations(now)) {
|
||||
db.updateStackUpdateRecoveryGeneration(row.id, {
|
||||
status: 'recovery_required',
|
||||
operation_lease_expires_at: null,
|
||||
});
|
||||
flagged += 1;
|
||||
}
|
||||
let retired = 0;
|
||||
for (const row of db.listStackUpdateRecoveryGenerationsForArtifactRetirement(now)) {
|
||||
// Never retire an active/current or recovery_required hold target.
|
||||
if (row.is_current === 1 || row.status === 'recovery_required') continue;
|
||||
if (await this.retireGenerationArtifacts(row)) retired += 1;
|
||||
}
|
||||
if (abandoned > 0 || flagged > 0 || retired > 0) {
|
||||
console.log(
|
||||
`[StackUpdateRecovery] Reconciled ${abandoned} stale candidate(s), `
|
||||
+ `${flagged} stuck generation(s), retired ${retired} artifact set(s)`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'[StackUpdateRecovery] Reconcile failed: %s',
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns true when every tag is removed or already absent. */
|
||||
private async removeRollbackTags(nodeId: number, tags: string[]): Promise<boolean> {
|
||||
if (tags.length === 0) return true;
|
||||
try {
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
let allOk = true;
|
||||
for (const tag of tags) {
|
||||
try {
|
||||
await docker.getImage(tag).remove({ force: true });
|
||||
} catch (error) {
|
||||
const status = (error as { statusCode?: number }).statusCode;
|
||||
const message = getErrorMessage(error, 'unknown').toLowerCase();
|
||||
if (status === 404 || message.includes('no such image') || message.includes('not found')) {
|
||||
continue;
|
||||
}
|
||||
allOk = false;
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] Failed to remove rollback tag %s: %s',
|
||||
sanitizeForLog(tag),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
return allOk;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] Docker unavailable while removing tags: %s',
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async bestEffortRemoveTags(nodeId: number, tags: string[]): Promise<void> {
|
||||
await this.removeRollbackTags(nodeId, tags);
|
||||
}
|
||||
|
||||
private activeRecoveryTtlMs(): number {
|
||||
return Math.max(getComposeCommandTimeoutMs(), this.readRecoveryWindowSeconds() * 1000);
|
||||
}
|
||||
|
||||
private readRecoveryWindowSeconds(): number {
|
||||
try {
|
||||
const raw = parseInt(
|
||||
DatabaseService.getInstance().getGlobalSettings()['health_gate_window_seconds'] ?? '',
|
||||
10,
|
||||
);
|
||||
return Number.isFinite(raw) ? Math.max(raw, MIN_RECOVERY_WINDOW_SECONDS) : MIN_RECOVERY_WINDOW_SECONDS;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[StackUpdateRecovery] Settings read failed; using default window: %s',
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return MIN_RECOVERY_WINDOW_SECONDS;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function splitOpaqueTag(tag: string): { repo: string; tagName: string } {
|
||||
const lastColon = tag.lastIndexOf(':');
|
||||
if (lastColon > 0) {
|
||||
return { repo: tag.slice(0, lastColon), tagName: tag.slice(lastColon + 1) };
|
||||
}
|
||||
return { repo: tag, tagName: 'hold' };
|
||||
}
|
||||
@@ -174,18 +174,24 @@ export class WebhookService {
|
||||
case 'start':
|
||||
await compose.runCommand(stackName, 'start');
|
||||
break;
|
||||
case 'pull':
|
||||
case 'pull': {
|
||||
await assertPolicyGateAllows(
|
||||
stackName,
|
||||
nodeId,
|
||||
buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }),
|
||||
);
|
||||
await StackUpdateOrchestrator.getInstance().execute(
|
||||
const orchResult = await StackUpdateOrchestrator.getInstance().execute(
|
||||
{ nodeId, stackName, target: { scope: 'stack' }, trigger: 'webhook', actor: 'system:webhook' },
|
||||
{ atomic: atomic ?? false, terminalWs: null },
|
||||
);
|
||||
HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook');
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook');
|
||||
const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
|
||||
if (recoveryId) {
|
||||
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
||||
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* Thin Compose project context for safe full-stack updates.
|
||||
*
|
||||
* Wraps the current authored compose argument path and atomic file backup/restore.
|
||||
* When a richer shared Compose project context lands, migrate callers to that type;
|
||||
* this module must not become a competing full-manifest resolver.
|
||||
*/
|
||||
import path from 'path';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { ComposeService } from './ComposeService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { buildEffectiveServiceModel } from './effectiveServiceModel';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
|
||||
export type ImageReferenceKind = 'moving_tag' | 'digest_pinned' | 'none';
|
||||
|
||||
const DIGEST_PIN_PATTERN = /@sha256:[a-f0-9]{64}$/i;
|
||||
|
||||
export function classifyReferenceKind(declaredImageRef: string | null): ImageReferenceKind {
|
||||
if (!declaredImageRef) return 'none';
|
||||
if (DIGEST_PIN_PATTERN.test(declaredImageRef)) return 'digest_pinned';
|
||||
return 'moving_tag';
|
||||
}
|
||||
|
||||
async function requireRenderableModel(nodeId: number, stackName: string) {
|
||||
const model = await buildEffectiveServiceModel(nodeId, stackName);
|
||||
if (!model.renderable) {
|
||||
throw new Error(model.error || 'Effective Compose model failed to render');
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
export interface ComposeProjectContext {
|
||||
readonly nodeId: number;
|
||||
readonly stackName: string;
|
||||
readonly stackDir: string;
|
||||
backupSlotId: string | null;
|
||||
toComposeArgs(action: string[]): Promise<string[]>;
|
||||
validateForMutation(): Promise<void>;
|
||||
backupFromContext(operation: 'update' | 'deployment'): Promise<string>;
|
||||
restoreFromContext(): Promise<void>;
|
||||
resolveServiceImageMap(): Promise<Map<string, string | null>>;
|
||||
}
|
||||
|
||||
class AuthoredComposeProjectContext implements ComposeProjectContext {
|
||||
backupSlotId: string | null = null;
|
||||
|
||||
constructor(
|
||||
readonly nodeId: number,
|
||||
readonly stackName: string,
|
||||
readonly stackDir: string,
|
||||
) {}
|
||||
|
||||
async toComposeArgs(action: string[]): Promise<string[]> {
|
||||
return ComposeService.getInstance(this.nodeId).buildAuthoredComposeArgs(this.stackName, action);
|
||||
}
|
||||
|
||||
async validateForMutation(): Promise<void> {
|
||||
await ComposeService.getInstance(this.nodeId).validateStackForMutation(this.stackName);
|
||||
await requireRenderableModel(this.nodeId, this.stackName);
|
||||
}
|
||||
|
||||
async backupFromContext(_operation: 'update' | 'deployment'): Promise<string> {
|
||||
await FileSystemService.getInstance(this.nodeId).backupStackFiles(this.stackName);
|
||||
const slotId = randomUUID();
|
||||
this.backupSlotId = slotId;
|
||||
return slotId;
|
||||
}
|
||||
|
||||
async restoreFromContext(): Promise<void> {
|
||||
await FileSystemService.getInstance(this.nodeId).restoreStackFiles(this.stackName);
|
||||
}
|
||||
|
||||
async resolveServiceImageMap(): Promise<Map<string, string | null>> {
|
||||
const model = await requireRenderableModel(this.nodeId, this.stackName);
|
||||
const map = new Map<string, string | null>();
|
||||
for (const svc of model.services) {
|
||||
map.set(svc.name, svc.declaredImage);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveComposeProjectContext(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
): Promise<ComposeProjectContext> {
|
||||
const stackDir = path.join(FileSystemService.getInstance(nodeId).getBaseDir(), stackName);
|
||||
return new AuthoredComposeProjectContext(nodeId, stackName, stackDir);
|
||||
}
|
||||
|
||||
export function describeContextError(error: unknown): string {
|
||||
return getErrorMessage(error, 'Compose project context failed');
|
||||
}
|
||||
@@ -304,11 +304,11 @@ export function buildRollbackItems(inputs: RollbackInputs, now: number): Rollbac
|
||||
if (inputs.rollbackTarget === 'error') {
|
||||
items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The update preview is unavailable.' });
|
||||
} else if (inputs.rollbackTarget.target && inputs.rollbackTarget.moving) {
|
||||
items.push({ id: 'previous_images', state: 'not_covered', label: 'Previous image tag', detail: `Rollback target ${inputs.rollbackTarget.target}. This stack uses a moving image tag, so restoring the compose and env files does not revert the image: the local tag still resolves to the newer digest. Pin every image to an immutable version tag for a true image rollback.` });
|
||||
items.push({ id: 'previous_images', state: 'ready', label: 'Previous image tag', detail: `Rollback target ${inputs.rollbackTarget.target}. This stack uses a moving image tag. Full-stack updates capture the running image ID before pull/build and retain it through the recovery window so an immediate restore can retarget the exact prior image, not only the tag string.` });
|
||||
} else if (inputs.rollbackTarget.target) {
|
||||
items.push({ id: 'previous_images', state: 'ready', label: 'Previous image tag', detail: `Known rollback target: ${inputs.rollbackTarget.target}. The compose file pins an immutable tag, so restoring files also restores the image.` });
|
||||
} else {
|
||||
items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The previous image tag could not be determined. A rollback restores compose and env files; a moving tag may keep the newer image.' });
|
||||
items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The previous image tag could not be determined. When no prior image is recorded, a restore may fall back to compose and env files alone; a moving tag can still resolve to a newer digest.' });
|
||||
}
|
||||
|
||||
if (inputs.lastDeployAt === 'error') {
|
||||
|
||||
@@ -5,7 +5,7 @@ export default defineConfig({
|
||||
environment: 'node',
|
||||
// Only run TypeScript sources - exclude the compiled dist/ output.
|
||||
include: ['src/__tests__/**/*.test.ts'],
|
||||
exclude: ['dist/**', 'node_modules/**'],
|
||||
exclude: ['dist/**', 'node_modules/**', 'src/__tests__/docker-integration/**'],
|
||||
// Build the baseline DB (schema + migrations + admin seed) once; each
|
||||
// test file's setupTestDb copies it instead of re-running migrations.
|
||||
globalSetup: ['./src/__tests__/helpers/vitestGlobalSetup.ts'],
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
environment: 'node',
|
||||
include: ['src/__tests__/docker-integration/**/*.test.ts'],
|
||||
exclude: ['dist/**', 'node_modules/**'],
|
||||
globalSetup: ['./src/__tests__/helpers/vitestGlobalSetup.ts'],
|
||||
pool: 'forks',
|
||||
maxWorkers: 1,
|
||||
minWorkers: 1,
|
||||
testTimeout: 180_000,
|
||||
hookTimeout: 180_000,
|
||||
sequence: { concurrent: false },
|
||||
},
|
||||
});
|
||||
@@ -19,6 +19,7 @@ vi.mock('@/components/ui/toast-store', () => ({
|
||||
}));
|
||||
|
||||
import { apiFetch } from '@/lib/api';
|
||||
import { toast } from '@/components/ui/toast-store';
|
||||
|
||||
type EditorState = ReturnType<typeof useEditorViewState>;
|
||||
type StackListState = ReturnType<typeof useStackListState>;
|
||||
@@ -314,6 +315,18 @@ describe('useStackActions policy-block dialog wiring', () => {
|
||||
expect(overlayState.setPolicyBlock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('toasts a deletion-in-progress message when the conflict action is delete', async () => {
|
||||
const inProgress = JSON.stringify({
|
||||
code: 'stack_op_in_progress',
|
||||
inProgress: { action: 'delete', startedAt: Date.now(), user: 'admin' },
|
||||
});
|
||||
vi.mocked(apiFetch).mockResolvedValueOnce(new Response(inProgress, { status: 409 }));
|
||||
const { result, overlayState } = setup();
|
||||
await act(async () => { await result.current.updateStack(mouseEvent); });
|
||||
expect(overlayState.setPolicyBlock).not.toHaveBeenCalled();
|
||||
expect(toast.error).toHaveBeenCalledWith(expect.stringMatching(/already deleting/i));
|
||||
});
|
||||
|
||||
it('opens the dialog with action "update" via the sidebar update entry point', async () => {
|
||||
vi.mocked(apiFetch).mockResolvedValueOnce(new Response(JSON.stringify(policyPayload), { status: 409 }));
|
||||
const { result, overlayState } = setup();
|
||||
|
||||
@@ -107,7 +107,7 @@ const parseFailureClassification = (value: unknown): FailureClassification | und
|
||||
return undefined;
|
||||
};
|
||||
|
||||
type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update';
|
||||
type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'delete';
|
||||
|
||||
interface StackOpInProgressInfo {
|
||||
action: StackOpAction;
|
||||
@@ -122,6 +122,7 @@ const STACK_OP_PRESENT_PARTICIPLE: Record<StackOpAction, string> = {
|
||||
stop: 'stopping',
|
||||
start: 'starting',
|
||||
update: 'updating',
|
||||
delete: 'deleting',
|
||||
};
|
||||
|
||||
const VALID_STACK_OP_ACTIONS: ReadonlySet<string> = new Set(
|
||||
|
||||
@@ -38,7 +38,7 @@ export function classifyOperationPhase(rows: ParsedLogRow[], action: ActionVerb)
|
||||
if (stage === 'BUILD') {
|
||||
return 'Building images';
|
||||
}
|
||||
if (message.includes('Backup created for atomic') || message.includes('Cleaning up existing containers')) {
|
||||
if (message.includes('Backup created for atomic') || message.includes('Validating stack for update') || message.includes('Capturing rollback generation')) {
|
||||
return 'Preparing';
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user