Files
sencho/backend/src/__tests__/deployed-stack-deletion-service.test.ts
T
Anso 3f1f15a6f4 fix: keep running containers until stack pull/build succeeds (#1657)
* fix: keep running containers until stack pull/build succeeds

Acquire images before reconcile, capture a recovery generation for
compensation, and only remove classified orphans after handoff.

* fix: address recovery audit blockers for safe stack updates

Retire abandoned and expired recovery artifacts, probe compensated
runtimes before reporting rollback success, preserve local Docker when
deleting a node, validate the exact Compose invocation before capture,
and repair updateStack return-contract fixtures.

* fix: resolve ESLint errors blocking CI on this branch

Unused-import and unused-variable errors left over from the stack
deletion refactor: MeshService in stacks.ts (its opt-out cascade moved
into DeployedStackDeletionService), a redundant pruneVolumes
destructure in deleteDeployedStack (the real one is re-derived from
the same input object inside runDeletionBody), and an unused beforeAll
import in a Docker-integration test stub. Also scopes the webhook
pull-action case body in a block to satisfy no-case-declarations;
purely syntactic, no behavior change.

* fix: harden recovery probe, cleanup retry, and failed-pull Docker test

Reject absent or unhealthy expected replicas before reporting rollback
success, keep cleanup records until artifacts are actually removed, fail
closed when a mesh override cannot be generated, and assert a real
failed pull leaves the original container running.

* fix: verify recovery probe image identity and stack-scoped override paths

Reject recovered runtimes that use the wrong image or leave scale-zero
services running, and confine tombstone override deletion to the intent
stack directory so forged cross-stack paths cannot be swept.

* test: batch notification cap fixtures in a SQLite transaction

Unbatched 1200-row inserts were timing out at the default 30s under
CI load even though the same assertions pass in under 2s when green.
2026-07-21 12:18:01 -04:00

157 lines
5.2 KiB
TypeScript

/**
* Deployed-stack deletion: ready transaction retires both recovery models;
* blocking intents gate same-name create.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
import { randomUUID } from 'crypto';
import path from 'path';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import type {
StackUpdateRecoveryGenerationRow,
ServiceUpdateRecoveryRow,
StackUpdateCleanupPendingRow,
} from '../services/DatabaseService';
let tmpDir: string;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let DeployedStackDeletionService: typeof import('../services/DeployedStackDeletionService').DeployedStackDeletionService;
let overrideDeletionContainmentBase: typeof import('../services/DeployedStackDeletionService').overrideDeletionContainmentBase;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
({ DeployedStackDeletionService, overrideDeletionContainmentBase } = await import('../services/DeployedStackDeletionService'));
});
afterAll(() => cleanupTestDb(tmpDir));
function db() {
return DatabaseService.getInstance();
}
beforeEach(() => {
const raw = (db() as unknown as { db: { prepare: (s: string) => { run: () => void } } }).db;
raw.prepare('DELETE FROM stack_update_recovery_generations').run();
raw.prepare('DELETE FROM service_update_recovery').run();
raw.prepare('DELETE FROM stack_update_cleanup_pending').run();
});
const NODE = 1;
describe('DeployedStackDeletionService ready transaction', () => {
it('commitStackDeletionReadyTransaction deletes full-stack and service recovery rows', () => {
const now = Date.now();
const stackName = 'del-stack';
const gen: StackUpdateRecoveryGenerationRow = {
id: randomUUID(),
node_id: NODE,
stack_name: stackName,
status: 'active',
phase: 'immediate_verified',
is_current: 1,
backup_slot_id: null,
override_path: null,
services_json: '[]',
health_gate_id: null,
gate_retain_until: null,
artifact_expires_at: null,
operation_lease_expires_at: null,
created_at: now,
updated_at: now,
created_by: null,
artifacts_retired: 0,
};
db().insertStackUpdateRecoveryGeneration(gen);
const svc: ServiceUpdateRecoveryRow = {
id: randomUUID(),
node_id: NODE,
stack_name: stackName,
service_name: 'web',
replicas_json: '[]',
majority_image_id: 'sha256:abc',
declared_image_ref: 'nginx:latest',
weak_floating_tag: 0,
health_gate_id: null,
status: 'active',
expires_at: now + 60_000,
claim_expires_at: null,
created_at: now,
created_by: null,
};
db().insertServiceUpdateRecovery(svc);
const intentId = randomUUID();
const intent: StackUpdateCleanupPendingRow = {
id: intentId,
node_id: NODE,
stack_name: stackName,
status: 'prepared',
target_kind: 'local_socket',
rollback_tags_json: '[]',
override_paths_json: '[]',
prune_volumes_requested: 0,
created_at: now,
updated_at: now,
};
db().insertCleanupPending(intent);
expect(db().commitStackDeletionReadyTransaction(intentId, NODE, stackName)).toBe(true);
expect(db().listStackUpdateRecoveryForStack(NODE, stackName)).toHaveLength(0);
expect(db().listActiveServiceUpdateRecoveries(NODE, stackName, 'web')).toHaveLength(0);
expect(db().getCleanupPending(intentId)?.status).toBe('ready');
});
it('hasBlockingDeletionIntent is true for prepared intents', () => {
const now = Date.now();
db().insertCleanupPending({
id: randomUUID(),
node_id: NODE,
stack_name: 'blocked',
status: 'prepared',
target_kind: 'local_socket',
rollback_tags_json: '[]',
override_paths_json: '[]',
prune_volumes_requested: 0,
created_at: now,
updated_at: now,
});
expect(db().hasBlockingDeletionIntent(NODE, 'blocked')).toBe(true);
expect(db().hasBlockingDeletionIntent(NODE, 'other')).toBe(false);
});
it('assertNoBlockingDeletionIntent throws for prepared stacks', () => {
const now = Date.now();
db().insertCleanupPending({
id: randomUUID(),
node_id: NODE,
stack_name: 'prep',
status: 'prepared',
target_kind: 'local_socket',
rollback_tags_json: '[]',
override_paths_json: '[]',
prune_volumes_requested: 0,
created_at: now,
updated_at: now,
});
expect(() => {
DeployedStackDeletionService.getInstance().assertNoBlockingDeletionIntent(NODE, 'prep');
}).toThrow(/deletion in progress/i);
});
});
describe('overrideDeletionContainmentBase', () => {
it('confines stack-scoped intents to the stack directory', () => {
expect(overrideDeletionContainmentBase('/app/compose', 'my-stack')).toBe(
path.resolve('/app/compose', 'my-stack'),
);
expect(overrideDeletionContainmentBase('/app/compose', null)).toBe(
path.resolve('/app/compose'),
);
});
it('rejects invalid stack names that could traverse', () => {
expect(overrideDeletionContainmentBase('/app/compose', '../other')).toBeNull();
expect(overrideDeletionContainmentBase('/app/compose', 'bad/name')).toBeNull();
});
});