Files
sencho/backend/src/__tests__/update-guard-rollback.test.ts
T
Anso 3f1f15a6f4 fix: keep running containers until stack pull/build succeeds (#1657)
* fix: keep running containers until stack pull/build succeeds

Acquire images before reconcile, capture a recovery generation for
compensation, and only remove classified orphans after handoff.

* fix: address recovery audit blockers for safe stack updates

Retire abandoned and expired recovery artifacts, probe compensated
runtimes before reporting rollback success, preserve local Docker when
deleting a node, validate the exact Compose invocation before capture,
and repair updateStack return-contract fixtures.

* fix: resolve ESLint errors blocking CI on this branch

Unused-import and unused-variable errors left over from the stack
deletion refactor: MeshService in stacks.ts (its opt-out cascade moved
into DeployedStackDeletionService), a redundant pruneVolumes
destructure in deleteDeployedStack (the real one is re-derived from
the same input object inside runDeletionBody), and an unused beforeAll
import in a Docker-integration test stub. Also scopes the webhook
pull-action case body in a block to satisfy no-case-declarations;
purely syntactic, no behavior change.

* fix: harden recovery probe, cleanup retry, and failed-pull Docker test

Reject absent or unhealthy expected replicas before reporting rollback
success, keep cleanup records until artifacts are actually removed, fail
closed when a mesh override cannot be generated, and assert a real
failed pull leaves the original container running.

* fix: verify recovery probe image identity and stack-scoped override paths

Reject recovered runtimes that use the wrong image or leave scale-zero
services running, and confine tombstone override deletion to the intent
stack directory so forged cross-stack paths cannot be swept.

* test: batch notification cap fixtures in a SQLite transaction

Unbatched 1200-row inserts were timing out at the default 30s under
CI load even though the same assertions pass in under 2s when green.
2026-07-21 12:18:01 -04:00

233 lines
9.2 KiB
TypeScript

import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import fs from 'fs';
import os from 'os';
import path from 'path';
// Mutable state the mocked NodeRegistry reads (same harness as
// filesystem-backup.test.ts) so FileSystemService can be constructed against
// temp directories.
const mockState = { composeDir: '' };
vi.mock('../services/NodeRegistry', () => ({
NodeRegistry: {
getInstance: () => ({
getComposeDir: () => mockState.composeDir,
getDefaultNodeId: () => 1,
}),
},
}));
import {
aggregateRollbackOverall,
buildRollbackItems,
type RollbackInputs,
} from '../services/updateGuard/readiness';
import type { ContainerProbe } from '../services/updateGuard/types';
const NOW = 1_750_000_000_000;
const baseInputs = (over: Partial<RollbackInputs> = {}): RollbackInputs => ({
backup: { exists: true, timestamp: NOW - 3_600_000 },
envSummary: { exists: true, envPresent: true, keys: ['DB_HOST', 'DB_PASS'] },
stackHasEnv: true,
rollbackTarget: { target: 'nginx:1.27.1', moving: false },
lastDeployAt: NOW - 3_600_000,
containers: [{
name: 'app-web-1', state: 'running', health: 'healthy', exitCode: null,
hasHealthcheck: true, restartPolicy: 'unless-stopped', mounts: ['volume app_data'],
}],
...over,
});
const itemById = (inputs: RollbackInputs, id: string) =>
buildRollbackItems(inputs, NOW).find(i => i.id === id)!;
describe('buildRollbackItems', () => {
it('reports a full set of six items', () => {
const items = buildRollbackItems(baseInputs(), NOW);
expect(items.map(i => i.id)).toEqual([
'compose_source', 'env_keys', 'previous_images', 'last_deploy', 'healthchecks', 'volume_data',
]);
});
it('marks the volume row not_covered unconditionally and names the mounts', () => {
const item = itemById(baseInputs(), 'volume_data');
expect(item.state).toBe('not_covered');
expect(item.detail).toContain('not included in file backups');
expect(item.detail).toContain('volume app_data');
const noMounts = itemById(baseInputs({ containers: [] }), 'volume_data');
expect(noMounts.state).toBe('not_covered');
const dockerDown = itemById(baseInputs({ containers: 'error' }), 'volume_data');
expect(dockerDown.state).toBe('not_covered');
});
it('exposes env coverage as names only', () => {
const item = itemById(baseInputs(), 'env_keys');
expect(item.state).toBe('ready');
expect(item.detail).toContain('2 variable names');
expect(item.detail).not.toContain('DB_HOST');
expect(item.detail).not.toContain('DB_PASS');
});
it('treats a stack without an env file as covered', () => {
const item = itemById(baseInputs({
envSummary: { exists: true, envPresent: false, keys: [] },
stackHasEnv: false,
}), 'env_keys');
expect(item.state).toBe('ready');
expect(item.detail).toContain('no env file');
});
it('flags an env file the backup predates', () => {
const item = itemById(baseInputs({
envSummary: { exists: true, envPresent: false, keys: [] },
stackHasEnv: true,
}), 'env_keys');
expect(item.state).toBe('missing');
});
it('marks the previous image unknown when no rollback target is known', () => {
expect(itemById(baseInputs({ rollbackTarget: { target: null, moving: false } }), 'previous_images').state).toBe('unknown');
expect(itemById(baseInputs({ rollbackTarget: 'error' }), 'previous_images').state).toBe('unknown');
const known = itemById(baseInputs(), 'previous_images');
expect(known.state).toBe('ready');
expect(known.detail).toContain('nginx:1.27.1');
});
it('treats a moving tag as ready because full-stack updates retain the prior image ID', () => {
const item = itemById(baseInputs({ rollbackTarget: { target: 'nginx:latest', moving: true } }), 'previous_images');
expect(item.state).toBe('ready');
expect(item.detail).toContain('moving image tag');
expect(item.detail).toContain('nginx:latest');
expect(item.detail).toContain('recovery window');
});
it('does not mistake an image literally named error for a failed preview', () => {
const item = itemById(baseInputs({ rollbackTarget: { target: 'error', moving: false } }), 'previous_images');
expect(item.state).toBe('ready');
expect(item.detail).toContain('error');
});
it('reports last deploy and healthcheck coverage', () => {
expect(itemById(baseInputs(), 'last_deploy').state).toBe('ready');
expect(itemById(baseInputs({ lastDeployAt: null }), 'last_deploy').state).toBe('missing');
expect(itemById(baseInputs(), 'healthchecks').state).toBe('ready');
const none: ContainerProbe[] = [{
name: 'a', state: 'running', health: null, exitCode: null,
hasHealthcheck: false, restartPolicy: null, mounts: [],
}];
expect(itemById(baseInputs({ containers: none }), 'healthchecks').state).toBe('missing');
});
});
describe('aggregateRollbackOverall', () => {
it('is ready when compose, env, and previous image are all covered', () => {
expect(aggregateRollbackOverall(buildRollbackItems(baseInputs(), NOW))).toBe('ready');
});
it('is not_ready without a backup slot', () => {
const items = buildRollbackItems(baseInputs({
backup: { exists: false, timestamp: null },
envSummary: { exists: false, envPresent: false, keys: [] },
}), NOW);
expect(aggregateRollbackOverall(items)).toBe('not_ready');
});
it('is partial when the previous image tag is unknown', () => {
const items = buildRollbackItems(baseInputs({ rollbackTarget: { target: null, moving: false } }), NOW);
expect(aggregateRollbackOverall(items)).toBe('partial');
});
it('is ready when the rollback target is a moving tag (image ID capture covers it)', () => {
const items = buildRollbackItems(baseInputs({ rollbackTarget: { target: 'nginx:latest', moving: true } }), NOW);
expect(aggregateRollbackOverall(items)).toBe('ready');
});
it('is partial when env coverage is missing', () => {
const items = buildRollbackItems(baseInputs({
envSummary: { exists: true, envPresent: false, keys: [] },
stackHasEnv: true,
}), NOW);
expect(aggregateRollbackOverall(items)).toBe('partial');
});
it('never gates on the volume or healthcheck disclosures', () => {
const items = buildRollbackItems(baseInputs({ containers: 'error' }), NOW);
expect(aggregateRollbackOverall(items)).toBe('ready');
});
});
describe('FileSystemService.getBackupEnvSummary', () => {
let tmpDir: string;
let composeDir: string;
let originalDataDir: string | undefined;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-envsum-'));
composeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-envsum-compose-'));
mockState.composeDir = composeDir;
originalDataDir = process.env.DATA_DIR;
process.env.DATA_DIR = tmpDir;
});
afterEach(() => {
if (originalDataDir === undefined) delete process.env.DATA_DIR;
else process.env.DATA_DIR = originalDataDir;
fs.rmSync(tmpDir, { recursive: true, force: true });
fs.rmSync(composeDir, { recursive: true, force: true });
});
async function getService() {
const { FileSystemService } = await import('../services/FileSystemService');
return FileSystemService.getInstance();
}
function writeBackupEnv(stackName: string, content: string | null) {
const dir = path.join(tmpDir, 'backups', '1', stackName);
fs.mkdirSync(dir, { recursive: true });
if (content !== null) fs.writeFileSync(path.join(dir, '.env'), content, 'utf-8');
}
it('returns key names only, never values', async () => {
writeBackupEnv('web', 'DB_HOST=db.internal\nDB_PASS=s3cret-value\n# comment\nEMPTY=\n INDENTED=ok\nnot a var line\n');
const svc = await getService();
const summary = await svc.getBackupEnvSummary('web');
expect(summary).toEqual({
exists: true,
envPresent: true,
keys: ['DB_HOST', 'DB_PASS', 'EMPTY', 'INDENTED'],
});
expect(JSON.stringify(summary)).not.toContain('s3cret-value');
expect(JSON.stringify(summary)).not.toContain('db.internal');
});
it('reports a backup without an env file', async () => {
writeBackupEnv('web', null);
const svc = await getService();
expect(await svc.getBackupEnvSummary('web')).toEqual({ exists: true, envPresent: false, keys: [] });
});
it('reports a missing backup slot', async () => {
const svc = await getService();
expect(await svc.getBackupEnvSummary('web')).toEqual({ exists: false, envPresent: false, keys: [] });
});
it('rejects traversal-shaped stack names without touching the filesystem', async () => {
const svc = await getService();
expect(await svc.getBackupEnvSummary('../../etc')).toEqual({ exists: false, envPresent: false, keys: [] });
expect(await svc.getBackupEnvSummary('..')).toEqual({ exists: false, envPresent: false, keys: [] });
});
it('propagates a non-ENOENT env read failure instead of reporting "no env in backup"', async () => {
// An unreadable .env (here: a directory, EISDIR) must throw so callers
// degrade the item to unknown rather than falsely claiming the backup
// contains no env file.
const dir = path.join(tmpDir, 'backups', '1', 'web');
fs.mkdirSync(path.join(dir, '.env'), { recursive: true });
const svc = await getService();
await expect(svc.getBackupEnvSummary('web')).rejects.toThrow();
});
});