mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-22 08:06:42 +00:00
fix: keep running containers until stack pull/build succeeds (#1657)
* fix: keep running containers until stack pull/build succeeds Acquire images before reconcile, capture a recovery generation for compensation, and only remove classified orphans after handoff. * fix: address recovery audit blockers for safe stack updates Retire abandoned and expired recovery artifacts, probe compensated runtimes before reporting rollback success, preserve local Docker when deleting a node, validate the exact Compose invocation before capture, and repair updateStack return-contract fixtures. * fix: resolve ESLint errors blocking CI on this branch Unused-import and unused-variable errors left over from the stack deletion refactor: MeshService in stacks.ts (its opt-out cascade moved into DeployedStackDeletionService), a redundant pruneVolumes destructure in deleteDeployedStack (the real one is re-derived from the same input object inside runDeletionBody), and an unused beforeAll import in a Docker-integration test stub. Also scopes the webhook pull-action case body in a block to satisfy no-case-declarations; purely syntactic, no behavior change. * fix: harden recovery probe, cleanup retry, and failed-pull Docker test Reject absent or unhealthy expected replicas before reporting rollback success, keep cleanup records until artifacts are actually removed, fail closed when a mesh override cannot be generated, and assert a real failed pull leaves the original container running. * fix: verify recovery probe image identity and stack-scoped override paths Reject recovered runtimes that use the wrong image or leave scale-zero services running, and confine tombstone override deletion to the intent stack directory so forged cross-stack paths cannot be swept. * test: batch notification cap fixtures in a SQLite transaction Unbatched 1200-row inserts were timing out at the default 30s under CI load even though the same assertions pass in under 2s when green.
This commit is contained in:
Anso
@@ -24,6 +24,17 @@ const {
|
||||
mockResolveMissingExternalNetworks,
|
||||
mockCreateNetwork,
|
||||
mockAddNotificationHistory,
|
||||
mockIsMeshStackEnabled,
|
||||
mockClassifyLegacyOrphansForUpdate,
|
||||
mockCaptureCandidate,
|
||||
mockMarkAcquired,
|
||||
mockHandoff,
|
||||
mockMarkReconciling,
|
||||
mockMarkImmediateVerified,
|
||||
mockAbandon,
|
||||
mockCompensateWithCandidate,
|
||||
mockBuildUnifiedHeldImagePredicate,
|
||||
mockGetRecovery,
|
||||
} = vi.hoisted(() => ({
|
||||
mockSpawn: vi.fn(),
|
||||
mockGetContainersByStack: vi.fn().mockResolvedValue([]),
|
||||
@@ -58,6 +69,24 @@ const {
|
||||
}),
|
||||
mockCreateNetwork: vi.fn().mockResolvedValue({ id: 'net-1' }),
|
||||
mockAddNotificationHistory: vi.fn(),
|
||||
mockIsMeshStackEnabled: vi.fn().mockReturnValue(false),
|
||||
mockClassifyLegacyOrphansForUpdate: vi.fn().mockResolvedValue({ status: 'none' }),
|
||||
mockCaptureCandidate: vi.fn().mockResolvedValue({
|
||||
id: 'recovery-1',
|
||||
node_id: 1,
|
||||
stack_name: 'my-stack',
|
||||
status: 'candidate',
|
||||
phase: 'captured',
|
||||
override_path: '/test/compose/my-stack/.sencho-recovery-abc.yml',
|
||||
}),
|
||||
mockMarkAcquired: vi.fn().mockReturnValue(true),
|
||||
mockHandoff: vi.fn().mockReturnValue(true),
|
||||
mockMarkReconciling: vi.fn().mockReturnValue(true),
|
||||
mockMarkImmediateVerified: vi.fn().mockReturnValue(true),
|
||||
mockAbandon: vi.fn().mockResolvedValue(true),
|
||||
mockCompensateWithCandidate: vi.fn().mockResolvedValue(true),
|
||||
mockBuildUnifiedHeldImagePredicate: vi.fn().mockReturnValue(() => false),
|
||||
mockGetRecovery: vi.fn().mockReturnValue(undefined),
|
||||
}));
|
||||
|
||||
vi.mock('child_process', () => ({ spawn: mockSpawn, execFile: vi.fn() }));
|
||||
@@ -89,6 +118,7 @@ vi.mock('../services/DockerController', () => ({
|
||||
getInstance: () => ({
|
||||
getContainersByStack: mockGetContainersByStack,
|
||||
getLegacyOrphanContainersByStack: mockGetLegacyOrphanContainersByStack,
|
||||
classifyLegacyOrphansForUpdate: mockClassifyLegacyOrphansForUpdate,
|
||||
removeContainers: mockRemoveContainers,
|
||||
pruneDanglingImages: mockPruneDanglingImages,
|
||||
createNetwork: mockCreateNetwork,
|
||||
@@ -111,6 +141,7 @@ vi.mock('../services/DatabaseService', () => ({
|
||||
getGitSource: () => undefined,
|
||||
getStackProjectEnvFiles: () => [],
|
||||
addNotificationHistory: mockAddNotificationHistory,
|
||||
isMeshStackEnabled: (...args: unknown[]) => mockIsMeshStackEnabled(...args),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
@@ -150,6 +181,24 @@ vi.mock('../services/MeshService', () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/StackUpdateRecoveryService', () => ({
|
||||
StackUpdateRecoveryService: {
|
||||
getInstance: () => ({
|
||||
captureCandidate: mockCaptureCandidate,
|
||||
markAcquired: mockMarkAcquired,
|
||||
handoff: mockHandoff,
|
||||
markReconciling: mockMarkReconciling,
|
||||
markImmediateVerified: mockMarkImmediateVerified,
|
||||
abandon: mockAbandon,
|
||||
compensateWithCandidate: mockCompensateWithCandidate,
|
||||
buildUnifiedHeldImagePredicate: mockBuildUnifiedHeldImagePredicate,
|
||||
get: mockGetRecovery,
|
||||
linkGateOrRetain: vi.fn(),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
|
||||
vi.mock('../services/SelfIdentityService', () => ({
|
||||
default: {
|
||||
getInstance: () => ({ getBindMounts: mockGetBindMounts }),
|
||||
@@ -247,6 +296,23 @@ beforeEach(() => {
|
||||
declaredExternalCount: 0,
|
||||
});
|
||||
mockCreateNetwork.mockResolvedValue({ id: 'net-1' });
|
||||
mockClassifyLegacyOrphansForUpdate.mockResolvedValue({ status: 'none' });
|
||||
mockCaptureCandidate.mockResolvedValue({
|
||||
id: 'recovery-1',
|
||||
node_id: 1,
|
||||
stack_name: 'my-stack',
|
||||
status: 'candidate',
|
||||
phase: 'captured',
|
||||
override_path: '/test/compose/my-stack/.sencho-recovery-abc.yml',
|
||||
});
|
||||
mockMarkAcquired.mockReturnValue(true);
|
||||
mockHandoff.mockReturnValue(true);
|
||||
mockMarkReconciling.mockReturnValue(true);
|
||||
mockMarkImmediateVerified.mockReturnValue(true);
|
||||
mockAbandon.mockResolvedValue(true);
|
||||
mockIsMeshStackEnabled.mockReturnValue(false);
|
||||
mockCompensateWithCandidate.mockResolvedValue(true);
|
||||
mockBuildUnifiedHeldImagePredicate.mockReturnValue(() => false);
|
||||
delete process.env.SENCHO_MODE;
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true });
|
||||
});
|
||||
@@ -586,6 +652,16 @@ describe('ComposeService - authoredComposeArgs mesh override', () => {
|
||||
|
||||
// ── deployStack ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
it('fails closed when a mesh-enabled stack cannot generate its override', async () => {
|
||||
mockIsMeshStackEnabled.mockReturnValue(true);
|
||||
mockEnsureStackOverride.mockRejectedValue(new Error('mesh override write failed'));
|
||||
|
||||
const svc = ComposeService.getInstance(1);
|
||||
await expect(svc.validateExactComposeInvocation('my-stack')).rejects.toThrow(/mesh override write failed/i);
|
||||
expect(mockSpawn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe('ComposeService - deployStack', () => {
|
||||
it('blocks a pilot deploy with relative binds before replacing containers when path mapping differs', async () => {
|
||||
process.env.SENCHO_MODE = 'pilot';
|
||||
@@ -987,7 +1063,7 @@ describe('ComposeService - updateStack build-aware', () => {
|
||||
expect(spawnArgs.some(args => args.includes('pull') && !args.includes('--ignore-buildable'))).toBe(true);
|
||||
});
|
||||
|
||||
it('rolls back compose files when a build step fails during atomic update', async () => {
|
||||
it('abandons recovery and leaves runtime untouched when acquire fails before handoff', async () => {
|
||||
mockLoadStackBuildServices.mockResolvedValueOnce(['app']);
|
||||
let spawnCount = 0;
|
||||
mockSpawn.mockImplementation(() => {
|
||||
@@ -1004,8 +1080,12 @@ describe('ComposeService - updateStack build-aware', () => {
|
||||
const error = await result;
|
||||
|
||||
expect(error).not.toBeNull();
|
||||
expect(mockRestoreStackFiles).toHaveBeenCalled();
|
||||
expect(getComposeRollbackInfo(error)?.attempted).toBe(true);
|
||||
expect(mockAbandon).toHaveBeenCalledWith('recovery-1');
|
||||
expect(mockHandoff).not.toHaveBeenCalled();
|
||||
expect(mockRemoveContainers).not.toHaveBeenCalled();
|
||||
expect(mockCompensateWithCandidate).not.toHaveBeenCalled();
|
||||
const spawnArgs = mockSpawn.mock.calls.map(c => c[1] as string[]);
|
||||
expect(spawnArgs.some(args => args.includes('up'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1106,7 +1186,7 @@ describe('ComposeService - updateStack prune-on-update', () => {
|
||||
|
||||
// The update already succeeded before the prune ran, so a prune failure
|
||||
// must neither reject nor trigger the atomic restore.
|
||||
await expect(promise).resolves.toBeUndefined();
|
||||
await expect(promise).resolves.toEqual({ recoveryId: 'recovery-1' });
|
||||
expect(mockRestoreStackFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -1120,7 +1200,7 @@ describe('ComposeService - updateStack prune-on-update', () => {
|
||||
const promise = svc.updateStack('my-stack');
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
|
||||
await expect(promise).resolves.toBeUndefined();
|
||||
await expect(promise).resolves.toEqual({ recoveryId: 'recovery-1' });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1428,19 +1508,10 @@ describe('ComposeService - idle-output stall backstop', () => {
|
||||
await promise;
|
||||
});
|
||||
|
||||
it('preserves STACK_STALLED_OUTPUT through the atomic rollback wrapper', async () => {
|
||||
it('surfaces STACK_STALLED_OUTPUT on acquire stall without runtime compensation', async () => {
|
||||
process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS = '1000';
|
||||
const pullProc = createMockProcess();
|
||||
let call = 0;
|
||||
// First spawn is the stalling pull; later spawns (the rollback restore's
|
||||
// `up`) close cleanly, proving the restore is not idle-timeout armed.
|
||||
mockSpawn.mockImplementation(() => {
|
||||
call += 1;
|
||||
if (call === 1) return pullProc;
|
||||
const p = createMockProcess();
|
||||
Promise.resolve().then(() => p.emit('close', 0));
|
||||
return p;
|
||||
});
|
||||
mockSpawn.mockImplementation(() => pullProc);
|
||||
|
||||
const svc = ComposeService.getInstance(1);
|
||||
const result = svc.updateStack('my-stack', undefined, true).then(() => null, (e: Error) => e);
|
||||
@@ -1451,7 +1522,10 @@ describe('ComposeService - idle-output stall backstop', () => {
|
||||
const error = await result;
|
||||
expect(error).not.toBeNull();
|
||||
expect(error!.message).toContain('STACK_STALLED_OUTPUT');
|
||||
expect(getComposeRollbackInfo(error)).toMatchObject({ attempted: true });
|
||||
// Acquire failure is pre-handoff: abandon candidate, do not wrap as ComposeRollbackError.
|
||||
expect(getComposeRollbackInfo(error)).toBeNull();
|
||||
expect(mockAbandon).toHaveBeenCalledWith('recovery-1');
|
||||
expect(mockCompensateWithCandidate).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1580,3 +1654,41 @@ describe('ComposeService - streamLogs', () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe('ComposeService - updateStack safe ordering', () => {
|
||||
it('never broad-removes Compose-managed containers before up', async () => {
|
||||
setupAutoCloseSpawn();
|
||||
mockListContainers.mockResolvedValue([]);
|
||||
mockGetContainersByStack.mockResolvedValue([{ Id: 'should-not-remove' }]);
|
||||
|
||||
const svc = ComposeService.getInstance(1);
|
||||
const promise = svc.updateStack('my-stack');
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await promise;
|
||||
|
||||
expect(mockGetContainersByStack).not.toHaveBeenCalled();
|
||||
expect(mockRemoveContainers).not.toHaveBeenCalled();
|
||||
expect(mockCaptureCandidate).toHaveBeenCalled();
|
||||
expect(mockClassifyLegacyOrphansForUpdate).toHaveBeenCalledWith('my-stack');
|
||||
expect(mockHandoff).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('removes only classified orphans after handoff', async () => {
|
||||
setupAutoCloseSpawn();
|
||||
mockListContainers.mockResolvedValue([]);
|
||||
mockClassifyLegacyOrphansForUpdate.mockResolvedValueOnce({
|
||||
status: 'orphans',
|
||||
ids: ['orphan-1'],
|
||||
});
|
||||
mockRemoveContainers.mockResolvedValueOnce([{ id: 'orphan-1', success: true }]);
|
||||
|
||||
const svc = ComposeService.getInstance(1);
|
||||
const promise = svc.updateStack('my-stack');
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await promise;
|
||||
|
||||
expect(mockHandoff).toHaveBeenCalled();
|
||||
expect(mockRemoveContainers).toHaveBeenCalledWith(['orphan-1']);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -225,23 +225,25 @@ describe('DatabaseService - notification history cap (periodic)', () => {
|
||||
|
||||
// A chatty stack writes 600 events.
|
||||
const base = Date.now();
|
||||
for (let i = 0; i < 600; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `chatty-${i}`,
|
||||
timestamp: base + i,
|
||||
stack_name: 'chatty',
|
||||
});
|
||||
}
|
||||
// A quiet stack writes 3 events long before the chatty burst.
|
||||
for (let i = 0; i < 3; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `quiet-${i}`,
|
||||
timestamp: base - 10_000 + i,
|
||||
stack_name: 'quiet',
|
||||
});
|
||||
}
|
||||
db.transaction(() => {
|
||||
for (let i = 0; i < 600; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `chatty-${i}`,
|
||||
timestamp: base + i,
|
||||
stack_name: 'chatty',
|
||||
});
|
||||
}
|
||||
// A quiet stack writes 3 events long before the chatty burst.
|
||||
for (let i = 0; i < 3; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `quiet-${i}`,
|
||||
timestamp: base - 10_000 + i,
|
||||
stack_name: 'quiet',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// No per-insert prune: every row is present.
|
||||
const beforeCleanup = db.getNotificationHistory(0, 2000);
|
||||
@@ -261,13 +263,15 @@ describe('DatabaseService - notification history cap (periodic)', () => {
|
||||
db.deleteAllNotifications(0);
|
||||
|
||||
const base = Date.now();
|
||||
for (let i = 0; i < 1200; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `system-${i}`,
|
||||
timestamp: base + i,
|
||||
});
|
||||
}
|
||||
db.transaction(() => {
|
||||
for (let i = 0; i < 1200; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `system-${i}`,
|
||||
timestamp: base + i,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
db.cleanupOldNotifications(30, { perStackCap: 500, perNodeUnattachedCap: 1000 });
|
||||
|
||||
@@ -279,14 +283,16 @@ describe('DatabaseService - notification history cap (periodic)', () => {
|
||||
it('keeps the newest entries per (node, stack) after periodic cap', () => {
|
||||
db.deleteAllNotifications(0);
|
||||
const base = Date.now();
|
||||
for (let i = 0; i < 600; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `ordered-${i}`,
|
||||
timestamp: base + i * 10,
|
||||
stack_name: 'ordered',
|
||||
});
|
||||
}
|
||||
db.transaction(() => {
|
||||
for (let i = 0; i < 600; i++) {
|
||||
db.addNotificationHistory(0, {
|
||||
level: 'info',
|
||||
message: `ordered-${i}`,
|
||||
timestamp: base + i * 10,
|
||||
stack_name: 'ordered',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
db.cleanupOldNotifications(30, { perStackCap: 500, perNodeUnattachedCap: 1000 });
|
||||
|
||||
@@ -301,9 +307,11 @@ describe('DatabaseService - notification history cap (periodic)', () => {
|
||||
it('uses safe defaults when called with only the retention argument', () => {
|
||||
db.deleteAllNotifications(0);
|
||||
const base = Date.now();
|
||||
for (let i = 0; i < 600; i++) {
|
||||
db.addNotificationHistory(0, { level: 'info', message: `d-${i}`, timestamp: base + i, stack_name: 'default' });
|
||||
}
|
||||
db.transaction(() => {
|
||||
for (let i = 0; i < 600; i++) {
|
||||
db.addNotificationHistory(0, { level: 'info', message: `d-${i}`, timestamp: base + i, stack_name: 'default' });
|
||||
}
|
||||
});
|
||||
// Production caller (MonitorService) only passes daysToKeep; the cap defaults must enforce the per-stack 500 limit.
|
||||
const summary = db.cleanupOldNotifications(30);
|
||||
const after = db.getNotificationHistory(0, 2000).filter((n: any) => n.stack_name === 'default');
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
/**
|
||||
* Deployed-stack deletion: ready transaction retires both recovery models;
|
||||
* blocking intents gate same-name create.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
|
||||
import { randomUUID } from 'crypto';
|
||||
import path from 'path';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
import type {
|
||||
StackUpdateRecoveryGenerationRow,
|
||||
ServiceUpdateRecoveryRow,
|
||||
StackUpdateCleanupPendingRow,
|
||||
} from '../services/DatabaseService';
|
||||
|
||||
let tmpDir: string;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
let DeployedStackDeletionService: typeof import('../services/DeployedStackDeletionService').DeployedStackDeletionService;
|
||||
let overrideDeletionContainmentBase: typeof import('../services/DeployedStackDeletionService').overrideDeletionContainmentBase;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
({ DeployedStackDeletionService, overrideDeletionContainmentBase } = await import('../services/DeployedStackDeletionService'));
|
||||
});
|
||||
|
||||
afterAll(() => cleanupTestDb(tmpDir));
|
||||
|
||||
function db() {
|
||||
return DatabaseService.getInstance();
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
const raw = (db() as unknown as { db: { prepare: (s: string) => { run: () => void } } }).db;
|
||||
raw.prepare('DELETE FROM stack_update_recovery_generations').run();
|
||||
raw.prepare('DELETE FROM service_update_recovery').run();
|
||||
raw.prepare('DELETE FROM stack_update_cleanup_pending').run();
|
||||
});
|
||||
|
||||
const NODE = 1;
|
||||
|
||||
describe('DeployedStackDeletionService ready transaction', () => {
|
||||
it('commitStackDeletionReadyTransaction deletes full-stack and service recovery rows', () => {
|
||||
const now = Date.now();
|
||||
const stackName = 'del-stack';
|
||||
const gen: StackUpdateRecoveryGenerationRow = {
|
||||
id: randomUUID(),
|
||||
node_id: NODE,
|
||||
stack_name: stackName,
|
||||
status: 'active',
|
||||
phase: 'immediate_verified',
|
||||
is_current: 1,
|
||||
backup_slot_id: null,
|
||||
override_path: null,
|
||||
services_json: '[]',
|
||||
health_gate_id: null,
|
||||
gate_retain_until: null,
|
||||
artifact_expires_at: null,
|
||||
operation_lease_expires_at: null,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
};
|
||||
db().insertStackUpdateRecoveryGeneration(gen);
|
||||
const svc: ServiceUpdateRecoveryRow = {
|
||||
id: randomUUID(),
|
||||
node_id: NODE,
|
||||
stack_name: stackName,
|
||||
service_name: 'web',
|
||||
replicas_json: '[]',
|
||||
majority_image_id: 'sha256:abc',
|
||||
declared_image_ref: 'nginx:latest',
|
||||
weak_floating_tag: 0,
|
||||
health_gate_id: null,
|
||||
status: 'active',
|
||||
expires_at: now + 60_000,
|
||||
claim_expires_at: null,
|
||||
created_at: now,
|
||||
created_by: null,
|
||||
};
|
||||
db().insertServiceUpdateRecovery(svc);
|
||||
const intentId = randomUUID();
|
||||
const intent: StackUpdateCleanupPendingRow = {
|
||||
id: intentId,
|
||||
node_id: NODE,
|
||||
stack_name: stackName,
|
||||
status: 'prepared',
|
||||
target_kind: 'local_socket',
|
||||
rollback_tags_json: '[]',
|
||||
override_paths_json: '[]',
|
||||
prune_volumes_requested: 0,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
};
|
||||
db().insertCleanupPending(intent);
|
||||
|
||||
expect(db().commitStackDeletionReadyTransaction(intentId, NODE, stackName)).toBe(true);
|
||||
expect(db().listStackUpdateRecoveryForStack(NODE, stackName)).toHaveLength(0);
|
||||
expect(db().listActiveServiceUpdateRecoveries(NODE, stackName, 'web')).toHaveLength(0);
|
||||
expect(db().getCleanupPending(intentId)?.status).toBe('ready');
|
||||
});
|
||||
|
||||
it('hasBlockingDeletionIntent is true for prepared intents', () => {
|
||||
const now = Date.now();
|
||||
db().insertCleanupPending({
|
||||
id: randomUUID(),
|
||||
node_id: NODE,
|
||||
stack_name: 'blocked',
|
||||
status: 'prepared',
|
||||
target_kind: 'local_socket',
|
||||
rollback_tags_json: '[]',
|
||||
override_paths_json: '[]',
|
||||
prune_volumes_requested: 0,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
});
|
||||
expect(db().hasBlockingDeletionIntent(NODE, 'blocked')).toBe(true);
|
||||
expect(db().hasBlockingDeletionIntent(NODE, 'other')).toBe(false);
|
||||
});
|
||||
|
||||
it('assertNoBlockingDeletionIntent throws for prepared stacks', () => {
|
||||
const now = Date.now();
|
||||
db().insertCleanupPending({
|
||||
id: randomUUID(),
|
||||
node_id: NODE,
|
||||
stack_name: 'prep',
|
||||
status: 'prepared',
|
||||
target_kind: 'local_socket',
|
||||
rollback_tags_json: '[]',
|
||||
override_paths_json: '[]',
|
||||
prune_volumes_requested: 0,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
});
|
||||
expect(() => {
|
||||
DeployedStackDeletionService.getInstance().assertNoBlockingDeletionIntent(NODE, 'prep');
|
||||
}).toThrow(/deletion in progress/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe('overrideDeletionContainmentBase', () => {
|
||||
it('confines stack-scoped intents to the stack directory', () => {
|
||||
expect(overrideDeletionContainmentBase('/app/compose', 'my-stack')).toBe(
|
||||
path.resolve('/app/compose', 'my-stack'),
|
||||
);
|
||||
expect(overrideDeletionContainmentBase('/app/compose', null)).toBe(
|
||||
path.resolve('/app/compose'),
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects invalid stack names that could traverse', () => {
|
||||
expect(overrideDeletionContainmentBase('/app/compose', '../other')).toBeNull();
|
||||
expect(overrideDeletionContainmentBase('/app/compose', 'bad/name')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1714,3 +1714,23 @@ describe('DockerController - getLegacyOrphanContainersByStack', () => {
|
||||
fallbackSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('DockerController - classifyLegacyOrphansForUpdate', () => {
|
||||
it('returns none when compose ps already manages containers', async () => {
|
||||
// Reuse the same mocks as getLegacyOrphanContainersByStack tests in this file.
|
||||
const { default: DockerController } = await import('../services/DockerController');
|
||||
const dc = DockerController.getInstance(1);
|
||||
vi.spyOn(dc as any, 'fetchComposePsContainers').mockResolvedValue([{ ID: 'c1' }]);
|
||||
await expect(dc.classifyLegacyOrphansForUpdate('my-stack')).resolves.toEqual({ status: 'none' });
|
||||
});
|
||||
|
||||
it('returns classification_failed when compose ps and fallback both fail', async () => {
|
||||
const { default: DockerController } = await import('../services/DockerController');
|
||||
const dc = DockerController.getInstance(1);
|
||||
vi.spyOn(dc as any, 'fetchComposePsContainers').mockRejectedValue(new Error('compose boom'));
|
||||
vi.spyOn(dc as any, 'smartFallback').mockRejectedValue(new Error('fallback boom'));
|
||||
const result = await dc.classifyLegacyOrphansForUpdate('my-stack');
|
||||
expect(result.status).toBe('classification_failed');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* Docker-backed integration: a failed pull must leave the running stack untouched.
|
||||
* Skipped automatically when Docker is unavailable.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { execFileSync } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { setupTestDb, cleanupTestDb } from '../helpers/setupTestDb';
|
||||
|
||||
function dockerAvailable(): boolean {
|
||||
try {
|
||||
execFileSync('docker', ['info'], { stdio: 'ignore' });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const hasDocker = dockerAvailable();
|
||||
const STACK = 'fpkeep';
|
||||
|
||||
function compose(args: string[], cwd: string): string {
|
||||
return execFileSync('docker', ['compose', '-p', STACK, ...args], {
|
||||
cwd,
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
}
|
||||
|
||||
describe.skipIf(!hasDocker)('failed pull keeps stack running', () => {
|
||||
let tmpDir: string;
|
||||
let composeDir: string;
|
||||
let stackDir: string;
|
||||
let nodeId: number;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
composeDir = process.env.COMPOSE_DIR!;
|
||||
stackDir = path.join(composeDir, STACK);
|
||||
fs.mkdirSync(stackDir, { recursive: true });
|
||||
|
||||
const { DatabaseService } = await import('../../services/DatabaseService');
|
||||
const db = DatabaseService.getInstance();
|
||||
const local = db.getDefaultNode();
|
||||
if (!local?.id) throw new Error('Test DB has no default local node');
|
||||
nodeId = local.id;
|
||||
db.updateGlobalSetting('prune_on_update', '0');
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(stackDir, 'compose.yaml'),
|
||||
[
|
||||
'services:',
|
||||
' web:',
|
||||
' image: busybox:1.36.1',
|
||||
' command: ["sleep", "3600"]',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
compose(['up', '-d', '--pull', 'always'], stackDir);
|
||||
}, 180_000);
|
||||
|
||||
afterAll(async () => {
|
||||
try {
|
||||
if (stackDir && fs.existsSync(stackDir)) {
|
||||
compose(['down', '--remove-orphans'], stackDir);
|
||||
}
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
if (tmpDir) cleanupTestDb(tmpDir);
|
||||
}, 120_000);
|
||||
|
||||
it('leaves the original container running when ComposeService updateStack pull fails', async () => {
|
||||
const beforeId = compose(['ps', '-q'], stackDir).trim();
|
||||
expect(beforeId.length).toBeGreaterThan(0);
|
||||
|
||||
const beforeInspect = JSON.parse(
|
||||
execFileSync('docker', ['inspect', beforeId], { encoding: 'utf8' }),
|
||||
) as Array<{ State: { Running: boolean; Status: string } }>;
|
||||
expect(beforeInspect[0].State.Running).toBe(true);
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(stackDir, 'compose.yaml'),
|
||||
[
|
||||
'services:',
|
||||
' web:',
|
||||
' image: busybox:sencho-does-not-exist-fpkeep-xyz',
|
||||
' command: ["sleep", "3600"]',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
const { StackUpdateRecoveryService } = await import('../../services/StackUpdateRecoveryService');
|
||||
StackUpdateRecoveryService.resetForTests();
|
||||
const { ComposeService } = await import('../../services/ComposeService');
|
||||
await expect(
|
||||
ComposeService.getInstance(nodeId).updateStack(STACK, undefined, true),
|
||||
).rejects.toThrow();
|
||||
|
||||
const afterId = compose(['ps', '-q'], stackDir).trim();
|
||||
expect(afterId).toBe(beforeId);
|
||||
|
||||
const afterInspect = JSON.parse(
|
||||
execFileSync('docker', ['inspect', beforeId], { encoding: 'utf8' }),
|
||||
) as Array<{ State: { Running: boolean } }>;
|
||||
expect(afterInspect[0].State.Running).toBe(true);
|
||||
}, 180_000);
|
||||
});
|
||||
@@ -375,7 +375,7 @@ describe('POST /api/auto-update/execute', () => {
|
||||
.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never);
|
||||
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
|
||||
.mockResolvedValue({ hasUpdate: true } as never);
|
||||
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue();
|
||||
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
|
||||
const gateSpy = vi.spyOn(PolicyEnforcement, 'enforcePolicyPreDeploy').mockResolvedValue({
|
||||
ok: false,
|
||||
bypassed: false,
|
||||
@@ -421,7 +421,7 @@ describe('POST /api/auto-update/execute', () => {
|
||||
.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never);
|
||||
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
|
||||
.mockResolvedValue({ hasUpdate: true } as never);
|
||||
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue();
|
||||
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-au');
|
||||
try {
|
||||
const res = await request(app)
|
||||
|
||||
@@ -54,7 +54,7 @@ const {
|
||||
mockStartContainer: vi.fn().mockResolvedValue(undefined),
|
||||
mockStopContainer: vi.fn().mockResolvedValue(undefined),
|
||||
mockPruneSystem: vi.fn().mockResolvedValue({ success: true, reclaimedBytes: 0 }),
|
||||
mockUpdateStack: vi.fn().mockResolvedValue(undefined),
|
||||
mockUpdateStack: vi.fn().mockResolvedValue({ recoveryId: null }),
|
||||
mockGetStacks: vi.fn().mockResolvedValue([]),
|
||||
mockGetStackContent: vi.fn().mockResolvedValue(''),
|
||||
mockGetEnvContent: vi.fn().mockResolvedValue(''),
|
||||
|
||||
@@ -260,7 +260,7 @@ describe('POST /api/stacks/bulk execution', () => {
|
||||
});
|
||||
|
||||
it('handles update action (paid tier) by calling ComposeService.updateStack', async () => {
|
||||
mockUpdateStack.mockResolvedValue(undefined);
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
const { LicenseService } = await import('../services/LicenseService');
|
||||
const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
try {
|
||||
@@ -287,7 +287,7 @@ describe('POST /api/stacks/bulk execution', () => {
|
||||
policy: { id: 1, name: 'block-criticals', node_id: null, node_identity: '', stack_pattern: null, max_severity: 'HIGH', block_on_deploy: 1, block_on_severity: 1, block_on_kev: 0, block_on_fixable: 0, enabled: 1, replicated_from_control: 0, created_at: Date.now(), updated_at: Date.now() },
|
||||
violations: [{ imageRef: 'nginx:latest', severity: 'CRITICAL', criticalCount: 3, highCount: 0, kevCount: 0, fixableCount: 0, reasons: ['severity'], scanId: 1 }],
|
||||
});
|
||||
mockUpdateStack.mockResolvedValue(undefined);
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
try {
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/bulk')
|
||||
|
||||
@@ -124,7 +124,9 @@ describe('DELETE /api/stacks/:stackName mesh opt-out cascade (F-1 / F-14)', () =
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const sawCascadeWarning = warnSpy.mock.calls.some(args =>
|
||||
typeof args[0] === 'string' && args[0].includes('Mesh opt-out cascade failed')
|
||||
typeof args[0] === 'string'
|
||||
&& (args[0].includes('Mesh opt-out cascade failed')
|
||||
|| args[0].includes('Mesh opt-out failed'))
|
||||
);
|
||||
expect(sawCascadeWarning).toBe(true);
|
||||
});
|
||||
|
||||
@@ -164,7 +164,7 @@ describe('self stack lifecycle refusal', () => {
|
||||
});
|
||||
|
||||
it('allows update on a non-self stack', async () => {
|
||||
mockUpdateStack.mockResolvedValue(undefined);
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/web/update')
|
||||
.set('Cookie', authCookie);
|
||||
@@ -177,7 +177,7 @@ describe('self stack lifecycle refusal', () => {
|
||||
describe('POST /api/stacks/bulk self stack skip', () => {
|
||||
beforeEach(() => {
|
||||
stubSelfProject('sencho');
|
||||
mockUpdateStack.mockResolvedValue(undefined);
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1' }]);
|
||||
});
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ function spec(name: string): EffectiveServiceSpec {
|
||||
|
||||
beforeEach(() => {
|
||||
state.updateStack.mockReset();
|
||||
state.updateStack.mockResolvedValue(undefined);
|
||||
state.updateStack.mockResolvedValue({ recoveryId: null });
|
||||
state.model = null;
|
||||
});
|
||||
|
||||
@@ -85,8 +85,9 @@ describe('StackUpdateOrchestrator stack branch', () => {
|
||||
{ nodeId: 0, stackName: 'web', target: { scope: 'stack' }, trigger: 'manual', actor: 'tester' },
|
||||
{ atomic: true, terminalWs: null },
|
||||
);
|
||||
expect(result).toEqual({ kind: 'stack_compose_done' });
|
||||
expect(result).toEqual({ kind: 'stack_compose_done', recoveryId: null });
|
||||
expect(state.updateStack).toHaveBeenCalledWith('web', undefined, true);
|
||||
// recoveryId is forwarded from ComposeService.updateStack
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
/**
|
||||
* Direct tests for StackUpdateRecoveryService artifact lifecycle and compensation probe.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const mockTag = vi.fn().mockResolvedValue(undefined);
|
||||
const mockRemove = vi.fn().mockResolvedValue(undefined);
|
||||
const mockListContainers = vi.fn().mockResolvedValue([]);
|
||||
const mockInspectContainer = vi.fn();
|
||||
const mockGetContainer = vi.fn(() => ({ inspect: mockInspectContainer }));
|
||||
const mockGetImage = vi.fn((ref: string) => ({
|
||||
tag: mockTag,
|
||||
remove: mockRemove,
|
||||
inspect: vi.fn().mockResolvedValue({ RepoDigests: [] }),
|
||||
_ref: ref,
|
||||
}));
|
||||
|
||||
vi.mock('../services/DockerController', () => ({
|
||||
default: {
|
||||
getInstance: () => ({
|
||||
getDocker: () => ({
|
||||
listContainers: mockListContainers,
|
||||
getContainer: mockGetContainer,
|
||||
getImage: mockGetImage,
|
||||
}),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/FileSystemService', () => ({
|
||||
FileSystemService: {
|
||||
getInstance: () => ({
|
||||
getBaseDir: () => '/test/compose',
|
||||
backupStackFiles: vi.fn().mockResolvedValue(undefined),
|
||||
restoreStackFiles: vi.fn().mockResolvedValue(undefined),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/composeProjectContext', () => ({
|
||||
classifyReferenceKind: () => 'moving_tag',
|
||||
resolveComposeProjectContext: vi.fn().mockResolvedValue({
|
||||
validateForMutation: vi.fn().mockResolvedValue(undefined),
|
||||
backupFromContext: vi.fn().mockResolvedValue('backup-1'),
|
||||
restoreFromContext: vi.fn().mockResolvedValue(undefined),
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock('../services/effectiveServiceModel', () => ({
|
||||
buildEffectiveServiceModel: vi.fn().mockResolvedValue({
|
||||
renderable: true,
|
||||
services: [{ name: 'web', declaredImage: 'nginx:latest', hasBuild: false, expectedReplicas: 1 }],
|
||||
}),
|
||||
}));
|
||||
|
||||
const mockValidateExact = vi.fn().mockResolvedValue(undefined);
|
||||
vi.mock('../services/ComposeService', async () => {
|
||||
const actual = await vi.importActual<typeof import('../services/ComposeService')>('../services/ComposeService');
|
||||
return {
|
||||
...actual,
|
||||
ComposeService: {
|
||||
getInstance: () => ({
|
||||
validateExactComposeInvocation: mockValidateExact,
|
||||
buildAuthoredComposeArgs: vi.fn(),
|
||||
}),
|
||||
},
|
||||
getComposeCommandTimeoutMs: () => 60_000,
|
||||
};
|
||||
});
|
||||
|
||||
const mockUnlink = vi.fn().mockResolvedValue(undefined);
|
||||
const mockWriteFile = vi.fn().mockResolvedValue(undefined);
|
||||
const mockRealpath = vi.fn(async (p: string) => p);
|
||||
vi.mock('fs/promises', () => ({
|
||||
default: {
|
||||
unlink: (p: string) => mockUnlink(p),
|
||||
writeFile: (p: string, data: string, enc?: string) => mockWriteFile(p, data, enc),
|
||||
realpath: (p: string) => mockRealpath(p),
|
||||
},
|
||||
unlink: (p: string) => mockUnlink(p),
|
||||
writeFile: (p: string, data: string, enc?: string) => mockWriteFile(p, data, enc),
|
||||
realpath: (p: string) => mockRealpath(p),
|
||||
}));
|
||||
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService';
|
||||
|
||||
describe('StackUpdateRecoveryService', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
StackUpdateRecoveryService.resetForTests();
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'c1', State: 'running', Labels: {} },
|
||||
]);
|
||||
mockInspectContainer.mockResolvedValue({
|
||||
State: { Status: 'running', ExitCode: 0 },
|
||||
Image: 'sha256:abc',
|
||||
});
|
||||
mockValidateExact.mockResolvedValue(undefined);
|
||||
mockUnlink.mockResolvedValue(undefined);
|
||||
mockWriteFile.mockResolvedValue(undefined);
|
||||
mockRealpath.mockImplementation(async (p: string) => p);
|
||||
mockRemove.mockResolvedValue(undefined);
|
||||
mockTag.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
it('validates exact invocation before tagging images', async () => {
|
||||
const order: string[] = [];
|
||||
mockValidateExact.mockImplementation(async () => { order.push('validate'); });
|
||||
mockTag.mockImplementation(async () => { order.push('tag'); });
|
||||
mockWriteFile.mockImplementation(async () => { order.push('write'); });
|
||||
|
||||
const spyInsert = vi.spyOn(DatabaseService.prototype, 'insertStackUpdateRecoveryGeneration')
|
||||
.mockImplementation(() => { order.push('insert'); });
|
||||
vi.spyOn(DatabaseService.prototype, 'getGlobalSettings').mockReturnValue({});
|
||||
|
||||
await StackUpdateRecoveryService.getInstance().captureCandidate({
|
||||
nodeId: 1,
|
||||
stackName: 'my-stack',
|
||||
createdBy: 'test',
|
||||
});
|
||||
|
||||
expect(order.indexOf('validate')).toBeLessThan(order.indexOf('tag'));
|
||||
expect(order.indexOf('tag')).toBeLessThan(order.indexOf('write'));
|
||||
expect(order.indexOf('write')).toBeLessThan(order.indexOf('insert'));
|
||||
spyInsert.mockRestore();
|
||||
});
|
||||
|
||||
it('retires opaque tags and override on abandon', async () => {
|
||||
const row = {
|
||||
id: 'gen-1',
|
||||
node_id: 1,
|
||||
stack_name: 'my-stack',
|
||||
status: 'candidate' as const,
|
||||
phase: 'captured' as const,
|
||||
is_current: 0,
|
||||
backup_slot_id: null,
|
||||
override_path: '/test/compose/my-stack/.sencho-recovery-aaaaaaaaaaaa.yml',
|
||||
services_json: JSON.stringify([{
|
||||
serviceName: 'web',
|
||||
scale: 1,
|
||||
hasBuild: false,
|
||||
declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag',
|
||||
replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold' }],
|
||||
}]),
|
||||
health_gate_id: null,
|
||||
gate_retain_until: null,
|
||||
artifact_expires_at: null,
|
||||
operation_lease_expires_at: null,
|
||||
created_at: Date.now(),
|
||||
updated_at: Date.now(),
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
};
|
||||
|
||||
vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row);
|
||||
vi.spyOn(DatabaseService.prototype, 'abandonStackUpdateRecoveryGeneration').mockReturnValue(true);
|
||||
const markRetired = vi.spyOn(DatabaseService.prototype, 'markStackUpdateRecoveryArtifactsRetired')
|
||||
.mockReturnValue(true);
|
||||
|
||||
const ok = await StackUpdateRecoveryService.getInstance().abandon('gen-1');
|
||||
expect(ok).toBe(true);
|
||||
expect(mockRemove).toHaveBeenCalled();
|
||||
expect(mockUnlink).toHaveBeenCalledWith(row.override_path);
|
||||
expect(markRetired).toHaveBeenCalledWith('gen-1');
|
||||
});
|
||||
|
||||
it('does not mark restored_current when recovery probe finds crashed containers', async () => {
|
||||
const servicesJson = JSON.stringify([{
|
||||
serviceName: 'web',
|
||||
scale: 1,
|
||||
hasBuild: false,
|
||||
declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag',
|
||||
replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/x/web:hold' }],
|
||||
}]);
|
||||
const row = {
|
||||
id: 'gen-2',
|
||||
node_id: 1,
|
||||
stack_name: 'my-stack',
|
||||
status: 'active' as const,
|
||||
phase: 'reconciling' as const,
|
||||
is_current: 1,
|
||||
backup_slot_id: 'b1',
|
||||
override_path: '/test/compose/my-stack/.sencho-recovery-bbbbbbbbbbbb.yml',
|
||||
services_json: servicesJson,
|
||||
health_gate_id: null,
|
||||
gate_retain_until: null,
|
||||
artifact_expires_at: null,
|
||||
operation_lease_expires_at: null,
|
||||
created_at: Date.now(),
|
||||
updated_at: Date.now(),
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
};
|
||||
vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row);
|
||||
const update = vi.spyOn(DatabaseService.prototype, 'updateStackUpdateRecoveryGeneration')
|
||||
.mockImplementation(() => undefined);
|
||||
|
||||
mockListContainers.mockResolvedValue([{ Id: 'c1', State: 'exited', Labels: { 'com.docker.compose.service': 'web' } }]);
|
||||
mockInspectContainer.mockResolvedValue({ State: { ExitCode: 0 } });
|
||||
|
||||
vi.useFakeTimers();
|
||||
const promise = StackUpdateRecoveryService.getInstance().compensateWithCandidate(
|
||||
'gen-2',
|
||||
async () => undefined,
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
const ok = await promise;
|
||||
vi.useRealTimers();
|
||||
|
||||
expect(ok).toBe(false);
|
||||
expect(update).toHaveBeenCalledWith('gen-2', expect.objectContaining({ status: 'recovery_required' }));
|
||||
expect(update).not.toHaveBeenCalledWith(
|
||||
'gen-2',
|
||||
expect.objectContaining({ status: 'restored_current' }),
|
||||
);
|
||||
});
|
||||
|
||||
const capturedWebReplica = {
|
||||
containerId: 'c1',
|
||||
imageId: 'sha256:oldimg',
|
||||
repoDigest: null,
|
||||
state: 'running' as const,
|
||||
rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold',
|
||||
};
|
||||
|
||||
it('probeRecoveredStack rejects empty runtime when expected replicas were running', async () => {
|
||||
const servicesJson = JSON.stringify([{
|
||||
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag', replicas: [capturedWebReplica],
|
||||
}]);
|
||||
mockListContainers.mockResolvedValue([]);
|
||||
vi.useFakeTimers();
|
||||
const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await expect(promise).resolves.toBe(false);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('probeRecoveredStack rejects restarting and unhealthy expected containers', async () => {
|
||||
const servicesJson = JSON.stringify([{
|
||||
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag', replicas: [capturedWebReplica],
|
||||
}]);
|
||||
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'c1', State: 'restarting', Labels: { 'com.docker.compose.service': 'web' } },
|
||||
]);
|
||||
vi.useFakeTimers();
|
||||
let promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await expect(promise).resolves.toBe(false);
|
||||
vi.useRealTimers();
|
||||
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } },
|
||||
]);
|
||||
mockInspectContainer.mockResolvedValue({
|
||||
Image: 'sha256:oldimg',
|
||||
State: { Status: 'running', Health: { Status: 'unhealthy' } },
|
||||
});
|
||||
vi.useFakeTimers();
|
||||
promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await expect(promise).resolves.toBe(false);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('probeRecoveredStack rejects healthy containers running a mismatched image', async () => {
|
||||
const servicesJson = JSON.stringify([{
|
||||
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag', replicas: [capturedWebReplica],
|
||||
}]);
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } },
|
||||
]);
|
||||
mockInspectContainer.mockResolvedValue({
|
||||
Image: 'sha256:newfailed',
|
||||
Config: { Image: 'nginx:alpine' },
|
||||
State: { Status: 'running' },
|
||||
});
|
||||
vi.useFakeTimers();
|
||||
const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await expect(promise).resolves.toBe(false);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('probeRecoveredStack rejects a running replica for a service captured at scale zero', async () => {
|
||||
const servicesJson = JSON.stringify([{
|
||||
serviceName: 'worker', scale: 0, hasBuild: false, declaredImageRef: 'busybox:latest',
|
||||
referenceKind: 'moving_tag',
|
||||
replicas: [{
|
||||
containerId: 'c0', imageId: 'sha256:worker', repoDigest: null, state: 'stopped',
|
||||
rollbackTag: 'sencho-rb/aaaaaaaaaaaa/worker:hold',
|
||||
}],
|
||||
}]);
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'c0', State: 'running', Labels: { 'com.docker.compose.service': 'worker' } },
|
||||
]);
|
||||
mockInspectContainer.mockResolvedValue({
|
||||
Image: 'sha256:worker',
|
||||
State: { Status: 'running' },
|
||||
});
|
||||
vi.useFakeTimers();
|
||||
const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await expect(promise).resolves.toBe(false);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('probeRecoveredStack accepts healthy running replicas matching capture scale and image', async () => {
|
||||
const servicesJson = JSON.stringify([{
|
||||
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag', replicas: [capturedWebReplica],
|
||||
}]);
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } },
|
||||
]);
|
||||
mockInspectContainer.mockResolvedValue({
|
||||
Image: 'sha256:oldimg',
|
||||
Config: { Image: 'sencho-rb/aaaaaaaaaaaa/web:hold' },
|
||||
State: { Status: 'running' },
|
||||
});
|
||||
vi.useFakeTimers();
|
||||
const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await expect(promise).resolves.toBe(true);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('does not mark artifacts retired when tag removal fails', async () => {
|
||||
const row = {
|
||||
id: 'gen-3',
|
||||
node_id: 1,
|
||||
stack_name: 'my-stack',
|
||||
status: 'abandoned' as const,
|
||||
phase: 'captured' as const,
|
||||
is_current: 0,
|
||||
backup_slot_id: null,
|
||||
override_path: null,
|
||||
services_json: JSON.stringify([{
|
||||
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag',
|
||||
replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/cccccccccccc/web:hold' }],
|
||||
}]),
|
||||
health_gate_id: null,
|
||||
gate_retain_until: null,
|
||||
artifact_expires_at: Date.now() - 1,
|
||||
operation_lease_expires_at: null,
|
||||
created_at: Date.now(),
|
||||
updated_at: Date.now(),
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
};
|
||||
mockRemove.mockRejectedValueOnce(Object.assign(new Error('docker busy'), { statusCode: 500 }));
|
||||
const markRetired = vi.spyOn(DatabaseService.prototype, 'markStackUpdateRecoveryArtifactsRetired')
|
||||
.mockReturnValue(true);
|
||||
const ok = await StackUpdateRecoveryService.getInstance().retireGenerationArtifacts(row);
|
||||
expect(ok).toBe(false);
|
||||
expect(markRetired).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -270,7 +270,7 @@ describe('health gate begin call sites', () => {
|
||||
});
|
||||
|
||||
it('begins a gate after a manual update and returns its id', async () => {
|
||||
mockUpdateStack.mockResolvedValue(undefined);
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/myapp/update')
|
||||
.set('Cookie', authCookie)
|
||||
@@ -281,7 +281,7 @@ describe('health gate begin call sites', () => {
|
||||
});
|
||||
|
||||
it('begins a gate per stack in a bulk update and carries ids in the results', async () => {
|
||||
mockUpdateStack.mockResolvedValue(undefined);
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/bulk')
|
||||
.set('Cookie', authCookie)
|
||||
@@ -490,7 +490,7 @@ describe('deploy_failure notification on /update error', () => {
|
||||
});
|
||||
|
||||
it('uses trusted proxy tier headers for remote atomic updates', async () => {
|
||||
mockUpdateStack.mockResolvedValue(undefined);
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
const token = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' });
|
||||
|
||||
const res = await request(app)
|
||||
|
||||
@@ -96,11 +96,12 @@ describe('buildRollbackItems', () => {
|
||||
expect(known.detail).toContain('nginx:1.27.1');
|
||||
});
|
||||
|
||||
it('downgrades the previous image to not_covered for a moving tag', () => {
|
||||
it('treats a moving tag as ready because full-stack updates retain the prior image ID', () => {
|
||||
const item = itemById(baseInputs({ rollbackTarget: { target: 'nginx:latest', moving: true } }), 'previous_images');
|
||||
expect(item.state).toBe('not_covered');
|
||||
expect(item.state).toBe('ready');
|
||||
expect(item.detail).toContain('moving image tag');
|
||||
expect(item.detail).toContain('nginx:latest');
|
||||
expect(item.detail).toContain('recovery window');
|
||||
});
|
||||
|
||||
it('does not mistake an image literally named error for a failed preview', () => {
|
||||
@@ -139,9 +140,9 @@ describe('aggregateRollbackOverall', () => {
|
||||
expect(aggregateRollbackOverall(items)).toBe('partial');
|
||||
});
|
||||
|
||||
it('is partial when the rollback target is a moving tag', () => {
|
||||
it('is ready when the rollback target is a moving tag (image ID capture covers it)', () => {
|
||||
const items = buildRollbackItems(baseInputs({ rollbackTarget: { target: 'nginx:latest', moving: true } }), NOW);
|
||||
expect(aggregateRollbackOverall(items)).toBe('partial');
|
||||
expect(aggregateRollbackOverall(items)).toBe('ready');
|
||||
});
|
||||
|
||||
it('is partial when env coverage is missing', () => {
|
||||
|
||||
@@ -295,12 +295,11 @@ describe('UpdateGuardService.computeRollbackReadiness moving-tag wiring', () =>
|
||||
mockListContainers.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
it('marks previous_images not_covered (overall partial) when any image uses a moving tag', async () => {
|
||||
// Primary pinned, sidecar on a moving tag: a file rollback cannot revert it.
|
||||
it('marks previous_images ready when any image uses a moving tag (full-stack image ID capture)', async () => {
|
||||
mockGetPreview.mockResolvedValue(preview([{ current_tag: '1.2.3' }, { current_tag: 'latest' }]));
|
||||
const report = await UpdateGuardService.getInstance().computeRollbackReadiness(0, 'app');
|
||||
expect(report.items.find(i => i.id === 'previous_images')?.state).toBe('not_covered');
|
||||
expect(report.overall).toBe('partial');
|
||||
expect(report.items.find(i => i.id === 'previous_images')?.state).toBe('ready');
|
||||
expect(report.overall).toBe('ready');
|
||||
});
|
||||
|
||||
it('marks previous_images ready (overall ready) when every image is pinned', async () => {
|
||||
|
||||
@@ -481,7 +481,7 @@ describe('WebhookService.execute: health gate begin call sites', () => {
|
||||
const { HealthGateService } = await import('../services/HealthGateService');
|
||||
vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined);
|
||||
vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]);
|
||||
vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue(undefined);
|
||||
vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook');
|
||||
|
||||
const result = await WebhookService.getInstance().execute(webhook, 'pull', 'test', true);
|
||||
|
||||
Reference in New Issue
Block a user