mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-25 09:46:47 +00:00
feat: add service-scoped Compose update and restore (#1648)
* feat: add service-scoped Compose update and restore Allow updating or rebuilding one declared Compose service on multi-service stacks without recreating siblings, with recovery snapshots, health-gate observation, and prune holds for rollback images. Full-stack update paths and single-service UX stay unchanged. * fix: sanitize service-scoped update log messages for CodeQL * fix: address service-scoped update audit findings B-01 through B-07 * fix: complete service-scoped update audit metadata and surfaces * test: wrap Updates readiness tests for deploy-feedback context * fix: keep service recovery reachable without Deploy Progress Make failed service-gate recovery discoverable when Deploy Progress is disabled or dismissed, suppress stale image-scan notification side effects, normalize ComposeService line endings, and add focused regression coverage. * fix: resurface ContainersHealth density and expand on multi-service stacks Service grouping hid the summary strip and Compact/Detailed/Expand controls that still applied to multi-container stacks.
This commit is contained in:
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* Ref-counted Auto-Heal suppression for overlapping service-scoped updates.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { AutoHealService } from '../services/AutoHealService';
|
||||
|
||||
describe('AutoHealService suppress refcount', () => {
|
||||
beforeEach(() => {
|
||||
AutoHealService.getInstance().stop();
|
||||
});
|
||||
|
||||
it('keeps suppression until every overlapping owner clears', () => {
|
||||
const svc = AutoHealService.getInstance();
|
||||
svc.suppress(0, 'web', 'api');
|
||||
svc.suppress(0, 'web', 'api');
|
||||
expect(svc.isSuppressed(0, 'web', 'api')).toBe(true);
|
||||
svc.clearSuppress(0, 'web', 'api');
|
||||
expect(svc.isSuppressed(0, 'web', 'api')).toBe(true);
|
||||
svc.clearSuppress(0, 'web', 'api');
|
||||
expect(svc.isSuppressed(0, 'web', 'api')).toBe(false);
|
||||
});
|
||||
|
||||
it('scopes suppression per service on the same stack', () => {
|
||||
const svc = AutoHealService.getInstance();
|
||||
svc.suppress(0, 'web', 'api');
|
||||
expect(svc.isSuppressed(0, 'web', 'api')).toBe(true);
|
||||
expect(svc.isSuppressed(0, 'web', 'db')).toBe(false);
|
||||
svc.clearSuppress(0, 'web', 'api');
|
||||
expect(svc.isSuppressed(0, 'web', 'api')).toBe(false);
|
||||
});
|
||||
|
||||
it('ignores clearSuppress when nothing is suppressed', () => {
|
||||
const svc = AutoHealService.getInstance();
|
||||
svc.clearSuppress(0, 'web', 'api');
|
||||
expect(svc.isSuppressed(0, 'web', 'api')).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -660,6 +660,38 @@ describe('ComposeService - deployStack', () => {
|
||||
expect(mockGetLegacyOrphanContainersByStack).toHaveBeenCalledWith('my-stack');
|
||||
});
|
||||
|
||||
it('blocks a pilot service restore with relative binds before Compose up', async () => {
|
||||
process.env.SENCHO_MODE = 'pilot';
|
||||
const composeDir = path.resolve('/test/compose');
|
||||
mockGetBindMounts.mockResolvedValue([
|
||||
{ source: '/opt/docker/sencho', destination: composeDir },
|
||||
]);
|
||||
const configProc = createMockProcess();
|
||||
mockSpawn.mockReturnValue(configProc);
|
||||
|
||||
const svc = ComposeService.getInstance(1);
|
||||
const result = svc.recreateServiceFromLocal('my-stack', 'db').then(() => null, (error: Error) => error);
|
||||
await waitForSpawn();
|
||||
configProc.stdout.emit('data', Buffer.from(JSON.stringify({
|
||||
name: 'my-stack',
|
||||
services: {
|
||||
db: {
|
||||
image: 'postgres:17',
|
||||
volumes: [{
|
||||
type: 'bind',
|
||||
source: path.join(composeDir, 'my-stack', 'data'),
|
||||
target: '/var/lib/postgresql/data',
|
||||
}],
|
||||
},
|
||||
},
|
||||
})));
|
||||
configProc.emit('close', 0);
|
||||
|
||||
const error = await result;
|
||||
expect(error?.message).toContain('1:1');
|
||||
expect(mockSpawn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not remove compose-managed containers before deploy (#1565)', async () => {
|
||||
setupAutoCloseSpawn();
|
||||
mockListContainers.mockResolvedValue([]);
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
/**
|
||||
* Coverage for the service_update_recovery accessors on the real
|
||||
* DatabaseService (against a temp DB, so the CAS WHERE clauses run against
|
||||
* actual SQLite semantics rather than a mock):
|
||||
* - insert / get / list active
|
||||
* - atomic claim CAS (active -> restoring) and its race/expiry guards
|
||||
* - renewal CAS never revives a terminal or already-active row
|
||||
* - mark consumed / reactivate / invalidate transitions
|
||||
* - sweep of abandoned restoring claims vs a still-live claim
|
||||
* - held image id projection and stack-delete cleanup
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
import type { ServiceUpdateRecoveryRow } from '../services/DatabaseService';
|
||||
|
||||
let tmpDir: string;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
});
|
||||
|
||||
afterAll(() => cleanupTestDb(tmpDir));
|
||||
|
||||
function db() {
|
||||
return DatabaseService.getInstance();
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
const raw = (db() as unknown as { db: { prepare: (s: string) => { run: () => void } } }).db;
|
||||
raw.prepare('DELETE FROM service_update_recovery').run();
|
||||
});
|
||||
|
||||
const NODE = 1;
|
||||
|
||||
function makeRow(overrides: Partial<ServiceUpdateRecoveryRow> = {}): ServiceUpdateRecoveryRow {
|
||||
return {
|
||||
id: overrides.id ?? 'rec-1',
|
||||
node_id: NODE,
|
||||
stack_name: 'web',
|
||||
service_name: 'api',
|
||||
replicas_json: JSON.stringify([{ imageId: 'sha256:aaa', repoDigest: 'sha256:digest-aaa' }]),
|
||||
majority_image_id: 'sha256:aaa',
|
||||
declared_image_ref: 'ghcr.io/acme/api:v1',
|
||||
weak_floating_tag: 0,
|
||||
health_gate_id: null,
|
||||
status: 'active',
|
||||
expires_at: 10_000,
|
||||
claim_expires_at: null,
|
||||
created_at: 1_000,
|
||||
created_by: 'tester',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('service_update_recovery accessors', () => {
|
||||
it('inserts and reads a row back by id', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow());
|
||||
const row = db().getServiceUpdateRecovery('rec-1');
|
||||
expect(row).toMatchObject({ id: 'rec-1', stack_name: 'web', service_name: 'api', status: 'active' });
|
||||
});
|
||||
|
||||
it('lists only active rows for a service, most recent first', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-old', created_at: 1_000 }));
|
||||
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-new', created_at: 2_000 }));
|
||||
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-consumed', status: 'consumed' }));
|
||||
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-other-service', service_name: 'db' }));
|
||||
|
||||
const active = db().listActiveServiceUpdateRecoveries(NODE, 'web', 'api');
|
||||
expect(active.map(r => r.id)).toEqual(['rec-new', 'rec-old']);
|
||||
});
|
||||
|
||||
it('links a health gate id only while the row is still active', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow());
|
||||
db().linkServiceUpdateRecoveryHealthGate('rec-1', 'gate-1');
|
||||
expect(db().getServiceUpdateRecovery('rec-1')?.health_gate_id).toBe('gate-1');
|
||||
|
||||
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-consumed', status: 'consumed' }));
|
||||
db().linkServiceUpdateRecoveryHealthGate('rec-consumed', 'gate-2');
|
||||
expect(db().getServiceUpdateRecovery('rec-consumed')?.health_gate_id).toBeNull();
|
||||
});
|
||||
|
||||
describe('claim CAS', () => {
|
||||
it('claims an active, unexpired row and moves it to restoring', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ expires_at: 50_000 }));
|
||||
const claimed = db().claimServiceUpdateRecovery('rec-1', 100_000, 20_000);
|
||||
expect(claimed).toMatchObject({ id: 'rec-1', status: 'restoring', claim_expires_at: 100_000 });
|
||||
});
|
||||
|
||||
it('refuses to claim an already-expired active row', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ expires_at: 10_000 }));
|
||||
const claimed = db().claimServiceUpdateRecovery('rec-1', 100_000, 20_000);
|
||||
expect(claimed).toBeUndefined();
|
||||
expect(db().getServiceUpdateRecovery('rec-1')?.status).toBe('active');
|
||||
});
|
||||
|
||||
it('refuses a second claim on an already-restoring row (no double claim)', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ expires_at: 50_000 }));
|
||||
const first = db().claimServiceUpdateRecovery('rec-1', 100_000, 20_000);
|
||||
expect(first).toBeTruthy();
|
||||
const second = db().claimServiceUpdateRecovery('rec-1', 200_000, 21_000);
|
||||
expect(second).toBeUndefined();
|
||||
});
|
||||
|
||||
it('refuses to claim a consumed/expired/invalidated row', () => {
|
||||
for (const status of ['consumed', 'expired', 'invalidated'] as const) {
|
||||
db().insertServiceUpdateRecovery(makeRow({ id: `rec-${status}`, status, expires_at: 50_000 }));
|
||||
const claimed = db().claimServiceUpdateRecovery(`rec-${status}`, 100_000, 20_000);
|
||||
expect(claimed).toBeUndefined();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('renewal CAS (no revive of terminal)', () => {
|
||||
it('extends claim_expires_at while restoring', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', claim_expires_at: 100_000 }));
|
||||
const renewed = db().renewServiceUpdateRecoveryClaim('rec-1', 200_000);
|
||||
expect(renewed).toBe(true);
|
||||
expect(db().getServiceUpdateRecovery('rec-1')?.claim_expires_at).toBe(200_000);
|
||||
});
|
||||
|
||||
it.each(['active', 'consumed', 'expired', 'invalidated'] as const)(
|
||||
'never revives a %s row via renewal',
|
||||
(status) => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ status, claim_expires_at: status === 'active' ? null : 100_000 }));
|
||||
const renewed = db().renewServiceUpdateRecoveryClaim('rec-1', 999_000);
|
||||
expect(renewed).toBe(false);
|
||||
expect(db().getServiceUpdateRecovery('rec-1')?.status).toBe(status);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('terminal transitions', () => {
|
||||
it('marks a restoring row consumed and links the restore health gate id', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', claim_expires_at: 100_000, health_gate_id: 'update-gate' }));
|
||||
const ok = db().markServiceUpdateRecoveryConsumed('rec-1', 'restore-gate');
|
||||
expect(ok).toBe(true);
|
||||
const row = db().getServiceUpdateRecovery('rec-1');
|
||||
expect(row).toMatchObject({ status: 'consumed', claim_expires_at: null, health_gate_id: 'restore-gate' });
|
||||
});
|
||||
|
||||
it('keeps the prior health gate id when marking consumed with no runId', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', health_gate_id: 'update-gate' }));
|
||||
db().markServiceUpdateRecoveryConsumed('rec-1', null);
|
||||
expect(db().getServiceUpdateRecovery('rec-1')?.health_gate_id).toBe('update-gate');
|
||||
});
|
||||
|
||||
it('cannot mark an already-active row consumed', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ status: 'active' }));
|
||||
expect(db().markServiceUpdateRecoveryConsumed('rec-1')).toBe(false);
|
||||
});
|
||||
|
||||
it('reactivates a restoring row on a mid-flight failure with the image still local', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', claim_expires_at: 100_000 }));
|
||||
const ok = db().reactivateServiceUpdateRecovery('rec-1');
|
||||
expect(ok).toBe(true);
|
||||
expect(db().getServiceUpdateRecovery('rec-1')).toMatchObject({ status: 'active', claim_expires_at: null });
|
||||
});
|
||||
|
||||
it('invalidates a restoring row on a mid-flight failure with the image gone', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', claim_expires_at: 100_000 }));
|
||||
const ok = db().invalidateServiceUpdateRecovery('rec-1');
|
||||
expect(ok).toBe(true);
|
||||
expect(db().getServiceUpdateRecovery('rec-1')).toMatchObject({ status: 'invalidated', claim_expires_at: null });
|
||||
});
|
||||
|
||||
it('cannot reactivate or invalidate a row that is not restoring', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ status: 'active' }));
|
||||
expect(db().reactivateServiceUpdateRecovery('rec-1')).toBe(false);
|
||||
expect(db().invalidateServiceUpdateRecovery('rec-1')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sweep', () => {
|
||||
it('expires an active row whose TTL has lapsed', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ expires_at: 10_000 }));
|
||||
const count = db().sweepExpiredActiveServiceUpdateRecoveries(20_000);
|
||||
expect(count).toBe(1);
|
||||
expect(db().getServiceUpdateRecovery('rec-1')?.status).toBe('expired');
|
||||
});
|
||||
|
||||
it('leaves an unexpired active row alone', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ expires_at: 50_000 }));
|
||||
expect(db().sweepExpiredActiveServiceUpdateRecoveries(20_000)).toBe(0);
|
||||
expect(db().getServiceUpdateRecovery('rec-1')?.status).toBe('active');
|
||||
});
|
||||
|
||||
it('expires an abandoned restoring row (dead claim) but never a live one, even past the original expires_at', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({
|
||||
id: 'rec-abandoned', status: 'restoring', expires_at: 10_000, claim_expires_at: 15_000,
|
||||
}));
|
||||
db().insertServiceUpdateRecovery(makeRow({
|
||||
id: 'rec-live', status: 'restoring', expires_at: 10_000, claim_expires_at: 100_000,
|
||||
}));
|
||||
|
||||
const count = db().sweepAbandonedRestoringServiceUpdateRecoveries(20_000);
|
||||
expect(count).toBe(1);
|
||||
expect(db().getServiceUpdateRecovery('rec-abandoned')?.status).toBe('expired');
|
||||
// Original expires_at (10_000) is long past, but the claim is still live: not swept.
|
||||
expect(db().getServiceUpdateRecovery('rec-live')?.status).toBe('restoring');
|
||||
});
|
||||
|
||||
it('never touches an active row from the restoring sweep, and vice versa', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-active', status: 'active', expires_at: 10_000 }));
|
||||
db().insertServiceUpdateRecovery(makeRow({
|
||||
id: 'rec-restoring', status: 'restoring', expires_at: 999_000, claim_expires_at: 10_000,
|
||||
}));
|
||||
|
||||
expect(db().sweepAbandonedRestoringServiceUpdateRecoveries(20_000)).toBe(1);
|
||||
expect(db().getServiceUpdateRecovery('rec-active')?.status).toBe('active');
|
||||
|
||||
expect(db().sweepExpiredActiveServiceUpdateRecoveries(20_000)).toBe(1);
|
||||
expect(db().getServiceUpdateRecovery('rec-active')?.status).toBe('expired');
|
||||
});
|
||||
});
|
||||
|
||||
describe('held image ids', () => {
|
||||
it('holds the image for an active row and a restoring row with a live claim, not a terminal or abandoned one', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-active', majority_image_id: 'sha256:held-1', status: 'active', expires_at: 999_000 }));
|
||||
db().insertServiceUpdateRecovery(makeRow({
|
||||
id: 'rec-live-claim', majority_image_id: 'sha256:held-2', status: 'restoring', claim_expires_at: 100_000,
|
||||
}));
|
||||
db().insertServiceUpdateRecovery(makeRow({
|
||||
id: 'rec-dead-claim', majority_image_id: 'sha256:not-held-1', status: 'restoring', claim_expires_at: 5_000,
|
||||
}));
|
||||
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-consumed', majority_image_id: 'sha256:not-held-2', status: 'consumed' }));
|
||||
|
||||
const held = db().listHeldServiceUpdateRecoveryImageIds(NODE, 20_000);
|
||||
expect(new Set(held)).toEqual(new Set(['sha256:held-1', 'sha256:held-2']));
|
||||
});
|
||||
|
||||
it('scopes held image ids per node', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-node-1', node_id: 1, majority_image_id: 'sha256:node-1', expires_at: 999_000 }));
|
||||
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-node-2', node_id: 2, majority_image_id: 'sha256:node-2', expires_at: 999_000 }));
|
||||
|
||||
expect(db().listHeldServiceUpdateRecoveryImageIds(1, 0)).toEqual(['sha256:node-1']);
|
||||
expect(db().listHeldServiceUpdateRecoveryImageIds(2, 0)).toEqual(['sha256:node-2']);
|
||||
});
|
||||
});
|
||||
|
||||
it('deletes every row for a stack on stack delete, leaving other stacks untouched', () => {
|
||||
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-web', stack_name: 'web' }));
|
||||
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-api', stack_name: 'api' }));
|
||||
|
||||
db().deleteServiceUpdateRecoveries(NODE, 'web');
|
||||
|
||||
expect(db().getServiceUpdateRecovery('rec-web')).toBeUndefined();
|
||||
expect(db().getServiceUpdateRecovery('rec-api')).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -78,3 +78,86 @@ describe('stack_update_status tri-state accessors', () => {
|
||||
expect(db().getStackUpdateDetail(2).web.checkStatus).toBe('failed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('stack_update_status services_json (per-service reduction)', () => {
|
||||
const SERVICES = [
|
||||
{ service: 'web', image: 'web:latest', hasUpdate: true, checkStatus: 'ok' as const, lastError: null },
|
||||
{ service: 'worker', image: 'worker:latest', hasUpdate: false, checkStatus: 'ok' as const, lastError: null },
|
||||
];
|
||||
|
||||
it('persists and returns a per-service breakdown via getStackUpdateDetail', () => {
|
||||
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null, SERVICES, 3);
|
||||
|
||||
const detail = db().getStackUpdateDetail(NODE).stackA;
|
||||
expect(detail.services).toEqual(SERVICES);
|
||||
});
|
||||
|
||||
it('omits the services field entirely for a stack with no persisted per-service data', () => {
|
||||
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null);
|
||||
|
||||
const detail = db().getStackUpdateDetail(NODE).stackA;
|
||||
expect(detail.services).toBeUndefined();
|
||||
expect('services' in detail).toBe(false);
|
||||
});
|
||||
|
||||
it('leaves a prior services_json untouched when a later write omits services (COALESCE)', () => {
|
||||
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null, SERVICES, 1);
|
||||
// A legacy-path caller (or the fallback tally) writes without a services array.
|
||||
db().upsertStackUpdateStatus(NODE, 'stackA', true, 2000, 'ok', null);
|
||||
|
||||
expect(db().getStackUpdateDetail(NODE).stackA.services).toEqual(SERVICES);
|
||||
});
|
||||
|
||||
it('getStackServicesJson returns [] for a stack with no persisted row', () => {
|
||||
expect(db().getStackServicesJson(NODE, 'missing')).toEqual([]);
|
||||
});
|
||||
|
||||
it('getStackServicesJson returns [] for a stack row with no services_json', () => {
|
||||
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null);
|
||||
expect(db().getStackServicesJson(NODE, 'stackA')).toEqual([]);
|
||||
});
|
||||
|
||||
it('getStackServicesJson round-trips a persisted per-service breakdown', () => {
|
||||
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null, SERVICES, 5);
|
||||
expect(db().getStackServicesJson(NODE, 'stackA')).toEqual(SERVICES);
|
||||
});
|
||||
|
||||
it('recordStackCheckFailure persists a per-service breakdown alongside the failure', () => {
|
||||
db().recordStackCheckFailure(NODE, 'stackA', 'Registry unreachable', 4000, SERVICES, 2);
|
||||
|
||||
const detail = db().getStackUpdateDetail(NODE).stackA;
|
||||
expect(detail.checkStatus).toBe('failed');
|
||||
expect(detail.services).toEqual(SERVICES);
|
||||
});
|
||||
|
||||
it('treats corrupt services_json as an empty list rather than throwing (empty model invariant)', () => {
|
||||
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null);
|
||||
const raw = (db() as unknown as { db: { prepare: (s: string) => { run: (...args: unknown[]) => void } } }).db;
|
||||
raw.prepare('UPDATE stack_update_status SET services_json = ? WHERE node_id = ? AND stack_name = ?')
|
||||
.run('{not valid json', NODE, 'stackA');
|
||||
|
||||
expect(db().getStackServicesJson(NODE, 'stackA')).toEqual([]);
|
||||
const detail = db().getStackUpdateDetail(NODE).stackA;
|
||||
expect(detail.services).toBeUndefined();
|
||||
expect(detail.hasUpdate).toBe(true); // aggregate columns are unaffected by corrupt services_json
|
||||
});
|
||||
|
||||
it('treats a services_json shape-version mismatch as an empty list', () => {
|
||||
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null);
|
||||
const raw = (db() as unknown as { db: { prepare: (s: string) => { run: (...args: unknown[]) => void } } }).db;
|
||||
raw.prepare('UPDATE stack_update_status SET services_json = ? WHERE node_id = ? AND stack_name = ?')
|
||||
.run(JSON.stringify({ version: 999, generation: 1, services: SERVICES }), NODE, 'stackA');
|
||||
|
||||
expect(db().getStackServicesJson(NODE, 'stackA')).toEqual([]);
|
||||
});
|
||||
|
||||
it('filters out malformed entries within an otherwise valid services_json array', () => {
|
||||
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null);
|
||||
const raw = (db() as unknown as { db: { prepare: (s: string) => { run: (...args: unknown[]) => void } } }).db;
|
||||
const malformed = [...SERVICES, { service: 'broken' /* missing required fields */ }];
|
||||
raw.prepare('UPDATE stack_update_status SET services_json = ? WHERE node_id = ? AND stack_name = ?')
|
||||
.run(JSON.stringify({ version: 1, generation: 1, services: malformed }), NODE, 'stackA');
|
||||
|
||||
expect(db().getStackServicesJson(NODE, 'stackA')).toEqual(SERVICES);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -428,6 +428,83 @@ describe('DockerController - pruneDanglingImages', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── Prune holds (): images held for a pending service-update
|
||||
// recovery must never be deleted by any image-prune path. ──────────────
|
||||
|
||||
describe('DockerController - prune holds (isImageHeld)', () => {
|
||||
it('pruneSystem("images") falls back to enumerate-and-delete when isImageHeld is given, skipping the held image', async () => {
|
||||
mockDocker.listImages.mockResolvedValue([
|
||||
{ Id: 'img-held', RepoTags: ['app:1'], Containers: 0 },
|
||||
{ Id: 'img-free', RepoTags: ['app:2'], Containers: 0 },
|
||||
]);
|
||||
mockDocker.df
|
||||
.mockResolvedValueOnce({ LayersSize: 5000 })
|
||||
.mockResolvedValueOnce({ LayersSize: 3000 });
|
||||
const removeFn = vi.fn().mockResolvedValue(undefined);
|
||||
mockDocker.getImage.mockReturnValue({ remove: removeFn });
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const result = await dc.pruneSystem('images', undefined, (id: string) => id === 'img-held');
|
||||
|
||||
expect(mockDocker.pruneImages).not.toHaveBeenCalled();
|
||||
expect(removeFn).toHaveBeenCalledTimes(1);
|
||||
expect(mockDocker.getImage).toHaveBeenCalledWith('img-free');
|
||||
expect(result).toEqual({ success: true, reclaimedBytes: 2000 });
|
||||
});
|
||||
|
||||
it('pruneDanglingImages falls back to enumerate-and-delete when isImageHeld is given, skipping the held image', async () => {
|
||||
mockDocker.listImages.mockResolvedValue([
|
||||
{ Id: 'img-held', RepoTags: [], Containers: 0 },
|
||||
{ Id: 'img-free', RepoTags: ['<none>:<none>'], Containers: 0 },
|
||||
{ Id: 'img-tagged', RepoTags: ['app:1'], Containers: 0 }, // not dangling; excluded regardless of hold
|
||||
]);
|
||||
mockDocker.df
|
||||
.mockResolvedValueOnce({ LayersSize: 5000 })
|
||||
.mockResolvedValueOnce({ LayersSize: 4000 });
|
||||
const removeFn = vi.fn().mockResolvedValue(undefined);
|
||||
mockDocker.getImage.mockReturnValue({ remove: removeFn });
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const result = await dc.pruneDanglingImages((id: string) => id === 'img-held');
|
||||
|
||||
expect(mockDocker.pruneImages).not.toHaveBeenCalled();
|
||||
expect(removeFn).toHaveBeenCalledTimes(1);
|
||||
expect(mockDocker.getImage).toHaveBeenCalledWith('img-free');
|
||||
expect(result).toEqual({ success: true, reclaimedBytes: 1000 });
|
||||
});
|
||||
|
||||
it('pruneManagedOnly("images") re-checks isImageHeld immediately before each delete', async () => {
|
||||
mockDocker.df
|
||||
.mockResolvedValueOnce({ LayersSize: 5000, Images: [] })
|
||||
.mockResolvedValueOnce({ LayersSize: 4000, Images: [] });
|
||||
mockDocker.listImages.mockResolvedValue([
|
||||
{ Id: 'img-held', Containers: 0, Size: 1000, Labels: { 'com.docker.compose.project': 'any-stack' } },
|
||||
{ Id: 'img-free', Containers: 0, Size: 1000, Labels: { 'com.docker.compose.project': 'any-stack' } },
|
||||
]);
|
||||
mockDocker.listContainers.mockResolvedValue([]);
|
||||
const removeFn = vi.fn().mockResolvedValue(undefined);
|
||||
mockDocker.getImage.mockReturnValue({ remove: removeFn });
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const result = await dc.pruneManagedOnly('images', ['any-stack'], (id: string) => id === 'img-held');
|
||||
|
||||
expect(removeFn).toHaveBeenCalledTimes(1);
|
||||
expect(mockDocker.getImage).toHaveBeenCalledWith('img-free');
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('pruneSystem("images") keeps the bulk Docker prune API when no isImageHeld predicate is given', async () => {
|
||||
mockDocker.pruneImages.mockResolvedValue({ SpaceReclaimed: 999 });
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const result = await dc.pruneSystem('images');
|
||||
|
||||
expect(mockDocker.pruneImages).toHaveBeenCalled();
|
||||
expect(mockDocker.listImages).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({ success: true, reclaimedBytes: 999 });
|
||||
});
|
||||
});
|
||||
|
||||
// ── getClassifiedResources ─────────────────────────────────────────────
|
||||
|
||||
describe('DockerController - getClassifiedResources', () => {
|
||||
|
||||
@@ -246,6 +246,74 @@ describe('DriftLedgerService.reconcile', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('DriftLedgerService.reconcileService', () => {
|
||||
beforeEach(() => clearLedger('svcrec'));
|
||||
const ledger = () => DriftLedgerService.getInstance();
|
||||
|
||||
it('inserts only the selected service\'s findings, ignoring other services in the same report', () => {
|
||||
const report = reportWith([finding('image-mismatch', 'web'), finding('service-missing', 'db')], { stack: 'svcrec' });
|
||||
const res = ledger().reconcileService(nodeId, 'svcrec', 'web', report);
|
||||
expect(res).toEqual({ detected: 1, resolved: 0 });
|
||||
const open = db().getOpenDriftFindings(nodeId, 'svcrec');
|
||||
expect(open).toHaveLength(1);
|
||||
expect(open[0].service).toBe('web');
|
||||
});
|
||||
|
||||
it('resolves only the selected service\'s open finding, leaving other open services untouched', () => {
|
||||
ledger().reconcile(nodeId, 'svcrec', reportWith([finding('image-mismatch', 'web'), finding('service-missing', 'db')], { stack: 'svcrec' }));
|
||||
// 'web' cleared, 'db' absent from this service-scoped report too, but only 'web' is in scope.
|
||||
const res = ledger().reconcileService(nodeId, 'svcrec', 'web', reportWith([], { stack: 'svcrec', status: 'in-sync' }));
|
||||
expect(res).toEqual({ detected: 0, resolved: 1 });
|
||||
const open = db().getOpenDriftFindings(nodeId, 'svcrec');
|
||||
expect(open).toHaveLength(1);
|
||||
expect(open[0].service).toBe('db');
|
||||
});
|
||||
|
||||
it('preserves an empty-service (network-level) finding when reconciling one named service', () => {
|
||||
ledger().reconcile(nodeId, 'svcrec', reportWith([finding('image-mismatch', 'web'), finding('network-missing', '')], { stack: 'svcrec' }));
|
||||
const res = ledger().reconcileService(nodeId, 'svcrec', 'web', reportWith([], { stack: 'svcrec', status: 'in-sync' }));
|
||||
expect(res).toEqual({ detected: 0, resolved: 1 });
|
||||
const open = db().getOpenDriftFindings(nodeId, 'svcrec');
|
||||
expect(open).toHaveLength(1);
|
||||
expect(open[0].service).toBe('');
|
||||
});
|
||||
|
||||
it('does not touch findings when the same service is idempotently rechecked', () => {
|
||||
const report = reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' });
|
||||
ledger().reconcileService(nodeId, 'svcrec', 'web', report);
|
||||
const res = ledger().reconcileService(nodeId, 'svcrec', 'web', report);
|
||||
expect(res).toEqual({ detected: 0, resolved: 0 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'svcrec')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not reconcile an unreachable report (open findings are not falsely resolved)', () => {
|
||||
ledger().reconcile(nodeId, 'svcrec', reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' }));
|
||||
const res = ledger().reconcileService(nodeId, 'svcrec', 'web', { stack: 'svcrec', status: 'unreachable', hasComposeFile: true, hasContainers: false, findings: [] });
|
||||
expect(res).toEqual({ detected: 0, resolved: 0 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'svcrec')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not reconcile a parse-error report', () => {
|
||||
ledger().reconcile(nodeId, 'svcrec', reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' }));
|
||||
const res = ledger().reconcileService(nodeId, 'svcrec', 'web', { stack: 'svcrec', status: 'drifted', hasComposeFile: false, hasContainers: false, findings: [], parseError: 'bad yaml' });
|
||||
expect(res).toEqual({ detected: 0, resolved: 0 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'svcrec')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('names the service in the recorded activity message', () => {
|
||||
ledger().reconcileService(nodeId, 'svcrec', 'web', reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' }));
|
||||
const activity = driftActivity('svcrec');
|
||||
expect(activity).toHaveLength(1);
|
||||
expect(activity[0].message).toContain('web');
|
||||
});
|
||||
|
||||
it('stamps the dossier last-checked time', () => {
|
||||
expect(db().getStackDossier(nodeId, 'svcrec')?.last_drift_check_at ?? null).toBeNull();
|
||||
ledger().reconcileService(nodeId, 'svcrec', 'web', reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' }));
|
||||
expect(typeof db().getStackDossier(nodeId, 'svcrec')?.last_drift_check_at).toBe('number');
|
||||
});
|
||||
});
|
||||
|
||||
describe('DriftLedgerService.reconcileStack', () => {
|
||||
const STACK_A = 'recstacka';
|
||||
let dirA: string;
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* buildEffectiveServiceModel: turns `docker compose config --format json`
|
||||
* output into the per-service facts (declared image, build presence,
|
||||
* expected replicas, dependencies, healthcheck) that service-scoped update
|
||||
* and restore key off of. Fail-closed is the critical property: a render
|
||||
* failure must leave `renderable: false` with no services, never a
|
||||
* root-file fallback.
|
||||
*/
|
||||
import { describe, it, expect, afterEach, vi } from 'vitest';
|
||||
import { ComposeService } from '../services/ComposeService';
|
||||
import { buildEffectiveServiceModel } from '../services/effectiveServiceModel';
|
||||
|
||||
const ENV_SECRET = 'env-secret-9d4a-value';
|
||||
|
||||
function stubRender(rendered: string | null, stderr = '') {
|
||||
vi.spyOn(ComposeService, 'getInstance').mockReturnValue({
|
||||
renderConfig: vi.fn().mockResolvedValue({ rendered, stderr, code: rendered === null ? 1 : 0, timedOut: false }),
|
||||
} as unknown as ComposeService);
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('buildEffectiveServiceModel', () => {
|
||||
it('extracts declared image, dependsOn, and healthcheck for an image-only service', async () => {
|
||||
stubRender(JSON.stringify({
|
||||
services: {
|
||||
web: {
|
||||
image: 'nginx:latest',
|
||||
depends_on: { db: { condition: 'service_healthy', required: true } },
|
||||
healthcheck: { test: ['CMD', 'true'] },
|
||||
},
|
||||
db: { image: 'postgres:16' },
|
||||
},
|
||||
}));
|
||||
const result = await buildEffectiveServiceModel(1, 'mystack');
|
||||
expect(result.renderable).toBe(true);
|
||||
if (!result.renderable) throw new Error('expected renderable');
|
||||
expect(result.services).toEqual([
|
||||
{ name: 'web', declaredImage: 'nginx:latest', hasBuild: false, expectedReplicas: 1, dependsOn: ['db'], hasHealthcheck: true },
|
||||
{ name: 'db', declaredImage: 'postgres:16', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false },
|
||||
]);
|
||||
});
|
||||
|
||||
it('flags hasBuild for a build-backed service, including image+build', async () => {
|
||||
stubRender(JSON.stringify({
|
||||
services: {
|
||||
api: { build: { context: '.' } },
|
||||
worker: { image: 'app:latest', build: { context: '.' } },
|
||||
},
|
||||
}));
|
||||
const result = await buildEffectiveServiceModel(1, 'mystack');
|
||||
if (!result.renderable) throw new Error('expected renderable');
|
||||
expect(result.services[0]).toMatchObject({ name: 'api', declaredImage: null, hasBuild: true });
|
||||
expect(result.services[1]).toMatchObject({ name: 'worker', declaredImage: 'app:latest', hasBuild: true });
|
||||
});
|
||||
|
||||
it('reads expectedReplicas from the top-level scale field, deploy.replicas, or defaults to 1', async () => {
|
||||
stubRender(JSON.stringify({
|
||||
services: {
|
||||
scaled: { image: 'a', scale: 3 },
|
||||
deployed: { image: 'b', deploy: { replicas: 2 } },
|
||||
zeroed: { image: 'c', scale: 0 },
|
||||
unset: { image: 'd' },
|
||||
},
|
||||
}));
|
||||
const result = await buildEffectiveServiceModel(1, 'mystack');
|
||||
if (!result.renderable) throw new Error('expected renderable');
|
||||
expect(result.services.map(s => s.expectedReplicas)).toEqual([3, 2, 0, 1]);
|
||||
});
|
||||
|
||||
it('treats a disabled healthcheck as none', async () => {
|
||||
stubRender(JSON.stringify({
|
||||
services: { web: { image: 'a', healthcheck: { disable: true } } },
|
||||
}));
|
||||
const result = await buildEffectiveServiceModel(1, 'mystack');
|
||||
if (!result.renderable) throw new Error('expected renderable');
|
||||
expect(result.services[0].hasHealthcheck).toBe(false);
|
||||
});
|
||||
|
||||
it('parses depends_on given in the short list form', async () => {
|
||||
stubRender(JSON.stringify({
|
||||
services: { web: { image: 'a', depends_on: ['db', 'cache'] } },
|
||||
}));
|
||||
const result = await buildEffectiveServiceModel(1, 'mystack');
|
||||
if (!result.renderable) throw new Error('expected renderable');
|
||||
expect(result.services[0].dependsOn).toEqual(['db', 'cache']);
|
||||
});
|
||||
|
||||
it('fails closed with a redacted error and no services when the render errors', async () => {
|
||||
stubRender(null, `error: the "${ENV_SECRET}" variable is not set`);
|
||||
const result = await buildEffectiveServiceModel(1, 'mystack');
|
||||
expect(result.renderable).toBe(false);
|
||||
if (result.renderable) throw new Error('expected not renderable');
|
||||
expect(result.code).toBe('effective_model_render_failed');
|
||||
expect(result.error).not.toContain(ENV_SECRET);
|
||||
});
|
||||
|
||||
it('fails closed when the rendered output is not valid JSON', async () => {
|
||||
stubRender('this is not json {');
|
||||
const result = await buildEffectiveServiceModel(1, 'mystack');
|
||||
expect(result.renderable).toBe(false);
|
||||
if (result.renderable) throw new Error('expected not renderable');
|
||||
expect(result.code).toBe('effective_model_render_failed');
|
||||
});
|
||||
|
||||
it('fails closed and redacts the message when renderConfig throws (docker unavailable)', async () => {
|
||||
vi.spyOn(ComposeService, 'getInstance').mockReturnValue({
|
||||
renderConfig: vi.fn().mockRejectedValue(new Error(`spawn failed: password=${ENV_SECRET}`)),
|
||||
} as unknown as ComposeService);
|
||||
const result = await buildEffectiveServiceModel(1, 'mystack');
|
||||
expect(result.renderable).toBe(false);
|
||||
if (result.renderable) throw new Error('expected not renderable');
|
||||
expect(result.code).toBe('effective_model_render_failed');
|
||||
expect(result.error).not.toContain(ENV_SECRET);
|
||||
});
|
||||
|
||||
it('yields an empty service list for a garbage or empty rendered model', async () => {
|
||||
stubRender(JSON.stringify({ services: {} }));
|
||||
const result = await buildEffectiveServiceModel(1, 'mystack');
|
||||
if (!result.renderable) throw new Error('expected renderable');
|
||||
expect(result.services).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1168,7 +1168,7 @@ describe('GitSourceService.apply', () => {
|
||||
const { HealthGateService } = await import('../services/HealthGateService');
|
||||
const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue();
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-git');
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-git');
|
||||
const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!;
|
||||
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
/**
|
||||
* Restart-safe rebuild of health_gate_runs when widening the trigger CHECK and
|
||||
* adding target/failure columns.
|
||||
*/
|
||||
import path from 'path';
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import Database from 'better-sqlite3';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
|
||||
describe('migrateHealthGateTargetSchema', () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
it('recovers from a stale health_gate_runs_new and preserves existing rows', async () => {
|
||||
const dbPath = path.join(tmpDir, 'sencho.db');
|
||||
const raw = new Database(dbPath);
|
||||
raw.exec(`
|
||||
DROP TABLE IF EXISTS health_gate_runs;
|
||||
DROP TABLE IF EXISTS health_gate_runs_new;
|
||||
CREATE TABLE health_gate_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
node_id INTEGER NOT NULL,
|
||||
stack_name TEXT NOT NULL,
|
||||
trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy')),
|
||||
status TEXT NOT NULL CHECK (status IN ('observing','passed','failed','unknown')),
|
||||
reason TEXT,
|
||||
window_seconds INTEGER NOT NULL,
|
||||
containers_json TEXT NOT NULL DEFAULT '[]',
|
||||
started_at INTEGER NOT NULL,
|
||||
ended_at INTEGER,
|
||||
created_by TEXT
|
||||
);
|
||||
CREATE TABLE health_gate_runs_new (id TEXT PRIMARY KEY);
|
||||
INSERT INTO health_gate_runs (
|
||||
id, node_id, stack_name, trigger_action, status, reason, window_seconds, containers_json, started_at, ended_at, created_by
|
||||
) VALUES ('gate-1', 1, 'web', 'update', 'passed', null, 90, '[]', 1, 2, 'tester');
|
||||
`);
|
||||
raw.close();
|
||||
|
||||
const { DatabaseService } = await import('../services/DatabaseService');
|
||||
(DatabaseService as unknown as { instance?: typeof DatabaseService }).instance = undefined;
|
||||
const db = DatabaseService.getInstance();
|
||||
const row = db.getDb().prepare('SELECT * FROM health_gate_runs WHERE id = ?').get('gate-1') as {
|
||||
target_scope: string;
|
||||
service_name: string | null;
|
||||
failure_source: string | null;
|
||||
trigger_action: string;
|
||||
};
|
||||
expect(row.trigger_action).toBe('update');
|
||||
expect(row.target_scope).toBe('stack');
|
||||
expect(row.service_name).toBeNull();
|
||||
expect(row.failure_source).toBeNull();
|
||||
|
||||
const stale = db.getDb().prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'health_gate_runs_new'",
|
||||
).get();
|
||||
expect(stale).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,293 @@
|
||||
/**
|
||||
* Tests for the service-scoped health gate: prepare/attach/beginPrepared
|
||||
* nullability (disabled, unknown token, concurrency cap, armed), the prepare
|
||||
* TTL, and primary vs collateral failure attribution (regression-eligible
|
||||
* siblings can fail the gate; pre-existing unhealthy siblings stay advisory).
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
|
||||
interface StoredRun {
|
||||
id: string;
|
||||
node_id: number;
|
||||
stack_name: string;
|
||||
trigger_action: 'update' | 'deploy' | 'service_update' | 'service_restore';
|
||||
status: 'observing' | 'passed' | 'failed' | 'unknown';
|
||||
reason: string | null;
|
||||
window_seconds: number;
|
||||
containers_json: string;
|
||||
started_at: number;
|
||||
ended_at: number | null;
|
||||
created_by: string | null;
|
||||
target_scope: 'stack' | 'service';
|
||||
service_name: string | null;
|
||||
failure_source: 'primary' | 'collateral' | null;
|
||||
}
|
||||
|
||||
const { state } = vi.hoisted(() => ({
|
||||
state: {
|
||||
runs: new Map<string, StoredRun>(),
|
||||
settings: {} as Record<string, string>,
|
||||
listContainers: vi.fn(),
|
||||
inspect: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/DatabaseService', () => ({
|
||||
DatabaseService: {
|
||||
getInstance: () => ({
|
||||
getGlobalSettings: () => state.settings,
|
||||
insertHealthGateRun: (run: StoredRun) => { state.runs.set(run.id, { ...run }); },
|
||||
finalizeHealthGateRun: (id: string, status: StoredRun['status'], reason: string | null, endedAt: number, containersJson: string, failureSource: StoredRun['failure_source'] = null) => {
|
||||
const run = state.runs.get(id);
|
||||
if (run) Object.assign(run, { status, reason, ended_at: endedAt, containers_json: containersJson, failure_source: failureSource });
|
||||
},
|
||||
getHealthGateRun: (nodeId: number, stackName: string, id: string) => {
|
||||
const run = state.runs.get(id);
|
||||
return run && run.node_id === nodeId && run.stack_name === stackName ? { ...run } : undefined;
|
||||
},
|
||||
getLatestHealthGateRun: (nodeId: number, stackName: string) => {
|
||||
const matches = [...state.runs.values()]
|
||||
.filter(r => r.node_id === nodeId && r.stack_name === stackName)
|
||||
.sort((a, b) => b.started_at - a.started_at);
|
||||
return matches[0] ? { ...matches[0] } : undefined;
|
||||
},
|
||||
markInterruptedHealthGateRuns: () => 0,
|
||||
addNotificationHistory: (_nodeId: number, item: { category?: string; message: string; level: string }) => ({ ...item, id: 1, is_read: false }),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/DockerController', () => ({
|
||||
default: {
|
||||
getInstance: () => ({
|
||||
getDocker: () => ({
|
||||
listContainers: state.listContainers,
|
||||
getContainer: (id: string) => ({ inspect: () => state.inspect(id) }),
|
||||
}),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
// AutoHeal suppression is exercised elsewhere; here we only need the calls to
|
||||
// not throw when the gate finalizes a service run.
|
||||
vi.mock('../services/AutoHealService', () => ({
|
||||
AutoHealService: { getInstance: () => ({ clearSuppress: vi.fn() }) },
|
||||
}));
|
||||
|
||||
import { HealthGateService } from '../services/HealthGateService';
|
||||
|
||||
type Fixture = {
|
||||
id: string;
|
||||
name: string;
|
||||
service: string;
|
||||
state?: string;
|
||||
health?: string | null;
|
||||
restartCount?: number;
|
||||
startedAt?: string;
|
||||
imageId?: string;
|
||||
};
|
||||
|
||||
function setContainers(fixtures: Fixture[]): void {
|
||||
state.listContainers.mockResolvedValue(fixtures.map(f => ({
|
||||
Id: f.id,
|
||||
Names: [`/${f.name}`],
|
||||
State: f.state ?? 'running',
|
||||
Labels: { 'com.docker.compose.service': f.service },
|
||||
})));
|
||||
state.inspect.mockImplementation((id: string) => {
|
||||
const f = fixtures.find(c => c.id === id);
|
||||
if (!f) return Promise.reject(Object.assign(new Error('no such container'), { statusCode: 404 }));
|
||||
return Promise.resolve({
|
||||
State: {
|
||||
Status: f.state ?? 'running',
|
||||
Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined,
|
||||
StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z',
|
||||
},
|
||||
RestartCount: f.restartCount ?? 0,
|
||||
Image: f.imageId ?? 'sha256:app',
|
||||
Config: { Labels: { 'com.docker.compose.service': f.service } },
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const svc = () => HealthGateService.getInstance();
|
||||
|
||||
async function ticks(n: number): Promise<void> {
|
||||
for (let i = 0; i < n; i++) await vi.advanceTimersByTimeAsync(5_000);
|
||||
}
|
||||
|
||||
async function prepareService(
|
||||
fixtures: Fixture[],
|
||||
opts: { serviceName?: string; expectedReplicas?: number; trigger?: 'service_update' | 'service_restore' } = {},
|
||||
): Promise<string> {
|
||||
setContainers(fixtures);
|
||||
const { prepareToken } = await svc().prepare({
|
||||
nodeId: 0,
|
||||
stackName: 'web',
|
||||
target: { scope: 'service', serviceName: opts.serviceName ?? 'app' },
|
||||
trigger: opts.trigger ?? 'service_update',
|
||||
expectedReplicas: opts.expectedReplicas ?? 1,
|
||||
});
|
||||
return prepareToken;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
state.runs.clear();
|
||||
state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' };
|
||||
state.listContainers.mockReset();
|
||||
state.inspect.mockReset();
|
||||
svc().start();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
svc().stop();
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe('prepare / beginPrepared nullability', () => {
|
||||
it('prepare returns a token with a TTL that outlives the Compose timeout', async () => {
|
||||
const before = Date.now();
|
||||
setContainers([{ id: 'p1', name: 'web-app-1', service: 'app' }]);
|
||||
const { prepareToken, expiresAt } = await svc().prepare({
|
||||
nodeId: 0, stackName: 'web', target: { scope: 'service', serviceName: 'app' },
|
||||
trigger: 'service_update', expectedReplicas: 1,
|
||||
});
|
||||
expect(prepareToken).toBeTruthy();
|
||||
// Default compose timeout is 30m; prepare TTL is max(timeout, 30m) + 5m.
|
||||
expect(expiresAt).toBe(before + 35 * 60_000);
|
||||
});
|
||||
|
||||
it('returns runId null / observing false when gating is disabled', async () => {
|
||||
const token = await prepareService([{ id: 'p1', name: 'web-app-1', service: 'app' }]);
|
||||
state.settings.health_gate_enabled = '0';
|
||||
expect(svc().beginPrepared({ prepareToken: token, actor: 'tester' })).toEqual({ runId: null, observing: false });
|
||||
});
|
||||
|
||||
it('returns runId null / observing false for an unknown or consumed token', async () => {
|
||||
const token = await prepareService([{ id: 'p1', name: 'web-app-1', service: 'app' }]);
|
||||
// First consume arms a gate; the second call sees a consumed token.
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
const first = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
expect(first.observing).toBe(true);
|
||||
expect(svc().beginPrepared({ prepareToken: token, actor: 'tester' })).toEqual({ runId: null, observing: false });
|
||||
});
|
||||
|
||||
it('persists an immediate unknown past the concurrency cap', async () => {
|
||||
for (let i = 0; i < 25; i++) svc().beginStack(0, `stack-${i}`, 'update', 'tester');
|
||||
const token = await prepareService([{ id: 'p1', name: 'web-app-1', service: 'app' }]);
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
const result = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
expect(result.runId).toBeTruthy();
|
||||
expect(result.observing).toBe(false);
|
||||
const report = svc().getReport(0, 'web', result.runId!);
|
||||
expect(report.status).toBe('unknown');
|
||||
expect(report.reason).toContain('concurrent');
|
||||
});
|
||||
});
|
||||
|
||||
describe('primary vs collateral attribution', () => {
|
||||
it('fails with failureSource primary when a replica reports unhealthy', async () => {
|
||||
const token = await prepareService([
|
||||
{ id: 'p1', name: 'web-app-1', service: 'app' },
|
||||
{ id: 's1', name: 'web-db-1', service: 'db' },
|
||||
]);
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
await ticks(1); // arm expected set
|
||||
setContainers([
|
||||
{ id: 'p1', name: 'web-app-1', service: 'app', health: 'unhealthy' },
|
||||
{ id: 's1', name: 'web-db-1', service: 'db' },
|
||||
]);
|
||||
await ticks(1);
|
||||
const report = svc().getReport(0, 'web', runId!);
|
||||
expect(report.status).toBe('failed');
|
||||
expect(report.failureSource).toBe('primary');
|
||||
});
|
||||
|
||||
it('fails with failureSource collateral when a regression-eligible sibling exits', async () => {
|
||||
const token = await prepareService([
|
||||
{ id: 'p1', name: 'web-app-1', service: 'app' },
|
||||
{ id: 's1', name: 'web-db-1', service: 'db' },
|
||||
]);
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
await ticks(1);
|
||||
setContainers([
|
||||
{ id: 'p1', name: 'web-app-1', service: 'app' },
|
||||
{ id: 's1', name: 'web-db-1', service: 'db', state: 'exited' },
|
||||
]);
|
||||
await ticks(1);
|
||||
const report = svc().getReport(0, 'web', runId!);
|
||||
expect(report.status).toBe('failed');
|
||||
expect(report.failureSource).toBe('collateral');
|
||||
});
|
||||
|
||||
it('fails with failureSource collateral when a healthy sibling vanishes before the first poll', async () => {
|
||||
// Sibling is healthy at prepare, then gone before arming. Seeding expected
|
||||
// from the prepare baseline must still track it so the gate fails.
|
||||
const token = await prepareService([
|
||||
{ id: 'p1', name: 'web-app-1', service: 'app' },
|
||||
{ id: 's1', name: 'web-db-1', service: 'db' },
|
||||
]);
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
setContainers([
|
||||
{ id: 'p1', name: 'web-app-1', service: 'app' },
|
||||
]);
|
||||
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
await ticks(1); // arm expected from primary + prepare collateral baseline
|
||||
await ticks(1); // first miss on the vanished sibling
|
||||
await ticks(1); // second miss fails the gate
|
||||
const report = svc().getReport(0, 'web', runId!);
|
||||
expect(report.status).toBe('failed');
|
||||
expect(report.failureSource).toBe('collateral');
|
||||
expect(report.reason ?? '').toMatch(/disappeared|sibling/i);
|
||||
});
|
||||
|
||||
it('keeps a pre-existing unhealthy sibling advisory: the gate can still pass', async () => {
|
||||
// The sibling is unhealthy at prepare, so it is not regression-eligible and
|
||||
// cannot fail the service gate; the healthy primary carries it to a pass.
|
||||
const token = await prepareService([
|
||||
{ id: 'p1', name: 'web-app-1', service: 'app' },
|
||||
{ id: 's1', name: 'web-db-1', service: 'db', health: 'unhealthy' },
|
||||
]);
|
||||
svc().attachExpectedImage(token, 'sha256:app');
|
||||
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
|
||||
await ticks(8); // through the 30s window
|
||||
const report = svc().getReport(0, 'web', runId!);
|
||||
expect(report.status).toBe('passed');
|
||||
expect(report.failureSource).toBeNull();
|
||||
});
|
||||
|
||||
it('lets different-service gates coexist while same-service begin supersedes', async () => {
|
||||
const appToken = await prepareService(
|
||||
[{ id: 'p1', name: 'web-app-1', service: 'app' }, { id: 's1', name: 'web-db-1', service: 'db' }],
|
||||
{ serviceName: 'app' },
|
||||
);
|
||||
svc().attachExpectedImage(appToken, 'sha256:app');
|
||||
const appFirst = svc().beginPrepared({ prepareToken: appToken, actor: 'tester' });
|
||||
expect(appFirst.observing).toBe(true);
|
||||
|
||||
const dbToken = await prepareService(
|
||||
[{ id: 'p1', name: 'web-app-1', service: 'app' }, { id: 's1', name: 'web-db-1', service: 'db' }],
|
||||
{ serviceName: 'db' },
|
||||
);
|
||||
svc().attachExpectedImage(dbToken, 'sha256:db');
|
||||
const dbGate = svc().beginPrepared({ prepareToken: dbToken, actor: 'tester' });
|
||||
expect(dbGate.observing).toBe(true);
|
||||
expect(svc().getReport(0, 'web', appFirst.runId!).status).toBe('observing');
|
||||
expect(svc().getReport(0, 'web', dbGate.runId!).status).toBe('observing');
|
||||
|
||||
const appToken2 = await prepareService(
|
||||
[{ id: 'p1', name: 'web-app-1', service: 'app' }, { id: 's1', name: 'web-db-1', service: 'db' }],
|
||||
{ serviceName: 'app' },
|
||||
);
|
||||
svc().attachExpectedImage(appToken2, 'sha256:app2');
|
||||
const appSecond = svc().beginPrepared({ prepareToken: appToken2, actor: 'tester' });
|
||||
expect(appSecond.observing).toBe(true);
|
||||
expect(svc().getReport(0, 'web', appFirst.runId!).status).toBe('unknown');
|
||||
expect(svc().getReport(0, 'web', dbGate.runId!).status).toBe('observing');
|
||||
expect(svc().getReport(0, 'web', appSecond.runId!).status).toBe('observing');
|
||||
});
|
||||
});
|
||||
@@ -9,7 +9,7 @@ interface StoredRun {
|
||||
id: string;
|
||||
node_id: number;
|
||||
stack_name: string;
|
||||
trigger_action: 'update' | 'deploy';
|
||||
trigger_action: 'update' | 'deploy' | 'service_update' | 'service_restore';
|
||||
status: 'observing' | 'passed' | 'failed' | 'unknown';
|
||||
reason: string | null;
|
||||
window_seconds: number;
|
||||
@@ -17,6 +17,9 @@ interface StoredRun {
|
||||
started_at: number;
|
||||
ended_at: number | null;
|
||||
created_by: string | null;
|
||||
target_scope: 'stack' | 'service';
|
||||
service_name: string | null;
|
||||
failure_source: 'primary' | 'collateral' | null;
|
||||
}
|
||||
|
||||
const { state } = vi.hoisted(() => ({
|
||||
@@ -34,9 +37,9 @@ vi.mock('../services/DatabaseService', () => ({
|
||||
getInstance: () => ({
|
||||
getGlobalSettings: () => state.settings,
|
||||
insertHealthGateRun: (run: StoredRun) => { state.runs.set(run.id, { ...run }); },
|
||||
finalizeHealthGateRun: (id: string, status: StoredRun['status'], reason: string | null, endedAt: number, containersJson: string) => {
|
||||
finalizeHealthGateRun: (id: string, status: StoredRun['status'], reason: string | null, endedAt: number, containersJson: string, failureSource: StoredRun['failure_source'] = null) => {
|
||||
const run = state.runs.get(id);
|
||||
if (run) Object.assign(run, { status, reason, ended_at: endedAt, containers_json: containersJson });
|
||||
if (run) Object.assign(run, { status, reason, ended_at: endedAt, containers_json: containersJson, failure_source: failureSource });
|
||||
},
|
||||
getHealthGateRun: (nodeId: number, stackName: string, id: string) => {
|
||||
const run = state.runs.get(id);
|
||||
@@ -134,7 +137,7 @@ afterEach(() => {
|
||||
|
||||
describe('HealthGateService verdicts', () => {
|
||||
it('passes at the window end when containers stay running', async () => {
|
||||
const id = svc().begin(0, 'web', 'update', 'tester');
|
||||
const id = svc().beginStack(0, 'web', 'update', 'tester');
|
||||
expect(id).toBeTruthy();
|
||||
await ticks(3); // 15s: still observing
|
||||
expect(latest().status).toBe('observing');
|
||||
@@ -144,7 +147,7 @@ describe('HealthGateService verdicts', () => {
|
||||
});
|
||||
|
||||
it('fails fast when a container exits', async () => {
|
||||
svc().begin(0, 'web', 'update', 'tester');
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1); // baseline
|
||||
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited' }]);
|
||||
await ticks(1);
|
||||
@@ -155,7 +158,7 @@ describe('HealthGateService verdicts', () => {
|
||||
});
|
||||
|
||||
it('fails fast when a healthcheck reports unhealthy', async () => {
|
||||
svc().begin(0, 'web', 'update', 'tester');
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
setContainers([{ id: 'aaa', name: 'web-app-1', health: 'unhealthy' }]);
|
||||
await ticks(1);
|
||||
@@ -164,7 +167,7 @@ describe('HealthGateService verdicts', () => {
|
||||
});
|
||||
|
||||
it('detects a restart loop via container replacement (new id)', async () => {
|
||||
svc().begin(0, 'web', 'update', 'tester');
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
setContainers([{ id: 'bbb', name: 'web-app-1' }]);
|
||||
await ticks(1); // restart 1 observed; carried as new baseline
|
||||
@@ -180,7 +183,7 @@ describe('HealthGateService verdicts', () => {
|
||||
});
|
||||
|
||||
it('detects a restart loop via RestartCount and StartedAt movement', async () => {
|
||||
svc().begin(0, 'web', 'update', 'tester');
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
setContainers([{ id: 'aaa', name: 'web-app-1', restartCount: 1 }]);
|
||||
await ticks(1);
|
||||
@@ -195,7 +198,7 @@ describe('HealthGateService verdicts', () => {
|
||||
});
|
||||
|
||||
it('tolerates a one-poll disappearance but fails on two consecutive misses', async () => {
|
||||
svc().begin(0, 'web', 'update', 'tester');
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1); // baseline
|
||||
setContainers([]); // one missed poll: tolerated
|
||||
await ticks(1);
|
||||
@@ -203,7 +206,7 @@ describe('HealthGateService verdicts', () => {
|
||||
await ticks(5); // through the 30s window
|
||||
expect(latest().status).toBe('passed');
|
||||
|
||||
const second = svc().begin(0, 'web', 'update', 'tester')!;
|
||||
const second = svc().beginStack(0, 'web', 'update', 'tester')!;
|
||||
await ticks(1);
|
||||
setContainers([]);
|
||||
await ticks(2); // two consecutive misses: disappeared
|
||||
@@ -213,7 +216,7 @@ describe('HealthGateService verdicts', () => {
|
||||
});
|
||||
|
||||
it('fails when a container is stuck restarting across consecutive polls', async () => {
|
||||
svc().begin(0, 'web', 'update', 'tester');
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'restarting' }]);
|
||||
await ticks(2);
|
||||
@@ -222,7 +225,7 @@ describe('HealthGateService verdicts', () => {
|
||||
});
|
||||
|
||||
it('goes unknown after three consecutive docker errors', async () => {
|
||||
svc().begin(0, 'web', 'update', 'tester');
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(1);
|
||||
state.listContainers.mockRejectedValue(new Error('socket gone'));
|
||||
await ticks(3);
|
||||
@@ -231,7 +234,7 @@ describe('HealthGateService verdicts', () => {
|
||||
});
|
||||
|
||||
it('resolves unknown when every docker observe hangs', async () => {
|
||||
svc().begin(0, 'web', 'update', 'tester');
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
// A wedged socket never settles. The per-observe timeout turns each poll
|
||||
// into an error, and three in a row finalize the gate unknown instead of
|
||||
// observing forever on a pending promise.
|
||||
@@ -244,7 +247,7 @@ describe('HealthGateService verdicts', () => {
|
||||
});
|
||||
|
||||
it('recovers from a transient observe timeout instead of finalizing', async () => {
|
||||
svc().begin(0, 'web', 'update', 'tester');
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
// One observe wedges and times out (a single strike), then the socket
|
||||
// recovers; the gate must keep observing, not give up at one error.
|
||||
state.listContainers.mockImplementationOnce(() => new Promise<never>(() => {}));
|
||||
@@ -257,7 +260,7 @@ describe('HealthGateService verdicts', () => {
|
||||
});
|
||||
|
||||
it('runs polls single-flight: no second observe until the first settles', async () => {
|
||||
svc().begin(0, 'web', 'update', 'tester');
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
let release: (value: Array<{ Id: string; Names: string[]; State: string }>) => void = () => {};
|
||||
state.listContainers.mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
|
||||
// Advance past a second poll interval while the first observe is still
|
||||
@@ -272,7 +275,7 @@ describe('HealthGateService verdicts', () => {
|
||||
});
|
||||
|
||||
it('ends unknown when a healthcheck is still starting at the window end', async () => {
|
||||
svc().begin(0, 'web', 'update', 'tester');
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
setContainers([{ id: 'aaa', name: 'web-app-1', health: 'starting' }]);
|
||||
await ticks(7);
|
||||
expect(latest().status).toBe('unknown');
|
||||
@@ -281,7 +284,7 @@ describe('HealthGateService verdicts', () => {
|
||||
|
||||
it('goes unknown when no containers ever appear', async () => {
|
||||
setContainers([]);
|
||||
svc().begin(0, 'web', 'update', 'tester');
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
await ticks(4);
|
||||
expect(latest().status).toBe('unknown');
|
||||
expect(latest().reason).toContain('no containers');
|
||||
@@ -293,7 +296,7 @@ describe('HealthGateService lifecycle', () => {
|
||||
// A poll is mid-await on Docker when a newer update supersedes the gate;
|
||||
// when the await resolves with healthy containers, the superseded run
|
||||
// must keep its terminal unknown verdict.
|
||||
const first = svc().begin(0, 'web', 'update', 'tester')!;
|
||||
const first = svc().beginStack(0, 'web', 'update', 'tester')!;
|
||||
await ticks(2); // baseline established, healthy
|
||||
|
||||
let releasePoll: (value: Array<{ Id: string; Names: string[]; State: string }>) => void = () => {};
|
||||
@@ -302,7 +305,7 @@ describe('HealthGateService lifecycle', () => {
|
||||
);
|
||||
const straddlingPoll = vi.advanceTimersByTimeAsync(5_000); // poll now awaiting Docker
|
||||
|
||||
const second = svc().begin(0, 'web', 'update', 'tester')!;
|
||||
const second = svc().beginStack(0, 'web', 'update', 'tester')!;
|
||||
expect(svc().getReport(0, 'web', first).status).toBe('unknown');
|
||||
|
||||
releasePoll([{ Id: 'aaa', Names: ['/web-app-1'], State: 'running' }]);
|
||||
@@ -317,10 +320,10 @@ describe('HealthGateService lifecycle', () => {
|
||||
});
|
||||
|
||||
it('supersede finalizes the old run as unknown, clears its timer, and getRun still resolves it', async () => {
|
||||
const first = svc().begin(0, 'web', 'update', 'tester')!;
|
||||
const first = svc().beginStack(0, 'web', 'update', 'tester')!;
|
||||
await ticks(1);
|
||||
const timersBefore = vi.getTimerCount();
|
||||
const second = svc().begin(0, 'web', 'update', 'tester')!;
|
||||
const second = svc().beginStack(0, 'web', 'update', 'tester')!;
|
||||
expect(vi.getTimerCount()).toBe(timersBefore); // old interval cleared, new one added
|
||||
|
||||
const superseded = svc().getReport(0, 'web', first);
|
||||
@@ -337,6 +340,7 @@ describe('HealthGateService lifecycle', () => {
|
||||
state.runs.set('stale', {
|
||||
id: 'stale', node_id: 0, stack_name: 'web', trigger_action: 'update', status: 'observing',
|
||||
reason: null, window_seconds: 30, containers_json: '[]', started_at: 1, ended_at: null, created_by: null,
|
||||
target_scope: 'stack', service_name: null, failure_source: null,
|
||||
});
|
||||
svc().start();
|
||||
expect(state.runs.get('stale')!.status).toBe('unknown');
|
||||
@@ -345,7 +349,7 @@ describe('HealthGateService lifecycle', () => {
|
||||
|
||||
it('no-ops when disabled but still records the update_started event', () => {
|
||||
state.settings.health_gate_enabled = '0';
|
||||
const id = svc().begin(0, 'web', 'update', 'tester');
|
||||
const id = svc().beginStack(0, 'web', 'update', 'tester');
|
||||
expect(id).toBeNull();
|
||||
expect(state.runs.size).toBe(0);
|
||||
expect(state.activity.some(a => a.category === 'update_started')).toBe(true);
|
||||
@@ -353,24 +357,24 @@ describe('HealthGateService lifecycle', () => {
|
||||
});
|
||||
|
||||
it('records update_started for update triggers but not deploy triggers', () => {
|
||||
svc().begin(0, 'web', 'deploy', 'tester');
|
||||
svc().beginStack(0, 'web', 'deploy', 'tester');
|
||||
expect(state.activity.some(a => a.category === 'update_started')).toBe(false);
|
||||
svc().begin(0, 'web', 'update', 'tester');
|
||||
svc().beginStack(0, 'web', 'update', 'tester');
|
||||
expect(state.activity.some(a => a.category === 'update_started')).toBe(true);
|
||||
});
|
||||
|
||||
it('refuses to begin before start() so shutdown cannot leak timers', () => {
|
||||
svc().stop();
|
||||
expect(svc().begin(0, 'web', 'update', 'tester')).toBeNull();
|
||||
expect(svc().beginStack(0, 'web', 'update', 'tester')).toBeNull();
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
svc().start();
|
||||
});
|
||||
|
||||
it('persists an immediate unknown past the concurrency cap', () => {
|
||||
for (let i = 0; i < 25; i++) {
|
||||
svc().begin(0, `stack-${i}`, 'update', 'tester');
|
||||
svc().beginStack(0, `stack-${i}`, 'update', 'tester');
|
||||
}
|
||||
const overCap = svc().begin(0, 'one-too-many', 'update', 'tester')!;
|
||||
const overCap = svc().beginStack(0, 'one-too-many', 'update', 'tester')!;
|
||||
const report = svc().getReport(0, 'one-too-many', overCap);
|
||||
expect(report.status).toBe('unknown');
|
||||
expect(report.reason).toContain('concurrent');
|
||||
@@ -378,10 +382,10 @@ describe('HealthGateService lifecycle', () => {
|
||||
|
||||
it('clamps the configured window into its valid range and falls back on garbage', () => {
|
||||
state.settings.health_gate_window_seconds = '99999';
|
||||
const a = svc().begin(0, 'web', 'update', 'tester')!;
|
||||
const a = svc().beginStack(0, 'web', 'update', 'tester')!;
|
||||
expect(svc().getReport(0, 'web', a).windowSeconds).toBe(600);
|
||||
state.settings.health_gate_window_seconds = 'banana';
|
||||
const b = svc().begin(0, 'web', 'update', 'tester')!;
|
||||
const b = svc().beginStack(0, 'web', 'update', 'tester')!;
|
||||
expect(svc().getReport(0, 'web', b).windowSeconds).toBe(90);
|
||||
});
|
||||
|
||||
@@ -392,7 +396,7 @@ describe('HealthGateService lifecycle', () => {
|
||||
});
|
||||
|
||||
it('stop() finalizes in-flight gates as unknown with zero timers left', async () => {
|
||||
const id = svc().begin(0, 'web', 'update', 'tester')!;
|
||||
const id = svc().beginStack(0, 'web', 'update', 'tester')!;
|
||||
await ticks(1);
|
||||
svc().stop();
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
/**
|
||||
* Unit tests for the §5 per-service reduction pure functions:
|
||||
* reduceServiceStatus, aggregateServiceCheckStatus, and
|
||||
* buildAvailabilityNotifyMessage. These operate on plain data (an
|
||||
* imageUpdateMap and a prior per-service snapshot), so they are tested
|
||||
* directly without mocking DatabaseService, Docker, or Compose.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
reduceServiceStatus,
|
||||
aggregateServiceCheckStatus,
|
||||
buildAvailabilityNotifyMessage,
|
||||
type ImageCheckResult,
|
||||
} from '../services/ImageUpdateService';
|
||||
import type { StackServiceStatus } from '../services/DatabaseService';
|
||||
|
||||
function ok(hasUpdate: boolean): ImageCheckResult {
|
||||
return { hasUpdate };
|
||||
}
|
||||
function errored(message: string): ImageCheckResult {
|
||||
return { hasUpdate: false, error: message };
|
||||
}
|
||||
function notCheckable(): ImageCheckResult {
|
||||
return { hasUpdate: false, notCheckable: true };
|
||||
}
|
||||
|
||||
describe('reduceServiceStatus', () => {
|
||||
it('marks a service not_checkable when it has no checkable ref (STATUS-NOT-CHECKABLE-3)', () => {
|
||||
const map = new Map([['app:latest', notCheckable()]]);
|
||||
const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', [], map, undefined);
|
||||
expect(status).toEqual({ service: 'app', image: 'app:latest', hasUpdate: false, checkStatus: 'not_checkable', lastError: null });
|
||||
expect(confirmedUpdateThisRun).toBe(false);
|
||||
});
|
||||
|
||||
it('marks a service not_checkable when it declares no image and has no running container', () => {
|
||||
const map = new Map<string, ImageCheckResult>();
|
||||
const { status, confirmedUpdateThisRun } = reduceServiceStatus('worker', null, [], map, undefined);
|
||||
expect(status).toEqual({ service: 'worker', image: null, hasUpdate: false, checkStatus: 'not_checkable', lastError: null });
|
||||
expect(confirmedUpdateThisRun).toBe(false);
|
||||
});
|
||||
|
||||
it('reports ok + hasUpdate=true when every checkable ref succeeds and one confirms an update', () => {
|
||||
const map = new Map([['app:latest', ok(true)]]);
|
||||
const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', [], map, undefined);
|
||||
expect(status).toEqual({ service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null });
|
||||
expect(confirmedUpdateThisRun).toBe(true);
|
||||
});
|
||||
|
||||
it('reports ok + hasUpdate=false when every checkable ref succeeds with no update', () => {
|
||||
const map = new Map([['app:latest', ok(false)]]);
|
||||
const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', [], map, undefined);
|
||||
expect(status.checkStatus).toBe('ok');
|
||||
expect(status.hasUpdate).toBe(false);
|
||||
expect(confirmedUpdateThisRun).toBe(false);
|
||||
});
|
||||
|
||||
it('STATUS-CONFIRMATION-PARTIAL-5: one newly outdated success + one error -> partial, hasUpdate=true, confirmedUpdateThisRun=true', () => {
|
||||
// Two refs on the same declared service (declared image + a divergent runtime image).
|
||||
const map = new Map([
|
||||
['app:latest', ok(true)],
|
||||
['app:1.2.3', errored('Registry unreachable for app:1.2.3')],
|
||||
]);
|
||||
const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', ['app:1.2.3'], map, undefined);
|
||||
expect(status.checkStatus).toBe('partial');
|
||||
expect(status.hasUpdate).toBe(true);
|
||||
expect(status.lastError).toBe('Registry unreachable for app:1.2.3');
|
||||
expect(confirmedUpdateThisRun).toBe(true);
|
||||
});
|
||||
|
||||
it('partial with only a preserved prior (no successful outdated check this run) -> confirmedUpdateThisRun=false', () => {
|
||||
const map = new Map([
|
||||
['app:latest', ok(false)], // succeeds, but reports no update
|
||||
['app:1.2.3', errored('timeout')],
|
||||
]);
|
||||
const prior: StackServiceStatus = { service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null };
|
||||
const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', ['app:1.2.3'], map, prior);
|
||||
expect(status.checkStatus).toBe('partial');
|
||||
expect(status.hasUpdate).toBe(true); // preserved from prior, not newly confirmed
|
||||
expect(confirmedUpdateThisRun).toBe(false);
|
||||
});
|
||||
|
||||
it('all checkable refs erroring -> failed, preserves prior hasUpdate, confirmedUpdateThisRun=false', () => {
|
||||
const map = new Map([['app:latest', errored('Registry unreachable')]]);
|
||||
const prior: StackServiceStatus = { service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null };
|
||||
const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', [], map, prior);
|
||||
expect(status).toEqual({ service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'failed', lastError: 'Registry unreachable' });
|
||||
expect(confirmedUpdateThisRun).toBe(false);
|
||||
});
|
||||
|
||||
it('all checkable refs erroring with no prior row -> failed, hasUpdate=false', () => {
|
||||
const map = new Map([['app:latest', errored('boom')]]);
|
||||
const { status } = reduceServiceStatus('app', 'app:latest', [], map, undefined);
|
||||
expect(status.hasUpdate).toBe(false);
|
||||
expect(status.checkStatus).toBe('failed');
|
||||
});
|
||||
|
||||
it('a shared image gets the same check result on every declared service that references it', () => {
|
||||
const map = new Map([['shared:latest', ok(true)]]);
|
||||
const web = reduceServiceStatus('web', 'shared:latest', [], map, undefined);
|
||||
const worker = reduceServiceStatus('worker', 'shared:latest', [], map, undefined);
|
||||
expect(web.status.hasUpdate).toBe(true);
|
||||
expect(worker.status.hasUpdate).toBe(true);
|
||||
expect(web.status.checkStatus).toBe(worker.status.checkStatus);
|
||||
});
|
||||
|
||||
it('records sorted, deduplicated runtimeImages only when a runtime container was observed', () => {
|
||||
const map = new Map([['app:latest', ok(false)], ['app:1.0', ok(false)]]);
|
||||
const withRuntime = reduceServiceStatus('app', 'app:latest', ['app:1.0', 'app:1.0'], map, undefined);
|
||||
expect(withRuntime.status.runtimeImages).toEqual(['app:1.0']);
|
||||
|
||||
const noRuntime = reduceServiceStatus('app', 'app:latest', [], map, undefined);
|
||||
expect(noRuntime.status.runtimeImages).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('aggregateServiceCheckStatus', () => {
|
||||
const svc = (checkStatus: StackServiceStatus['checkStatus'], hasUpdate = false): StackServiceStatus => ({
|
||||
service: `svc-${checkStatus}-${hasUpdate}`, image: null, hasUpdate, checkStatus, lastError: null,
|
||||
});
|
||||
|
||||
it('is ok for an empty services array (empty effective model)', () => {
|
||||
expect(aggregateServiceCheckStatus([])).toBe('ok');
|
||||
});
|
||||
|
||||
it('is ok when every service is not_checkable (nothing checkable)', () => {
|
||||
expect(aggregateServiceCheckStatus([svc('not_checkable'), svc('not_checkable')])).toBe('ok');
|
||||
});
|
||||
|
||||
it('is ok when every checkable service succeeded', () => {
|
||||
expect(aggregateServiceCheckStatus([svc('ok'), svc('ok'), svc('not_checkable')])).toBe('ok');
|
||||
});
|
||||
|
||||
it('is failed when every checkable service failed, even with a preserved hasUpdate=true', () => {
|
||||
expect(aggregateServiceCheckStatus([svc('failed', true), svc('failed', true)])).toBe('failed');
|
||||
});
|
||||
|
||||
it('is partial when checkable services mix ok and failed', () => {
|
||||
expect(aggregateServiceCheckStatus([svc('ok'), svc('failed')])).toBe('partial');
|
||||
});
|
||||
|
||||
it('is partial when a checkable service is itself partial', () => {
|
||||
expect(aggregateServiceCheckStatus([svc('ok'), svc('partial')])).toBe('partial');
|
||||
});
|
||||
|
||||
it('is not failed when only some (not all) checkable services failed', () => {
|
||||
expect(aggregateServiceCheckStatus([svc('ok'), svc('failed'), svc('not_checkable')])).not.toBe('failed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('aggregate invariant: has_update === services.some(hasUpdate)', () => {
|
||||
it('holds for an empty services array', () => {
|
||||
const services: StackServiceStatus[] = [];
|
||||
expect(services.some(s => s.hasUpdate)).toBe(false);
|
||||
});
|
||||
|
||||
it('holds when every checkable service failed but one preserves hasUpdate=true', () => {
|
||||
const map = new Map([['app:latest', errored('unreachable')]]);
|
||||
const prior: StackServiceStatus = { service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null };
|
||||
const { status } = reduceServiceStatus('app', 'app:latest', [], map, prior);
|
||||
const services = [status];
|
||||
expect(services.some(s => s.hasUpdate)).toBe(true);
|
||||
expect(aggregateServiceCheckStatus(services)).toBe('failed');
|
||||
});
|
||||
|
||||
it('holds when one service confirms an update and a sibling service fails outright', () => {
|
||||
const map = new Map([
|
||||
['web:latest', ok(true)],
|
||||
['worker:latest', errored('timeout')],
|
||||
]);
|
||||
const web = reduceServiceStatus('web', 'web:latest', [], map, undefined);
|
||||
const worker = reduceServiceStatus('worker', 'worker:latest', [], map, undefined);
|
||||
const services = [web.status, worker.status];
|
||||
// web confirmed an update; worker's sole ref errored, so worker alone is 'failed'.
|
||||
expect(services.some(s => s.hasUpdate)).toBe(true);
|
||||
expect(worker.status.checkStatus).toBe('failed');
|
||||
// At the stack level, one ok service and one failed service is a mix, not a unanimous failure.
|
||||
expect(aggregateServiceCheckStatus(services)).toBe('partial');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildAvailabilityNotifyMessage', () => {
|
||||
it('uses the generic line for a single-service (or model-unavailable) stack', () => {
|
||||
expect(buildAvailabilityNotifyMessage('stackA', [])).toBe('Stack "stackA" has image updates available.');
|
||||
const single: StackServiceStatus[] = [{ service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null }];
|
||||
expect(buildAvailabilityNotifyMessage('stackA', single)).toBe('Stack "stackA" has image updates available.');
|
||||
});
|
||||
|
||||
it('names sorted services with hasUpdate for a multi-service stack', () => {
|
||||
const services: StackServiceStatus[] = [
|
||||
{ service: 'worker', image: 'worker:latest', hasUpdate: true, checkStatus: 'ok', lastError: null },
|
||||
{ service: 'api', image: 'api:latest', hasUpdate: true, checkStatus: 'ok', lastError: null },
|
||||
{ service: 'db', image: 'db:latest', hasUpdate: false, checkStatus: 'ok', lastError: null },
|
||||
];
|
||||
expect(buildAvailabilityNotifyMessage('stackA', services)).toBe('Stack "stackA" has image updates available for services: api, worker.');
|
||||
});
|
||||
|
||||
it('falls back to the generic line for a multi-service stack with no service currently showing hasUpdate', () => {
|
||||
const services: StackServiceStatus[] = [
|
||||
{ service: 'api', image: 'api:latest', hasUpdate: false, checkStatus: 'ok', lastError: null },
|
||||
{ service: 'db', image: 'db:latest', hasUpdate: false, checkStatus: 'ok', lastError: null },
|
||||
];
|
||||
expect(buildAvailabilityNotifyMessage('stackA', services)).toBe('Stack "stackA" has image updates available.');
|
||||
});
|
||||
|
||||
it('uses singular wording for exactly one named service', () => {
|
||||
const services: StackServiceStatus[] = [
|
||||
{ service: 'api', image: 'api:latest', hasUpdate: true, checkStatus: 'ok', lastError: null },
|
||||
{ service: 'db', image: 'db:latest', hasUpdate: false, checkStatus: 'ok', lastError: null },
|
||||
];
|
||||
expect(buildAvailabilityNotifyMessage('stackA', services)).toBe('Stack "stackA" has image updates available for service: api.');
|
||||
});
|
||||
});
|
||||
@@ -9,17 +9,19 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
const {
|
||||
mockGetAuthForRegistry,
|
||||
mockGetStackUpdateStatus, mockUpsertStackUpdateStatus, mockClearStackUpdateStatus,
|
||||
mockRecordStackCheckFailure,
|
||||
mockRecordStackCheckFailure, mockGetStackServicesJson,
|
||||
mockGetSystemState, mockSetSystemState, mockAddNotificationHistory,
|
||||
mockDispatchAlert,
|
||||
mockGetStacks, mockGetStackContent, mockGetEnvContent, mockEnvExists,
|
||||
mockGetAllContainers, mockGetGlobalSettings, mockInspect,
|
||||
mockBuildEffectiveServiceModel,
|
||||
} = vi.hoisted(() => ({
|
||||
mockGetAuthForRegistry: vi.fn().mockResolvedValue(null),
|
||||
mockGetStackUpdateStatus: vi.fn().mockReturnValue({}),
|
||||
mockUpsertStackUpdateStatus: vi.fn(),
|
||||
mockClearStackUpdateStatus: vi.fn(),
|
||||
mockRecordStackCheckFailure: vi.fn(),
|
||||
mockGetStackServicesJson: vi.fn().mockReturnValue([]),
|
||||
mockGetSystemState: vi.fn().mockReturnValue('1'), // default: backfilled
|
||||
mockSetSystemState: vi.fn(),
|
||||
mockAddNotificationHistory: vi.fn(),
|
||||
@@ -33,6 +35,10 @@ const {
|
||||
// Backs DockerController.getInstance().getDocker().getImage().inspect() for tests
|
||||
// that exercise the real checkImage (rather than stubbing it) through checkNode.
|
||||
mockInspect: vi.fn().mockResolvedValue({ RepoDigests: [] }),
|
||||
// Defaults to non-renderable so checkNode's per-service reduction is a no-op
|
||||
// (falls back to the legacy whole-stack tally) unless a test overrides this,
|
||||
// matching how no real Compose model exists in this unit-test environment.
|
||||
mockBuildEffectiveServiceModel: vi.fn().mockResolvedValue({ renderable: false, code: 'effective_model_render_failed', error: 'no model in test' }),
|
||||
}));
|
||||
|
||||
vi.mock('../services/RegistryService', () => ({
|
||||
@@ -54,6 +60,7 @@ vi.mock('../services/DatabaseService', () => ({
|
||||
getStackUpdateStatus: mockGetStackUpdateStatus,
|
||||
clearStackUpdateStatus: mockClearStackUpdateStatus,
|
||||
recordStackCheckFailure: mockRecordStackCheckFailure,
|
||||
getStackServicesJson: mockGetStackServicesJson,
|
||||
getSystemState: mockGetSystemState,
|
||||
setSystemState: mockSetSystemState,
|
||||
addNotificationHistory: mockAddNotificationHistory,
|
||||
@@ -61,6 +68,10 @@ vi.mock('../services/DatabaseService', () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/effectiveServiceModel', () => ({
|
||||
buildEffectiveServiceModel: mockBuildEffectiveServiceModel,
|
||||
}));
|
||||
|
||||
vi.mock('../services/NotificationService', () => ({
|
||||
NotificationService: {
|
||||
getInstance: () => ({
|
||||
@@ -1437,3 +1448,206 @@ describe('ImageUpdateService cron scheduling', () => {
|
||||
expect(delay).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Model-based per-service reduction wiring (§5) ─────────────────
|
||||
|
||||
describe('ImageUpdateService - model-based per-service reduction', () => {
|
||||
const TWO_SERVICE_COMPOSE = `
|
||||
services:
|
||||
web:
|
||||
image: web:latest
|
||||
worker:
|
||||
image: worker:latest
|
||||
`;
|
||||
|
||||
const THREE_SERVICE_COMPOSE = `
|
||||
services:
|
||||
worker:
|
||||
image: worker:latest
|
||||
api:
|
||||
image: api:latest
|
||||
db:
|
||||
image: db:latest
|
||||
`;
|
||||
|
||||
const fakeDb = () => ({
|
||||
getStackUpdateStatus: mockGetStackUpdateStatus,
|
||||
upsertStackUpdateStatus: mockUpsertStackUpdateStatus,
|
||||
clearStackUpdateStatus: mockClearStackUpdateStatus,
|
||||
recordStackCheckFailure: mockRecordStackCheckFailure,
|
||||
getSystemState: mockGetSystemState,
|
||||
setSystemState: mockSetSystemState,
|
||||
addNotificationHistory: mockAddNotificationHistory,
|
||||
});
|
||||
|
||||
function specFor(name: string, image: string): { name: string; declaredImage: string; hasBuild: boolean; expectedReplicas: number; dependsOn: string[]; hasHealthcheck: boolean } {
|
||||
return { name, declaredImage: image, hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false };
|
||||
}
|
||||
|
||||
/** Stubs checkImage to report hasUpdate only for the given image refs. */
|
||||
function stubCheckImagePerRef(service: ImageUpdateService, updatedRefs: string[]) {
|
||||
(service as any).checkImage = vi.fn().mockImplementation(async (_docker: unknown, ref: string) => ({ hasUpdate: updatedRefs.includes(ref) }));
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
(ImageUpdateService as any).instance = undefined;
|
||||
mockGetSystemState.mockReturnValue('1');
|
||||
mockGetAllContainers.mockResolvedValue([]);
|
||||
mockEnvExists.mockResolvedValue(false);
|
||||
});
|
||||
|
||||
it('reduces per-service status through the effective model and persists services_json with a generation', async () => {
|
||||
mockGetStacks.mockResolvedValue(['stackA']);
|
||||
mockGetStackContent.mockResolvedValue(TWO_SERVICE_COMPOSE);
|
||||
mockGetStackUpdateStatus.mockReturnValue({});
|
||||
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
|
||||
renderable: true,
|
||||
services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')],
|
||||
});
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImagePerRef(service, ['web:latest']);
|
||||
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(
|
||||
1, 'stackA', true, expect.any(Number), 'ok', null,
|
||||
[
|
||||
{ service: 'web', image: 'web:latest', hasUpdate: true, checkStatus: 'ok', lastError: null },
|
||||
{ service: 'worker', image: 'worker:latest', hasUpdate: false, checkStatus: 'ok', lastError: null },
|
||||
],
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to the legacy whole-stack tally when the effective model is not renderable', async () => {
|
||||
mockGetStacks.mockResolvedValue(['stackA']);
|
||||
mockGetStackContent.mockResolvedValue(TWO_SERVICE_COMPOSE);
|
||||
mockGetStackUpdateStatus.mockReturnValue({});
|
||||
mockBuildEffectiveServiceModel.mockResolvedValueOnce({ renderable: false, code: 'effective_model_render_failed', error: 'render failed' });
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImagePerRef(service, ['web:latest']);
|
||||
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
// Legacy path never passes a services/generation payload.
|
||||
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(1, 'stackA', true, expect.any(Number), 'ok', null);
|
||||
});
|
||||
|
||||
it('names sorted services with hasUpdate in the availability notification for a multi-service stack', async () => {
|
||||
mockGetStacks.mockResolvedValue(['stackA']);
|
||||
mockGetStackContent.mockResolvedValue(THREE_SERVICE_COMPOSE);
|
||||
mockGetStackUpdateStatus.mockReturnValue({ stackA: false });
|
||||
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
|
||||
renderable: true,
|
||||
services: [specFor('worker', 'worker:latest'), specFor('api', 'api:latest'), specFor('db', 'db:latest')],
|
||||
});
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImagePerRef(service, ['worker:latest', 'api:latest']);
|
||||
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'info',
|
||||
'image_update_available',
|
||||
'Stack "stackA" has image updates available for services: api, worker.',
|
||||
{ stackName: 'stackA', actor: 'system:image-update' },
|
||||
);
|
||||
});
|
||||
|
||||
it('does not re-notify a multi-service stack already known to have updates', async () => {
|
||||
mockGetStacks.mockResolvedValue(['stackA']);
|
||||
mockGetStackContent.mockResolvedValue(TWO_SERVICE_COMPOSE);
|
||||
mockGetStackUpdateStatus.mockReturnValue({ stackA: true });
|
||||
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
|
||||
renderable: true,
|
||||
services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')],
|
||||
});
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImagePerRef(service, ['web:latest']);
|
||||
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('suppresses notification and count side effects when a newer recheck discards a stale full-scan write', async () => {
|
||||
mockGetStacks.mockResolvedValue(['stackA']);
|
||||
mockGetStackContent.mockResolvedValue(TWO_SERVICE_COMPOSE);
|
||||
mockGetStackUpdateStatus.mockReturnValue({});
|
||||
// Full-scan write path (after registry work) and recheck both need a model.
|
||||
mockBuildEffectiveServiceModel.mockResolvedValue({
|
||||
renderable: true,
|
||||
services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')],
|
||||
});
|
||||
|
||||
const service = ImageUpdateService.getInstance();
|
||||
let resolveFullScan: ((value: { hasUpdate: boolean }) => void) | undefined;
|
||||
let fullScanPending = true;
|
||||
(service as any).checkImage = vi.fn().mockImplementation(async () => {
|
||||
if (fullScanPending) {
|
||||
return new Promise<{ hasUpdate: boolean }>((resolve) => {
|
||||
resolveFullScan = resolve;
|
||||
});
|
||||
}
|
||||
return { hasUpdate: false };
|
||||
});
|
||||
|
||||
const fullScan = (service as any).checkNode(1, fakeDb());
|
||||
await vi.waitFor(() => expect((service as any).checkImage).toHaveBeenCalled());
|
||||
|
||||
// A newer service recheck reserves a higher generation and commits "no update".
|
||||
fullScanPending = false;
|
||||
await service.recheckStack(1, 'stackA');
|
||||
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(
|
||||
1, 'stackA', false, expect.any(Number), 'ok', null,
|
||||
expect.any(Array),
|
||||
expect.any(Number),
|
||||
);
|
||||
const upsertsAfterRecheck = mockUpsertStackUpdateStatus.mock.calls.length;
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
||||
|
||||
// Stale full scan finishes with hasUpdate=true but must not commit or notify.
|
||||
resolveFullScan?.({ hasUpdate: true });
|
||||
await fullScan;
|
||||
|
||||
expect(mockUpsertStackUpdateStatus.mock.calls.length).toBe(upsertsAfterRecheck);
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe('recheckStack', () => {
|
||||
it('persists a fresh per-service reduction and returns no warning on success', async () => {
|
||||
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
|
||||
renderable: true,
|
||||
services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')],
|
||||
});
|
||||
mockGetAllContainers.mockResolvedValue([
|
||||
{ Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } },
|
||||
]);
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImagePerRef(service, ['web:latest']);
|
||||
|
||||
const result = await service.recheckStack(1, 'stackA');
|
||||
|
||||
expect(result).toEqual({ warning: null });
|
||||
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(
|
||||
1, 'stackA', true, expect.any(Number), 'ok', null,
|
||||
[
|
||||
{ service: 'web', image: 'web:latest', runtimeImages: ['web:latest'], hasUpdate: true, checkStatus: 'ok', lastError: null },
|
||||
{ service: 'worker', image: 'worker:latest', hasUpdate: false, checkStatus: 'ok', lastError: null },
|
||||
],
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns a warning and leaves the prior row untouched when the model cannot render', async () => {
|
||||
mockBuildEffectiveServiceModel.mockResolvedValueOnce({ renderable: false, code: 'effective_model_render_failed', error: 'no model in test' });
|
||||
const service = ImageUpdateService.getInstance();
|
||||
|
||||
const result = await service.recheckStack(1, 'stackA');
|
||||
|
||||
expect(result).toEqual({ warning: 'no model in test' });
|
||||
expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -422,7 +422,7 @@ describe('POST /api/auto-update/execute', () => {
|
||||
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
|
||||
.mockResolvedValue({ hasUpdate: true } as never);
|
||||
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue();
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-au');
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-au');
|
||||
try {
|
||||
const res = await request(app)
|
||||
.post('/api/auto-update/execute')
|
||||
|
||||
@@ -259,6 +259,18 @@ describe('DockerController.buildPrunePlan', () => {
|
||||
// Unique bytes per image = Size - SharedSize = 100MB each, not full 1GB each.
|
||||
expect(plan.reclaimableBytes).toBe(200_000_000);
|
||||
});
|
||||
|
||||
it('excludes an image held for a pending service-update recovery from the plan', async () => {
|
||||
mockDocker.listImages.mockResolvedValue([
|
||||
{ Id: 'img-held', RepoTags: ['app:1'], Size: 100, Containers: 0 },
|
||||
{ Id: 'img-free', RepoTags: ['app:2'], Size: 100, Containers: 0 },
|
||||
]);
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const plan = await dc.buildPrunePlan(['images'], 'all', [], 1, (id) => id === 'img-held');
|
||||
|
||||
expect(plan.items.map((i) => i.id)).toEqual(['img-free']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DockerController.executePrunePlan', () => {
|
||||
@@ -428,4 +440,27 @@ describe('DockerController.executePrunePlan', () => {
|
||||
|
||||
expect(remove).toHaveBeenCalledWith({ force: false });
|
||||
});
|
||||
|
||||
it('re-checks isImageHeld immediately before deleting, skipping a hold that started after the plan was built', async () => {
|
||||
mockDocker.listImages.mockResolvedValue([
|
||||
{ Id: 'img1', RepoTags: ['app:latest'], Size: 1000, Containers: 0 },
|
||||
]);
|
||||
const imageRemove = vi.fn().mockResolvedValue(undefined);
|
||||
mockDocker.getImage.mockReturnValue({ remove: imageRemove });
|
||||
|
||||
const dc = DockerController.getInstance(1);
|
||||
const plan = await dc.buildPrunePlan(['images'], 'all', [], 1);
|
||||
expect(plan.items.map((i) => i.id)).toEqual(['img1']);
|
||||
|
||||
// Bypass the rebuild-based staleness check so this isolates the
|
||||
// immediate per-item recheck (prune holds), simulating a
|
||||
// recovery snapshot created after the plan was built but before execute.
|
||||
vi.spyOn(dc, 'assertPlanFresh').mockResolvedValue(plan);
|
||||
const result = await dc.executePrunePlan(plan, [], (id) => id === 'img1');
|
||||
|
||||
expect(imageRemove).not.toHaveBeenCalled();
|
||||
expect(result.outcomes).toEqual([
|
||||
expect.objectContaining({ id: 'img1', target: 'images', status: 'skipped', reason: expect.stringMatching(/held/i) }),
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -690,8 +690,8 @@ describe('SchedulerService - executePrune', () => {
|
||||
await svc.triggerTask(71);
|
||||
|
||||
expect(mockPruneSystem).toHaveBeenCalledTimes(2);
|
||||
expect(mockPruneSystem).toHaveBeenCalledWith('images', undefined);
|
||||
expect(mockPruneSystem).toHaveBeenCalledWith('volumes', undefined);
|
||||
expect(mockPruneSystem).toHaveBeenCalledWith('images', undefined, expect.any(Function));
|
||||
expect(mockPruneSystem).toHaveBeenCalledWith('volumes', undefined, expect.any(Function));
|
||||
});
|
||||
|
||||
it('includes label filter when configured', async () => {
|
||||
@@ -711,7 +711,7 @@ describe('SchedulerService - executePrune', () => {
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(72);
|
||||
|
||||
expect(mockPruneSystem).toHaveBeenCalledWith('containers', 'env=staging');
|
||||
expect(mockPruneSystem).toHaveBeenCalledWith('containers', 'env=staging', expect.any(Function));
|
||||
});
|
||||
|
||||
it('marks scheduled prune runs as failed when a target prune fails', async () => {
|
||||
@@ -828,7 +828,7 @@ describe('SchedulerService - executeUpdate', () => {
|
||||
|
||||
it('begins a health gate after a scheduled update succeeds', async () => {
|
||||
const { HealthGateService } = await import('../services/HealthGateService');
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-1');
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-1');
|
||||
try {
|
||||
mockGetScheduledTask.mockReturnValue({
|
||||
id: 83,
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
/**
|
||||
* Route smoke tests for the nested service update/restore endpoints: auth and
|
||||
* permission gates, the restore recoveryId requirement, and the mapping from
|
||||
* OrchestratorResult to HTTP status. The orchestrator itself is mocked so these
|
||||
* tests exercise only the route wiring.
|
||||
*/
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
|
||||
|
||||
const { mockExecute } = vi.hoisted(() => ({ mockExecute: vi.fn() }));
|
||||
|
||||
vi.mock('../services/StackUpdateOrchestrator', () => ({
|
||||
StackUpdateOrchestrator: { getInstance: () => ({ execute: mockExecute }) },
|
||||
}));
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
let adminCookie: string;
|
||||
let viewerCookie: string;
|
||||
|
||||
function writeStack(name: string) {
|
||||
const dir = path.join(process.env.COMPOSE_DIR!, name);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'compose.yaml'), 'services:\n app:\n image: nginx\n db:\n image: postgres\n');
|
||||
}
|
||||
|
||||
async function loginAsViewer(): Promise<string> {
|
||||
const bcrypt = (await import('bcrypt')).default;
|
||||
const { DatabaseService } = await import('../services/DatabaseService');
|
||||
const hash = await bcrypt.hash('viewerpass', 1);
|
||||
DatabaseService.getInstance().addUser({ username: 'viewer1', password_hash: hash, role: 'viewer' });
|
||||
const res = await request(app).post('/api/auth/login').send({ username: 'viewer1', password: 'viewerpass' });
|
||||
const cookies = res.headers['set-cookie'] as string | string[];
|
||||
return Array.isArray(cookies) ? cookies[0] : cookies;
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ app } = await import('../index'));
|
||||
adminCookie = await loginAsTestAdmin(app);
|
||||
viewerCookie = await loginAsViewer();
|
||||
writeStack('web');
|
||||
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
vi.restoreAllMocks();
|
||||
cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
mockExecute.mockReset();
|
||||
const { StackOpLockService } = await import('../services/StackOpLockService');
|
||||
StackOpLockService.resetForTests();
|
||||
});
|
||||
|
||||
describe('nested service route auth and permission gates', () => {
|
||||
it('rejects an unauthenticated update with 401', async () => {
|
||||
const res = await request(app).post('/api/stacks/web/services/app/update');
|
||||
expect(res.status).toBe(401);
|
||||
expect(mockExecute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects an unauthenticated restore with 401', async () => {
|
||||
const res = await request(app).post('/api/stacks/web/services/app/restore').send({ recoveryId: 'r1' });
|
||||
expect(res.status).toBe(401);
|
||||
expect(mockExecute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects a viewer without stack:deploy with 403', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/web/services/app/update')
|
||||
.set('Cookie', viewerCookie);
|
||||
expect(res.status).toBe(403);
|
||||
expect(mockExecute).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('restore recoveryId requirement', () => {
|
||||
it('rejects a restore with no recoveryId with 400 recovery_id_required', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/web/services/app/restore')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({});
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.code).toBe('recovery_id_required');
|
||||
expect(mockExecute).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('OrchestratorResult to HTTP mapping', () => {
|
||||
it('maps service_done to 200 with the result body', async () => {
|
||||
mockExecute.mockResolvedValue({
|
||||
kind: 'service_done',
|
||||
serviceName: 'app',
|
||||
healthGateId: 'hg-1',
|
||||
observing: true,
|
||||
recoveryId: 'rec-1',
|
||||
recoveryAvailable: true,
|
||||
});
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/web/services/app/update')
|
||||
.set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toMatchObject({ serviceName: 'app', healthGateId: 'hg-1', recoveryId: 'rec-1', recoveryAvailable: true });
|
||||
expect(mockExecute).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('maps service_update_single_service to 400', async () => {
|
||||
mockExecute.mockResolvedValue({ kind: 'service_failed', code: 'service_update_single_service', error: 'only one service' });
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/web/services/app/update')
|
||||
.set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.code).toBe('service_update_single_service');
|
||||
});
|
||||
|
||||
it('maps a policy block to 409', async () => {
|
||||
mockExecute.mockResolvedValue({ kind: 'service_failed', code: 'policy_blocked', error: 'blocked' });
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/web/services/app/update')
|
||||
.set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.code).toBe('policy_blocked');
|
||||
});
|
||||
|
||||
it('maps a service compose failure to 500', async () => {
|
||||
mockExecute.mockResolvedValue({ kind: 'service_failed', code: 'service_update_compose_failed', error: 'compose exploded' });
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/web/services/app/update')
|
||||
.set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(500);
|
||||
expect(res.body.code).toBe('service_update_compose_failed');
|
||||
});
|
||||
|
||||
it('maps a restore service_done to 200', async () => {
|
||||
mockExecute.mockResolvedValue({
|
||||
kind: 'service_done',
|
||||
serviceName: 'app',
|
||||
healthGateId: 'hg-2',
|
||||
observing: true,
|
||||
recoveryId: 'rec-2',
|
||||
recoveryAvailable: false,
|
||||
});
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/web/services/app/restore')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ recoveryId: 'rec-2' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toMatchObject({ serviceName: 'app', healthGateId: 'hg-2', recoveryId: 'rec-2' });
|
||||
expect(mockExecute).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/stacks/:stackName/services/:serviceName/recovery', () => {
|
||||
it('returns null when no active recovery exists', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/stacks/web/services/app/recovery')
|
||||
.set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({ recovery: null });
|
||||
});
|
||||
|
||||
it('returns the newest active recovery row', async () => {
|
||||
const { DatabaseService } = await import('../services/DatabaseService');
|
||||
const { NodeRegistry } = await import('../services/NodeRegistry');
|
||||
const db = DatabaseService.getInstance();
|
||||
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const now = Date.now();
|
||||
db.insertServiceUpdateRecovery({
|
||||
id: 'rec-old',
|
||||
node_id: nodeId,
|
||||
stack_name: 'web',
|
||||
service_name: 'app',
|
||||
replicas_json: '[]',
|
||||
majority_image_id: 'sha256:old',
|
||||
declared_image_ref: 'nginx:latest',
|
||||
weak_floating_tag: 0,
|
||||
health_gate_id: null,
|
||||
status: 'active',
|
||||
expires_at: now + 60_000,
|
||||
claim_expires_at: null,
|
||||
created_at: now - 1_000,
|
||||
created_by: 'tester',
|
||||
});
|
||||
db.insertServiceUpdateRecovery({
|
||||
id: 'rec-new',
|
||||
node_id: nodeId,
|
||||
stack_name: 'web',
|
||||
service_name: 'app',
|
||||
replicas_json: '[]',
|
||||
majority_image_id: 'sha256:new',
|
||||
declared_image_ref: 'nginx:latest',
|
||||
weak_floating_tag: 0,
|
||||
health_gate_id: 'gate-1',
|
||||
status: 'active',
|
||||
expires_at: now + 60_000,
|
||||
claim_expires_at: null,
|
||||
created_at: now,
|
||||
created_by: 'tester',
|
||||
});
|
||||
const res = await request(app)
|
||||
.get('/api/stacks/web/services/app/recovery')
|
||||
.set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.recovery).toMatchObject({
|
||||
id: 'rec-new',
|
||||
healthGateId: 'gate-1',
|
||||
majorityImageId: 'sha256:new',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,309 @@
|
||||
/**
|
||||
* Coverage for ServiceUpdateRecoveryService against a real DatabaseService
|
||||
* (temp DB) with fake timers: eligibility computation (§8 "Unavailable"
|
||||
* cases), the claim/renewal lifecycle (RECOVERY-CLAIM-2, no revive of a
|
||||
* terminal row), the sweep timer, held image ids, and start/stop timer
|
||||
* hygiene.
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeAll, afterAll, beforeEach, afterEach } from 'vitest';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
import type { ServiceReplicaSnapshot } from '../services/ServiceUpdateRecoveryService';
|
||||
|
||||
let tmpDir: string;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
let ServiceUpdateRecoveryService: typeof import('../services/ServiceUpdateRecoveryService').ServiceUpdateRecoveryService;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
({ ServiceUpdateRecoveryService } = await import('../services/ServiceUpdateRecoveryService'));
|
||||
});
|
||||
|
||||
afterAll(() => cleanupTestDb(tmpDir));
|
||||
|
||||
function db() {
|
||||
return DatabaseService.getInstance();
|
||||
}
|
||||
|
||||
const svc = () => ServiceUpdateRecoveryService.getInstance();
|
||||
|
||||
const BASE_INPUT = {
|
||||
nodeId: 1,
|
||||
stackName: 'web',
|
||||
serviceName: 'api',
|
||||
declaredImageRef: 'ghcr.io/acme/api:v1',
|
||||
createdBy: 'tester',
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
const raw = (db() as unknown as { db: { prepare: (s: string) => { run: () => void } } }).db;
|
||||
raw.prepare('DELETE FROM service_update_recovery').run();
|
||||
raw.prepare("DELETE FROM global_settings WHERE key = 'health_gate_window_seconds'").run();
|
||||
// updateGlobalSetting() invalidates this cache; the raw DELETE above bypasses
|
||||
// it, so a value set by a previous test would otherwise leak into this one.
|
||||
(db() as unknown as { cachedGlobalSettings: unknown }).cachedGlobalSettings = null;
|
||||
svc().start();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
svc().stop();
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe('createIfEligible - unavailable cases', () => {
|
||||
it('rejects a service with no replicas', () => {
|
||||
const result = svc().createIfEligible({ ...BASE_INPUT, replicas: [] });
|
||||
expect(result).toEqual({ eligible: false, reason: 'no_replicas' });
|
||||
});
|
||||
|
||||
it('rejects replicas with no local image id', () => {
|
||||
const replicas: ServiceReplicaSnapshot[] = [{ imageId: '', repoDigest: null }, { imageId: ' ', repoDigest: null }];
|
||||
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
|
||||
expect(result).toEqual({ eligible: false, reason: 'missing_local_id' });
|
||||
});
|
||||
|
||||
it('rejects a majority tie between two equally-common images', () => {
|
||||
const replicas: ServiceReplicaSnapshot[] = [
|
||||
{ imageId: 'sha256:aaa', repoDigest: null },
|
||||
{ imageId: 'sha256:bbb', repoDigest: null },
|
||||
];
|
||||
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
|
||||
expect(result).toEqual({ eligible: false, reason: 'majority_tie' });
|
||||
});
|
||||
|
||||
it('rejects a pure-build service with no declared image ref', () => {
|
||||
const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }];
|
||||
const result = svc().createIfEligible({ ...BASE_INPUT, declaredImageRef: null, replicas });
|
||||
expect(result).toEqual({ eligible: false, reason: 'build_only' });
|
||||
});
|
||||
|
||||
it('rejects a digest-pinned declared image ref', () => {
|
||||
const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }];
|
||||
const result = svc().createIfEligible({
|
||||
...BASE_INPUT,
|
||||
declaredImageRef: 'ghcr.io/acme/api@sha256:' + 'a'.repeat(64),
|
||||
replicas,
|
||||
});
|
||||
expect(result).toEqual({ eligible: false, reason: 'digest_pinned_declared_ref' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('createIfEligible - eligible cases', () => {
|
||||
it('persists the majority image and flags a weak floating tag when no replica has a captured digest', () => {
|
||||
const replicas: ServiceReplicaSnapshot[] = [
|
||||
{ imageId: 'sha256:aaa', repoDigest: null },
|
||||
{ imageId: 'sha256:aaa', repoDigest: null },
|
||||
{ imageId: 'sha256:bbb', repoDigest: null },
|
||||
];
|
||||
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
|
||||
expect(result.eligible).toBe(true);
|
||||
if (!result.eligible) throw new Error('expected eligible');
|
||||
expect(result.row).toMatchObject({
|
||||
node_id: 1, stack_name: 'web', service_name: 'api',
|
||||
majority_image_id: 'sha256:aaa', declared_image_ref: 'ghcr.io/acme/api:v1',
|
||||
weak_floating_tag: 1, status: 'active', health_gate_id: null,
|
||||
});
|
||||
expect(db().getServiceUpdateRecovery(result.row.id)).toBeTruthy();
|
||||
});
|
||||
|
||||
it('clears the weak floating tag flag when a majority replica has a captured digest', () => {
|
||||
const replicas: ServiceReplicaSnapshot[] = [
|
||||
{ imageId: 'sha256:aaa', repoDigest: 'sha256:digest-aaa' },
|
||||
{ imageId: 'sha256:aaa', repoDigest: 'sha256:digest-aaa' },
|
||||
{ imageId: 'sha256:bbb', repoDigest: null },
|
||||
];
|
||||
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
|
||||
if (!result.eligible) throw new Error('expected eligible');
|
||||
expect(result.row.weak_floating_tag).toBe(0);
|
||||
});
|
||||
|
||||
it('sets expires_at from max(compose timeout, health window) plus buffer', () => {
|
||||
db().updateGlobalSetting('health_gate_window_seconds', '120');
|
||||
const now = Date.now();
|
||||
const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }];
|
||||
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
|
||||
if (!result.eligible) throw new Error('expected eligible');
|
||||
// Default compose timeout is 30m; health window 120s is smaller, so compose wins.
|
||||
expect(result.row.expires_at).toBe(now + 30 * 60_000 + 30 * 60_000);
|
||||
});
|
||||
|
||||
it('uses a raised health window when it exceeds the compose command timeout', () => {
|
||||
db().updateGlobalSetting('health_gate_window_seconds', String(45 * 60));
|
||||
const now = Date.now();
|
||||
const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }];
|
||||
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
|
||||
if (!result.eligible) throw new Error('expected eligible');
|
||||
expect(result.row.expires_at).toBe(now + 45 * 60_000 + 30 * 60_000);
|
||||
});
|
||||
|
||||
it('floors the health window to 90s but still honors the compose timeout floor', () => {
|
||||
db().updateGlobalSetting('health_gate_window_seconds', '15');
|
||||
const now = Date.now();
|
||||
const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }];
|
||||
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
|
||||
if (!result.eligible) throw new Error('expected eligible');
|
||||
expect(result.row.expires_at).toBe(now + 30 * 60_000 + 30 * 60_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('claim + mandatory renewal (RECOVERY-CLAIM-2)', () => {
|
||||
function seedEligible(): string {
|
||||
const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: 'sha256:digest-aaa' }];
|
||||
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
|
||||
if (!result.eligible) throw new Error('expected eligible');
|
||||
return result.row.id;
|
||||
}
|
||||
|
||||
it('claims an active row and moves it to restoring with a claim window at least 30m + 5m out', () => {
|
||||
const id = seedEligible();
|
||||
const now = Date.now();
|
||||
const claimed = svc().claim(id);
|
||||
expect(claimed?.status).toBe('restoring');
|
||||
expect(claimed?.claim_expires_at).toBeGreaterThanOrEqual(now + 30 * 60_000 + 5 * 60_000);
|
||||
});
|
||||
|
||||
it('renews the claim every 5 minutes while restoring, keeping a long restore held', () => {
|
||||
const id = seedEligible();
|
||||
svc().claim(id);
|
||||
const firstExpiry = db().getServiceUpdateRecovery(id)?.claim_expires_at ?? 0;
|
||||
|
||||
svc().startClaimRenewal(id);
|
||||
vi.advanceTimersByTime(5 * 60_000);
|
||||
|
||||
const renewedExpiry = db().getServiceUpdateRecovery(id)?.claim_expires_at ?? 0;
|
||||
expect(renewedExpiry).toBeGreaterThan(firstExpiry);
|
||||
expect(db().getServiceUpdateRecovery(id)?.status).toBe('restoring');
|
||||
|
||||
svc().stopClaimRenewal(id);
|
||||
});
|
||||
|
||||
it('does not revive a row that left restoring mid-loop (renewal self-stops, never overwrites the terminal status)', () => {
|
||||
const id = seedEligible();
|
||||
svc().claim(id);
|
||||
svc().startClaimRenewal(id);
|
||||
|
||||
// Simulate the restore finishing (consumed) between renewal ticks.
|
||||
db().markServiceUpdateRecoveryConsumed(id, 'restore-gate');
|
||||
const claimExpiresBefore = db().getServiceUpdateRecovery(id)?.claim_expires_at;
|
||||
|
||||
vi.advanceTimersByTime(5 * 60_000);
|
||||
|
||||
const row = db().getServiceUpdateRecovery(id);
|
||||
expect(row?.status).toBe('consumed');
|
||||
expect(row?.claim_expires_at).toBe(claimExpiresBefore);
|
||||
|
||||
svc().stopClaimRenewal(id);
|
||||
});
|
||||
|
||||
it('stops renewing once stopClaimRenewal is called (finally-block contract)', () => {
|
||||
const id = seedEligible();
|
||||
svc().claim(id);
|
||||
svc().startClaimRenewal(id);
|
||||
svc().stopClaimRenewal(id);
|
||||
|
||||
const expiryAtStop = db().getServiceUpdateRecovery(id)?.claim_expires_at;
|
||||
vi.advanceTimersByTime(15 * 60_000);
|
||||
expect(db().getServiceUpdateRecovery(id)?.claim_expires_at).toBe(expiryAtStop);
|
||||
});
|
||||
|
||||
it('is idempotent: a second startClaimRenewal call for the same id does not schedule a duplicate timer', () => {
|
||||
const id = seedEligible();
|
||||
svc().claim(id);
|
||||
const before = vi.getTimerCount();
|
||||
svc().startClaimRenewal(id);
|
||||
svc().startClaimRenewal(id);
|
||||
expect(vi.getTimerCount()).toBe(before + 1);
|
||||
svc().stopClaimRenewal(id);
|
||||
});
|
||||
|
||||
it('refuses to start a renewal loop once the service has been stopped', () => {
|
||||
const id = seedEligible();
|
||||
svc().claim(id);
|
||||
svc().stop();
|
||||
const before = vi.getTimerCount();
|
||||
svc().startClaimRenewal(id);
|
||||
expect(vi.getTimerCount()).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sweep()', () => {
|
||||
it('expires an abandoned restoring claim but leaves a live one and an unexpired active one alone', () => {
|
||||
const now = Date.now();
|
||||
db().insertServiceUpdateRecovery({
|
||||
id: 'rec-abandoned', node_id: 1, stack_name: 'web', service_name: 'api',
|
||||
replicas_json: '[]', majority_image_id: 'sha256:aaa', declared_image_ref: 'x:1',
|
||||
weak_floating_tag: 0, health_gate_id: null, status: 'restoring',
|
||||
expires_at: now - 1_000, claim_expires_at: now - 1_000, created_at: now - 60_000, created_by: null,
|
||||
});
|
||||
db().insertServiceUpdateRecovery({
|
||||
id: 'rec-live', node_id: 1, stack_name: 'web', service_name: 'db',
|
||||
replicas_json: '[]', majority_image_id: 'sha256:bbb', declared_image_ref: 'x:2',
|
||||
weak_floating_tag: 0, health_gate_id: null, status: 'restoring',
|
||||
expires_at: now - 1_000, claim_expires_at: now + 60 * 60_000, created_at: now - 60_000, created_by: null,
|
||||
});
|
||||
db().insertServiceUpdateRecovery({
|
||||
id: 'rec-active', node_id: 1, stack_name: 'web', service_name: 'cache',
|
||||
replicas_json: '[]', majority_image_id: 'sha256:ccc', declared_image_ref: 'x:3',
|
||||
weak_floating_tag: 0, health_gate_id: null, status: 'active',
|
||||
expires_at: now + 60 * 60_000, claim_expires_at: null, created_at: now, created_by: null,
|
||||
});
|
||||
|
||||
svc().sweep();
|
||||
|
||||
expect(db().getServiceUpdateRecovery('rec-abandoned')?.status).toBe('expired');
|
||||
expect(db().getServiceUpdateRecovery('rec-live')?.status).toBe('restoring');
|
||||
expect(db().getServiceUpdateRecovery('rec-active')?.status).toBe('active');
|
||||
});
|
||||
|
||||
it('runs automatically on the interval timer after start()', () => {
|
||||
const now = Date.now();
|
||||
db().insertServiceUpdateRecovery({
|
||||
id: 'rec-ttl', node_id: 1, stack_name: 'web', service_name: 'api',
|
||||
replicas_json: '[]', majority_image_id: 'sha256:aaa', declared_image_ref: 'x:1',
|
||||
weak_floating_tag: 0, health_gate_id: null, status: 'active',
|
||||
expires_at: now + 1_000, claim_expires_at: null, created_at: now, created_by: null,
|
||||
});
|
||||
|
||||
vi.advanceTimersByTime(30_000 + 5 * 60_000 + 1_000);
|
||||
|
||||
expect(db().getServiceUpdateRecovery('rec-ttl')?.status).toBe('expired');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getHeldImageIds', () => {
|
||||
it('wraps the DB projection for one node', () => {
|
||||
const now = Date.now();
|
||||
db().insertServiceUpdateRecovery({
|
||||
id: 'rec-held', node_id: 1, stack_name: 'web', service_name: 'api',
|
||||
replicas_json: '[]', majority_image_id: 'sha256:held', declared_image_ref: 'x:1',
|
||||
weak_floating_tag: 0, health_gate_id: null, status: 'active',
|
||||
expires_at: now + 60_000, claim_expires_at: null, created_at: now, created_by: null,
|
||||
});
|
||||
expect(svc().getHeldImageIds(1)).toEqual(new Set(['sha256:held']));
|
||||
});
|
||||
|
||||
it('fails closed (null) when the DB read throws', () => {
|
||||
const spy = vi.spyOn(db(), 'listHeldServiceUpdateRecoveryImageIds').mockImplementation(() => {
|
||||
throw new Error('boom');
|
||||
});
|
||||
expect(svc().getHeldImageIds(1)).toBeNull();
|
||||
spy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('start() / stop() timer hygiene', () => {
|
||||
it('clears the sweep timer and stops firing after stop()', () => {
|
||||
const spy = vi.spyOn(svc(), 'sweep');
|
||||
svc().stop();
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
svc().start();
|
||||
vi.advanceTimersByTime(30_000);
|
||||
expect(spy).toHaveBeenCalledTimes(1);
|
||||
svc().stop();
|
||||
vi.advanceTimersByTime(5 * 60_000);
|
||||
expect(spy).toHaveBeenCalledTimes(1);
|
||||
spy.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -34,6 +34,23 @@ describe('StackOpMetricsService', () => {
|
||||
successCount: 2,
|
||||
errorCount: 1,
|
||||
avgMs: 200,
|
||||
targetScope: 'stack',
|
||||
serviceName: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('records the latest service-scoped metadata on the action bucket', () => {
|
||||
const svc = StackOpMetricsService.getInstance();
|
||||
svc.record(1, 'update', 50, true, { targetScope: 'service', serviceName: 'web' });
|
||||
expect(svc.snapshot()[0]).toMatchObject({
|
||||
action: 'update',
|
||||
targetScope: 'service',
|
||||
serviceName: 'web',
|
||||
});
|
||||
svc.record(1, 'update', 40, true, { targetScope: 'stack' });
|
||||
expect(svc.snapshot()[0]).toMatchObject({
|
||||
targetScope: 'stack',
|
||||
serviceName: null,
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
/**
|
||||
* Unit tests for StackUpdateOrchestrator dispatch: the stack branch runs
|
||||
* ComposeService.updateStack (side effects stay with callers), the service
|
||||
* branch hard-rejects any non-manual trigger, and a single-service stack is
|
||||
* refused with the stable `service_update_single_service` code before any
|
||||
* mutation.
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import type { EffectiveServiceModelResult, EffectiveServiceSpec } from '../services/effectiveServiceModel';
|
||||
|
||||
const { state } = vi.hoisted(() => ({
|
||||
state: {
|
||||
updateStack: vi.fn(),
|
||||
model: null as EffectiveServiceModelResult | null,
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/ComposeService', () => ({
|
||||
ComposeService: {
|
||||
getInstance: () => ({ updateStack: state.updateStack }),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/effectiveServiceModel', () => ({
|
||||
buildEffectiveServiceModel: vi.fn(async () => state.model),
|
||||
}));
|
||||
|
||||
import { StackUpdateOrchestrator, evaluateServiceReplicaConvergence, shortImageId } from '../services/StackUpdateOrchestrator';
|
||||
|
||||
const orch = () => StackUpdateOrchestrator.getInstance();
|
||||
|
||||
function spec(name: string): EffectiveServiceSpec {
|
||||
return { name, declaredImage: 'nginx:latest', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false };
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
state.updateStack.mockReset();
|
||||
state.updateStack.mockResolvedValue(undefined);
|
||||
state.model = null;
|
||||
});
|
||||
|
||||
describe('evaluateServiceReplicaConvergence', () => {
|
||||
it('converges when the exact expected running replicas share one image', () => {
|
||||
expect(evaluateServiceReplicaConvergence('api', 3, ['sha:a', 'sha:a', 'sha:a'], 0)).toEqual({
|
||||
kind: 'converged', imageId: 'sha:a',
|
||||
});
|
||||
});
|
||||
|
||||
it('reports divergent when running replica count is below expected', () => {
|
||||
const result = evaluateServiceReplicaConvergence('api', 3, ['sha:a'], 0);
|
||||
expect(result).toMatchObject({ kind: 'divergent' });
|
||||
expect((result as { error: string }).error).toMatch(/1 running.*expected 3/i);
|
||||
});
|
||||
|
||||
it('reports divergent when no running replicas remain', () => {
|
||||
const result = evaluateServiceReplicaConvergence('api', 2, [], 0);
|
||||
expect(result).toMatchObject({ kind: 'divergent' });
|
||||
expect((result as { error: string }).error).toMatch(/no running replicas/i);
|
||||
});
|
||||
|
||||
it('reports divergent when replicas disagree on image', () => {
|
||||
const result = evaluateServiceReplicaConvergence('api', 2, ['sha:a', 'sha:b'], 0);
|
||||
expect(result).toMatchObject({ kind: 'divergent' });
|
||||
expect((result as { error: string }).error).toMatch(/did not converge/i);
|
||||
});
|
||||
|
||||
it('reports inspect_failed when every inspect failed', () => {
|
||||
expect(evaluateServiceReplicaConvergence('api', 2, [], 2)).toMatchObject({ kind: 'inspect_failed' });
|
||||
});
|
||||
|
||||
it('treats scale-zero as converged with a null image', () => {
|
||||
expect(evaluateServiceReplicaConvergence('api', 0, [], 0)).toEqual({ kind: 'converged', imageId: null });
|
||||
});
|
||||
});
|
||||
|
||||
describe('shortImageId', () => {
|
||||
it('strips the sha256 prefix and truncates', () => {
|
||||
expect(shortImageId('sha256:abcdef0123456789')).toBe('abcdef012345');
|
||||
});
|
||||
});
|
||||
|
||||
describe('StackUpdateOrchestrator stack branch', () => {
|
||||
it('runs ComposeService.updateStack and returns stack_compose_done', async () => {
|
||||
const result = await orch().execute(
|
||||
{ nodeId: 0, stackName: 'web', target: { scope: 'stack' }, trigger: 'manual', actor: 'tester' },
|
||||
{ atomic: true, terminalWs: null },
|
||||
);
|
||||
expect(result).toEqual({ kind: 'stack_compose_done' });
|
||||
expect(state.updateStack).toHaveBeenCalledWith('web', undefined, true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('StackUpdateOrchestrator service branch guards', () => {
|
||||
it('rejects a service-scoped update with a non-manual trigger', async () => {
|
||||
await expect(
|
||||
orch().execute(
|
||||
{ nodeId: 0, stackName: 'web', target: { scope: 'service', serviceName: 'app' }, trigger: 'scheduled', actor: 'system' },
|
||||
{ policyOptions: { bypass: false, actor: 'system' } },
|
||||
),
|
||||
).rejects.toThrow(/manual/i);
|
||||
expect(state.updateStack).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('refuses a single-service stack with code service_update_single_service', async () => {
|
||||
state.model = { renderable: true, services: [spec('only')] };
|
||||
const result = await orch().execute(
|
||||
{ nodeId: 0, stackName: 'web', target: { scope: 'service', serviceName: 'only' }, trigger: 'manual', actor: 'tester' },
|
||||
{ policyOptions: { bypass: false, actor: 'tester' } },
|
||||
);
|
||||
expect(result).toMatchObject({ kind: 'service_failed', code: 'service_update_single_service' });
|
||||
expect(state.updateStack).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('fails closed when the effective model cannot render', async () => {
|
||||
state.model = { renderable: false, code: 'effective_model_render_failed', error: 'boom' };
|
||||
const result = await orch().execute(
|
||||
{ nodeId: 0, stackName: 'web', target: { scope: 'service', serviceName: 'app' }, trigger: 'manual', actor: 'tester' },
|
||||
{ policyOptions: { bypass: false, actor: 'tester' } },
|
||||
);
|
||||
expect(result).toMatchObject({ kind: 'service_failed', code: 'effective_model_render_failed' });
|
||||
});
|
||||
});
|
||||
@@ -251,7 +251,7 @@ describe('health gate begin call sites', () => {
|
||||
|
||||
beforeEach(async () => {
|
||||
const { HealthGateService } = await import('../services/HealthGateService');
|
||||
beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-123') as ReturnType<typeof vi.spyOn>;
|
||||
beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-123') as ReturnType<typeof vi.spyOn>;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
|
||||
@@ -12,10 +12,15 @@ import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTes
|
||||
const {
|
||||
mockComputeUpdateReadiness,
|
||||
mockComputeRollbackReadiness,
|
||||
} = vi.hoisted(() => ({
|
||||
mockComputeUpdateReadiness: vi.fn(),
|
||||
mockComputeRollbackReadiness: vi.fn(),
|
||||
}));
|
||||
MockSingleServiceUpdateReadinessError,
|
||||
} = vi.hoisted(() => {
|
||||
class MockSingleServiceUpdateReadinessError extends Error {}
|
||||
return {
|
||||
mockComputeUpdateReadiness: vi.fn(),
|
||||
mockComputeRollbackReadiness: vi.fn(),
|
||||
MockSingleServiceUpdateReadinessError,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('../services/UpdateGuardService', () => ({
|
||||
UpdateGuardService: {
|
||||
@@ -24,6 +29,7 @@ vi.mock('../services/UpdateGuardService', () => ({
|
||||
computeRollbackReadiness: mockComputeRollbackReadiness,
|
||||
}),
|
||||
},
|
||||
SingleServiceUpdateReadinessError: MockSingleServiceUpdateReadinessError,
|
||||
}));
|
||||
|
||||
vi.mock('../services/FileSystemService', () => ({
|
||||
@@ -85,6 +91,28 @@ describe('GET /api/stacks/:stackName/update-readiness', () => {
|
||||
expect(res.status).toBe(500);
|
||||
expect(res.body).toEqual({ error: 'Failed to compute update readiness' });
|
||||
});
|
||||
|
||||
it('forwards the service query param to computeUpdateReadiness', async () => {
|
||||
const report = { stack: 'web', computedAt: 1, verdict: 'ready', signals: [], serviceName: 'api', advisories: [] };
|
||||
mockComputeUpdateReadiness.mockResolvedValue(report);
|
||||
const res = await request(app).get('/api/stacks/web/update-readiness?service=api').set('Cookie', authCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual(report);
|
||||
expect(mockComputeUpdateReadiness).toHaveBeenCalledWith(expect.any(Number), 'web', 'api');
|
||||
});
|
||||
|
||||
it('omits the service argument when no service query param is given', async () => {
|
||||
mockComputeUpdateReadiness.mockResolvedValue({ stack: 'web', computedAt: 1, verdict: 'ready', signals: [] });
|
||||
await request(app).get('/api/stacks/web/update-readiness').set('Cookie', authCookie);
|
||||
expect(mockComputeUpdateReadiness).toHaveBeenCalledWith(expect.any(Number), 'web', undefined);
|
||||
});
|
||||
|
||||
it('returns 400 when the service query targets a single-service stack', async () => {
|
||||
mockComputeUpdateReadiness.mockRejectedValue(new MockSingleServiceUpdateReadinessError('single service'));
|
||||
const res = await request(app).get('/api/stacks/web/update-readiness?service=api').set('Cookie', authCookie);
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body).toMatchObject({ code: 'service_update_single_service' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/stacks/:stackName/rollback-readiness', () => {
|
||||
@@ -120,6 +148,7 @@ describe('GET /api/stacks/:stackName/health-gate', () => {
|
||||
db.insertHealthGateRun({
|
||||
id, node_id: defaultNodeId, stack_name: 'web', trigger_action: 'update', status, reason,
|
||||
window_seconds: 90, containers_json: '[]', started_at: startedAt, ended_at: status === 'observing' ? null : startedAt + 1000, created_by: 'tester',
|
||||
target_scope: 'stack', service_name: null, failure_source: null,
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ const {
|
||||
mockGetOpenDriftFindings,
|
||||
mockGetGlobalSettings,
|
||||
mockFsSize,
|
||||
mockBuildEffectiveServiceModel,
|
||||
} = vi.hoisted(() => ({
|
||||
mockListContainers: vi.fn(),
|
||||
mockGetContainer: vi.fn(),
|
||||
@@ -27,6 +28,7 @@ const {
|
||||
mockGetOpenDriftFindings: vi.fn(),
|
||||
mockGetGlobalSettings: vi.fn(),
|
||||
mockFsSize: vi.fn(),
|
||||
mockBuildEffectiveServiceModel: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../services/DockerController', () => ({
|
||||
@@ -75,7 +77,11 @@ vi.mock('systeminformation', () => ({
|
||||
default: { fsSize: mockFsSize },
|
||||
}));
|
||||
|
||||
import { UpdateGuardService } from '../services/UpdateGuardService';
|
||||
vi.mock('../services/effectiveServiceModel', () => ({
|
||||
buildEffectiveServiceModel: mockBuildEffectiveServiceModel,
|
||||
}));
|
||||
|
||||
import { UpdateGuardService, SingleServiceUpdateReadinessError } from '../services/UpdateGuardService';
|
||||
|
||||
const inspectResult = (over: Record<string, unknown> = {}) => ({
|
||||
State: { Status: 'running', ExitCode: 0 },
|
||||
@@ -166,6 +172,110 @@ describe('UpdateGuardService.computeUpdateReadiness wiring', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('UpdateGuardService.computeUpdateReadiness with a serviceName', () => {
|
||||
const twoServiceModel = {
|
||||
renderable: true as const,
|
||||
services: [
|
||||
{ name: 'web', declaredImage: 'nginx:1.27', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: true },
|
||||
{ name: 'db', declaredImage: 'postgres:16', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false },
|
||||
],
|
||||
};
|
||||
const multiServicePreview = {
|
||||
stack_name: 'app',
|
||||
images: [
|
||||
{ service: 'web', image: 'nginx', current_tag: '1.27.0', next_tag: '1.27.1', has_update: true, semver_bump: 'patch' },
|
||||
{ service: 'db', image: 'postgres', current_tag: '16.0', next_tag: null, has_update: false, semver_bump: 'none' },
|
||||
],
|
||||
build_services: [],
|
||||
summary: {
|
||||
has_update: true, primary_image: 'nginx', current_tag: '1.27.0', next_tag: '1.27.1',
|
||||
semver_bump: 'patch', update_kind: 'tag', blocked: false, blocked_reason: null,
|
||||
has_build_services: false, rebuild_available: false,
|
||||
},
|
||||
rollback_target: 'nginx:1.27.0',
|
||||
changelog: null,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
mockGetLatest.mockReturnValue({ activeStatus: 'pass' });
|
||||
mockGetOpenDriftFindings.mockReturnValue([]);
|
||||
mockGetPreview.mockResolvedValue(multiServicePreview);
|
||||
mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: Date.now() });
|
||||
mockFsSize.mockResolvedValue([{ mount: '/', use: 42 }]);
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'web1', Names: ['/web'], State: 'running' },
|
||||
{ Id: 'db1', Names: ['/db'], State: 'running' },
|
||||
]);
|
||||
mockGetContainer.mockReturnValue({ inspect: vi.fn().mockResolvedValue(inspectResult()) });
|
||||
});
|
||||
|
||||
it('scopes the verdict to the selected service, includes a service signal, and filters the preview', async () => {
|
||||
mockBuildEffectiveServiceModel.mockResolvedValue(twoServiceModel);
|
||||
|
||||
const report = await UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'web');
|
||||
|
||||
expect(report.serviceName).toBe('web');
|
||||
expect(report.verdict).toBe('ready');
|
||||
const serviceSig = report.signals.find(s => s.id === 'service');
|
||||
expect(serviceSig?.status).toBe('ok');
|
||||
// update_preview signal is derived from the filtered summary (recomputed for 'web' only).
|
||||
const previewSig = report.signals.find(s => s.id === 'update_preview');
|
||||
expect(previewSig?.status).toBe('ok');
|
||||
});
|
||||
|
||||
it('throws SingleServiceUpdateReadinessError for a single-service stack', async () => {
|
||||
mockBuildEffectiveServiceModel.mockResolvedValue({
|
||||
renderable: true,
|
||||
services: [{ name: 'web', declaredImage: 'nginx:1.27', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: true }],
|
||||
});
|
||||
|
||||
await expect(UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'web'))
|
||||
.rejects.toBeInstanceOf(SingleServiceUpdateReadinessError);
|
||||
});
|
||||
|
||||
it('fails closed (blocked service signal) when the effective model fails to render', async () => {
|
||||
mockBuildEffectiveServiceModel.mockResolvedValue({
|
||||
renderable: false,
|
||||
code: 'effective_model_render_failed',
|
||||
error: 'bad yaml',
|
||||
});
|
||||
|
||||
const report = await UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'web');
|
||||
|
||||
const serviceSig = report.signals.find(s => s.id === 'service');
|
||||
expect(serviceSig?.status).toBe('blocked');
|
||||
expect(report.verdict).toBe('blocked');
|
||||
});
|
||||
|
||||
it('fails closed when the selected service is not declared in the model', async () => {
|
||||
mockBuildEffectiveServiceModel.mockResolvedValue(twoServiceModel);
|
||||
|
||||
const report = await UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'cache');
|
||||
|
||||
const serviceSig = report.signals.find(s => s.id === 'service');
|
||||
expect(serviceSig?.status).toBe('blocked');
|
||||
expect(report.verdict).toBe('blocked');
|
||||
});
|
||||
|
||||
it('reports a sibling advisory without affecting the verdict', async () => {
|
||||
mockBuildEffectiveServiceModel.mockResolvedValue(twoServiceModel);
|
||||
mockListContainers.mockResolvedValue([
|
||||
{ Id: 'web1', Names: ['/web'], State: 'running' },
|
||||
{ Id: 'db1', Names: ['/db'], State: 'exited', ExitCode: 1 },
|
||||
]);
|
||||
mockGetContainer.mockImplementation((id: string) => ({
|
||||
inspect: vi.fn().mockResolvedValue(id === 'db1'
|
||||
? inspectResult({ State: { Status: 'exited', ExitCode: 1 } })
|
||||
: inspectResult()),
|
||||
}));
|
||||
|
||||
const report = await UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'web');
|
||||
|
||||
expect(report.verdict).toBe('ready');
|
||||
expect(report.advisories.some(a => a.includes('db'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('UpdateGuardService.computeRollbackReadiness moving-tag wiring', () => {
|
||||
const preview = (images: Array<{ current_tag: string }>) => ({
|
||||
stack_name: 'app',
|
||||
|
||||
@@ -462,7 +462,7 @@ describe('WebhookService.execute: health gate begin call sites', () => {
|
||||
vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined);
|
||||
vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]);
|
||||
vi.spyOn(compose.ComposeService.prototype, 'deployStack').mockResolvedValue(undefined);
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-hook');
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook');
|
||||
|
||||
const result = await WebhookService.getInstance().execute(webhook, 'deploy', 'test', true);
|
||||
expect(result.success).toBe(true);
|
||||
@@ -482,7 +482,7 @@ describe('WebhookService.execute: health gate begin call sites', () => {
|
||||
vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined);
|
||||
vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]);
|
||||
vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue(undefined);
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-hook');
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook');
|
||||
|
||||
const result = await WebhookService.getInstance().execute(webhook, 'pull', 'test', true);
|
||||
expect(result.success).toBe(true);
|
||||
|
||||
@@ -4,6 +4,7 @@ import { LicenseService } from '../services/LicenseService';
|
||||
import { MonitorService } from '../services/MonitorService';
|
||||
import { AutoHealService } from '../services/AutoHealService';
|
||||
import { HealthGateService } from '../services/HealthGateService';
|
||||
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
|
||||
import { FleetSyncRetryService } from '../services/FleetSyncRetryService';
|
||||
import { DockerEventManager } from '../services/DockerEventManager';
|
||||
import { ImageUpdateService } from '../services/ImageUpdateService';
|
||||
@@ -34,6 +35,7 @@ export function installShutdownHandlers(server: Server): void {
|
||||
}
|
||||
try { AutoHealService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] AutoHealService cleanup failed:', (e as Error).message); }
|
||||
try { HealthGateService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] HealthGateService cleanup failed:', (e as Error).message); }
|
||||
try { ServiceUpdateRecoveryService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] ServiceUpdateRecoveryService cleanup failed:', (e as Error).message); }
|
||||
try { FleetSyncRetryService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] FleetSyncRetryService cleanup failed:', (e as Error).message); }
|
||||
try { DockerEventManager.getInstance().stop(); } catch (e) {
|
||||
console.warn('[Shutdown] DockerEventManager cleanup failed:', (e as Error).message);
|
||||
|
||||
@@ -11,6 +11,7 @@ import SelfIdentityService from '../services/SelfIdentityService';
|
||||
import { MonitorService } from '../services/MonitorService';
|
||||
import { AutoHealService } from '../services/AutoHealService';
|
||||
import { HealthGateService } from '../services/HealthGateService';
|
||||
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
|
||||
import { FleetSyncRetryService } from '../services/FleetSyncRetryService';
|
||||
import { DockerEventManager } from '../services/DockerEventManager';
|
||||
import TrivyService, { sweepStaleTrivyTempDirs } from '../services/TrivyService';
|
||||
@@ -139,6 +140,7 @@ export async function startServer(server: Server): Promise<void> {
|
||||
MonitorService.getInstance().start();
|
||||
AutoHealService.getInstance().start();
|
||||
HealthGateService.getInstance().start();
|
||||
ServiceUpdateRecoveryService.getInstance().start();
|
||||
FleetSyncRetryService.getInstance().start();
|
||||
ImageUpdateService.getInstance().start();
|
||||
SchedulerService.getInstance().start();
|
||||
|
||||
@@ -26,6 +26,8 @@ const DEPLOY_ALLOWED_PATTERNS: RegExp[] = [
|
||||
/^\/api\/stacks\/[^/]+\/stop$/,
|
||||
/^\/api\/stacks\/[^/]+\/start$/,
|
||||
/^\/api\/stacks\/[^/]+\/update$/,
|
||||
/^\/api\/stacks\/[^/]+\/services\/[^/]+\/update$/,
|
||||
/^\/api\/stacks\/[^/]+\/services\/[^/]+\/restore$/,
|
||||
];
|
||||
|
||||
const deny = (res: Response, req: Request, error: string, scope: ApiTokenScope | 'unknown'): void => {
|
||||
|
||||
@@ -5,7 +5,7 @@ import { PROXY_TIER_HEADER, PROXY_ROLE_HEADER, PROXY_DEPLOY_SOURCE_HEADER, PROXY
|
||||
import { LicenseService } from '../services/LicenseService';
|
||||
import { isProxyExemptPath } from '../helpers/proxyExemptPaths';
|
||||
import { remoteSupportsCrossNodeRbac, remoteAdvertisesCapability } from '../helpers/remoteCapabilities';
|
||||
import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY } from '../services/CapabilityRegistry';
|
||||
import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { redactSensitiveText } from '../utils/safeLog';
|
||||
@@ -171,7 +171,7 @@ export function createRemoteProxyMiddleware(): RequestHandler {
|
||||
finalizeProxyTiming(req, 'error');
|
||||
console.error('[Proxy] Remote node error:', getErrorMessage(err, 'unknown'));
|
||||
const path = req.originalUrl || req.url;
|
||||
if (req.method === 'POST' && /^\/api\/stacks\/[^/]+\/(?:deploy|update)(?:\?|$)/.test(path)) {
|
||||
if (req.method === 'POST' && /^\/api\/stacks\/[^/]+\/(?:deploy|update|services\/[^/]+\/(?:update|restore))(?:\?|$)/.test(path)) {
|
||||
try {
|
||||
DatabaseService.getInstance().insertAuditLog({
|
||||
timestamp: Date.now(),
|
||||
@@ -238,6 +238,14 @@ export function createRemoteProxyMiddleware(): RequestHandler {
|
||||
}
|
||||
}
|
||||
|
||||
if (isServiceScopedUpdateRoute(req)) {
|
||||
const supported = await remoteAdvertisesCapability(req.nodeId, SERVICE_SCOPED_UPDATE_CAPABILITY);
|
||||
if (!supported) {
|
||||
res.status(400).json({ error: 'Service-scoped updates are not supported on this node', code: 'capability_unavailable' });
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Mixed-version RBAC gate (non-admin only).
|
||||
if (req.user?.role !== 'admin') {
|
||||
const rbacSupported = await remoteSupportsCrossNodeRbac(req.nodeId);
|
||||
@@ -264,3 +272,14 @@ function isStackDownWithRemoveVolumes(req: Request): boolean {
|
||||
if (!/^\/stacks\/[^/]+\/down$/.test(req.path)) return false;
|
||||
return req.query.removeVolumes === 'true';
|
||||
}
|
||||
|
||||
/** Nested service update/restore/recovery routes (path is post-/api strip). */
|
||||
function isServiceScopedUpdateRoute(req: Request): boolean {
|
||||
if (req.method === 'GET') {
|
||||
return /^\/stacks\/[^/]+\/services\/[^/]+\/recovery$/.test(req.path);
|
||||
}
|
||||
if (req.method === 'POST') {
|
||||
return /^\/stacks\/[^/]+\/services\/[^/]+\/(?:update|restore)$/.test(req.path);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import { FleetUpdateTrackerService, type UpdateTracker, type TerminalStatus, UPD
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import { computeNodeNetworkingSummary, type NodeNetworkingSummary } from '../services/network/networkingSummary';
|
||||
import DockerController from '../services/DockerController';
|
||||
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
|
||||
import { getHostMemory } from '../helpers/hostMemory';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { ComposeService } from '../services/ComposeService';
|
||||
@@ -2115,9 +2116,10 @@ fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res
|
||||
targetResults.push({ target, success: true, reclaimedBytes: estimate.reclaimableBytes, dryRun: true });
|
||||
continue;
|
||||
}
|
||||
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(node.id);
|
||||
const result = scope === 'managed'
|
||||
? await dockerController.pruneManagedOnly(target, knownStacks)
|
||||
: await dockerController.pruneSystem(target);
|
||||
? await dockerController.pruneManagedOnly(target, knownStacks, isImageHeld)
|
||||
: await dockerController.pruneSystem(target, undefined, isImageHeld);
|
||||
targetResults.push({ target, success: true, reclaimedBytes: result.reclaimedBytes });
|
||||
if (result.reclaimedBytes > 0 || result.success) anySuccess = true;
|
||||
} catch (err) {
|
||||
|
||||
@@ -7,7 +7,7 @@ import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import { CacheService } from '../services/CacheService';
|
||||
import { ImageUpdateService } from '../services/ImageUpdateService';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { ComposeService } from '../services/ComposeService';
|
||||
import { StackUpdateOrchestrator } from '../services/StackUpdateOrchestrator';
|
||||
import { StackOpLockService, stackOpSkipMessage } from '../services/StackOpLockService';
|
||||
import { NotificationService } from '../services/NotificationService';
|
||||
import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement';
|
||||
@@ -342,7 +342,6 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
|
||||
|
||||
const docker = DockerController.getInstance(req.nodeId);
|
||||
const imageUpdateService = ImageUpdateService.getInstance();
|
||||
const compose = ComposeService.getInstance(req.nodeId);
|
||||
const db = DatabaseService.getInstance();
|
||||
const atomic = true;
|
||||
const results: string[] = [];
|
||||
@@ -417,14 +416,17 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
|
||||
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
req.nodeId, stackName, 'update', 'system',
|
||||
() => compose.updateStack(stackName, undefined, atomic),
|
||||
() => StackUpdateOrchestrator.getInstance().execute(
|
||||
{ nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'automatic', actor: `auto-update:${req.user?.username ?? 'scheduler'}` },
|
||||
{ atomic, terminalWs: null },
|
||||
),
|
||||
);
|
||||
if (!lock.ran) {
|
||||
results.push(stackOpSkipMessage(stackName, lock.existing.action));
|
||||
continue;
|
||||
}
|
||||
db.clearStackUpdateStatus(req.nodeId, stackName);
|
||||
HealthGateService.getInstance().begin(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`);
|
||||
HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`);
|
||||
|
||||
NotificationService.getInstance().broadcastEvent({
|
||||
type: 'state-invalidate',
|
||||
|
||||
@@ -13,6 +13,7 @@ import { FileSystemService } from '../services/FileSystemService';
|
||||
import { StackFileRootsService, STACK_SOURCE_ROOT_ID, stackSourceFileRoot, type StackFileRoot } from '../services/StackFileRootsService';
|
||||
import { FileRootGateway } from '../services/FileRootGateway';
|
||||
import { ComposeService, getComposeRollbackInfo } from '../services/ComposeService';
|
||||
import { StackUpdateOrchestrator, shortImageId, type OrchestratorResult } from '../services/StackUpdateOrchestrator';
|
||||
import DockerController, { type BulkStackInfo } from '../services/DockerController';
|
||||
import { DatabaseService, type StackDossierFields } from '../services/DatabaseService';
|
||||
import { MeshService } from '../services/MeshService';
|
||||
@@ -30,11 +31,12 @@ import { buildStackNetworkFacts } from '../services/network/composeNetworkInspec
|
||||
import { buildStorageInventory } from '../services/storage/inventory';
|
||||
import { probeComposeDiscovery } from '../services/ComposeDiscoveryService';
|
||||
import { buildEffectiveAnatomy } from '../services/effectiveAnatomy';
|
||||
import { buildEffectiveServiceModel } from '../services/effectiveServiceModel';
|
||||
import { buildEnvInventory } from '../services/EnvInventoryService';
|
||||
import { buildStackLabelInventory } from '../services/LabelInventoryService';
|
||||
import { labelInventoryOptionsFromRequest, requireRevealAdmin } from '../helpers/labelInventoryRequest';
|
||||
import { EXPOSURE_INTENTS, type ExposureIntent } from '../services/network/types';
|
||||
import { UpdateGuardService } from '../services/UpdateGuardService';
|
||||
import { UpdateGuardService, SingleServiceUpdateReadinessError } from '../services/UpdateGuardService';
|
||||
import { HealthGateService } from '../services/HealthGateService';
|
||||
import { classifyFailure } from '../services/updateGuard/failureClassifier';
|
||||
import { requirePermission, checkPermission } from '../middleware/permissions';
|
||||
@@ -58,7 +60,8 @@ import { resolveStackEnvSources, discoverStackLocalEnvFiles } from '../helpers/e
|
||||
import { STACK_STATUSES_CACHE_TTL_MS } from '../helpers/constants';
|
||||
import { getTerminalWs, DEPLOY_SESSION_HEADER } from '../websocket/generic';
|
||||
import { isSelfStack, refuseIfSelfStack, selfStackProtectedBulkResult } from '../helpers/selfStackGuard';
|
||||
import { getActiveCapabilities, STACK_DOWN_REMOVE_VOLUMES_CAPABILITY } from '../services/CapabilityRegistry';
|
||||
import { getActiveCapabilities, STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry';
|
||||
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
|
||||
|
||||
// Authenticated users with edit permission can write arbitrarily large compose
|
||||
// files. Refuse to YAML.parse anything beyond this bound so a malformed (or
|
||||
@@ -471,7 +474,10 @@ async function runStackBulkOp(
|
||||
};
|
||||
}
|
||||
const atomic = true;
|
||||
await ComposeService.getInstance(req.nodeId).updateStack(stackName, getTerminalWs(req.get(DEPLOY_SESSION_HEADER)), atomic);
|
||||
await StackUpdateOrchestrator.getInstance().execute(
|
||||
{ nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'bulk', actor: user },
|
||||
{ atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) },
|
||||
);
|
||||
DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName);
|
||||
NotificationService.getInstance().broadcastEvent({
|
||||
type: 'state-invalidate',
|
||||
@@ -485,7 +491,7 @@ async function runStackBulkOp(
|
||||
triggerPostDeployScan(stackName, req.nodeId).catch(err =>
|
||||
console.error('[Security] Post-deploy scan failed for %s:', sanitizeForLog(stackName), err),
|
||||
);
|
||||
const healthGateId = HealthGateService.getInstance().begin(req.nodeId, stackName, 'update', req.user?.username ?? null);
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
|
||||
return { stackName, ok: true, healthGateId };
|
||||
} else {
|
||||
const outcome = await containerActionForStack(req.nodeId, stackName, action);
|
||||
@@ -1143,6 +1149,7 @@ stacksRouter.delete('/:stackName', async (req: Request, res: Response) => {
|
||||
try {
|
||||
DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName);
|
||||
DatabaseService.getInstance().clearStackScanAttempts(req.nodeId, stackName);
|
||||
DatabaseService.getInstance().deleteServiceUpdateRecoveries(req.nodeId, stackName);
|
||||
DatabaseService.getInstance().deleteRoleAssignmentsByResource('stack', stackName);
|
||||
DatabaseService.getInstance().deleteGitSource(stackName);
|
||||
DatabaseService.getInstance().deleteStackDossier(req.nodeId, stackName);
|
||||
@@ -1558,6 +1565,24 @@ stacksRouter.get('/:stackName/effective-anatomy', async (req: Request, res: Resp
|
||||
}
|
||||
});
|
||||
|
||||
// Effective Service Model: per-service facts (declared image, build presence,
|
||||
// expected replica count, dependencies, healthcheck) that service-scoped
|
||||
// update/restore key off of, from the fully-merged effective model. Read-only
|
||||
// and advisory; auto-proxies to the active node. Never returns raw render
|
||||
// stderr or any environment/label/command value.
|
||||
stacksRouter.get('/:stackName/effective-services', async (req: Request, res: Response) => {
|
||||
const stackName = req.params.stackName as string;
|
||||
if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return;
|
||||
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
|
||||
try {
|
||||
res.json(await buildEffectiveServiceModel(req.nodeId, stackName));
|
||||
} catch (error) {
|
||||
console.error('[Stacks] Failed to build effective service model for %s:', sanitizeForLog(stackName),
|
||||
sanitizeForLog(inspect(error, { depth: 4 })));
|
||||
res.status(500).json({ error: 'Failed to build effective service model' });
|
||||
}
|
||||
});
|
||||
|
||||
// Environment inventory: per-stack env vars with their source, scope (Compose
|
||||
// interpolation vs container injection), and status (present/missing/unused/
|
||||
// duplicate/unpersisted), plus likely-secret classification. Read-only and
|
||||
@@ -1656,12 +1681,17 @@ stacksRouter.put('/:stackName/exposure', async (req: Request, res: Response) =>
|
||||
// that owns it. Read-only, so stack:read is the correct gate.
|
||||
stacksRouter.get('/:stackName/update-readiness', async (req: Request, res: Response) => {
|
||||
const stackName = req.params.stackName as string;
|
||||
const serviceName = typeof req.query.service === 'string' && req.query.service.length > 0 ? req.query.service : undefined;
|
||||
if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return;
|
||||
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
|
||||
try {
|
||||
const report = await UpdateGuardService.getInstance().computeUpdateReadiness(req.nodeId, stackName);
|
||||
const report = await UpdateGuardService.getInstance().computeUpdateReadiness(req.nodeId, stackName, serviceName);
|
||||
res.json(report);
|
||||
} catch (error) {
|
||||
if (error instanceof SingleServiceUpdateReadinessError) {
|
||||
res.status(400).json({ error: error.message, code: 'service_update_single_service' });
|
||||
return;
|
||||
}
|
||||
console.error('[Stacks] Failed to compute update readiness for %s:', sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
res.status(500).json({ error: 'Failed to compute update readiness' });
|
||||
@@ -1724,7 +1754,7 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => {
|
||||
dlog(`[Stacks] Deploy completed: ${sanitizeForLog(stackName)}`);
|
||||
if (debug) console.debug(`[Stacks:debug] Deploy finished in ${Date.now() - t0}ms`);
|
||||
ok = true;
|
||||
const healthGateId = HealthGateService.getInstance().begin(req.nodeId, stackName, 'deploy', req.user?.username ?? null);
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'deploy', req.user?.username ?? null);
|
||||
res.json({ message: 'Deployed successfully', healthGateId });
|
||||
notifyActionSuccess('deploy_success', `${stackName} deployed`, stackName, req.user?.username ?? 'system');
|
||||
if (!skipScan) {
|
||||
@@ -1987,6 +2017,230 @@ stacksRouter.post('/:stackName/services/:serviceName/stop', (req, res) =>
|
||||
stacksRouter.post('/:stackName/services/:serviceName/start', (req, res) =>
|
||||
handleServiceAction(req, res, 'start'));
|
||||
|
||||
/** Map an orchestrator `service_failed` code to an HTTP status. */
|
||||
function serviceFailureStatus(code: string): number {
|
||||
switch (code) {
|
||||
case 'service_update_single_service':
|
||||
case 'service_not_updatable':
|
||||
case 'effective_model_render_failed':
|
||||
case 'recovery_id_required':
|
||||
return 400;
|
||||
case 'service_not_found':
|
||||
case 'recovery_not_found':
|
||||
return 404;
|
||||
case 'policy_blocked':
|
||||
case 'recovery_not_restorable':
|
||||
case 'recovery_claim_failed':
|
||||
return 409;
|
||||
default:
|
||||
// Compose, retag, inspect, and replica-divergence failures are server-side.
|
||||
return 500;
|
||||
}
|
||||
}
|
||||
|
||||
/** Send an orchestrator service result as an HTTP response. Returns success. */
|
||||
function sendServiceResult(res: Response, result: OrchestratorResult, serviceName: string): boolean {
|
||||
if (result.kind === 'service_done') {
|
||||
res.json({
|
||||
serviceName: result.serviceName,
|
||||
healthGateId: result.healthGateId,
|
||||
observing: result.observing,
|
||||
recoveryId: result.recoveryId,
|
||||
recoveryAvailable: result.recoveryAvailable,
|
||||
...(result.previousImageId ? { previousImageId: result.previousImageId } : {}),
|
||||
...(result.newImageId ? { newImageId: result.newImageId } : {}),
|
||||
...(result.recheckWarning ? { recheckWarning: result.recheckWarning } : {}),
|
||||
});
|
||||
return true;
|
||||
}
|
||||
if (result.kind === 'service_failed') {
|
||||
res.status(serviceFailureStatus(result.code)).json({
|
||||
error: result.error,
|
||||
code: result.code,
|
||||
serviceName: result.serviceName ?? serviceName,
|
||||
...(result.mutationStage ? { mutationStage: result.mutationStage } : {}),
|
||||
...(result.recoveryId ? { recoveryId: result.recoveryId } : {}),
|
||||
});
|
||||
return false;
|
||||
}
|
||||
// Stack results never reach the service routes; treat defensively.
|
||||
res.status(500).json({ error: 'Unexpected orchestrator result', code: 'unexpected_result' });
|
||||
return false;
|
||||
}
|
||||
|
||||
function requireServiceScopedUpdateCapability(res: Response): boolean {
|
||||
if (getActiveCapabilities().includes(SERVICE_SCOPED_UPDATE_CAPABILITY)) return true;
|
||||
res.status(400).json({ error: 'Service-scoped updates are not supported on this node', code: 'capability_unavailable' });
|
||||
return false;
|
||||
}
|
||||
|
||||
async function handleServiceScopedMutation(
|
||||
req: Request,
|
||||
res: Response,
|
||||
options: {
|
||||
lockAction: 'update' | 'rollback';
|
||||
recoveryId?: string;
|
||||
notifyFailureAction: 'update' | 'rollback';
|
||||
failureCode: string;
|
||||
failureMessage: string;
|
||||
onSuccess: (
|
||||
stackName: string,
|
||||
serviceName: string,
|
||||
actor: string,
|
||||
meta: { previousImageId?: string | null; newImageId?: string | null },
|
||||
) => void;
|
||||
},
|
||||
): Promise<void> {
|
||||
const stackName = req.params.stackName as string;
|
||||
const serviceName = req.params.serviceName as string;
|
||||
if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
|
||||
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
|
||||
if (await refuseIfSelfStack(req, res, stackName)) return;
|
||||
if (!requireServiceScopedUpdateCapability(res)) return;
|
||||
if (!tryAcquireStackOpLock(req, res, stackName, options.lockAction)) return;
|
||||
|
||||
const t0 = Date.now();
|
||||
let ok = false;
|
||||
try {
|
||||
const result = await StackUpdateOrchestrator.getInstance().execute(
|
||||
{
|
||||
nodeId: req.nodeId,
|
||||
stackName,
|
||||
target: { scope: 'service', serviceName },
|
||||
trigger: 'manual',
|
||||
actor: req.user?.username ?? null,
|
||||
},
|
||||
{
|
||||
policyOptions: buildPolicyGateOptions(req),
|
||||
terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)),
|
||||
...(options.recoveryId ? { recoveryId: options.recoveryId } : {}),
|
||||
},
|
||||
);
|
||||
invalidateNodeCaches(req.nodeId);
|
||||
ok = sendServiceResult(res, result, serviceName);
|
||||
if (ok && result.kind === 'service_done') {
|
||||
options.onSuccess(stackName, serviceName, req.user?.username ?? 'system', {
|
||||
previousImageId: result.previousImageId,
|
||||
newImageId: result.newImageId,
|
||||
});
|
||||
NotificationService.getInstance().broadcastEvent({
|
||||
type: 'state-invalidate',
|
||||
scope: 'image-updates',
|
||||
nodeId: req.nodeId,
|
||||
stackName,
|
||||
action: 'stack-updated',
|
||||
ts: Date.now(),
|
||||
});
|
||||
} else if (result.kind === 'service_failed') {
|
||||
notifyActionFailure(
|
||||
options.notifyFailureAction,
|
||||
stackName,
|
||||
new Error(result.error),
|
||||
req.user?.username ?? 'system',
|
||||
);
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
console.error(
|
||||
'[Stacks] Service %s failed: %s/%s: %s',
|
||||
sanitizeForLog(options.notifyFailureAction),
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(serviceName),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
notifyActionFailure(options.notifyFailureAction, stackName, error, req.user?.username ?? 'system');
|
||||
if (!res.headersSent) {
|
||||
res.status(500).json({ error: getErrorMessage(error, options.failureMessage), code: options.failureCode });
|
||||
}
|
||||
} finally {
|
||||
StackOpMetricsService.getInstance().record(req.nodeId, 'update', Date.now() - t0, ok, {
|
||||
targetScope: 'service',
|
||||
serviceName,
|
||||
});
|
||||
releaseStackOpLock(req, stackName);
|
||||
}
|
||||
}
|
||||
|
||||
stacksRouter.post('/:stackName/services/:serviceName/update', async (req: Request, res: Response) => {
|
||||
await handleServiceScopedMutation(req, res, {
|
||||
lockAction: 'update',
|
||||
notifyFailureAction: 'update',
|
||||
failureCode: 'service_update_failed',
|
||||
failureMessage: 'Failed to update service',
|
||||
onSuccess: (stackName, serviceName, actor, meta) => {
|
||||
const from = shortImageId(meta.previousImageId);
|
||||
const to = shortImageId(meta.newImageId);
|
||||
const transition = from && to && from !== to ? ` (${from} -> ${to})` : '';
|
||||
notifyActionSuccess(
|
||||
'image_update_applied',
|
||||
`${stackName}/${serviceName} updated${transition}`,
|
||||
stackName,
|
||||
actor,
|
||||
);
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
/** Latest restorable service recovery snapshot (active, unexpired), for UI discovery outside Deploy Progress. */
|
||||
stacksRouter.get('/:stackName/services/:serviceName/recovery', async (req: Request, res: Response) => {
|
||||
const stackName = req.params.stackName as string;
|
||||
const serviceName = req.params.serviceName as string;
|
||||
if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
|
||||
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
|
||||
if (!requireServiceScopedUpdateCapability(res)) return;
|
||||
try {
|
||||
const row = ServiceUpdateRecoveryService.getInstance().listActive(req.nodeId, stackName, serviceName)[0];
|
||||
if (!row) {
|
||||
res.json({ recovery: null });
|
||||
return;
|
||||
}
|
||||
res.json({
|
||||
recovery: {
|
||||
id: row.id,
|
||||
status: row.status,
|
||||
healthGateId: row.health_gate_id,
|
||||
expiresAt: row.expires_at,
|
||||
createdAt: row.created_at,
|
||||
majorityImageId: row.majority_image_id,
|
||||
declaredImageRef: row.declared_image_ref,
|
||||
},
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
console.error(
|
||||
'[Stacks] Failed to list service recovery for %s/%s: %s',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(serviceName),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
res.status(500).json({ error: 'Failed to load service recovery', code: 'service_recovery_lookup_failed' });
|
||||
}
|
||||
});
|
||||
|
||||
stacksRouter.post('/:stackName/services/:serviceName/restore', async (req: Request, res: Response) => {
|
||||
const recoveryId = typeof req.body?.recoveryId === 'string' ? req.body.recoveryId : '';
|
||||
if (!recoveryId) {
|
||||
res.status(400).json({ error: 'A recoveryId is required to restore a service.', code: 'recovery_id_required' });
|
||||
return;
|
||||
}
|
||||
await handleServiceScopedMutation(req, res, {
|
||||
lockAction: 'rollback',
|
||||
recoveryId,
|
||||
notifyFailureAction: 'rollback',
|
||||
failureCode: 'service_restore_failed',
|
||||
failureMessage: 'Failed to restore service',
|
||||
onSuccess: (stackName, serviceName, actor, meta) => {
|
||||
const from = shortImageId(meta.previousImageId);
|
||||
const to = shortImageId(meta.newImageId);
|
||||
const transition = from && to && from !== to ? ` (${from} -> ${to})` : '';
|
||||
notifyActionSuccess(
|
||||
'deploy_success',
|
||||
`${stackName}/${serviceName} restored${transition}`,
|
||||
stackName,
|
||||
actor,
|
||||
);
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
stacksRouter.get('/:stackName/update-preview', async (req: Request, res: Response) => {
|
||||
const stackName = req.params.stackName as string;
|
||||
try {
|
||||
@@ -2013,7 +2267,10 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
|
||||
const debug = isDebugEnabled();
|
||||
const atomic = true;
|
||||
if (debug) console.debug('[Stacks:debug] Update starting', { stackName, atomic, nodeId: req.nodeId });
|
||||
await ComposeService.getInstance(req.nodeId).updateStack(stackName, getTerminalWs(req.get(DEPLOY_SESSION_HEADER)), atomic);
|
||||
await StackUpdateOrchestrator.getInstance().execute(
|
||||
{ nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'manual', actor: req.user?.username ?? null },
|
||||
{ atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) },
|
||||
);
|
||||
DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName);
|
||||
invalidateNodeCaches(req.nodeId);
|
||||
NotificationService.getInstance().broadcastEvent({
|
||||
@@ -2027,7 +2284,7 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
|
||||
dlog(`[Stacks] Update completed: ${sanitizeForLog(stackName)}`);
|
||||
if (debug) console.debug(`[Stacks:debug] Update finished in ${Date.now() - t0}ms`);
|
||||
ok = true;
|
||||
const healthGateId = HealthGateService.getInstance().begin(req.nodeId, stackName, 'update', req.user?.username ?? null);
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
|
||||
res.json({ status: 'Update completed', healthGateId });
|
||||
notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, req.user?.username ?? 'system');
|
||||
if (!skipScan) {
|
||||
|
||||
@@ -8,6 +8,7 @@ import DockerController, {
|
||||
} from '../services/DockerController';
|
||||
import { isPruneTarget } from '../services/prunePlan';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
|
||||
import SelfIdentityService from '../services/SelfIdentityService';
|
||||
import { requireAdmin } from '../middleware/tierGates';
|
||||
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
|
||||
@@ -127,8 +128,9 @@ systemMaintenanceRouter.post('/prune/plan', async (req: Request, res: Response)
|
||||
const pruneScope = parsePruneScope((req.body as { scope?: unknown }).scope);
|
||||
const dockerController = DockerController.getInstance(req.nodeId);
|
||||
const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks();
|
||||
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(req.nodeId);
|
||||
const plan = await withTimeout(
|
||||
dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId),
|
||||
dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId, isImageHeld),
|
||||
PRUNE_ESTIMATE_TIMEOUT_MS,
|
||||
'docker prune plan',
|
||||
);
|
||||
@@ -171,11 +173,12 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
|
||||
const planFingerprint = typeof body.planFingerprint === 'string' ? body.planFingerprint : null;
|
||||
const dockerController = DockerController.getInstance(req.nodeId);
|
||||
const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks();
|
||||
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(req.nodeId);
|
||||
|
||||
if (isDryRun) {
|
||||
// Dry-run returns the same itemized plan shape Resources uses for preview.
|
||||
const plan = await withTimeout(
|
||||
dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId),
|
||||
dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId, isImageHeld),
|
||||
PRUNE_ESTIMATE_TIMEOUT_MS,
|
||||
'docker prune plan',
|
||||
);
|
||||
@@ -201,7 +204,7 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
|
||||
// fingerprint and keeps the legacy pruneManagedOnly / pruneSystem path.
|
||||
if (planFingerprint) {
|
||||
const built = await withTimeout(
|
||||
dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId),
|
||||
dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId, isImageHeld),
|
||||
PRUNE_ESTIMATE_TIMEOUT_MS,
|
||||
'docker prune plan',
|
||||
);
|
||||
@@ -215,7 +218,7 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
|
||||
`[Resources] System prune (plan): ${built.targets.join(',')} (scope: ${pruneScope}, items: ${built.items.length})`,
|
||||
);
|
||||
const pruneStartedAt = Date.now();
|
||||
const result = await dockerController.executePrunePlan(built, knownStacks);
|
||||
const result = await dockerController.executePrunePlan(built, knownStacks, isImageHeld);
|
||||
console.log(
|
||||
`[Resources] System prune completed: reclaimed ${result.reclaimedBytes} bytes, outcomes=${result.outcomes.length}`,
|
||||
);
|
||||
@@ -254,14 +257,15 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
|
||||
result = await dockerController.pruneManagedOnly(
|
||||
target as 'images' | 'volumes' | 'networks',
|
||||
knownStacks,
|
||||
isImageHeld,
|
||||
);
|
||||
} else if (pruneScope === 'managed' && target === 'containers') {
|
||||
// Managed containers must never fall through to system prune. Build and
|
||||
// execute an itemized plan for this single target instead.
|
||||
const plan = await dockerController.buildPrunePlan(['containers'], 'managed', knownStacks, req.nodeId);
|
||||
result = await dockerController.executePrunePlan(plan, knownStacks);
|
||||
const plan = await dockerController.buildPrunePlan(['containers'], 'managed', knownStacks, req.nodeId, isImageHeld);
|
||||
result = await dockerController.executePrunePlan(plan, knownStacks, isImageHeld);
|
||||
} else {
|
||||
result = await dockerController.pruneSystem(target);
|
||||
result = await dockerController.pruneSystem(target, undefined, isImageHeld);
|
||||
}
|
||||
|
||||
console.log(`[Resources] System prune completed: ${target}, reclaimed ${result.reclaimedBytes} bytes`);
|
||||
|
||||
@@ -71,6 +71,15 @@ export class AutoHealService {
|
||||
private observedUnhealthySince = new Map<string, number>();
|
||||
private historyTimestamps = new Map<string, number>();
|
||||
private leaseRefreshFailures = new Map<number, number>();
|
||||
/**
|
||||
* Ref-counted suppressions for service-scoped updates/restores. While a
|
||||
* service is being recreated and its post-mutation health gate observes it,
|
||||
* auto-heal must not restart the churning containers out from under the gate.
|
||||
* Keyed `${nodeId}:${stackName}:${serviceName}`. Nested owners (overlapping
|
||||
* same-service ops, or a gate that outlives the orchestrator finally) each
|
||||
* increment; clearSuppress only unmutes at zero. Process death clears it.
|
||||
*/
|
||||
private suppressedServices = new Map<string, number>();
|
||||
|
||||
private constructor() {}
|
||||
|
||||
@@ -108,6 +117,33 @@ export class AutoHealService {
|
||||
clearInterval(this.leaseRefreshTimer);
|
||||
this.leaseRefreshTimer = null;
|
||||
}
|
||||
this.suppressedServices.clear();
|
||||
}
|
||||
|
||||
private suppressKey(nodeId: number, stackName: string, serviceName: string): string {
|
||||
return `${nodeId}:${stackName}:${serviceName}`;
|
||||
}
|
||||
|
||||
/** Suppress auto-heal for one service while a service-scoped update/restore + its health gate run. */
|
||||
suppress(nodeId: number, stackName: string, serviceName: string): void {
|
||||
const key = this.suppressKey(nodeId, stackName, serviceName);
|
||||
this.suppressedServices.set(key, (this.suppressedServices.get(key) ?? 0) + 1);
|
||||
}
|
||||
|
||||
/** Release one suppression owner; Auto-Heal resumes only when the count hits zero. */
|
||||
clearSuppress(nodeId: number, stackName: string, serviceName: string): void {
|
||||
const key = this.suppressKey(nodeId, stackName, serviceName);
|
||||
const current = this.suppressedServices.get(key) ?? 0;
|
||||
if (current <= 1) {
|
||||
this.suppressedServices.delete(key);
|
||||
return;
|
||||
}
|
||||
this.suppressedServices.set(key, current - 1);
|
||||
}
|
||||
|
||||
/** True while the given service is suppressed. A stack-wide policy still heals other services. */
|
||||
isSuppressed(nodeId: number, stackName: string, serviceName: string): boolean {
|
||||
return (this.suppressedServices.get(this.suppressKey(nodeId, stackName, serviceName)) ?? 0) > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -283,6 +319,12 @@ export class AutoHealService {
|
||||
const containerName =
|
||||
container.Names?.[0]?.replace(/^\//, '') ?? container.Id.slice(0, 12);
|
||||
const serviceOverride = container.Labels?.['com.docker.compose.service'] ?? null;
|
||||
// Skip containers whose service is suppressed by an in-flight
|
||||
// service-scoped update/restore, so the post-mutation health gate
|
||||
// owns the churn without auto-heal racing it.
|
||||
if (serviceOverride && this.isSuppressed(nodeId, policy.stack_name, serviceOverride)) {
|
||||
continue;
|
||||
}
|
||||
const signal = this.getHealSignal(nodeId, container, eventSvc?.getContainerState(container.Id), now);
|
||||
const decision = this.shouldHeal(signal, policy, this.containerKey(nodeId, container.Id), now);
|
||||
|
||||
|
||||
@@ -56,6 +56,7 @@ export const CAPABILITIES = [
|
||||
'cross-node-rbac',
|
||||
'stack-down-remove-volumes',
|
||||
'guided-external-network-preflight',
|
||||
'service-scoped-update',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
@@ -72,6 +73,9 @@ export type Capability = (typeof CAPABILITIES)[number];
|
||||
/** Capability for optional `?removeVolumes=true` on POST /stacks/:name/down. */
|
||||
export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes' as const satisfies Capability;
|
||||
|
||||
/** Capability for the nested per-service update/restore routes and the `effective-services` model they read. */
|
||||
export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability;
|
||||
|
||||
/** Returns true when the string is a usable semver version. */
|
||||
export function isValidVersion(v: string | null | undefined): v is string {
|
||||
return !!v && v !== 'unknown' && v !== '0.0.0-dev' && !!semver.valid(v);
|
||||
|
||||
@@ -85,7 +85,8 @@ export function getComposeRollbackInfo(error: unknown): { attempted: boolean; ro
|
||||
|
||||
const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000;
|
||||
|
||||
function getComposeCommandTimeoutMs(): number {
|
||||
/** Public so other services (e.g. recovery claim leases) can size their own timers off the same ceiling without depending on a private module-local. */
|
||||
export function getComposeCommandTimeoutMs(): number {
|
||||
const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS);
|
||||
if (Number.isFinite(configured) && configured > 0) {
|
||||
return configured;
|
||||
@@ -873,7 +874,11 @@ export class ComposeService {
|
||||
try {
|
||||
const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1';
|
||||
if (pruneOnUpdate) {
|
||||
const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages();
|
||||
// Dynamic import: ServiceUpdateRecoveryService imports getComposeCommandTimeoutMs
|
||||
// from this module, so a static import here would be circular.
|
||||
const { ServiceUpdateRecoveryService } = await import('./ServiceUpdateRecoveryService');
|
||||
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(this.nodeId);
|
||||
const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld);
|
||||
// The Docker prune API does not report SpaceReclaimed on the containerd
|
||||
// image store, so only show the figure when the daemon actually returns one.
|
||||
const reclaimed = result.reclaimedBytes > 0
|
||||
@@ -906,6 +911,53 @@ export class ComposeService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Service-scoped update: pull (or `build --pull` for a build-backed service)
|
||||
* and recreate a single service's replicas in place. Always
|
||||
* `--no-deps --force-recreate`, never `--remove-orphans`, so sibling services
|
||||
* keep their container ids and StartedAt. No drift re-baseline and no dangling
|
||||
* prune here; the orchestrator owns per-service post-update reconciliation.
|
||||
*/
|
||||
async updateService(stackName: string, serviceName: string, hasBuild: boolean, ws?: WebSocket): Promise<void> {
|
||||
await this.assertRequiredEnvPresent(stackName);
|
||||
await this.assertSafePilotBindMapping(stackName);
|
||||
const stackDir = path.join(this.baseDir, stackName);
|
||||
const sendOutput = (data: string) => {
|
||||
if (ws && ws.readyState === WebSocket.OPEN) ws.send(data);
|
||||
};
|
||||
await this.withRegistryAuth(async (env) => {
|
||||
if (hasBuild) {
|
||||
sendOutput(`=== Building ${serviceName} ===\n`);
|
||||
await this.execute('docker', await this.authoredComposeArgs(stackName, ['build', '--pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs());
|
||||
} else {
|
||||
sendOutput(`=== Pulling ${serviceName} ===\n`);
|
||||
await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs());
|
||||
}
|
||||
sendOutput(`=== Recreating ${serviceName} ===\n`);
|
||||
await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs());
|
||||
}, sendOutput);
|
||||
}
|
||||
|
||||
/**
|
||||
* Recreate a single service from the image already present locally, without
|
||||
* pulling or building. Used by service restore after the recovery image id has
|
||||
* been retagged onto the declared ref (`--pull never --no-build` so Compose
|
||||
* uses the just-retagged local image). Always `--no-deps --force-recreate`,
|
||||
* never `--remove-orphans`.
|
||||
*/
|
||||
async recreateServiceFromLocal(stackName: string, serviceName: string, ws?: WebSocket): Promise<void> {
|
||||
await this.assertRequiredEnvPresent(stackName);
|
||||
await this.assertSafePilotBindMapping(stackName);
|
||||
const stackDir = path.join(this.baseDir, stackName);
|
||||
const sendOutput = (data: string) => {
|
||||
if (ws && ws.readyState === WebSocket.OPEN) ws.send(data);
|
||||
};
|
||||
await this.withRegistryAuth(async (env) => {
|
||||
sendOutput(`=== Restoring ${serviceName} ===\n`);
|
||||
await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', '--pull', 'never', '--no-build', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs());
|
||||
}, sendOutput);
|
||||
}
|
||||
|
||||
public async downStack(stackName: string): Promise<void> {
|
||||
const stackPath = path.join(this.baseDir, stackName);
|
||||
try {
|
||||
|
||||
@@ -35,11 +35,58 @@ export interface GlobalSetting {
|
||||
*/
|
||||
export type StackCheckStatus = 'ok' | 'partial' | 'failed';
|
||||
|
||||
/**
|
||||
* Per-service check outcome persisted in stack_update_status.services_json.
|
||||
* Distinct from StackCheckStatus: a service with no checkable image ref is
|
||||
* `not_checkable`, which never counts as a check failure at the aggregate
|
||||
* (stack-level) status.
|
||||
*/
|
||||
export type ServiceCheckStatus = 'ok' | 'partial' | 'failed' | 'not_checkable';
|
||||
|
||||
export interface StackServiceStatus {
|
||||
service: string;
|
||||
image: string | null;
|
||||
runtimeImages?: string[];
|
||||
hasUpdate: boolean;
|
||||
checkStatus: ServiceCheckStatus;
|
||||
lastError: string | null;
|
||||
}
|
||||
|
||||
export interface StackUpdateDetail {
|
||||
hasUpdate: boolean;
|
||||
checkStatus: StackCheckStatus;
|
||||
lastError: string | null;
|
||||
checkedAt: number;
|
||||
/** Per-service breakdown; omitted when the stack has no persisted per-service data (no effective model available yet, or corrupt JSON). */
|
||||
services?: StackServiceStatus[];
|
||||
}
|
||||
|
||||
const SERVICES_JSON_VERSION = 1;
|
||||
|
||||
function isStackServiceStatus(value: unknown): value is StackServiceStatus {
|
||||
if (!value || typeof value !== 'object') return false;
|
||||
const v = value as Record<string, unknown>;
|
||||
return typeof v.service === 'string'
|
||||
&& (v.image === null || typeof v.image === 'string')
|
||||
&& typeof v.hasUpdate === 'boolean'
|
||||
&& (v.checkStatus === 'ok' || v.checkStatus === 'partial' || v.checkStatus === 'failed' || v.checkStatus === 'not_checkable')
|
||||
&& (v.lastError === null || typeof v.lastError === 'string');
|
||||
}
|
||||
|
||||
/** Parses stack_update_status.services_json safely; a missing, corrupt, or version-mismatched value yields an empty list rather than throwing. */
|
||||
function parseServicesJson(raw: string | null | undefined): StackServiceStatus[] {
|
||||
if (!raw) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as { version?: unknown; services?: unknown };
|
||||
if (parsed?.version !== SERVICES_JSON_VERSION || !Array.isArray(parsed.services)) return [];
|
||||
return parsed.services.filter(isStackServiceStatus);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function stringifyServicesJson(services: StackServiceStatus[], generation: number): string {
|
||||
return JSON.stringify({ version: SERVICES_JSON_VERSION, generation, services });
|
||||
}
|
||||
|
||||
export interface StackAlert {
|
||||
@@ -146,7 +193,7 @@ export interface HealthGateRunRow {
|
||||
node_id: number;
|
||||
stack_name: string;
|
||||
/** Named trigger_action because TRIGGER is reserved in SQLite. */
|
||||
trigger_action: 'update' | 'deploy';
|
||||
trigger_action: 'update' | 'deploy' | 'service_update' | 'service_restore';
|
||||
status: 'observing' | 'passed' | 'failed' | 'unknown';
|
||||
reason: string | null;
|
||||
window_seconds: number;
|
||||
@@ -155,6 +202,30 @@ export interface HealthGateRunRow {
|
||||
started_at: number;
|
||||
ended_at: number | null;
|
||||
created_by: string | null;
|
||||
/** Additive; legacy rows default to stack. */
|
||||
target_scope: 'stack' | 'service';
|
||||
service_name: string | null;
|
||||
failure_source: 'primary' | 'collateral' | null;
|
||||
}
|
||||
|
||||
/** Pre-update image snapshot enabling a manual per-service restore after a service-scoped update. */
|
||||
export interface ServiceUpdateRecoveryRow {
|
||||
id: string;
|
||||
node_id: number;
|
||||
stack_name: string;
|
||||
service_name: string;
|
||||
/** JSON array of pre-update replica image snapshots (imageId + repoDigest when known). */
|
||||
replicas_json: string;
|
||||
majority_image_id: string;
|
||||
/** Audit/UI only: the tag the majority image is retagged onto during restore, never a policy scan target. */
|
||||
declared_image_ref: string;
|
||||
weak_floating_tag: number;
|
||||
health_gate_id: string | null;
|
||||
status: 'active' | 'restoring' | 'consumed' | 'expired' | 'invalidated';
|
||||
expires_at: number;
|
||||
claim_expires_at: number | null;
|
||||
created_at: number;
|
||||
created_by: string | null;
|
||||
}
|
||||
|
||||
/** One finding within a stored preflight run. Never carries an environment value. */
|
||||
@@ -1005,6 +1076,7 @@ export class DatabaseService {
|
||||
has_update INTEGER DEFAULT 0,
|
||||
check_status TEXT NOT NULL DEFAULT 'ok',
|
||||
last_error TEXT,
|
||||
services_json TEXT,
|
||||
checked_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (node_id, stack_name)
|
||||
);
|
||||
@@ -1525,18 +1597,44 @@ export class DatabaseService {
|
||||
id TEXT PRIMARY KEY,
|
||||
node_id INTEGER NOT NULL,
|
||||
stack_name TEXT NOT NULL,
|
||||
trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy')),
|
||||
trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy','service_update','service_restore')),
|
||||
status TEXT NOT NULL CHECK (status IN ('observing','passed','failed','unknown')),
|
||||
reason TEXT,
|
||||
window_seconds INTEGER NOT NULL,
|
||||
containers_json TEXT NOT NULL DEFAULT '[]',
|
||||
started_at INTEGER NOT NULL,
|
||||
ended_at INTEGER,
|
||||
created_by TEXT
|
||||
created_by TEXT,
|
||||
target_scope TEXT NOT NULL DEFAULT 'stack' CHECK (target_scope IN ('stack','service')),
|
||||
service_name TEXT,
|
||||
failure_source TEXT CHECK (failure_source IS NULL OR failure_source IN ('primary','collateral'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_health_gate_runs_node_stack
|
||||
ON health_gate_runs(node_id, stack_name, started_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS service_update_recovery (
|
||||
id TEXT PRIMARY KEY,
|
||||
node_id INTEGER NOT NULL,
|
||||
stack_name TEXT NOT NULL,
|
||||
service_name TEXT NOT NULL,
|
||||
replicas_json TEXT NOT NULL,
|
||||
majority_image_id TEXT NOT NULL,
|
||||
declared_image_ref TEXT NOT NULL,
|
||||
weak_floating_tag INTEGER NOT NULL DEFAULT 0,
|
||||
health_gate_id TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','restoring','consumed','expired','invalidated')),
|
||||
expires_at INTEGER NOT NULL,
|
||||
claim_expires_at INTEGER,
|
||||
created_at INTEGER NOT NULL,
|
||||
created_by TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_service_update_recovery_node_stack_service
|
||||
ON service_update_recovery(node_id, stack_name, service_name, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_service_update_recovery_status_expires
|
||||
ON service_update_recovery(status, expires_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_service_update_recovery_status_claim_expires
|
||||
ON service_update_recovery(status, claim_expires_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS secrets (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
@@ -1664,6 +1762,11 @@ export class DatabaseService {
|
||||
// failed check is no longer indistinguishable from "up to date".
|
||||
maybeAddCol('stack_update_status', 'check_status', "TEXT NOT NULL DEFAULT 'ok'");
|
||||
maybeAddCol('stack_update_status', 'last_error', 'TEXT');
|
||||
// Per-service image-update snapshot. Must run AFTER the composite-PK
|
||||
// recreate above so it is never silently dropped on old installs.
|
||||
maybeAddCol('stack_update_status', 'services_json', 'TEXT');
|
||||
|
||||
this.migrateHealthGateTargetSchema();
|
||||
|
||||
// Drop legacy SSH/TLS columns from pre-0.7 databases (no longer read or written)
|
||||
const legacyCols = ['host', 'port', 'ssh_port', 'ssh_user', 'ssh_password', 'ssh_key', 'tls_ca', 'tls_cert', 'tls_key'];
|
||||
@@ -1791,6 +1894,66 @@ export class DatabaseService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild health_gate_runs when the installed CHECK still only allows
|
||||
* update|deploy or target/failure columns are missing. Idempotent and
|
||||
* restart-safe: drops a stale temporary table, then rebuilds in one
|
||||
* better-sqlite3 transaction so an interrupted startup cannot leave
|
||||
* CREATE TABLE health_gate_runs_new blocking the next boot.
|
||||
*/
|
||||
private migrateHealthGateTargetSchema(): void {
|
||||
const tableSql = (this.db.prepare(
|
||||
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'health_gate_runs'"
|
||||
).get() as { sql: string } | undefined)?.sql ?? '';
|
||||
const cols = this.db.pragma('table_info(health_gate_runs)') as Array<{ name: string }>;
|
||||
const colNames = new Set(cols.map(c => c.name));
|
||||
const hasTarget = colNames.has('target_scope');
|
||||
const hasFailure = colNames.has('failure_source');
|
||||
const hasWideTrigger = tableSql.includes('service_update') && tableSql.includes('service_restore');
|
||||
if (hasTarget && hasFailure && hasWideTrigger) return;
|
||||
|
||||
// A previous crash between CREATE and RENAME leaves this temp table behind.
|
||||
this.db.exec('DROP TABLE IF EXISTS health_gate_runs_new');
|
||||
|
||||
const targetExpr = hasTarget ? 'target_scope' : "'stack'";
|
||||
const serviceExpr = colNames.has('service_name') ? 'service_name' : 'NULL';
|
||||
const failureExpr = hasFailure ? 'failure_source' : 'NULL';
|
||||
|
||||
this.db.transaction(() => {
|
||||
this.db.exec(`
|
||||
CREATE TABLE health_gate_runs_new (
|
||||
id TEXT PRIMARY KEY,
|
||||
node_id INTEGER NOT NULL,
|
||||
stack_name TEXT NOT NULL,
|
||||
trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy','service_update','service_restore')),
|
||||
status TEXT NOT NULL CHECK (status IN ('observing','passed','failed','unknown')),
|
||||
reason TEXT,
|
||||
window_seconds INTEGER NOT NULL,
|
||||
containers_json TEXT NOT NULL DEFAULT '[]',
|
||||
started_at INTEGER NOT NULL,
|
||||
ended_at INTEGER,
|
||||
created_by TEXT,
|
||||
target_scope TEXT NOT NULL DEFAULT 'stack' CHECK (target_scope IN ('stack','service')),
|
||||
service_name TEXT,
|
||||
failure_source TEXT CHECK (failure_source IS NULL OR failure_source IN ('primary','collateral'))
|
||||
);
|
||||
INSERT INTO health_gate_runs_new (
|
||||
id, node_id, stack_name, trigger_action, status, reason, window_seconds,
|
||||
containers_json, started_at, ended_at, created_by, target_scope, service_name, failure_source
|
||||
)
|
||||
SELECT
|
||||
id, node_id, stack_name, trigger_action, status, reason, window_seconds,
|
||||
containers_json, started_at, ended_at, created_by,
|
||||
${targetExpr}, ${serviceExpr}, ${failureExpr}
|
||||
FROM health_gate_runs;
|
||||
DROP TABLE health_gate_runs;
|
||||
ALTER TABLE health_gate_runs_new RENAME TO health_gate_runs;
|
||||
CREATE INDEX IF NOT EXISTS idx_health_gate_runs_node_stack
|
||||
ON health_gate_runs(node_id, stack_name, started_at);
|
||||
`);
|
||||
})();
|
||||
}
|
||||
|
||||
private migrateEncryptNodeTokens(): void {
|
||||
const crypto = CryptoService.getInstance();
|
||||
const rows = this.db.prepare("SELECT id, api_token FROM nodes WHERE api_token != '' AND api_token IS NOT NULL").all() as Array<{ id: number; api_token: string }>;
|
||||
@@ -3128,11 +3291,13 @@ export class DatabaseService {
|
||||
public insertHealthGateRun(run: HealthGateRunRow): void {
|
||||
this.db.prepare(
|
||||
`INSERT INTO health_gate_runs
|
||||
(id, node_id, stack_name, trigger_action, status, reason, window_seconds, containers_json, started_at, ended_at, created_by)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
(id, node_id, stack_name, trigger_action, status, reason, window_seconds, containers_json,
|
||||
started_at, ended_at, created_by, target_scope, service_name, failure_source)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
).run(
|
||||
run.id, run.node_id, run.stack_name, run.trigger_action, run.status, run.reason,
|
||||
run.window_seconds, run.containers_json, run.started_at, run.ended_at, run.created_by,
|
||||
run.target_scope, run.service_name, run.failure_source,
|
||||
);
|
||||
// Bounded history: keep only the 10 most recent runs per stack.
|
||||
this.db.prepare(
|
||||
@@ -3152,10 +3317,11 @@ export class DatabaseService {
|
||||
reason: string | null,
|
||||
endedAt: number,
|
||||
containersJson: string,
|
||||
failureSource: 'primary' | 'collateral' | null = null,
|
||||
): void {
|
||||
this.db.prepare(
|
||||
'UPDATE health_gate_runs SET status = ?, reason = ?, ended_at = ?, containers_json = ? WHERE id = ?'
|
||||
).run(status, reason, endedAt, containersJson, id);
|
||||
'UPDATE health_gate_runs SET status = ?, reason = ?, ended_at = ?, containers_json = ?, failure_source = ? WHERE id = ?'
|
||||
).run(status, reason, endedAt, containersJson, failureSource, id);
|
||||
}
|
||||
|
||||
public getHealthGateRun(nodeId: number, stackName: string, id: string): HealthGateRunRow | undefined {
|
||||
@@ -3178,6 +3344,128 @@ export class DatabaseService {
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
// --- Service Update Recovery ---
|
||||
|
||||
public insertServiceUpdateRecovery(row: ServiceUpdateRecoveryRow): void {
|
||||
this.db.prepare(
|
||||
`INSERT INTO service_update_recovery
|
||||
(id, node_id, stack_name, service_name, replicas_json, majority_image_id,
|
||||
declared_image_ref, weak_floating_tag, health_gate_id, status,
|
||||
expires_at, claim_expires_at, created_at, created_by)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
).run(
|
||||
row.id, row.node_id, row.stack_name, row.service_name, row.replicas_json, row.majority_image_id,
|
||||
row.declared_image_ref, row.weak_floating_tag, row.health_gate_id, row.status,
|
||||
row.expires_at, row.claim_expires_at, row.created_at, row.created_by,
|
||||
);
|
||||
}
|
||||
|
||||
public getServiceUpdateRecovery(id: string): ServiceUpdateRecoveryRow | undefined {
|
||||
return this.db.prepare(
|
||||
'SELECT * FROM service_update_recovery WHERE id = ?'
|
||||
).get(id) as ServiceUpdateRecoveryRow | undefined;
|
||||
}
|
||||
|
||||
/** Active, unexpired rows for one service, most recent first. */
|
||||
public listActiveServiceUpdateRecoveries(nodeId: number, stackName: string, serviceName: string): ServiceUpdateRecoveryRow[] {
|
||||
return this.db.prepare(
|
||||
`SELECT * FROM service_update_recovery
|
||||
WHERE node_id = ? AND stack_name = ? AND service_name = ? AND status = 'active'
|
||||
ORDER BY created_at DESC`
|
||||
).all(nodeId, stackName, serviceName) as ServiceUpdateRecoveryRow[];
|
||||
}
|
||||
|
||||
/** Attach the update flow's own health gate run id while the row is still active. */
|
||||
public linkServiceUpdateRecoveryHealthGate(id: string, healthGateId: string): void {
|
||||
this.db.prepare(
|
||||
`UPDATE service_update_recovery SET health_gate_id = ? WHERE id = ? AND status = 'active'`
|
||||
).run(healthGateId, id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomic claim CAS for restore: active -> restoring, only when the row has
|
||||
* not already expired. Returns the updated row, or undefined when another
|
||||
* claim, a sweep, or a terminal status won the race.
|
||||
*/
|
||||
public claimServiceUpdateRecovery(id: string, claimExpiresAt: number, now: number): ServiceUpdateRecoveryRow | undefined {
|
||||
const result = this.db.prepare(
|
||||
`UPDATE service_update_recovery
|
||||
SET status = 'restoring', claim_expires_at = ?
|
||||
WHERE id = ? AND status = 'active' AND expires_at > ?`
|
||||
).run(claimExpiresAt, id, now);
|
||||
if (result.changes === 0) return undefined;
|
||||
return this.getServiceUpdateRecovery(id);
|
||||
}
|
||||
|
||||
/** Renewal CAS: only while still restoring. Cannot revive a consumed/expired/invalidated/active row. */
|
||||
public renewServiceUpdateRecoveryClaim(id: string, claimExpiresAt: number): boolean {
|
||||
const result = this.db.prepare(
|
||||
`UPDATE service_update_recovery SET claim_expires_at = ? WHERE id = ? AND status = 'restoring'`
|
||||
).run(claimExpiresAt, id);
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
/** Restore succeeded: restoring -> consumed, optionally linking the restore's own health gate run. */
|
||||
public markServiceUpdateRecoveryConsumed(id: string, healthGateId: string | null = null): boolean {
|
||||
const result = this.db.prepare(
|
||||
`UPDATE service_update_recovery
|
||||
SET status = 'consumed', claim_expires_at = NULL, health_gate_id = COALESCE(?, health_gate_id)
|
||||
WHERE id = ? AND status = 'restoring'`
|
||||
).run(healthGateId, id);
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
/** Mid-flight restore failure with the image still local: restoring -> active, claim cleared. */
|
||||
public reactivateServiceUpdateRecovery(id: string): boolean {
|
||||
const result = this.db.prepare(
|
||||
`UPDATE service_update_recovery SET status = 'active', claim_expires_at = NULL WHERE id = ? AND status = 'restoring'`
|
||||
).run(id);
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
/** Mid-flight restore failure with the image gone: restoring -> invalidated. */
|
||||
public invalidateServiceUpdateRecovery(id: string): boolean {
|
||||
const result = this.db.prepare(
|
||||
`UPDATE service_update_recovery SET status = 'invalidated', claim_expires_at = NULL WHERE id = ? AND status = 'restoring'`
|
||||
).run(id);
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
/** Startup/interval sweep: active rows whose TTL has lapsed. */
|
||||
public sweepExpiredActiveServiceUpdateRecoveries(now: number): number {
|
||||
const result = this.db.prepare(
|
||||
`UPDATE service_update_recovery SET status = 'expired' WHERE status = 'active' AND expires_at <= ?`
|
||||
).run(now);
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Startup/interval sweep: restoring rows abandoned by a dead claim (process
|
||||
* died mid-restore). A restoring row with a live claim is never expired here,
|
||||
* even if its original expires_at has long passed.
|
||||
*/
|
||||
public sweepAbandonedRestoringServiceUpdateRecoveries(now: number): number {
|
||||
const result = this.db.prepare(
|
||||
`UPDATE service_update_recovery
|
||||
SET status = 'expired'
|
||||
WHERE status = 'restoring' AND claim_expires_at IS NOT NULL AND claim_expires_at <= ?`
|
||||
).run(now);
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
/** Image IDs currently protected from prune: active rows and restoring rows with a live claim. */
|
||||
public listHeldServiceUpdateRecoveryImageIds(nodeId: number, now: number): string[] {
|
||||
const rows = this.db.prepare(
|
||||
`SELECT DISTINCT majority_image_id FROM service_update_recovery
|
||||
WHERE node_id = ? AND (status = 'active' OR (status = 'restoring' AND claim_expires_at > ?))`
|
||||
).all(nodeId, now) as Array<{ majority_image_id: string }>;
|
||||
return rows.map(r => r.majority_image_id);
|
||||
}
|
||||
|
||||
public deleteServiceUpdateRecoveries(nodeId: number, stackName: string): void {
|
||||
this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
|
||||
}
|
||||
|
||||
// --- Notification History ---
|
||||
|
||||
private mapNotificationRow(row: any): NotificationHistory {
|
||||
@@ -3569,6 +3857,7 @@ export class DatabaseService {
|
||||
this.db.prepare('DELETE FROM preflight_acknowledgements WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM stack_exposure WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM health_gate_runs WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ?').run(id);
|
||||
this.db.prepare('UPDATE blueprints SET pinned_node_id = NULL WHERE pinned_node_id = ?').run(id);
|
||||
this.deleteRoleAssignmentsByResource('node', String(id));
|
||||
this.db.prepare('DELETE FROM fleet_sync_status WHERE node_id = ?').run(id);
|
||||
@@ -3644,6 +3933,12 @@ export class DatabaseService {
|
||||
|
||||
// --- Stack Update Status ---
|
||||
|
||||
/**
|
||||
* `services` is omitted by legacy (non-service-aware) callers; omitting it
|
||||
* leaves the stack's existing services_json untouched (COALESCE against the
|
||||
* current row) rather than erasing a per-service breakdown that a model-aware
|
||||
* caller persisted on a previous check.
|
||||
*/
|
||||
public upsertStackUpdateStatus(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
@@ -3651,16 +3946,20 @@ export class DatabaseService {
|
||||
checkedAt: number,
|
||||
checkStatus: StackCheckStatus = 'ok',
|
||||
lastError: string | null = null,
|
||||
services?: StackServiceStatus[],
|
||||
generation = 0,
|
||||
): void {
|
||||
const servicesJson = services !== undefined ? stringifyServicesJson(services, generation) : null;
|
||||
this.db.prepare(
|
||||
`INSERT INTO stack_update_status (node_id, stack_name, has_update, check_status, last_error, checked_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`INSERT INTO stack_update_status (node_id, stack_name, has_update, check_status, last_error, checked_at, services_json)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(node_id, stack_name) DO UPDATE SET
|
||||
has_update = excluded.has_update,
|
||||
check_status = excluded.check_status,
|
||||
last_error = excluded.last_error,
|
||||
checked_at = excluded.checked_at`
|
||||
).run(nodeId, stackName, hasUpdate ? 1 : 0, checkStatus, lastError, checkedAt);
|
||||
checked_at = excluded.checked_at,
|
||||
services_json = COALESCE(excluded.services_json, stack_update_status.services_json)`
|
||||
).run(nodeId, stackName, hasUpdate ? 1 : 0, checkStatus, lastError, checkedAt, servicesJson);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -3671,15 +3970,24 @@ export class DatabaseService {
|
||||
* first-ever failed check inserts a row with has_update = 0 so the stack
|
||||
* still appears with its failure reason.
|
||||
*/
|
||||
public recordStackCheckFailure(nodeId: number, stackName: string, lastError: string, checkedAt: number): void {
|
||||
public recordStackCheckFailure(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
lastError: string,
|
||||
checkedAt: number,
|
||||
services?: StackServiceStatus[],
|
||||
generation = 0,
|
||||
): void {
|
||||
const servicesJson = services !== undefined ? stringifyServicesJson(services, generation) : null;
|
||||
this.db.prepare(
|
||||
`INSERT INTO stack_update_status (node_id, stack_name, has_update, check_status, last_error, checked_at)
|
||||
VALUES (?, ?, 0, 'failed', ?, ?)
|
||||
`INSERT INTO stack_update_status (node_id, stack_name, has_update, check_status, last_error, checked_at, services_json)
|
||||
VALUES (?, ?, 0, 'failed', ?, ?, ?)
|
||||
ON CONFLICT(node_id, stack_name) DO UPDATE SET
|
||||
check_status = 'failed',
|
||||
last_error = excluded.last_error,
|
||||
checked_at = excluded.checked_at`
|
||||
).run(nodeId, stackName, lastError, checkedAt);
|
||||
checked_at = excluded.checked_at,
|
||||
services_json = COALESCE(excluded.services_json, stack_update_status.services_json)`
|
||||
).run(nodeId, stackName, lastError, checkedAt, servicesJson);
|
||||
}
|
||||
|
||||
public getStackUpdateStatus(nodeId?: number): Record<string, boolean> {
|
||||
@@ -3700,20 +4008,35 @@ export class DatabaseService {
|
||||
*/
|
||||
public getStackUpdateDetail(nodeId: number): Record<string, StackUpdateDetail> {
|
||||
const rows = this.db.prepare(
|
||||
'SELECT stack_name, has_update, check_status, last_error, checked_at FROM stack_update_status WHERE node_id = ?'
|
||||
).all(nodeId) as Array<{ stack_name: string; has_update: number; check_status: string | null; last_error: string | null; checked_at: number }>;
|
||||
'SELECT stack_name, has_update, check_status, last_error, checked_at, services_json FROM stack_update_status WHERE node_id = ?'
|
||||
).all(nodeId) as Array<{ stack_name: string; has_update: number; check_status: string | null; last_error: string | null; checked_at: number; services_json: string | null }>;
|
||||
const result: Record<string, StackUpdateDetail> = {};
|
||||
for (const row of rows) {
|
||||
const services = parseServicesJson(row.services_json);
|
||||
result[row.stack_name] = {
|
||||
hasUpdate: row.has_update === 1,
|
||||
checkStatus: (row.check_status === 'failed' || row.check_status === 'partial') ? row.check_status : 'ok',
|
||||
lastError: row.last_error,
|
||||
checkedAt: row.checked_at,
|
||||
...(services.length > 0 ? { services } : {}),
|
||||
};
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prior per-service breakdown for a single stack, used by ImageUpdateService
|
||||
* to look up each service's last-known hasUpdate before reducing a fresh
|
||||
* check (so a preserved value survives a partial/failed re-check). Corrupt
|
||||
* or missing JSON yields an empty list.
|
||||
*/
|
||||
public getStackServicesJson(nodeId: number, stackName: string): StackServiceStatus[] {
|
||||
const row = this.db.prepare(
|
||||
'SELECT services_json FROM stack_update_status WHERE node_id = ? AND stack_name = ?'
|
||||
).get(nodeId, stackName) as { services_json: string | null } | undefined;
|
||||
return parseServicesJson(row?.services_json);
|
||||
}
|
||||
|
||||
public clearStackUpdateStatus(nodeId: number, stackName: string): void {
|
||||
this.db.prepare('DELETE FROM stack_update_status WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
|
||||
}
|
||||
|
||||
@@ -403,13 +403,56 @@ class DockerController {
|
||||
};
|
||||
}
|
||||
|
||||
/** True when an image is unused and has no meaningful tag (dangling / untagged). */
|
||||
private static isDanglingImage(img: { RepoTags?: string[] | null; Containers?: number }): boolean {
|
||||
if ((img.Containers ?? 0) !== 0) return false;
|
||||
const tags = img.RepoTags ?? [];
|
||||
if (tags.length === 0) return true;
|
||||
return tags.every((t) => t === '<none>:<none>');
|
||||
}
|
||||
|
||||
/**
|
||||
* Enumerate candidate images and delete individually. Used when a recovery
|
||||
* hold predicate is supplied so held images are never removed by a bulk
|
||||
* Docker prune API. Reclaimed bytes come from a LayersSize before/after delta.
|
||||
*/
|
||||
private async pruneImagesEnumerated(
|
||||
eligible: (img: { Id: string; RepoTags?: string[] | null; Containers?: number }) => boolean,
|
||||
isImageHeld?: (imageId: string) => boolean,
|
||||
): Promise<{ success: boolean; reclaimedBytes: number }> {
|
||||
const beforeDf = await this.safeDfSnapshot();
|
||||
const rawImages = await this.docker.listImages({ all: false });
|
||||
const candidates = (rawImages as Array<{ Id: string; RepoTags?: string[] | null; Containers?: number }>)
|
||||
.filter((img) => eligible(img) && !isImageHeld?.(img.Id));
|
||||
|
||||
for (const img of candidates) {
|
||||
if (isImageHeld?.(img.Id)) continue;
|
||||
try {
|
||||
await this.docker.getImage(img.Id).remove({ force: true });
|
||||
} catch (e) {
|
||||
console.error(`[pruneImagesEnumerated] Failed to remove image ${sanitizeForLog(img.Id)}:`, e);
|
||||
}
|
||||
}
|
||||
|
||||
const afterDf = await this.safeDfSnapshot();
|
||||
let reclaimedBytes = 0;
|
||||
if (typeof beforeDf?.LayersSize === 'number' && typeof afterDf?.LayersSize === 'number') {
|
||||
reclaimedBytes = Math.max(0, beforeDf.LayersSize - afterDf.LayersSize);
|
||||
}
|
||||
return { success: true, reclaimedBytes };
|
||||
}
|
||||
|
||||
// Sencho's own image, networks, and named volumes are always in use by the
|
||||
// running container, so Docker's server-side prune APIs (pruneContainers,
|
||||
// pruneImages, pruneNetworks, pruneVolumes) skip them by definition. No
|
||||
// extra self-guard is needed at this layer; the `managed` scope path goes
|
||||
// through `pruneManagedOnly`, which adds an explicit self filter for
|
||||
// defense-in-depth.
|
||||
public async pruneSystem(target: 'containers' | 'images' | 'networks' | 'volumes', labelFilter?: string) {
|
||||
public async pruneSystem(
|
||||
target: 'containers' | 'images' | 'networks' | 'volumes',
|
||||
labelFilter?: string,
|
||||
isImageHeld?: (imageId: string) => boolean,
|
||||
) {
|
||||
let spaceReclaimed = 0;
|
||||
if (target === 'containers') {
|
||||
const filters: Record<string, string[]> = {};
|
||||
@@ -417,6 +460,9 @@ class DockerController {
|
||||
const r = await this.docker.pruneContainers({ filters });
|
||||
spaceReclaimed = r.SpaceReclaimed || 0;
|
||||
} else if (target === 'images') {
|
||||
if (isImageHeld) {
|
||||
return this.pruneImagesEnumerated((img) => (img.Containers ?? 0) === 0, isImageHeld);
|
||||
}
|
||||
// Remove all unused images, not just dangling ones
|
||||
const filters: Record<string, string[] | Record<string, boolean>> = { dangling: { 'false': true } };
|
||||
if (labelFilter) filters.label = [labelFilter];
|
||||
@@ -444,7 +490,10 @@ class DockerController {
|
||||
// which uses { dangling: { 'false': true } } to remove every unused image.
|
||||
// Used by the prune-on-update flow to reclaim the layers a pull/recreate
|
||||
// orphans, without touching tagged images for stopped stacks.
|
||||
public async pruneDanglingImages(): Promise<{ success: boolean; reclaimedBytes: number }> {
|
||||
public async pruneDanglingImages(isImageHeld?: (imageId: string) => boolean): Promise<{ success: boolean; reclaimedBytes: number }> {
|
||||
if (isImageHeld) {
|
||||
return this.pruneImagesEnumerated((img) => DockerController.isDanglingImage(img), isImageHeld);
|
||||
}
|
||||
const filters: Record<string, string[] | Record<string, boolean>> = { dangling: { 'true': true } };
|
||||
const r = await this.docker.pruneImages({ filters });
|
||||
return { success: true, reclaimedBytes: r.SpaceReclaimed || 0 };
|
||||
@@ -635,7 +684,8 @@ class DockerController {
|
||||
|
||||
public async pruneManagedOnly(
|
||||
target: 'images' | 'volumes' | 'networks',
|
||||
knownStackNames: string[]
|
||||
knownStackNames: string[],
|
||||
isImageHeld?: (imageId: string) => boolean,
|
||||
): Promise<{ success: boolean; reclaimedBytes: number }> {
|
||||
const knownSet = new Set(knownStackNames);
|
||||
const projectToStack = await DockerController.resolveProjectNameMap(knownStackNames);
|
||||
@@ -707,6 +757,7 @@ class DockerController {
|
||||
// once, but the per-image formula subtracts it from every referrer).
|
||||
const beforeDf = await this.safeDfSnapshot();
|
||||
await Promise.all(prunable.map(async (img) => {
|
||||
if (isImageHeld?.(img.Id)) return;
|
||||
try {
|
||||
await this.docker.getImage(img.Id).remove({ force: true });
|
||||
} catch (e) {
|
||||
@@ -832,6 +883,7 @@ class DockerController {
|
||||
scope: PruneScope,
|
||||
knownStackNames: string[],
|
||||
nodeId: number = this.nodeId,
|
||||
isImageHeld?: (imageId: string) => boolean,
|
||||
): Promise<PrunePlan> {
|
||||
const ordered = normalizePruneTargets(targets);
|
||||
const knownSet = new Set(knownStackNames);
|
||||
@@ -980,6 +1032,7 @@ class DockerController {
|
||||
const freeingImages = ordered.includes('containers');
|
||||
for (const img of rawImages) {
|
||||
if (selfIdentity.isOwnImage(img.Id)) continue;
|
||||
if (isImageHeld?.(img.Id)) continue;
|
||||
const containers = img.Containers ?? 0;
|
||||
const refs = imageToContainerIds.get(img.Id) ?? [];
|
||||
const becomesFree = freeingImages
|
||||
@@ -1038,8 +1091,9 @@ class DockerController {
|
||||
public async assertPlanFresh(
|
||||
plan: PrunePlan,
|
||||
knownStackNames: string[],
|
||||
isImageHeld?: (imageId: string) => boolean,
|
||||
): Promise<PrunePlan | null> {
|
||||
const rebuilt = await this.buildPrunePlan(plan.targets, plan.scope, knownStackNames, plan.nodeId);
|
||||
const rebuilt = await this.buildPrunePlan(plan.targets, plan.scope, knownStackNames, plan.nodeId, isImageHeld);
|
||||
if (rebuilt.fingerprint !== plan.fingerprint) return null;
|
||||
return rebuilt;
|
||||
}
|
||||
@@ -1051,8 +1105,9 @@ class DockerController {
|
||||
public async executePrunePlan(
|
||||
plan: PrunePlan,
|
||||
knownStackNames: string[],
|
||||
isImageHeld?: (imageId: string) => boolean,
|
||||
): Promise<{ outcomes: PruneItemOutcome[]; reclaimedBytes: number; success: boolean }> {
|
||||
const fresh = await this.assertPlanFresh(plan, knownStackNames);
|
||||
const fresh = await this.assertPlanFresh(plan, knownStackNames, isImageHeld);
|
||||
if (!fresh) throw new PrunePlanStaleError();
|
||||
|
||||
const knownSet = new Set(knownStackNames);
|
||||
@@ -1123,6 +1178,15 @@ class DockerController {
|
||||
}
|
||||
|
||||
if (target === 'images') {
|
||||
if (isImageHeld?.(item.id)) {
|
||||
outcomes.push({
|
||||
id: item.id,
|
||||
target: 'images',
|
||||
status: 'skipped',
|
||||
reason: 'Image held for pending service-update recovery',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (imageIdBlocked(item.id)) {
|
||||
const stillReferenced = await this.imageStillReferenced(item.id);
|
||||
if (stillReferenced) {
|
||||
|
||||
@@ -179,6 +179,65 @@ export class DriftLedgerService {
|
||||
return { detected: toInsert.length, resolved: toResolve.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Service-scoped counterpart to `reconcile`: only inserts/resolves findings
|
||||
* for `serviceName`, leaving every other service's (and the empty/`null`
|
||||
* service, i.e. network-level) open findings untouched. Used by the service
|
||||
* update orchestrator so a service-scoped update never resolves drift it
|
||||
* did not touch. Never pass a report already filtered to one service into
|
||||
* the stack-wide `reconcile`; that would falsely resolve every other
|
||||
* service's open findings.
|
||||
*/
|
||||
reconcileService(nodeId: number, stackName: string, serviceName: string, report: StackDriftReport): DriftReconcileResult {
|
||||
if (report.status === 'unreachable' || report.parseError) {
|
||||
return { detected: 0, resolved: 0 };
|
||||
}
|
||||
const db = DatabaseService.getInstance();
|
||||
const now = Date.now();
|
||||
const openForService = db.getOpenDriftFindings(nodeId, stackName).filter(r => r.service === serviceName);
|
||||
const openByKey = new Map(openForService.map(r => [findingKey(r.service, r.finding_type), r]));
|
||||
const currentForService = report.findings.filter(f => f.service === serviceName);
|
||||
const currentByKey = new Map(currentForService.map(f => [findingKey(f.service, f.kind), f]));
|
||||
|
||||
const toInsert: StackDriftFinding[] = [];
|
||||
for (const [key, f] of currentByKey) {
|
||||
if (!openByKey.has(key)) toInsert.push(f);
|
||||
}
|
||||
const toResolve: StackDriftFindingRow[] = [];
|
||||
for (const [key, row] of openByKey) {
|
||||
if (!currentByKey.has(key)) toResolve.push(row);
|
||||
}
|
||||
db.getDb().transaction(() => {
|
||||
db.setStackDossierDriftCheck(nodeId, stackName, now);
|
||||
for (const f of toInsert) {
|
||||
db.insertDriftFinding({
|
||||
node_id: nodeId,
|
||||
stack_name: stackName,
|
||||
service: f.service,
|
||||
finding_type: f.kind,
|
||||
severity: 'warning',
|
||||
message: f.detail,
|
||||
expected_json: f.expected !== undefined ? JSON.stringify(f.expected) : null,
|
||||
actual_json: f.actual !== undefined ? JSON.stringify(f.actual) : null,
|
||||
detected_at: now,
|
||||
});
|
||||
}
|
||||
for (const row of toResolve) {
|
||||
db.resolveDriftFinding(row.id, now);
|
||||
}
|
||||
})();
|
||||
|
||||
if (toInsert.length > 0) {
|
||||
this.recordActivity(nodeId, stackName, 'drift_detected', 'warning',
|
||||
`Drift detected on ${stackName} (${serviceName}): ${toInsert.length} new finding${toInsert.length === 1 ? '' : 's'}`, now);
|
||||
}
|
||||
if (toResolve.length > 0) {
|
||||
this.recordActivity(nodeId, stackName, 'drift_resolved', 'info',
|
||||
`Drift resolved on ${stackName} (${serviceName}): ${toResolve.length} finding${toResolve.length === 1 ? '' : 's'} cleared`, now);
|
||||
}
|
||||
return { detected: toInsert.length, resolved: toResolve.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the spatial report for one stack and reconcile it into the ledger.
|
||||
* Used by the deploy and update success hooks (and the rollback route, which
|
||||
@@ -196,6 +255,22 @@ export class DriftLedgerService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the spatial report for one stack and reconcile only `serviceName`'s
|
||||
* findings into the ledger. Used by the service update orchestrator's
|
||||
* success path. Best-effort: a build or reconcile failure is logged and
|
||||
* swallowed so it never fails the update that triggered it.
|
||||
*/
|
||||
async reconcileServiceForStack(nodeId: number, stackName: string, serviceName: string): Promise<DriftReconcileResult> {
|
||||
try {
|
||||
const report = await buildStackDriftReport(nodeId, stackName);
|
||||
return this.reconcileService(nodeId, stackName, serviceName, report);
|
||||
} catch (error) {
|
||||
console.error('[DriftLedger] reconcileServiceForStack failed for %s/%s:', sanitizeForLog(stackName), sanitizeForLog(serviceName), sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
return { detected: 0, resolved: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a drift transition to the stack activity timeline. History-only (no
|
||||
* external channel dispatch): a drift signal belongs in the activity feed, not
|
||||
|
||||
@@ -1337,7 +1337,7 @@ export class GitSourceService {
|
||||
console.warn(`[GitSource] ${busy}`);
|
||||
return { applied: true, deployed: false, deployError: busy };
|
||||
}
|
||||
HealthGateService.getInstance().begin(nodeId, stackName, 'deploy', 'system:git-source');
|
||||
HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:git-source');
|
||||
console.log(`[GitSource] Applied and deployed ${stackName} at ${commitSha.slice(0, 7)}`);
|
||||
return { applied: true, deployed: true };
|
||||
} catch (e) {
|
||||
|
||||
@@ -1,12 +1,22 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import DockerController from './DockerController';
|
||||
import { DatabaseService, type HealthGateRunRow } from './DatabaseService';
|
||||
import { AutoHealService } from './AutoHealService';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { withTimeout } from '../utils/withTimeout';
|
||||
import type { HealthGateContainer, HealthGateReport } from './updateGuard/types';
|
||||
import { getComposeCommandTimeoutMs } from './ComposeService';
|
||||
|
||||
const POLL_INTERVAL_MS = 5_000;
|
||||
// A prepared-but-never-begun token (prepare called, mutation then failed before
|
||||
// beginPrepared) is dropped after this long. Swept lazily on each prepare/
|
||||
// attach/beginPrepared call rather than on a timer, so the service leaves no
|
||||
// standing interval (the observation poll timers are the only timers it arms).
|
||||
// Must outlive the longest Compose mutation (compose timeout + 5m buffer).
|
||||
function prepareTtlMs(): number {
|
||||
return Math.max(getComposeCommandTimeoutMs(), 30 * 60_000) + 5 * 60_000;
|
||||
}
|
||||
// Per-observe ceiling so a hung Docker socket cannot leave a poll pending
|
||||
// forever. Above POLL_INTERVAL_MS so a slow-but-live probe is not cut short; a
|
||||
// timeout counts as a poll error and three in a row resolve the gate unknown.
|
||||
@@ -20,6 +30,8 @@ const MAX_WINDOW_SECONDS = 600;
|
||||
// updates). Gates past the cap finalize immediately as unknown.
|
||||
const MAX_CONCURRENT_GATES = 25;
|
||||
|
||||
type GateRole = 'primary' | 'collateral';
|
||||
|
||||
interface ObservedContainer {
|
||||
id: string;
|
||||
name: string;
|
||||
@@ -31,6 +43,10 @@ interface ObservedContainer {
|
||||
restarts: number;
|
||||
state: string;
|
||||
health: string | null;
|
||||
/** `com.docker.compose.service` label, when present (service gates only). */
|
||||
service: string | null;
|
||||
/** Image id the container is running (service gates check convergence on it). */
|
||||
imageId: string;
|
||||
}
|
||||
|
||||
interface ActiveGate {
|
||||
@@ -53,6 +69,90 @@ interface ActiveGate {
|
||||
* or stopped can never overwrite the terminal verdict with a stale one.
|
||||
*/
|
||||
finalized: boolean;
|
||||
/** 'stack' for the legacy post-mutation gate, 'service' for prepared gates. */
|
||||
targetScope: 'stack' | 'service';
|
||||
/** Named trigger persisted on the row. */
|
||||
trigger: 'update' | 'deploy' | 'service_update' | 'service_restore';
|
||||
/** Service gates only. */
|
||||
serviceName: string | null;
|
||||
/** Single image id every primary replica must converge on (service gates). */
|
||||
expectedImageId: string | null;
|
||||
/** Expected primary replica count from the effective model (service gates; may be 0). */
|
||||
expectedReplicas: number;
|
||||
/** Sibling names eligible for regression evaluation (service gates). */
|
||||
collateralEligibleNames: Set<string>;
|
||||
/**
|
||||
* Pre-mutation baselines for regression-eligible collateral siblings.
|
||||
* Used to seed `expected` at arm time so a sibling that vanishes during the
|
||||
* mutation gap is still tracked (and can fail) on the first poll.
|
||||
*/
|
||||
collateralBaselineByName: Map<string, ObservedContainer>;
|
||||
/** Role of each expected container (service gates), for failure attribution. */
|
||||
roleByName: Map<string, GateRole>;
|
||||
}
|
||||
|
||||
/** A pre-mutation baseline container captured at prepare time. */
|
||||
interface PreparedBaseline {
|
||||
id: string;
|
||||
name: string;
|
||||
service: string | null;
|
||||
state: string;
|
||||
health: string | null;
|
||||
restartCount: number;
|
||||
startedAt: string | null;
|
||||
imageId: string;
|
||||
}
|
||||
|
||||
function isRegressionEligibleSibling(baseline: PreparedBaseline): boolean {
|
||||
return baseline.state === 'running' && (baseline.health === null || baseline.health === 'healthy');
|
||||
}
|
||||
|
||||
function observedFromPreparedBaseline(baseline: PreparedBaseline): ObservedContainer {
|
||||
return {
|
||||
id: baseline.id,
|
||||
name: baseline.name,
|
||||
startedAt: baseline.startedAt,
|
||||
restartCount: baseline.restartCount,
|
||||
restarts: 0,
|
||||
state: baseline.state,
|
||||
health: baseline.health,
|
||||
service: baseline.service,
|
||||
imageId: baseline.imageId,
|
||||
};
|
||||
}
|
||||
|
||||
/** An in-memory prepare snapshot awaiting attachExpectedImage + beginPrepared. */
|
||||
interface PreparedGate {
|
||||
token: string;
|
||||
nodeId: number;
|
||||
stackName: string;
|
||||
serviceName: string;
|
||||
trigger: 'service_update' | 'service_restore';
|
||||
windowSeconds: number;
|
||||
expiresAt: number;
|
||||
expectedReplicas: number;
|
||||
expectedImageId: string | null;
|
||||
collateralEligibleNames: Set<string>;
|
||||
primaryBaseline: PreparedBaseline[];
|
||||
collateralBaseline: PreparedBaseline[];
|
||||
}
|
||||
|
||||
export interface PrepareInput {
|
||||
nodeId: number;
|
||||
stackName: string;
|
||||
target: { scope: 'service'; serviceName: string };
|
||||
trigger: 'service_update' | 'service_restore';
|
||||
/**
|
||||
* Expected primary replica count from the effective model (may be 0). Passed
|
||||
* by the orchestrator, which already rendered the model, so the gate stays a
|
||||
* pure observer and does not run a second `docker compose config`.
|
||||
*/
|
||||
expectedReplicas: number;
|
||||
}
|
||||
|
||||
export interface BeginPreparedResult {
|
||||
runId: string | null;
|
||||
observing: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -71,6 +171,8 @@ interface ActiveGate {
|
||||
export class HealthGateService {
|
||||
private static instance: HealthGateService;
|
||||
private readonly active = new Map<string, ActiveGate>();
|
||||
/** Prepare tokens awaiting beginPrepared (service update/restore only). */
|
||||
private readonly prepared = new Map<string, PreparedGate>();
|
||||
private started = false;
|
||||
|
||||
public static getInstance(): HealthGateService {
|
||||
@@ -101,6 +203,51 @@ export class HealthGateService {
|
||||
for (const gate of [...this.active.values()]) {
|
||||
this.finalize(gate, 'unknown', 'shutdown during observation', []);
|
||||
}
|
||||
this.prepared.clear();
|
||||
}
|
||||
|
||||
private gateKey(nodeId: number, stackName: string, scope: 'stack' | 'service', serviceName: string | null): string {
|
||||
return `${nodeId}:${stackName}:${scope}:${serviceName ?? '_'}`;
|
||||
}
|
||||
|
||||
private baselineMapFromPrepared(baselines: PreparedBaseline[]): Map<string, ObservedContainer> {
|
||||
const map = new Map<string, ObservedContainer>();
|
||||
for (const baseline of baselines) {
|
||||
map.set(baseline.name, observedFromPreparedBaseline(baseline));
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Finalize in-flight gates that a newer operation must replace:
|
||||
* - A stack-scoped begin supersedes every gate on the stack.
|
||||
* - A service-scoped begin supersedes only the same service's gate and any
|
||||
* stack-scoped gate; other services keep observing independently.
|
||||
*/
|
||||
private supersedeGatesForStack(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
options?: { serviceName?: string | null; stackOnly?: boolean },
|
||||
): void {
|
||||
const serviceName = options?.serviceName;
|
||||
const stackOnly = options?.stackOnly === true;
|
||||
for (const gate of [...this.active.values()]) {
|
||||
if (gate.nodeId !== nodeId || gate.stackName !== stackName) continue;
|
||||
if (serviceName !== undefined) {
|
||||
// Service begin: keep other services' gates.
|
||||
if (gate.targetScope === 'service' && gate.serviceName !== serviceName) continue;
|
||||
} else if (stackOnly) {
|
||||
if (gate.targetScope !== 'stack') continue;
|
||||
}
|
||||
this.finalize(gate, 'unknown', 'superseded by a newer operation', []);
|
||||
}
|
||||
}
|
||||
|
||||
/** Drop prepare tokens whose TTL elapsed without a beginPrepared (lazy, no standing timer). */
|
||||
private sweepExpiredPrepared(now: number): void {
|
||||
for (const [token, prep] of this.prepared) {
|
||||
if (now >= prep.expiresAt) this.prepared.delete(token);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -114,7 +261,7 @@ export class HealthGateService {
|
||||
* every gated update path gets the timeline marker even when the gate
|
||||
* itself is disabled.
|
||||
*/
|
||||
public begin(
|
||||
public beginStack(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
trigger: 'update' | 'deploy',
|
||||
@@ -132,12 +279,10 @@ export class HealthGateService {
|
||||
}
|
||||
if (!settings.enabled) return null;
|
||||
|
||||
// A newer operation supersedes an in-flight gate for the same stack.
|
||||
const key = `${nodeId}:${stackName}`;
|
||||
const existing = this.active.get(key);
|
||||
if (existing) {
|
||||
this.finalize(existing, 'unknown', 'superseded by a newer update', []);
|
||||
}
|
||||
// A newer stack operation supersedes every in-flight gate on this stack
|
||||
// (including service gates).
|
||||
const key = this.gateKey(nodeId, stackName, 'stack', null);
|
||||
this.supersedeGatesForStack(nodeId, stackName);
|
||||
|
||||
const runId = randomUUID();
|
||||
const startedAt = Date.now();
|
||||
@@ -153,6 +298,9 @@ export class HealthGateService {
|
||||
started_at: startedAt,
|
||||
ended_at: null,
|
||||
created_by: actor,
|
||||
target_scope: 'stack',
|
||||
service_name: null,
|
||||
failure_source: null,
|
||||
};
|
||||
|
||||
if (this.active.size >= MAX_CONCURRENT_GATES) {
|
||||
@@ -173,6 +321,14 @@ export class HealthGateService {
|
||||
missingLastPoll: new Set(),
|
||||
restartingLastPoll: new Set(),
|
||||
finalized: false,
|
||||
targetScope: 'stack',
|
||||
trigger,
|
||||
serviceName: null,
|
||||
expectedImageId: null,
|
||||
expectedReplicas: 0,
|
||||
collateralEligibleNames: new Set(),
|
||||
collateralBaselineByName: new Map(),
|
||||
roleByName: new Map(),
|
||||
};
|
||||
this.active.set(key, gate);
|
||||
this.scheduleNextPoll(gate);
|
||||
@@ -180,13 +336,169 @@ export class HealthGateService {
|
||||
} catch (error) {
|
||||
// The gate is an observer; its failure must never fail the operation.
|
||||
console.error(
|
||||
'[HealthGate] begin (%s) failed for %s on node %d:',
|
||||
'[HealthGate] beginStack (%s) failed for %s on node %d:',
|
||||
trigger, sanitizeForLog(stackName), nodeId, error,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** @deprecated Prefer beginStack; retained as a one-PR alias for callers under migration. */
|
||||
public begin(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
trigger: 'update' | 'deploy',
|
||||
actor: string | null,
|
||||
): string | null {
|
||||
return this.beginStack(nodeId, stackName, trigger, actor);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre-mutation snapshot for a service-scoped update/restore. Captures the
|
||||
* selected service's primary replicas and the sibling collateral set (marking
|
||||
* only currently-running, healthy or healthcheck-less siblings as
|
||||
* regression-eligible), and returns an opaque token the orchestrator carries
|
||||
* through attachExpectedImage and beginPrepared. Never throws: a Docker read
|
||||
* failure yields snapshotOk=false so the orchestrator can skip observation
|
||||
* rather than claim a gate with an empty baseline.
|
||||
*/
|
||||
public async prepare(input: PrepareInput): Promise<{ prepareToken: string; expiresAt: number; snapshotOk: boolean }> {
|
||||
const now = Date.now();
|
||||
this.sweepExpiredPrepared(now);
|
||||
const { nodeId, stackName, trigger } = input;
|
||||
const serviceName = input.target.serviceName;
|
||||
const settings = this.readSettings();
|
||||
|
||||
let baselines: PreparedBaseline[] = [];
|
||||
let snapshotOk = true;
|
||||
try {
|
||||
baselines = await this.snapshotBaselines(nodeId, stackName);
|
||||
} catch (error) {
|
||||
snapshotOk = false;
|
||||
console.warn('[HealthGate] prepare snapshot failed for %s/%s:',
|
||||
sanitizeForLog(stackName), sanitizeForLog(serviceName), getErrorMessage(error, 'unknown'));
|
||||
}
|
||||
|
||||
const primaryBaseline = baselines.filter(b => b.service === serviceName);
|
||||
const collateralBaseline = baselines.filter(b => b.service !== serviceName);
|
||||
const collateralEligibleNames = new Set(
|
||||
collateralBaseline.filter(isRegressionEligibleSibling).map(b => b.name),
|
||||
);
|
||||
|
||||
const token = randomUUID();
|
||||
const expiresAt = now + prepareTtlMs();
|
||||
this.prepared.set(token, {
|
||||
token,
|
||||
nodeId,
|
||||
stackName,
|
||||
serviceName,
|
||||
trigger,
|
||||
windowSeconds: settings.windowSeconds,
|
||||
expiresAt,
|
||||
expectedReplicas: input.expectedReplicas,
|
||||
expectedImageId: null,
|
||||
collateralEligibleNames,
|
||||
primaryBaseline,
|
||||
collateralBaseline,
|
||||
});
|
||||
return { prepareToken: token, expiresAt, snapshotOk };
|
||||
}
|
||||
|
||||
/** Record the single image id every primary replica must converge on. No-op for an unknown/expired token. */
|
||||
public attachExpectedImage(prepareToken: string, expectedImageId: string): void {
|
||||
this.sweepExpiredPrepared(Date.now());
|
||||
const prep = this.prepared.get(prepareToken);
|
||||
if (!prep) {
|
||||
console.warn('[HealthGate] attachExpectedImage: unknown or expired prepare token');
|
||||
return;
|
||||
}
|
||||
prep.expectedImageId = expectedImageId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Begin observing a prepared service gate after its mutation. Mirrors
|
||||
* beginStack's nullability: returns a null runId (and observing:false) when
|
||||
* the service is not started, gating is disabled, or the insert fails; a
|
||||
* non-null runId with observing:false past the concurrency cap; and a non-null
|
||||
* runId with observing:true once the poll is armed. The prepare token is
|
||||
* consumed either way. Never throws.
|
||||
*/
|
||||
public beginPrepared(input: { prepareToken: string; actor: string | null }): BeginPreparedResult {
|
||||
const now = Date.now();
|
||||
this.sweepExpiredPrepared(now);
|
||||
const prep = this.prepared.get(input.prepareToken);
|
||||
// Always consume the token so a caller cannot begin twice off one prepare.
|
||||
if (prep) this.prepared.delete(input.prepareToken);
|
||||
|
||||
if (!this.started || !prep) return { runId: null, observing: false };
|
||||
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const settings = this.readSettings();
|
||||
if (!settings.enabled) return { runId: null, observing: false };
|
||||
|
||||
const key = this.gateKey(prep.nodeId, prep.stackName, 'service', prep.serviceName);
|
||||
// Same-service + stack gates only; sibling service observations continue.
|
||||
this.supersedeGatesForStack(prep.nodeId, prep.stackName, { serviceName: prep.serviceName });
|
||||
|
||||
const runId = randomUUID();
|
||||
const startedAt = Date.now();
|
||||
const row: HealthGateRunRow = {
|
||||
id: runId,
|
||||
node_id: prep.nodeId,
|
||||
stack_name: prep.stackName,
|
||||
trigger_action: prep.trigger,
|
||||
status: 'observing',
|
||||
reason: null,
|
||||
window_seconds: prep.windowSeconds,
|
||||
containers_json: '[]',
|
||||
started_at: startedAt,
|
||||
ended_at: null,
|
||||
created_by: input.actor,
|
||||
target_scope: 'service',
|
||||
service_name: prep.serviceName,
|
||||
failure_source: null,
|
||||
};
|
||||
|
||||
if (this.active.size >= MAX_CONCURRENT_GATES) {
|
||||
db.insertHealthGateRun({ ...row, status: 'unknown', reason: 'too many concurrent observations', ended_at: startedAt });
|
||||
return { runId, observing: false };
|
||||
}
|
||||
|
||||
db.insertHealthGateRun(row);
|
||||
const gate: ActiveGate = {
|
||||
runId,
|
||||
nodeId: prep.nodeId,
|
||||
stackName: prep.stackName,
|
||||
windowSeconds: prep.windowSeconds,
|
||||
startedAt,
|
||||
timer: null,
|
||||
expected: null,
|
||||
consecutivePollErrors: 0,
|
||||
missingLastPoll: new Set(),
|
||||
restartingLastPoll: new Set(),
|
||||
finalized: false,
|
||||
targetScope: 'service',
|
||||
trigger: prep.trigger,
|
||||
serviceName: prep.serviceName,
|
||||
expectedImageId: prep.expectedImageId,
|
||||
expectedReplicas: prep.expectedReplicas,
|
||||
collateralEligibleNames: prep.collateralEligibleNames,
|
||||
collateralBaselineByName: this.baselineMapFromPrepared(
|
||||
prep.collateralBaseline.filter(b => prep.collateralEligibleNames.has(b.name)),
|
||||
),
|
||||
roleByName: new Map(),
|
||||
};
|
||||
this.active.set(key, gate);
|
||||
this.scheduleNextPoll(gate);
|
||||
return { runId, observing: true };
|
||||
} catch (error) {
|
||||
console.error('[HealthGate] beginPrepared failed for %s/%s:',
|
||||
sanitizeForLog(prep.stackName), sanitizeForLog(prep.serviceName), error);
|
||||
return { runId: null, observing: false };
|
||||
}
|
||||
}
|
||||
|
||||
/** A specific run by id, the latest run, or the never-run sentinel. */
|
||||
public getReport(nodeId: number, stackName: string, gateId?: string): HealthGateReport {
|
||||
const db = DatabaseService.getInstance();
|
||||
@@ -197,6 +509,7 @@ export class HealthGateService {
|
||||
return {
|
||||
stack: stackName, id: null, status: 'never-run', trigger: null, reason: null,
|
||||
windowSeconds: null, startedAt: null, endedAt: null, containers: [],
|
||||
targetScope: 'stack', serviceName: null, failureSource: null,
|
||||
};
|
||||
}
|
||||
let containers: HealthGateContainer[] = [];
|
||||
@@ -217,6 +530,9 @@ export class HealthGateService {
|
||||
startedAt: row.started_at,
|
||||
endedAt: row.ended_at,
|
||||
containers,
|
||||
targetScope: row.target_scope ?? 'stack',
|
||||
serviceName: row.service_name ?? null,
|
||||
failureSource: row.failure_source ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -227,11 +543,15 @@ export class HealthGateService {
|
||||
* or corrupt the restart/missing accounting that assumes one poll at a time.
|
||||
*/
|
||||
private async poll(gate: ActiveGate): Promise<void> {
|
||||
const key = `${gate.nodeId}:${gate.stackName}`;
|
||||
const key = this.gateKey(gate.nodeId, gate.stackName, gate.targetScope, gate.serviceName);
|
||||
// A late timer fire after supersede, stop, or finalize must do nothing.
|
||||
if (gate.finalized || this.active.get(key) !== gate) return;
|
||||
try {
|
||||
await this.runPollCycle(gate, key);
|
||||
if (gate.targetScope === 'service') {
|
||||
await this.runServicePollCycle(gate, key);
|
||||
} else {
|
||||
await this.runPollCycle(gate, key);
|
||||
}
|
||||
} finally {
|
||||
if (!gate.finalized && this.active.get(key) === gate) {
|
||||
this.scheduleNextPoll(gate);
|
||||
@@ -352,16 +672,201 @@ export class HealthGateService {
|
||||
this.finalize(gate, 'passed', null, summary);
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll cycle for a prepared service gate. Observes the selected service's
|
||||
* primary replicas (image convergence, replica count, health/restart) and the
|
||||
* regression-eligible collateral siblings, attributing a failure to
|
||||
* 'primary' or 'collateral'. Unrelated containers that appear after prepare do
|
||||
* not expand the observed set.
|
||||
*/
|
||||
private async runServicePollCycle(gate: ActiveGate, key: string): Promise<void> {
|
||||
let observed: ObservedContainer[];
|
||||
try {
|
||||
observed = await withTimeout(
|
||||
this.observeContainers(gate), OBSERVE_TIMEOUT_MS, 'health gate observe',
|
||||
);
|
||||
} catch (error) {
|
||||
gate.consecutivePollErrors += 1;
|
||||
console.warn(
|
||||
'[HealthGate] service poll error %d for %s/%s:',
|
||||
gate.consecutivePollErrors, sanitizeForLog(gate.stackName),
|
||||
sanitizeForLog(gate.serviceName ?? ''), getErrorMessage(error, 'unknown'),
|
||||
);
|
||||
if (gate.consecutivePollErrors >= 3) {
|
||||
this.finalize(gate, 'unknown', 'Docker became unreachable during observation', []);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (gate.finalized || this.active.get(key) !== gate) return;
|
||||
gate.consecutivePollErrors = 0;
|
||||
|
||||
const elapsedMs = Date.now() - gate.startedAt;
|
||||
const serviceName = gate.serviceName ?? '';
|
||||
|
||||
// Arm the expected set once from post-mutation primary replicas plus the
|
||||
// pre-mutation regression-eligible collateral baselines. Seeding collateral
|
||||
// from the prepare baseline (not only from the first post-mutation poll)
|
||||
// means a healthy sibling that vanished during the mutation gap is still
|
||||
// tracked and can fail the gate. Scale-0 arms immediately with an empty
|
||||
// primary set; scale>0 waits (up to the empty grace) for replicas.
|
||||
if (gate.expected === null) {
|
||||
const primary = observed.filter(c => c.service === serviceName);
|
||||
if (gate.expectedReplicas > 0 && primary.length === 0) {
|
||||
if (elapsedMs >= EMPTY_GRACE_MS) {
|
||||
this.finalize(gate, 'failed', `service ${serviceName} has no running replicas to observe`, [], 'primary');
|
||||
}
|
||||
return;
|
||||
}
|
||||
const expected = new Map<string, ObservedContainer>();
|
||||
for (const c of primary) {
|
||||
gate.roleByName.set(c.name, 'primary');
|
||||
expected.set(c.name, c);
|
||||
}
|
||||
const observedByName = new Map(observed.map(c => [c.name, c]));
|
||||
for (const [name, baseline] of gate.collateralBaselineByName) {
|
||||
if (gate.roleByName.has(name)) continue;
|
||||
gate.roleByName.set(name, 'collateral');
|
||||
// Prefer the live observation when the sibling is still present; otherwise
|
||||
// keep the prepare baseline so a missing sibling enters the miss path.
|
||||
expected.set(name, observedByName.get(name) ?? baseline);
|
||||
}
|
||||
// Also pick up eligible siblings that appeared under a new name after mutation
|
||||
// (unusual, but keeps prior behavior for containers still in the eligible set).
|
||||
for (const c of observed) {
|
||||
if (gate.collateralEligibleNames.has(c.name) && !gate.roleByName.has(c.name)) {
|
||||
gate.roleByName.set(c.name, 'collateral');
|
||||
expected.set(c.name, c);
|
||||
}
|
||||
}
|
||||
gate.expected = expected;
|
||||
return;
|
||||
}
|
||||
|
||||
const byName = new Map(observed.map(c => [c.name, c]));
|
||||
|
||||
for (const [name, baseline] of gate.expected) {
|
||||
const current = byName.get(name);
|
||||
if (!current) continue;
|
||||
const restarted =
|
||||
current.id !== baseline.id ||
|
||||
current.restartCount > baseline.restartCount ||
|
||||
(current.startedAt !== null && baseline.startedAt !== null && current.startedAt !== baseline.startedAt);
|
||||
current.restarts = baseline.restarts + (restarted ? 1 : 0);
|
||||
}
|
||||
const summary = this.summarizeService(gate, byName);
|
||||
|
||||
// Primary image convergence: a primary replica on any image id other than
|
||||
// the single attached expected id fails the gate.
|
||||
if (gate.expectedImageId) {
|
||||
const wrongImage = observed.find(
|
||||
c => c.service === serviceName && c.imageId && c.imageId !== gate.expectedImageId,
|
||||
);
|
||||
if (wrongImage) {
|
||||
this.finalize(gate, 'failed', `service ${serviceName} replica ${wrongImage.name} is running an unexpected image`, summary, 'primary');
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
for (const [name, baseline] of gate.expected) {
|
||||
const role = gate.roleByName.get(name) ?? 'collateral';
|
||||
const noun = role === 'primary' ? 'replica' : 'sibling';
|
||||
const current = byName.get(name);
|
||||
if (!current) {
|
||||
if (gate.missingLastPoll.has(name)) {
|
||||
this.finalize(gate, 'failed', `${noun} ${name} disappeared during observation`, summary, role);
|
||||
return;
|
||||
}
|
||||
gate.missingLastPoll.add(name);
|
||||
continue;
|
||||
}
|
||||
gate.missingLastPoll.delete(name);
|
||||
|
||||
if (current.state === 'exited' && baseline.restarts === current.restarts) {
|
||||
this.finalize(gate, 'failed', `${noun} ${name} exited during observation`, summary, role);
|
||||
return;
|
||||
}
|
||||
if (current.health === 'unhealthy') {
|
||||
this.finalize(gate, 'failed', `${noun} ${name} reported unhealthy`, summary, role);
|
||||
return;
|
||||
}
|
||||
if (current.restarts >= 2) {
|
||||
this.finalize(gate, 'failed', `${noun} ${name} is restart looping`, summary, role);
|
||||
return;
|
||||
}
|
||||
if (current.state === 'restarting') {
|
||||
if (gate.restartingLastPoll.has(name)) {
|
||||
this.finalize(gate, 'failed', `${noun} ${name} is stuck restarting`, summary, role);
|
||||
return;
|
||||
}
|
||||
gate.restartingLastPoll.add(name);
|
||||
} else {
|
||||
gate.restartingLastPoll.delete(name);
|
||||
}
|
||||
gate.expected.set(name, current);
|
||||
}
|
||||
|
||||
if (elapsedMs < gate.windowSeconds * 1000) return;
|
||||
|
||||
const stillStarting = observed.filter(
|
||||
c => c.health === 'starting' && (c.service === serviceName || gate.collateralEligibleNames.has(c.name)),
|
||||
);
|
||||
if (stillStarting.length > 0) {
|
||||
this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary);
|
||||
return;
|
||||
}
|
||||
const runningPrimary = observed.filter(c => c.service === serviceName && c.state === 'running');
|
||||
if (runningPrimary.length !== gate.expectedReplicas) {
|
||||
this.finalize(
|
||||
gate, 'failed',
|
||||
`service ${serviceName} has ${runningPrimary.length} running replica(s), expected ${gate.expectedReplicas}`,
|
||||
summary, 'primary',
|
||||
);
|
||||
return;
|
||||
}
|
||||
const collateralNotRunning = [...gate.expected.keys()]
|
||||
.filter(name => gate.roleByName.get(name) === 'collateral')
|
||||
.filter(name => byName.get(name)?.state !== 'running');
|
||||
if (collateralNotRunning.length > 0) {
|
||||
this.finalize(gate, 'failed', `sibling(s) not running at the end of the window: ${collateralNotRunning.join(', ')}`, summary, 'collateral');
|
||||
return;
|
||||
}
|
||||
this.finalize(gate, 'passed', null, summary);
|
||||
}
|
||||
|
||||
private summarizeService(
|
||||
gate: ActiveGate,
|
||||
current: Map<string, ObservedContainer>,
|
||||
): HealthGateContainer[] {
|
||||
if (!gate.expected) return [];
|
||||
return [...gate.expected.values()].map(baseline => {
|
||||
const now = current.get(baseline.name);
|
||||
return {
|
||||
name: baseline.name,
|
||||
state: now?.state ?? 'missing',
|
||||
health: now?.health ?? null,
|
||||
restarts: now?.restarts ?? baseline.restarts,
|
||||
service: (now ?? baseline).service ?? gate.serviceName,
|
||||
role: gate.roleByName.get(baseline.name) ?? null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private async observeContainers(gate: ActiveGate): Promise<ObservedContainer[]> {
|
||||
const docker = DockerController.getInstance(gate.nodeId).getDocker();
|
||||
return this.listStackContainers(gate.nodeId, gate.stackName);
|
||||
}
|
||||
|
||||
/** List and inspect a stack's containers into the gate's observation shape. */
|
||||
private async listStackContainers(nodeId: number, stackName: string): Promise<ObservedContainer[]> {
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
const listed = await docker.listContainers({
|
||||
all: true,
|
||||
filters: { label: [`com.docker.compose.project=${gate.stackName}`] },
|
||||
filters: { label: [`com.docker.compose.project=${stackName}`] },
|
||||
});
|
||||
const observed = await Promise.all(
|
||||
listed.map(async (info): Promise<ObservedContainer | null> => {
|
||||
try {
|
||||
const inspect = await docker.getContainer(info.Id).inspect();
|
||||
const labels = (info.Labels ?? inspect.Config?.Labels ?? {}) as Record<string, string>;
|
||||
return {
|
||||
id: info.Id,
|
||||
name: info.Names?.[0]?.replace(/^\//, '') ?? info.Id.slice(0, 12),
|
||||
@@ -370,6 +875,8 @@ export class HealthGateService {
|
||||
restarts: 0,
|
||||
state: inspect.State?.Status ?? info.State ?? 'unknown',
|
||||
health: inspect.State?.Health?.Status ?? null,
|
||||
service: labels['com.docker.compose.service'] ?? null,
|
||||
imageId: inspect.Image ?? '',
|
||||
};
|
||||
} catch (e: unknown) {
|
||||
// Removed between list and inspect; the missing-container logic will
|
||||
@@ -382,6 +889,23 @@ export class HealthGateService {
|
||||
return observed.filter((c): c is ObservedContainer => c !== null);
|
||||
}
|
||||
|
||||
/** Snapshot the current per-container facts for prepare's primary/collateral partition. */
|
||||
private async snapshotBaselines(nodeId: number, stackName: string): Promise<PreparedBaseline[]> {
|
||||
const observed = await withTimeout(
|
||||
this.listStackContainers(nodeId, stackName), OBSERVE_TIMEOUT_MS, 'health gate prepare snapshot',
|
||||
);
|
||||
return observed.map(c => ({
|
||||
id: c.id,
|
||||
name: c.name,
|
||||
service: c.service,
|
||||
state: c.state,
|
||||
health: c.health,
|
||||
restartCount: c.restartCount,
|
||||
startedAt: c.startedAt,
|
||||
imageId: c.imageId,
|
||||
}));
|
||||
}
|
||||
|
||||
private summarize(
|
||||
expected: Map<string, ObservedContainer>,
|
||||
current: Map<string, ObservedContainer>,
|
||||
@@ -402,16 +926,29 @@ export class HealthGateService {
|
||||
status: 'passed' | 'failed' | 'unknown',
|
||||
reason: string | null,
|
||||
containers: HealthGateContainer[],
|
||||
failureSource: 'primary' | 'collateral' | null = null,
|
||||
): void {
|
||||
if (gate.finalized) return;
|
||||
gate.finalized = true;
|
||||
if (gate.timer) clearTimeout(gate.timer);
|
||||
const key = `${gate.nodeId}:${gate.stackName}`;
|
||||
const key = this.gateKey(gate.nodeId, gate.stackName, gate.targetScope, gate.serviceName);
|
||||
if (this.active.get(key) === gate) this.active.delete(key);
|
||||
|
||||
// A service gate owns its service's Auto-Heal suppression while observing;
|
||||
// release it here so a failed/superseded/stopped gate never leaves auto-heal
|
||||
// muted for that service.
|
||||
if (gate.targetScope === 'service' && gate.serviceName) {
|
||||
try {
|
||||
AutoHealService.getInstance().clearSuppress(gate.nodeId, gate.stackName, gate.serviceName);
|
||||
} catch (error) {
|
||||
console.warn('[HealthGate] Failed to clear auto-heal suppression for %s/%s:',
|
||||
sanitizeForLog(gate.stackName), sanitizeForLog(gate.serviceName), getErrorMessage(error, 'unknown'));
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
DatabaseService.getInstance().finalizeHealthGateRun(
|
||||
gate.runId, status, reason, Date.now(), JSON.stringify(containers),
|
||||
gate.runId, status, reason, Date.now(), JSON.stringify(containers), failureSource,
|
||||
);
|
||||
} catch (error) {
|
||||
// The verdict is lost from the DB (the startup sweep will later rewrite
|
||||
|
||||
@@ -2,7 +2,7 @@ import path from 'path';
|
||||
import YAML from 'yaml';
|
||||
import { CronExpressionParser } from 'cron-parser';
|
||||
import DockerController from './DockerController';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import { DatabaseService, type StackCheckStatus, type StackServiceStatus } from './DatabaseService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { RegistryService } from './RegistryService';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
@@ -12,6 +12,7 @@ import { parseImageRef, selectLocalRepoDigest, compareLocalToRemoteTag } from '.
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { buildEffectiveServiceModel } from './effectiveServiceModel';
|
||||
|
||||
const BACKFILL_KEY = 'image_update_notifications_backfilled';
|
||||
|
||||
@@ -243,11 +244,124 @@ export async function loadStackBuildServices(nodeId: number, stackName: string):
|
||||
return extractBuildServicesFromCompose(composeContent);
|
||||
}
|
||||
|
||||
// ─── Per-service reduction (model-based status) ─────────────────────────────
|
||||
|
||||
export interface ServiceReduction {
|
||||
status: StackServiceStatus;
|
||||
confirmedUpdateThisRun: boolean;
|
||||
}
|
||||
|
||||
export function reduceServiceStatus(
|
||||
service: string,
|
||||
declaredImage: string | null,
|
||||
runtimeImages: string[],
|
||||
imageUpdateMap: Map<string, ImageCheckResult>,
|
||||
prior: StackServiceStatus | undefined,
|
||||
): ServiceReduction {
|
||||
const dedupedRuntime = [...new Set(runtimeImages)].sort();
|
||||
const refs = new Set<string>();
|
||||
if (declaredImage) refs.add(declaredImage);
|
||||
for (const ref of dedupedRuntime) refs.add(ref);
|
||||
|
||||
if (refs.size === 0) {
|
||||
return {
|
||||
status: { service, image: declaredImage, hasUpdate: false, checkStatus: 'not_checkable', lastError: null },
|
||||
confirmedUpdateThisRun: false,
|
||||
};
|
||||
}
|
||||
|
||||
const checkableResults: ImageCheckResult[] = [];
|
||||
for (const ref of refs) {
|
||||
const result = imageUpdateMap.get(ref);
|
||||
if (!result || result.notCheckable) continue;
|
||||
checkableResults.push(result);
|
||||
}
|
||||
|
||||
if (checkableResults.length === 0) {
|
||||
return {
|
||||
status: { service, image: declaredImage, hasUpdate: false, checkStatus: 'not_checkable', lastError: null },
|
||||
confirmedUpdateThisRun: false,
|
||||
};
|
||||
}
|
||||
|
||||
const errored = checkableResults.filter((r) => r.error !== undefined);
|
||||
const confirmedUpdateThisRun = checkableResults.some(
|
||||
(r) => r.error === undefined && r.hasUpdate === true,
|
||||
);
|
||||
|
||||
if (errored.length === checkableResults.length) {
|
||||
const priorHasUpdate = prior?.hasUpdate ?? false;
|
||||
return {
|
||||
status: {
|
||||
service,
|
||||
image: declaredImage,
|
||||
...(dedupedRuntime.length > 0 ? { runtimeImages: dedupedRuntime } : {}),
|
||||
hasUpdate: priorHasUpdate,
|
||||
checkStatus: 'failed',
|
||||
lastError: errored[0].error ?? 'Update check failed',
|
||||
},
|
||||
confirmedUpdateThisRun: false,
|
||||
};
|
||||
}
|
||||
|
||||
const checkStatus: StackServiceStatus['checkStatus'] = errored.length > 0 ? 'partial' : 'ok';
|
||||
const lastError = errored.length > 0 ? (errored[0].error ?? null) : null;
|
||||
const hasUpdate = checkStatus === 'partial'
|
||||
? (confirmedUpdateThisRun || (prior?.hasUpdate ?? false))
|
||||
: confirmedUpdateThisRun;
|
||||
|
||||
return {
|
||||
status: {
|
||||
service,
|
||||
image: declaredImage,
|
||||
...(dedupedRuntime.length > 0 ? { runtimeImages: dedupedRuntime } : {}),
|
||||
hasUpdate,
|
||||
checkStatus,
|
||||
lastError,
|
||||
},
|
||||
confirmedUpdateThisRun,
|
||||
};
|
||||
}
|
||||
|
||||
export function aggregateServiceCheckStatus(services: StackServiceStatus[]): StackCheckStatus {
|
||||
if (services.length === 0) return 'ok';
|
||||
const checkable = services.filter((s) => s.checkStatus !== 'not_checkable');
|
||||
if (checkable.length === 0) return 'ok';
|
||||
const failedCount = checkable.filter((s) => s.checkStatus === 'failed').length;
|
||||
if (failedCount === checkable.length) return 'failed';
|
||||
if (checkable.some((s) => s.checkStatus === 'partial')
|
||||
|| checkable.some((s) => s.checkStatus === 'failed')) {
|
||||
return 'partial';
|
||||
}
|
||||
return 'ok';
|
||||
}
|
||||
|
||||
export function buildAvailabilityNotifyMessage(stackName: string, services: StackServiceStatus[]): string {
|
||||
const named = services.filter((s) => s.hasUpdate).map((s) => s.service).sort();
|
||||
if (named.length === 0 || services.length <= 1) {
|
||||
return `Stack "${stackName}" has image updates available.`;
|
||||
}
|
||||
if (named.length === 1) {
|
||||
return `Stack "${stackName}" has image updates available for service: ${named[0]}.`;
|
||||
}
|
||||
return `Stack "${stackName}" has image updates available for services: ${named.join(', ')}.`;
|
||||
}
|
||||
|
||||
function stackStatusLastError(services: StackServiceStatus[]): string | null {
|
||||
for (const svc of services) {
|
||||
if (svc.checkStatus !== 'not_checkable' && svc.lastError) return svc.lastError;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─── Service ──────────────────────────────────────────────────────────────────
|
||||
|
||||
export class ImageUpdateService {
|
||||
private static instance: ImageUpdateService;
|
||||
|
||||
/** Per-stack write serialization and generation for stale-writer discard. */
|
||||
private stackWriteState = new Map<string, { chain: Promise<void>; generation: number }>();
|
||||
|
||||
private static readonly MIN_INTERVAL_MINUTES = 15;
|
||||
private static readonly MAX_INTERVAL_MINUTES = 1440; // 24 hours
|
||||
private static readonly DEFAULT_INTERVAL_MINUTES = 120; // 2 hours
|
||||
@@ -594,9 +708,10 @@ export class ImageUpdateService {
|
||||
}
|
||||
|
||||
// Phase 3: Container augmentation (captures actual deployed image tags)
|
||||
let allContainers: Awaited<ReturnType<DockerController['getAllContainers']>> = [];
|
||||
try {
|
||||
const containers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers');
|
||||
for (const c of containers) {
|
||||
allContainers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers');
|
||||
for (const c of allContainers) {
|
||||
// Prefer the pinned project label (== stackName for Sencho-deployed
|
||||
// stacks, including multi-file / context-dir ones where
|
||||
// --project-directory would otherwise change the working-dir
|
||||
@@ -635,6 +750,14 @@ export class ImageUpdateService {
|
||||
}
|
||||
|
||||
// Phase 4: Deduplicate and check all unique images
|
||||
// Reserve write generations before the slow registry checks so a concurrent
|
||||
// recheckStack that finishes first keeps its write (stale full-scan commits
|
||||
// are dropped in withStackWriteLock).
|
||||
const writeGenerations = new Map<string, number>();
|
||||
for (const stackName of stackImages.keys()) {
|
||||
writeGenerations.set(stackName, this.reserveStackWriteGeneration(nodeId, stackName));
|
||||
}
|
||||
|
||||
const allImages = new Set<string>();
|
||||
for (const imgs of stackImages.values()) for (const img of imgs) allImages.add(img);
|
||||
|
||||
@@ -662,54 +785,36 @@ export class ImageUpdateService {
|
||||
// Write status for ALL stacks (including those with no pullable images)
|
||||
const now = Date.now();
|
||||
let updatesFound = 0;
|
||||
const newlyUpdated: string[] = [];
|
||||
for (const [stackName, images] of stackImages) {
|
||||
// Tally only checkable images: a not-checkable image (locally built,
|
||||
// or a bare digest ref) is neither a pass nor a failure.
|
||||
const checkable = Array.from(images)
|
||||
.map(img => imageUpdateMap.get(img))
|
||||
.filter((r): r is ImageCheckResult => !!r && !r.notCheckable);
|
||||
const errored = checkable.filter(r => r.error !== undefined);
|
||||
const confirmedHasUpdate = checkable.some(r => r.error === undefined && r.hasUpdate === true);
|
||||
|
||||
// Every checkable image failed: status is undeterminable. Preserve the
|
||||
// last-known has_update so a transient registry outage neither erases a
|
||||
// real update nor flaps the notification state.
|
||||
if (checkable.length > 0 && errored.length === checkable.length) {
|
||||
db.recordStackCheckFailure(nodeId, stackName, errored[0].error ?? 'Update check failed', now);
|
||||
continue;
|
||||
}
|
||||
|
||||
const checkStatus = errored.length > 0 ? 'partial' : 'ok';
|
||||
const lastError = errored.length > 0 ? (errored[0].error ?? null) : null;
|
||||
// Only a fully-ok check is authoritative enough to lower has_update to
|
||||
// false. On a partial check some image could not be reached, so a
|
||||
// previously confirmed update is preserved rather than erased (which
|
||||
// would also re-fire the notification when that image recovers).
|
||||
const hasUpdate = checkStatus === 'partial'
|
||||
? (confirmedHasUpdate || previousState[stackName] === true)
|
||||
: confirmedHasUpdate;
|
||||
|
||||
if (hasUpdate) {
|
||||
const newlyUpdated: Array<{ stackName: string; message: string }> = [];
|
||||
for (const [stackName] of stackImages) {
|
||||
const outcome = await this.writeStackUpdateStatus(
|
||||
nodeId,
|
||||
stackName,
|
||||
stackImages.get(stackName) ?? new Set(),
|
||||
imageUpdateMap,
|
||||
previousState,
|
||||
now,
|
||||
allContainers,
|
||||
db,
|
||||
writeGenerations.get(stackName) ?? this.reserveStackWriteGeneration(nodeId, stackName),
|
||||
);
|
||||
if (outcome.committed && outcome.hasUpdate) {
|
||||
updatesFound++;
|
||||
// Notify only on state transition: was false/absent, now true
|
||||
if (!isBackfilled || !previousState[stackName]) {
|
||||
newlyUpdated.push(stackName);
|
||||
newlyUpdated.push({ stackName, message: outcome.notifyMessage });
|
||||
}
|
||||
}
|
||||
db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now, checkStatus, lastError);
|
||||
}
|
||||
|
||||
// Dispatch notifications for stacks that newly have updates
|
||||
if (newlyUpdated.length > 0) {
|
||||
const notifier = NotificationService.getInstance();
|
||||
for (const stackName of newlyUpdated) {
|
||||
for (const { stackName, message } of newlyUpdated) {
|
||||
try {
|
||||
await notifier.dispatchAlert(
|
||||
'info',
|
||||
'image_update_available',
|
||||
// Node-neutral body: the hub bell badge attributes remotes.
|
||||
`Stack "${stackName}" has image updates available.`,
|
||||
message,
|
||||
{ stackName, actor: 'system:image-update' },
|
||||
);
|
||||
} catch (e) {
|
||||
@@ -751,6 +856,212 @@ export class ImageUpdateService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-check a single stack after a service-scoped update or restore. On a
|
||||
* render failure the prior row is left untouched and a warning is returned.
|
||||
*/
|
||||
public async recheckStack(nodeId: number, stackName: string): Promise<{ warning: string | null }> {
|
||||
const generation = this.reserveStackWriteGeneration(nodeId, stackName);
|
||||
const db = DatabaseService.getInstance();
|
||||
const docker = DockerController.getInstance(nodeId);
|
||||
const model = await buildEffectiveServiceModel(nodeId, stackName);
|
||||
if (!model.renderable) {
|
||||
return { warning: model.error };
|
||||
}
|
||||
|
||||
let containers: Array<{ Image?: string; Labels?: Record<string, string> }> = [];
|
||||
try {
|
||||
containers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers');
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
'[ImageUpdateService] recheckStack container read failed for %s: %s',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(e, 'unknown')),
|
||||
);
|
||||
}
|
||||
|
||||
const refs = new Set<string>();
|
||||
const runtimeByService = this.runtimeImagesByService(stackName, containers);
|
||||
for (const spec of model.services) {
|
||||
if (spec.declaredImage) refs.add(spec.declaredImage);
|
||||
for (const ref of runtimeByService.get(spec.name) ?? []) refs.add(ref);
|
||||
}
|
||||
|
||||
const imageUpdateMap = new Map<string, ImageCheckResult>();
|
||||
for (const imageRef of refs) {
|
||||
try {
|
||||
imageUpdateMap.set(imageRef, await this.checkImage(docker, imageRef));
|
||||
} catch (e) {
|
||||
imageUpdateMap.set(imageRef, { hasUpdate: false, error: getErrorMessage(e, 'Update check failed') });
|
||||
}
|
||||
await sleep(ImageUpdateService.INTER_IMAGE_DELAY_MS);
|
||||
}
|
||||
|
||||
const priorByService = new Map(db.getStackServicesJson(nodeId, stackName).map((s) => [s.service, s]));
|
||||
const reductions = model.services.map((spec) => reduceServiceStatus(
|
||||
spec.name,
|
||||
spec.declaredImage,
|
||||
runtimeByService.get(spec.name) ?? [],
|
||||
imageUpdateMap,
|
||||
priorByService.get(spec.name),
|
||||
));
|
||||
const services = reductions.map((r) => r.status);
|
||||
const checkStatus = aggregateServiceCheckStatus(services);
|
||||
const hasUpdate = services.some((s) => s.hasUpdate);
|
||||
const lastError = stackStatusLastError(services);
|
||||
const now = Date.now();
|
||||
|
||||
await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => {
|
||||
if (checkStatus === 'failed') {
|
||||
db.recordStackCheckFailure(nodeId, stackName, lastError ?? 'Update check failed', now, services, gen);
|
||||
} else {
|
||||
db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now, checkStatus, lastError, services, gen);
|
||||
}
|
||||
});
|
||||
|
||||
return { warning: null };
|
||||
}
|
||||
|
||||
private stackWriteKey(nodeId: number, stackName: string): string {
|
||||
return `${nodeId}:${stackName}`;
|
||||
}
|
||||
|
||||
/** Bump the per-stack write generation before async registry work so a later
|
||||
* slower scan cannot commit after a newer recheck reserved a higher gen. */
|
||||
private reserveStackWriteGeneration(nodeId: number, stackName: string): number {
|
||||
const key = this.stackWriteKey(nodeId, stackName);
|
||||
let state = this.stackWriteState.get(key);
|
||||
if (!state) {
|
||||
state = { chain: Promise.resolve(), generation: 0 };
|
||||
this.stackWriteState.set(key, state);
|
||||
}
|
||||
state.generation += 1;
|
||||
return state.generation;
|
||||
}
|
||||
|
||||
private async withStackWriteLock(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
generation: number,
|
||||
write: (generation: number) => void | Promise<void>,
|
||||
): Promise<boolean> {
|
||||
const key = this.stackWriteKey(nodeId, stackName);
|
||||
let state = this.stackWriteState.get(key);
|
||||
if (!state) {
|
||||
state = { chain: Promise.resolve(), generation };
|
||||
this.stackWriteState.set(key, state);
|
||||
}
|
||||
let committed = false;
|
||||
state.chain = state.chain.then(async () => {
|
||||
const current = this.stackWriteState.get(key);
|
||||
// A newer reservation supersedes this writer; drop the stale commit.
|
||||
if (!current || generation < current.generation) return;
|
||||
await write(generation);
|
||||
committed = true;
|
||||
});
|
||||
await state.chain;
|
||||
return committed;
|
||||
}
|
||||
|
||||
private runtimeImagesByService(
|
||||
stackName: string,
|
||||
containers: Array<{ Image?: string; Labels?: Record<string, string> }>,
|
||||
): Map<string, string[]> {
|
||||
const out = new Map<string, string[]>();
|
||||
for (const c of containers) {
|
||||
if (c.Labels?.['com.docker.compose.project'] !== stackName) continue;
|
||||
const service = c.Labels?.['com.docker.compose.service'];
|
||||
if (!service) continue;
|
||||
const imageRef = c.Image ?? '';
|
||||
if (!imageRef || imageRef.startsWith('sha256:')) continue;
|
||||
const list = out.get(service) ?? [];
|
||||
list.push(imageRef);
|
||||
out.set(service, list);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private async writeStackUpdateStatus(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
images: Set<string>,
|
||||
imageUpdateMap: Map<string, ImageCheckResult>,
|
||||
previousState: Record<string, boolean>,
|
||||
checkedAt: number,
|
||||
containers: Array<{ Image?: string; Labels?: Record<string, string> }>,
|
||||
db: DatabaseService,
|
||||
generation: number,
|
||||
): Promise<{ hasUpdate: boolean; notifyMessage: string; committed: boolean }> {
|
||||
const model = await buildEffectiveServiceModel(nodeId, stackName);
|
||||
if (model.renderable) {
|
||||
const priorByService = new Map(
|
||||
DatabaseService.getInstance().getStackServicesJson(nodeId, stackName).map((s) => [s.service, s]),
|
||||
);
|
||||
const runtimeByService = this.runtimeImagesByService(stackName, containers);
|
||||
const reductions = model.services.map((spec) => reduceServiceStatus(
|
||||
spec.name,
|
||||
spec.declaredImage,
|
||||
runtimeByService.get(spec.name) ?? [],
|
||||
imageUpdateMap,
|
||||
priorByService.get(spec.name),
|
||||
));
|
||||
const services = reductions.map((r) => r.status);
|
||||
const checkStatus = aggregateServiceCheckStatus(services);
|
||||
const hasUpdate = services.some((s) => s.hasUpdate);
|
||||
const lastError = stackStatusLastError(services);
|
||||
const notifyMessage = buildAvailabilityNotifyMessage(stackName, services);
|
||||
|
||||
const committed = await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => {
|
||||
if (checkStatus === 'failed') {
|
||||
db.recordStackCheckFailure(
|
||||
nodeId, stackName, lastError ?? 'Update check failed', checkedAt, services, gen,
|
||||
);
|
||||
} else {
|
||||
db.upsertStackUpdateStatus(
|
||||
nodeId, stackName, hasUpdate, checkedAt, checkStatus, lastError, services, gen,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
return { hasUpdate, notifyMessage, committed };
|
||||
}
|
||||
|
||||
const checkable = Array.from(images)
|
||||
.map((img) => imageUpdateMap.get(img))
|
||||
.filter((r): r is ImageCheckResult => !!r && !r.notCheckable);
|
||||
const errored = checkable.filter((r) => r.error !== undefined);
|
||||
const confirmedHasUpdate = checkable.some((r) => r.error === undefined && r.hasUpdate === true);
|
||||
|
||||
if (checkable.length > 0 && errored.length === checkable.length) {
|
||||
const committed = await this.withStackWriteLock(nodeId, stackName, generation, async () => {
|
||||
db.recordStackCheckFailure(
|
||||
nodeId, stackName, errored[0].error ?? 'Update check failed', checkedAt,
|
||||
);
|
||||
});
|
||||
return {
|
||||
hasUpdate: previousState[stackName] === true,
|
||||
notifyMessage: buildAvailabilityNotifyMessage(stackName, []),
|
||||
committed,
|
||||
};
|
||||
}
|
||||
|
||||
const checkStatus: StackCheckStatus = errored.length > 0 ? 'partial' : 'ok';
|
||||
const lastError = errored.length > 0 ? (errored[0].error ?? null) : null;
|
||||
const hasUpdate = checkStatus === 'partial'
|
||||
? (confirmedHasUpdate || previousState[stackName] === true)
|
||||
: confirmedHasUpdate;
|
||||
|
||||
const committed = await this.withStackWriteLock(nodeId, stackName, generation, async () => {
|
||||
db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, checkedAt, checkStatus, lastError);
|
||||
});
|
||||
|
||||
return {
|
||||
hasUpdate,
|
||||
notifyMessage: buildAvailabilityNotifyMessage(stackName, []),
|
||||
committed,
|
||||
};
|
||||
}
|
||||
|
||||
public async checkImage(docker: DockerController, imageRef: string): Promise<ImageCheckResult> {
|
||||
const parsed = parseImageRef(imageRef);
|
||||
// A bare digest ref (sha256:...) has no tag to track upstream; not applicable.
|
||||
|
||||
@@ -5,9 +5,11 @@ import { LicenseService } from './LicenseService';
|
||||
import { PROXY_TIER_HEADER, deployProvenanceHeaders } from './license-headers';
|
||||
import DockerController from './DockerController';
|
||||
import { ComposeService } from './ComposeService';
|
||||
import { StackUpdateOrchestrator } from './StackUpdateOrchestrator';
|
||||
import { StackOpLockService, stackOpSkipMessage as skipMessage } from './StackOpLockService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { HealthGateService } from './HealthGateService';
|
||||
import { ServiceUpdateRecoveryService } from './ServiceUpdateRecoveryService';
|
||||
import { ImageUpdateService } from './ImageUpdateService';
|
||||
import type { ImageCheckResult } from './ImageUpdateService';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
@@ -706,12 +708,13 @@ export class SchedulerService {
|
||||
? (JSON.parse(task.prune_targets) as string[]).filter((t): t is PruneTarget => allTargets.includes(t as PruneTarget))
|
||||
: [...allTargets];
|
||||
const labelFilter = task.prune_label_filter || undefined;
|
||||
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(nodeId);
|
||||
const results: string[] = [];
|
||||
const failures: string[] = [];
|
||||
|
||||
for (const target of targets) {
|
||||
try {
|
||||
const result = await docker.pruneSystem(target, labelFilter);
|
||||
const result = await docker.pruneSystem(target, labelFilter, isImageHeld);
|
||||
results.push(`${target}: ${result.reclaimedBytes ?? 0} bytes reclaimed`);
|
||||
} catch (error: unknown) {
|
||||
const msg = error instanceof Error ? error.message : String(error);
|
||||
@@ -766,12 +769,11 @@ export class SchedulerService {
|
||||
const db = DatabaseService.getInstance();
|
||||
const docker = DockerController.getInstance(task.node_id);
|
||||
const imageUpdateService = ImageUpdateService.getInstance();
|
||||
const compose = ComposeService.getInstance(task.node_id);
|
||||
const results: string[] = [];
|
||||
|
||||
for (const stackName of stackNames) {
|
||||
try {
|
||||
const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, compose, db, isFleet || isWildcard);
|
||||
const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, db, isFleet || isWildcard);
|
||||
results.push(output);
|
||||
} catch (e) {
|
||||
const msg = getErrorMessage(e, String(e));
|
||||
@@ -1028,7 +1030,6 @@ export class SchedulerService {
|
||||
nodeId: number,
|
||||
docker: DockerController,
|
||||
imageUpdateService: ImageUpdateService,
|
||||
compose: ComposeService,
|
||||
db: DatabaseService,
|
||||
isWildcard = false
|
||||
): Promise<string> {
|
||||
@@ -1096,11 +1097,14 @@ export class SchedulerService {
|
||||
const atomic = true;
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
nodeId, stackName, 'update', 'system',
|
||||
() => compose.updateStack(stackName, undefined, atomic),
|
||||
() => StackUpdateOrchestrator.getInstance().execute(
|
||||
{ nodeId, stackName, target: { scope: 'stack' }, trigger: 'scheduled', actor: 'system:scheduler' },
|
||||
{ atomic, terminalWs: null },
|
||||
),
|
||||
);
|
||||
if (!lock.ran) return skipMessage(stackName, lock.existing.action);
|
||||
db.clearStackUpdateStatus(nodeId, stackName);
|
||||
HealthGateService.getInstance().begin(nodeId, stackName, 'update', 'system:scheduler');
|
||||
HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler');
|
||||
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import { DatabaseService, type ServiceUpdateRecoveryRow } from './DatabaseService';
|
||||
import { getComposeCommandTimeoutMs } from './ComposeService';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
|
||||
const SWEEP_INTERVAL_MS = 5 * 60_000;
|
||||
const INITIAL_SWEEP_DELAY_MS = 30_000;
|
||||
const CLAIM_RENEWAL_INTERVAL_MS = 5 * 60_000;
|
||||
const MIN_CLAIM_WINDOW_MS = 30 * 60_000;
|
||||
const CLAIM_RENEWAL_BUFFER_MS = 5 * 60_000;
|
||||
const RECOVERY_TTL_BUFFER_MS = 30 * 60_000;
|
||||
const MIN_RECOVERY_WINDOW_SECONDS = 90;
|
||||
const DIGEST_PIN_PATTERN = /@sha256:[a-f0-9]{64}$/i;
|
||||
|
||||
/** A single pre-update replica snapshot for one service. */
|
||||
export interface ServiceReplicaSnapshot {
|
||||
imageId: string;
|
||||
/** Repo digest captured from the running container, when known. */
|
||||
repoDigest: string | null;
|
||||
}
|
||||
|
||||
export interface CreateServiceUpdateRecoveryInput {
|
||||
nodeId: number;
|
||||
stackName: string;
|
||||
serviceName: string;
|
||||
replicas: ServiceReplicaSnapshot[];
|
||||
/** EffectiveServiceSpec.declaredImage; null for a pure-build service with no image field. */
|
||||
declaredImageRef: string | null;
|
||||
createdBy: string | null;
|
||||
}
|
||||
|
||||
export type ServiceUpdateRecoveryUnavailableReason =
|
||||
| 'no_replicas'
|
||||
| 'majority_tie'
|
||||
| 'build_only'
|
||||
| 'digest_pinned_declared_ref'
|
||||
| 'missing_local_id';
|
||||
|
||||
export type CreateServiceUpdateRecoveryResult =
|
||||
| { eligible: true; row: ServiceUpdateRecoveryRow }
|
||||
| { eligible: false; reason: ServiceUpdateRecoveryUnavailableReason };
|
||||
|
||||
/**
|
||||
* Pre-update image snapshots that let an operator manually restore a single
|
||||
* service after a service-scoped update, without touching its siblings.
|
||||
*
|
||||
* Purely a data/lifecycle layer: it decides whether a snapshot is eligible to
|
||||
* persist, holds the claimed row's image from prune while a restore is in
|
||||
* flight, and sweeps stale rows. It never runs Compose or touches Docker; the
|
||||
* update/restore orchestration (StackUpdateOrchestrator) calls into this
|
||||
* service at the right points and is responsible for policy, health gating,
|
||||
* and Auto-Heal suppression around those calls.
|
||||
*/
|
||||
export class ServiceUpdateRecoveryService {
|
||||
private static instance: ServiceUpdateRecoveryService;
|
||||
private started = false;
|
||||
private intervalId: NodeJS.Timeout | null = null;
|
||||
private initialTimer: NodeJS.Timeout | null = null;
|
||||
private readonly renewalTimers = new Map<string, NodeJS.Timeout>();
|
||||
|
||||
private constructor() {}
|
||||
|
||||
public static getInstance(): ServiceUpdateRecoveryService {
|
||||
if (!ServiceUpdateRecoveryService.instance) {
|
||||
ServiceUpdateRecoveryService.instance = new ServiceUpdateRecoveryService();
|
||||
}
|
||||
return ServiceUpdateRecoveryService.instance;
|
||||
}
|
||||
|
||||
public start(): void {
|
||||
this.started = true;
|
||||
if (this.initialTimer || this.intervalId) return;
|
||||
this.initialTimer = setTimeout(() => {
|
||||
this.sweep();
|
||||
this.intervalId = setInterval(() => this.sweep(), SWEEP_INTERVAL_MS);
|
||||
}, INITIAL_SWEEP_DELAY_MS);
|
||||
}
|
||||
|
||||
/** Clear the sweep timer and every in-flight claim renewal loop. No callbacks fire after this returns. */
|
||||
public stop(): void {
|
||||
this.started = false;
|
||||
if (this.initialTimer) {
|
||||
clearTimeout(this.initialTimer);
|
||||
this.initialTimer = null;
|
||||
}
|
||||
if (this.intervalId) {
|
||||
clearInterval(this.intervalId);
|
||||
this.intervalId = null;
|
||||
}
|
||||
for (const timer of this.renewalTimers.values()) clearInterval(timer);
|
||||
this.renewalTimers.clear();
|
||||
}
|
||||
|
||||
/** Expire active rows past their TTL and restoring rows abandoned by a dead claim. Never throws. */
|
||||
public sweep(): void {
|
||||
try {
|
||||
const now = Date.now();
|
||||
const db = DatabaseService.getInstance();
|
||||
const expiredActive = db.sweepExpiredActiveServiceUpdateRecoveries(now);
|
||||
const expiredAbandoned = db.sweepAbandonedRestoringServiceUpdateRecoveries(now);
|
||||
if (expiredActive > 0 || expiredAbandoned > 0) {
|
||||
console.log(
|
||||
`[ServiceUpdateRecovery] Swept ${expiredActive} expired active and ${expiredAbandoned} abandoned restoring record(s)`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[ServiceUpdateRecovery] Sweep failed:', getErrorMessage(error, 'unknown'));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a pre-update snapshot is eligible to persist and, if so,
|
||||
* insert it. Unavailable per §8: no replicas, a majority tie, a pure-build
|
||||
* service (no declared image ref), a digest-pinned declared ref, or a
|
||||
* replica missing its local image id.
|
||||
*/
|
||||
public createIfEligible(input: CreateServiceUpdateRecoveryInput): CreateServiceUpdateRecoveryResult {
|
||||
const validReplicas = input.replicas.filter(r => r.imageId.trim().length > 0);
|
||||
if (validReplicas.length === 0) {
|
||||
return { eligible: false, reason: input.replicas.length === 0 ? 'no_replicas' : 'missing_local_id' };
|
||||
}
|
||||
|
||||
const majority = this.computeMajorityImage(validReplicas);
|
||||
if (!majority) return { eligible: false, reason: 'majority_tie' };
|
||||
|
||||
if (!input.declaredImageRef) return { eligible: false, reason: 'build_only' };
|
||||
if (DIGEST_PIN_PATTERN.test(input.declaredImageRef)) return { eligible: false, reason: 'digest_pinned_declared_ref' };
|
||||
|
||||
const now = Date.now();
|
||||
const row: ServiceUpdateRecoveryRow = {
|
||||
id: randomUUID(),
|
||||
node_id: input.nodeId,
|
||||
stack_name: input.stackName,
|
||||
service_name: input.serviceName,
|
||||
replicas_json: JSON.stringify(input.replicas),
|
||||
majority_image_id: majority.imageId,
|
||||
declared_image_ref: input.declaredImageRef,
|
||||
weak_floating_tag: majority.weakFloatingTag ? 1 : 0,
|
||||
health_gate_id: null,
|
||||
status: 'active',
|
||||
expires_at: now + this.activeRecoveryTtlMs() + RECOVERY_TTL_BUFFER_MS,
|
||||
claim_expires_at: null,
|
||||
created_at: now,
|
||||
created_by: input.createdBy,
|
||||
};
|
||||
DatabaseService.getInstance().insertServiceUpdateRecovery(row);
|
||||
return { eligible: true, row };
|
||||
}
|
||||
|
||||
/** Active, unexpired recovery rows for one service (drives `recoveryAvailable` in the update/restore responses). */
|
||||
public listActive(nodeId: number, stackName: string, serviceName: string): ServiceUpdateRecoveryRow[] {
|
||||
return DatabaseService.getInstance().listActiveServiceUpdateRecoveries(nodeId, stackName, serviceName);
|
||||
}
|
||||
|
||||
public get(id: string): ServiceUpdateRecoveryRow | undefined {
|
||||
return DatabaseService.getInstance().getServiceUpdateRecovery(id);
|
||||
}
|
||||
|
||||
/** Attach the update flow's own health gate run id while the row is still active. */
|
||||
public linkHealthGate(id: string, healthGateId: string): void {
|
||||
DatabaseService.getInstance().linkServiceUpdateRecoveryHealthGate(id, healthGateId);
|
||||
}
|
||||
|
||||
/** Atomic claim: active -> restoring. Returns the claimed row, or undefined if another actor won the race. */
|
||||
public claim(id: string): ServiceUpdateRecoveryRow | undefined {
|
||||
const now = Date.now();
|
||||
return DatabaseService.getInstance().claimServiceUpdateRecovery(id, this.nextClaimExpiry(now), now);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the mandatory 5-minute renewal loop for a claimed row (RECOVERY-CLAIM-2).
|
||||
* Idempotent per id. Each tick CAS-extends claim_expires_at only while the row
|
||||
* is still restoring; a renewal that finds the row no longer restoring (superseded,
|
||||
* consumed, or force-expired) stops itself rather than reviving a terminal row.
|
||||
*/
|
||||
public startClaimRenewal(id: string): void {
|
||||
if (!this.started || this.renewalTimers.has(id)) return;
|
||||
const timer = setInterval(() => {
|
||||
try {
|
||||
const renewed = DatabaseService.getInstance().renewServiceUpdateRecoveryClaim(id, this.nextClaimExpiry(Date.now()));
|
||||
if (!renewed) this.stopClaimRenewal(id);
|
||||
} catch (error) {
|
||||
console.error('[ServiceUpdateRecovery] Claim renewal failed for %s:', id, getErrorMessage(error, 'unknown'));
|
||||
}
|
||||
}, CLAIM_RENEWAL_INTERVAL_MS);
|
||||
this.renewalTimers.set(id, timer);
|
||||
}
|
||||
|
||||
/** Stop the renewal loop for a row. Callers must call this in the restore operation's `finally` regardless of outcome. */
|
||||
public stopClaimRenewal(id: string): void {
|
||||
const timer = this.renewalTimers.get(id);
|
||||
if (timer) {
|
||||
clearInterval(timer);
|
||||
this.renewalTimers.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
/** Restore succeeded: restoring -> consumed, optionally linking the restore's own health gate run. */
|
||||
public markConsumed(id: string, healthGateId: string | null = null): boolean {
|
||||
return DatabaseService.getInstance().markServiceUpdateRecoveryConsumed(id, healthGateId);
|
||||
}
|
||||
|
||||
/** Mid-flight restore failure with the image still local: restoring -> active, claim cleared. */
|
||||
public reactivate(id: string): boolean {
|
||||
return DatabaseService.getInstance().reactivateServiceUpdateRecovery(id);
|
||||
}
|
||||
|
||||
/** Mid-flight restore failure with the image gone: restoring -> invalidated. */
|
||||
public invalidate(id: string): boolean {
|
||||
return DatabaseService.getInstance().invalidateServiceUpdateRecovery(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Image IDs currently held for this node (active rows and restoring rows
|
||||
* with a live claim). Returns null when the held set cannot be read so
|
||||
* callers can fail closed (skip prune) instead of treating the miss as empty.
|
||||
*/
|
||||
public getHeldImageIds(nodeId: number): Set<string> | null {
|
||||
try {
|
||||
return new Set(DatabaseService.getInstance().listHeldServiceUpdateRecoveryImageIds(nodeId, Date.now()));
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[ServiceUpdateRecovery] Failed to compute held image ids for node %d:', nodeId, getErrorMessage(error, 'unknown'),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A predicate a pruner can call immediately before deleting each candidate
|
||||
* image. Re-reads the held set on every call (rather than snapshotting it
|
||||
* once) so a snapshot that becomes eligible between plan and delete is
|
||||
* still honored. When the held set cannot be read, returns true for every
|
||||
* id so prune skips deletes (fail closed).
|
||||
*/
|
||||
public buildHeldImagePredicate(nodeId: number): (imageId: string) => boolean {
|
||||
return (imageId: string) => {
|
||||
const held = this.getHeldImageIds(nodeId);
|
||||
if (held === null) return true;
|
||||
return held.has(imageId);
|
||||
};
|
||||
}
|
||||
|
||||
private nextClaimExpiry(now: number): number {
|
||||
return now + Math.max(getComposeCommandTimeoutMs(), MIN_CLAIM_WINDOW_MS) + CLAIM_RENEWAL_BUFFER_MS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Active recovery must outlive both the Compose mutation and the post-update
|
||||
* health window. A raised SENCHO_COMPOSE_COMMAND_TIMEOUT_MS must not leave the
|
||||
* snapshot eligible for sweep while Compose is still running.
|
||||
*/
|
||||
private activeRecoveryTtlMs(): number {
|
||||
return Math.max(getComposeCommandTimeoutMs(), this.readRecoveryWindowSeconds() * 1000);
|
||||
}
|
||||
|
||||
private readRecoveryWindowSeconds(): number {
|
||||
try {
|
||||
const raw = parseInt(DatabaseService.getInstance().getGlobalSettings()['health_gate_window_seconds'] ?? '', 10);
|
||||
return Number.isFinite(raw) ? Math.max(raw, MIN_RECOVERY_WINDOW_SECONDS) : MIN_RECOVERY_WINDOW_SECONDS;
|
||||
} catch (error) {
|
||||
console.warn('[ServiceUpdateRecovery] Settings read failed; using default window:', getErrorMessage(error, 'unknown'));
|
||||
return MIN_RECOVERY_WINDOW_SECONDS;
|
||||
}
|
||||
}
|
||||
|
||||
/** Majority image id among replicas, or null on a tie (no single winner). */
|
||||
private computeMajorityImage(replicas: ServiceReplicaSnapshot[]): { imageId: string; weakFloatingTag: boolean } | null {
|
||||
const counts = new Map<string, number>();
|
||||
for (const replica of replicas) {
|
||||
counts.set(replica.imageId, (counts.get(replica.imageId) ?? 0) + 1);
|
||||
}
|
||||
const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1]);
|
||||
if (ranked.length === 0) return null;
|
||||
if (ranked.length > 1 && ranked[0][1] === ranked[1][1]) return null;
|
||||
const winner = ranked[0][0];
|
||||
const weakFloatingTag = !replicas.some(r => r.imageId === winner && r.repoDigest);
|
||||
return { imageId: winner, weakFloatingTag };
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,11 @@
|
||||
|
||||
export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update';
|
||||
|
||||
export interface StackOpRecordMeta {
|
||||
targetScope?: 'stack' | 'service';
|
||||
serviceName?: string | null;
|
||||
}
|
||||
|
||||
interface StackOpStats {
|
||||
count: number;
|
||||
successCount: number;
|
||||
@@ -22,6 +27,10 @@ interface StackOpStats {
|
||||
* computed from this window on demand.
|
||||
*/
|
||||
recentSamples: number[];
|
||||
/** Most recent target scope recorded for this bucket (additive; stack default). */
|
||||
lastTargetScope: 'stack' | 'service';
|
||||
/** Most recent service name when lastTargetScope is service; otherwise null. */
|
||||
lastServiceName: string | null;
|
||||
}
|
||||
|
||||
export interface StackOpSnapshotEntry {
|
||||
@@ -33,6 +42,8 @@ export interface StackOpSnapshotEntry {
|
||||
avgMs: number;
|
||||
p50Ms: number;
|
||||
p95Ms: number;
|
||||
targetScope: 'stack' | 'service';
|
||||
serviceName: string | null;
|
||||
}
|
||||
|
||||
const MAX_SAMPLES = 1000;
|
||||
@@ -66,12 +77,26 @@ export class StackOpMetricsService {
|
||||
* Record one completed op. Call from the route layer after the lifecycle
|
||||
* call resolves or rejects; `ok=false` for the rejection path.
|
||||
*/
|
||||
public record(nodeId: number, action: StackOpAction, durationMs: number, ok: boolean): void {
|
||||
public record(
|
||||
nodeId: number,
|
||||
action: StackOpAction,
|
||||
durationMs: number,
|
||||
ok: boolean,
|
||||
meta?: StackOpRecordMeta,
|
||||
): void {
|
||||
if (!Number.isFinite(durationMs) || durationMs < 0) return;
|
||||
const k = this.key(nodeId, action);
|
||||
let s = this.stats.get(k);
|
||||
if (!s) {
|
||||
s = { count: 0, successCount: 0, errorCount: 0, totalMs: 0, recentSamples: [] };
|
||||
s = {
|
||||
count: 0,
|
||||
successCount: 0,
|
||||
errorCount: 0,
|
||||
totalMs: 0,
|
||||
recentSamples: [],
|
||||
lastTargetScope: 'stack',
|
||||
lastServiceName: null,
|
||||
};
|
||||
this.stats.set(k, s);
|
||||
}
|
||||
s.count += 1;
|
||||
@@ -84,6 +109,12 @@ export class StackOpMetricsService {
|
||||
// and this path is once-per-stack-op (low cadence).
|
||||
s.recentSamples.shift();
|
||||
}
|
||||
if (meta?.targetScope) s.lastTargetScope = meta.targetScope;
|
||||
if (meta?.targetScope === 'service') {
|
||||
s.lastServiceName = meta.serviceName ?? null;
|
||||
} else if (meta?.targetScope === 'stack') {
|
||||
s.lastServiceName = null;
|
||||
}
|
||||
}
|
||||
|
||||
public snapshot(): StackOpSnapshotEntry[] {
|
||||
@@ -102,6 +133,8 @@ export class StackOpMetricsService {
|
||||
avgMs: s.count === 0 ? 0 : Math.round(s.totalMs / s.count),
|
||||
p50Ms: percentile(sorted, 0.5),
|
||||
p95Ms: percentile(sorted, 0.95),
|
||||
targetScope: s.lastTargetScope,
|
||||
serviceName: s.lastServiceName,
|
||||
});
|
||||
}
|
||||
// Stable ordering: nodeId asc, then action asc.
|
||||
|
||||
@@ -0,0 +1,684 @@
|
||||
import WebSocket from 'ws';
|
||||
import DockerController from './DockerController';
|
||||
import { ComposeService } from './ComposeService';
|
||||
import { HealthGateService } from './HealthGateService';
|
||||
import { AutoHealService } from './AutoHealService';
|
||||
import { ImageUpdateService } from './ImageUpdateService';
|
||||
import {
|
||||
ServiceUpdateRecoveryService,
|
||||
type ServiceReplicaSnapshot,
|
||||
} from './ServiceUpdateRecoveryService';
|
||||
import type { ServiceUpdateRecoveryRow } from './DatabaseService';
|
||||
import { buildEffectiveServiceModel } from './effectiveServiceModel';
|
||||
import { DriftLedgerService } from './DriftLedgerService';
|
||||
import { MeshService } from './MeshService';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import { enforcePolicyForImageRefs, type PolicyEnforcementOptions } from './PolicyEnforcement';
|
||||
import { describePolicyBlock } from '../helpers/policyGate';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import type { EffectiveServiceSpec } from './effectiveServiceModel';
|
||||
|
||||
export type UpdateTarget =
|
||||
| { scope: 'stack' }
|
||||
| { scope: 'service'; serviceName: string };
|
||||
|
||||
export type UpdateTrigger =
|
||||
| 'manual' | 'scheduled' | 'automatic' | 'blueprint' | 'webhook' | 'bulk';
|
||||
|
||||
export interface UpdateOperationContext {
|
||||
nodeId: number;
|
||||
stackName: string;
|
||||
target: UpdateTarget;
|
||||
trigger: UpdateTrigger;
|
||||
actor: string | null;
|
||||
}
|
||||
|
||||
export interface StackComposeOptions {
|
||||
atomic: boolean;
|
||||
terminalWs?: WebSocket | null;
|
||||
}
|
||||
|
||||
export interface ServiceUpdateOptions {
|
||||
policyOptions: PolicyEnforcementOptions;
|
||||
terminalWs?: WebSocket | null;
|
||||
/** Restore only: binds the operation to a recovery snapshot. */
|
||||
recoveryId?: string;
|
||||
}
|
||||
|
||||
export type OrchestratorResult =
|
||||
| { kind: 'stack_compose_done' }
|
||||
| {
|
||||
kind: 'service_done';
|
||||
serviceName: string;
|
||||
healthGateId: string | null;
|
||||
observing: boolean;
|
||||
recoveryId: string | null;
|
||||
recoveryAvailable: boolean;
|
||||
previousImageId?: string | null;
|
||||
newImageId?: string | null;
|
||||
recheckWarning?: string;
|
||||
}
|
||||
| {
|
||||
kind: 'service_failed';
|
||||
code: string;
|
||||
error: string;
|
||||
serviceName?: string;
|
||||
mutationStage?: string;
|
||||
recoveryId?: string | null;
|
||||
};
|
||||
|
||||
function serviceFailed(
|
||||
code: string,
|
||||
error: string,
|
||||
extra?: { serviceName?: string; mutationStage?: string; recoveryId?: string | null },
|
||||
): Extract<OrchestratorResult, { kind: 'service_failed' }> {
|
||||
return { kind: 'service_failed', code, error, ...extra };
|
||||
}
|
||||
|
||||
/** Shorten a local image id for activity copy (sha256:abcdef… → abcdef…). */
|
||||
export function shortImageId(imageId: string | null | undefined): string | null {
|
||||
if (!imageId) return null;
|
||||
const bare = imageId.startsWith('sha256:') ? imageId.slice(7) : imageId;
|
||||
return bare.slice(0, 12);
|
||||
}
|
||||
|
||||
/**
|
||||
* Post-mutation replica convergence rules shared by update and restore.
|
||||
* Requires the exact expected count of inspected running image ids and a
|
||||
* single shared image before success (independent of optional health gating).
|
||||
*/
|
||||
export function evaluateServiceReplicaConvergence(
|
||||
serviceName: string,
|
||||
expectedReplicas: number,
|
||||
ids: string[],
|
||||
inspectErrors: number,
|
||||
):
|
||||
| { kind: 'converged'; imageId: string | null }
|
||||
| { kind: 'divergent'; error: string }
|
||||
| { kind: 'inspect_failed'; error: string } {
|
||||
if (expectedReplicas === 0) {
|
||||
return { kind: 'converged', imageId: null };
|
||||
}
|
||||
if (inspectErrors > 0 && ids.length === 0) {
|
||||
return {
|
||||
kind: 'inspect_failed',
|
||||
error: `Could not inspect replicas for service "${serviceName}" after the update.`,
|
||||
};
|
||||
}
|
||||
const distinct = new Set(ids);
|
||||
if (distinct.size === 0) {
|
||||
return { kind: 'divergent', error: `Service "${serviceName}" has no running replicas after the update.` };
|
||||
}
|
||||
if (ids.length !== expectedReplicas) {
|
||||
return {
|
||||
kind: 'divergent',
|
||||
error: `Service "${serviceName}" has ${ids.length} running replica(s); expected ${expectedReplicas}.`,
|
||||
};
|
||||
}
|
||||
if (distinct.size > 1) {
|
||||
return { kind: 'divergent', error: `Service "${serviceName}" replicas did not converge on a single image.` };
|
||||
}
|
||||
return { kind: 'converged', imageId: [...distinct][0] };
|
||||
}
|
||||
|
||||
/** Parse a tag image ref into a Docker tag() repo + tag pair. */
|
||||
function splitImageRef(ref: string): { repo: string; tag: string } {
|
||||
const lastSlash = ref.lastIndexOf('/');
|
||||
const lastColon = ref.lastIndexOf(':');
|
||||
if (lastColon > lastSlash) {
|
||||
return { repo: ref.slice(0, lastColon), tag: ref.slice(lastColon + 1) };
|
||||
}
|
||||
return { repo: ref, tag: 'latest' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Single entry point for stack-scope Compose updates and the manual
|
||||
* service-scoped update/restore flows.
|
||||
*
|
||||
* Ownership boundaries (§3): the five stack update callers keep every
|
||||
* side effect (policy, lock, clear status, invalidate, broadcast, notify,
|
||||
* post-scan, and the post-success `beginStack`); this orchestrator's stack
|
||||
* branch only runs `ComposeService.updateStack`. The service branch (nested
|
||||
* manual routes only) owns policy, the recovery snapshot, the pre-mutation
|
||||
* health gate (prepare/attach/beginPrepared), Auto-Heal suppression, and the
|
||||
* per-service recheck; the route holds the stack lock and maps the result to
|
||||
* HTTP. Expected failures never throw past `execute`; they return a
|
||||
* `service_failed` result.
|
||||
*/
|
||||
export class StackUpdateOrchestrator {
|
||||
private static instance: StackUpdateOrchestrator;
|
||||
|
||||
public static getInstance(): StackUpdateOrchestrator {
|
||||
if (!StackUpdateOrchestrator.instance) {
|
||||
StackUpdateOrchestrator.instance = new StackUpdateOrchestrator();
|
||||
}
|
||||
return StackUpdateOrchestrator.instance;
|
||||
}
|
||||
|
||||
public async execute(
|
||||
ctx: UpdateOperationContext,
|
||||
options: StackComposeOptions | ServiceUpdateOptions,
|
||||
): Promise<OrchestratorResult> {
|
||||
if (ctx.target.scope === 'stack') {
|
||||
return this.executeStack(ctx, options as StackComposeOptions);
|
||||
}
|
||||
// Service scope: automation never selects this branch.
|
||||
if (ctx.trigger !== 'manual') {
|
||||
throw new Error('Service-scoped updates require a manual trigger');
|
||||
}
|
||||
const serviceOptions = options as ServiceUpdateOptions;
|
||||
if (serviceOptions.recoveryId) {
|
||||
return this.executeRestore(ctx, ctx.target.serviceName, serviceOptions);
|
||||
}
|
||||
return this.executeServiceUpdate(ctx, ctx.target.serviceName, serviceOptions);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stack branch: run the full-stack Compose update only. Side effects and the
|
||||
* post-success `beginStack(..., 'update')` stay with the caller.
|
||||
*/
|
||||
private async executeStack(
|
||||
ctx: UpdateOperationContext,
|
||||
options: StackComposeOptions,
|
||||
): Promise<OrchestratorResult> {
|
||||
await ComposeService.getInstance(ctx.nodeId).updateStack(
|
||||
ctx.stackName, options.terminalWs ?? undefined, options.atomic,
|
||||
);
|
||||
return { kind: 'stack_compose_done' };
|
||||
}
|
||||
|
||||
private async executeServiceUpdate(
|
||||
ctx: UpdateOperationContext,
|
||||
serviceName: string,
|
||||
options: ServiceUpdateOptions,
|
||||
): Promise<OrchestratorResult> {
|
||||
const { nodeId, stackName } = ctx;
|
||||
|
||||
const loaded = await this.loadServiceSpec(nodeId, stackName, serviceName);
|
||||
if (!loaded.ok) return loaded.result;
|
||||
const { spec, services } = loaded;
|
||||
|
||||
if (services.length <= 1) {
|
||||
return serviceFailed(
|
||||
'service_update_single_service',
|
||||
'Service-scoped updates require a stack with more than one service.',
|
||||
);
|
||||
}
|
||||
if (!spec.hasBuild && !spec.declaredImage) {
|
||||
return serviceFailed(
|
||||
'service_not_updatable',
|
||||
`Service "${serviceName}" has neither an image nor a build, so it cannot be updated.`,
|
||||
);
|
||||
}
|
||||
|
||||
AutoHealService.getInstance().suppress(nodeId, stackName, serviceName);
|
||||
let observing = false;
|
||||
let recoveryId: string | null = null;
|
||||
let recoveryAvailable = false;
|
||||
try {
|
||||
const policyFailure = await this.checkPolicyOrFail(
|
||||
stackName, nodeId, serviceName, spec.declaredImage ? [spec.declaredImage] : [], options.policyOptions, 'update',
|
||||
);
|
||||
if (policyFailure) return policyFailure;
|
||||
|
||||
const prepared = await HealthGateService.getInstance().prepare({
|
||||
nodeId, stackName,
|
||||
target: { scope: 'service', serviceName },
|
||||
trigger: 'service_update',
|
||||
expectedReplicas: spec.expectedReplicas,
|
||||
});
|
||||
const prepareToken = prepared.prepareToken;
|
||||
const prepareSnapshotOk = prepared.snapshotOk;
|
||||
|
||||
const replicas = await this.snapshotServiceReplicas(nodeId, stackName, serviceName);
|
||||
const recovery = ServiceUpdateRecoveryService.getInstance().createIfEligible({
|
||||
nodeId, stackName, serviceName,
|
||||
replicas,
|
||||
declaredImageRef: spec.declaredImage,
|
||||
createdBy: ctx.actor,
|
||||
});
|
||||
const previousImageId = recovery.eligible
|
||||
? recovery.row.majority_image_id
|
||||
: (replicas[0]?.imageId ?? null);
|
||||
if (recovery.eligible) {
|
||||
recoveryId = recovery.row.id;
|
||||
recoveryAvailable = true;
|
||||
}
|
||||
|
||||
try {
|
||||
await ComposeService.getInstance(nodeId).updateService(
|
||||
stackName, serviceName, spec.hasBuild, options.terminalWs ?? undefined,
|
||||
);
|
||||
} catch (composeError) {
|
||||
return serviceFailed(
|
||||
'service_update_compose_failed',
|
||||
getErrorMessage(composeError, 'Service update failed'),
|
||||
{ serviceName, mutationStage: 'compose', recoveryId },
|
||||
);
|
||||
}
|
||||
|
||||
const observed = await this.observePreparedService(
|
||||
prepareToken, nodeId, stackName, serviceName, spec.expectedReplicas, ctx.actor,
|
||||
'service_update_replica_divergence', recoveryId, prepareSnapshotOk,
|
||||
);
|
||||
if (!observed.ok) return observed.result;
|
||||
observing = observed.observing;
|
||||
const healthGateId = observed.healthGateId;
|
||||
|
||||
if (recoveryId && healthGateId) {
|
||||
try {
|
||||
ServiceUpdateRecoveryService.getInstance().linkHealthGate(recoveryId, healthGateId);
|
||||
} catch (linkError) {
|
||||
console.warn(
|
||||
'[StackUpdateOrchestrator] Failed to link recovery %s to gate %s: %s',
|
||||
sanitizeForLog(recoveryId), sanitizeForLog(healthGateId),
|
||||
sanitizeForLog(getErrorMessage(linkError, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
const recheck = await ImageUpdateService.getInstance().recheckStack(nodeId, stackName);
|
||||
await DriftLedgerService.getInstance().reconcileServiceForStack(nodeId, stackName, serviceName);
|
||||
await this.refreshMeshIfEnabled(nodeId, stackName);
|
||||
|
||||
const warnings = [observed.gateWarning, recheck.warning].filter((w): w is string => !!w);
|
||||
return {
|
||||
kind: 'service_done',
|
||||
serviceName,
|
||||
healthGateId,
|
||||
observing,
|
||||
recoveryId,
|
||||
recoveryAvailable,
|
||||
previousImageId,
|
||||
newImageId: observed.newImageId ?? null,
|
||||
recheckWarning: warnings.length > 0 ? warnings.join(' ') : undefined,
|
||||
};
|
||||
} finally {
|
||||
if (!observing) {
|
||||
AutoHealService.getInstance().clearSuppress(nodeId, stackName, serviceName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async executeRestore(
|
||||
ctx: UpdateOperationContext,
|
||||
serviceName: string,
|
||||
options: ServiceUpdateOptions,
|
||||
): Promise<OrchestratorResult> {
|
||||
const { nodeId, stackName } = ctx;
|
||||
const recoveryId = options.recoveryId as string;
|
||||
|
||||
const recovery = ServiceUpdateRecoveryService.getInstance().get(recoveryId);
|
||||
if (
|
||||
!recovery ||
|
||||
recovery.node_id !== nodeId ||
|
||||
recovery.stack_name !== stackName ||
|
||||
recovery.service_name !== serviceName
|
||||
) {
|
||||
return serviceFailed('recovery_not_found', 'No matching recovery snapshot for this service.');
|
||||
}
|
||||
if (recovery.status !== 'active') {
|
||||
return serviceFailed(
|
||||
'recovery_not_restorable',
|
||||
`This recovery snapshot is ${recovery.status} and can no longer be restored.`,
|
||||
{ recoveryId },
|
||||
);
|
||||
}
|
||||
|
||||
AutoHealService.getInstance().suppress(nodeId, stackName, serviceName);
|
||||
let observing = false;
|
||||
let claimed = false;
|
||||
let consumed = false;
|
||||
try {
|
||||
const loaded = await this.loadServiceSpec(nodeId, stackName, serviceName, recoveryId);
|
||||
if (!loaded.ok) return loaded.result;
|
||||
const { spec, services } = loaded;
|
||||
if (services.length <= 1) {
|
||||
return serviceFailed(
|
||||
'service_update_single_service',
|
||||
'Service-scoped restore requires a stack with more than one service.',
|
||||
{ recoveryId },
|
||||
);
|
||||
}
|
||||
|
||||
const scanTarget = await this.resolveRestoreScanTarget(nodeId, recovery);
|
||||
const policyFailure = await this.checkPolicyOrFail(
|
||||
stackName, nodeId, serviceName, [scanTarget], options.policyOptions, 'rollback', recoveryId,
|
||||
);
|
||||
if (policyFailure) return policyFailure;
|
||||
|
||||
const prepared = await HealthGateService.getInstance().prepare({
|
||||
nodeId, stackName,
|
||||
target: { scope: 'service', serviceName },
|
||||
trigger: 'service_restore',
|
||||
expectedReplicas: spec.expectedReplicas,
|
||||
});
|
||||
const prepareToken = prepared.prepareToken;
|
||||
const prepareSnapshotOk = prepared.snapshotOk;
|
||||
|
||||
const claimedRow = ServiceUpdateRecoveryService.getInstance().claim(recoveryId);
|
||||
if (!claimedRow) {
|
||||
return serviceFailed(
|
||||
'recovery_claim_failed',
|
||||
'This recovery snapshot is being restored by another operation.',
|
||||
{ recoveryId },
|
||||
);
|
||||
}
|
||||
claimed = true;
|
||||
ServiceUpdateRecoveryService.getInstance().startClaimRenewal(recoveryId);
|
||||
|
||||
const previousImageId = (await this.snapshotServiceReplicas(nodeId, stackName, serviceName))[0]?.imageId ?? null;
|
||||
|
||||
try {
|
||||
const { repo, tag } = splitImageRef(recovery.declared_image_ref);
|
||||
await DockerController.getInstance(nodeId).getDocker()
|
||||
.getImage(recovery.majority_image_id).tag({ repo, tag });
|
||||
} catch (retagError) {
|
||||
return serviceFailed(
|
||||
'restore_retag_failed',
|
||||
getErrorMessage(retagError, 'Could not retag the recovery image'),
|
||||
{ serviceName, mutationStage: 'retag', recoveryId },
|
||||
);
|
||||
}
|
||||
try {
|
||||
await ComposeService.getInstance(nodeId).recreateServiceFromLocal(
|
||||
stackName, serviceName, options.terminalWs ?? undefined,
|
||||
);
|
||||
} catch (composeError) {
|
||||
return serviceFailed(
|
||||
'restore_compose_failed',
|
||||
getErrorMessage(composeError, 'Service restore failed'),
|
||||
{ serviceName, mutationStage: 'compose', recoveryId },
|
||||
);
|
||||
}
|
||||
|
||||
const observed = await this.observePreparedService(
|
||||
prepareToken, nodeId, stackName, serviceName, spec.expectedReplicas, ctx.actor,
|
||||
'restore_replica_divergence', recoveryId, prepareSnapshotOk,
|
||||
);
|
||||
if (!observed.ok) return observed.result;
|
||||
observing = observed.observing;
|
||||
const healthGateId = observed.healthGateId;
|
||||
|
||||
try {
|
||||
ServiceUpdateRecoveryService.getInstance().markConsumed(recoveryId, healthGateId);
|
||||
consumed = true;
|
||||
} catch (consumeError) {
|
||||
console.warn(
|
||||
'[StackUpdateOrchestrator] Failed to mark recovery %s consumed: %s',
|
||||
sanitizeForLog(recoveryId), sanitizeForLog(getErrorMessage(consumeError, 'unknown')),
|
||||
);
|
||||
// Compose already succeeded; treat as consumed for lease/cleanup purposes so
|
||||
// we do not reactivate a snapshot that may no longer match the running image.
|
||||
consumed = true;
|
||||
}
|
||||
|
||||
const recheck = await ImageUpdateService.getInstance().recheckStack(nodeId, stackName);
|
||||
await this.refreshMeshIfEnabled(nodeId, stackName);
|
||||
|
||||
const warnings = [observed.gateWarning, recheck.warning].filter((w): w is string => !!w);
|
||||
return {
|
||||
kind: 'service_done',
|
||||
serviceName,
|
||||
healthGateId,
|
||||
observing,
|
||||
recoveryId,
|
||||
recoveryAvailable: false,
|
||||
previousImageId,
|
||||
newImageId: observed.newImageId ?? null,
|
||||
recheckWarning: warnings.length > 0 ? warnings.join(' ') : undefined,
|
||||
};
|
||||
} finally {
|
||||
ServiceUpdateRecoveryService.getInstance().stopClaimRenewal(recoveryId);
|
||||
if (claimed && !consumed) {
|
||||
const stillLocal = await this.imageIsLocal(nodeId, recovery.majority_image_id);
|
||||
if (stillLocal) {
|
||||
ServiceUpdateRecoveryService.getInstance().reactivate(recoveryId);
|
||||
} else {
|
||||
ServiceUpdateRecoveryService.getInstance().invalidate(recoveryId);
|
||||
}
|
||||
}
|
||||
if (!observing) {
|
||||
AutoHealService.getInstance().clearSuppress(nodeId, stackName, serviceName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Best-effort mesh alias refresh when this stack is opted into Mesh. */
|
||||
private async refreshMeshIfEnabled(nodeId: number, stackName: string): Promise<void> {
|
||||
try {
|
||||
if (!DatabaseService.getInstance().isMeshStackEnabled(nodeId, stackName)) return;
|
||||
await MeshService.getInstance().refreshAliasCache();
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[StackUpdateOrchestrator] Mesh alias refresh failed for %s on node %d: %s',
|
||||
sanitizeForLog(stackName), nodeId, sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async loadServiceSpec(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
serviceName: string,
|
||||
recoveryId?: string | null,
|
||||
): Promise<
|
||||
| { ok: true; spec: EffectiveServiceSpec; services: EffectiveServiceSpec[] }
|
||||
| { ok: false; result: Extract<OrchestratorResult, { kind: 'service_failed' }> }
|
||||
> {
|
||||
const model = await buildEffectiveServiceModel(nodeId, stackName);
|
||||
if (!model.renderable) {
|
||||
return { ok: false, result: serviceFailed(model.code, model.error, { recoveryId }) };
|
||||
}
|
||||
const spec = model.services.find(s => s.name === serviceName);
|
||||
if (!spec) {
|
||||
return {
|
||||
ok: false,
|
||||
result: serviceFailed(
|
||||
'service_not_found',
|
||||
`Service "${serviceName}" is not declared in this stack.`,
|
||||
{ recoveryId },
|
||||
),
|
||||
};
|
||||
}
|
||||
return { ok: true, spec, services: model.services };
|
||||
}
|
||||
|
||||
private async checkPolicyOrFail(
|
||||
stackName: string,
|
||||
nodeId: number,
|
||||
serviceName: string,
|
||||
refs: string[],
|
||||
policyOptions: PolicyEnforcementOptions,
|
||||
action: 'update' | 'rollback',
|
||||
recoveryId?: string | null,
|
||||
): Promise<Extract<OrchestratorResult, { kind: 'service_failed' }> | null> {
|
||||
if (refs.length === 0) return null;
|
||||
const gate = await enforcePolicyForImageRefs(stackName, nodeId, refs, policyOptions);
|
||||
if (gate.ok || gate.bypassed) return null;
|
||||
return serviceFailed(
|
||||
'policy_blocked',
|
||||
`Service "${serviceName}": ${describePolicyBlock(gate.policy, gate.violations, action)}`,
|
||||
{ serviceName, mutationStage: 'policy', recoveryId },
|
||||
);
|
||||
}
|
||||
|
||||
private async observePreparedService(
|
||||
prepareToken: string,
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
serviceName: string,
|
||||
expectedReplicas: number,
|
||||
actor: string | null,
|
||||
divergenceCode: string,
|
||||
recoveryId: string | null,
|
||||
prepareSnapshotOk: boolean,
|
||||
): Promise<
|
||||
| { ok: true; healthGateId: string | null; observing: boolean; newImageId: string | null; gateWarning?: string }
|
||||
| { ok: false; result: Extract<OrchestratorResult, { kind: 'service_failed' }> }
|
||||
> {
|
||||
const convergence = await this.resolvePrimaryImage(nodeId, stackName, serviceName, expectedReplicas);
|
||||
if (convergence.kind === 'inspect_failed') {
|
||||
return {
|
||||
ok: false,
|
||||
result: serviceFailed('replica_inspect_failed', convergence.error, {
|
||||
serviceName, mutationStage: 'inspect', recoveryId,
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (convergence.kind === 'divergent') {
|
||||
return {
|
||||
ok: false,
|
||||
result: serviceFailed(divergenceCode, convergence.error, {
|
||||
serviceName, mutationStage: 'inspect', recoveryId,
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (!prepareSnapshotOk) {
|
||||
return {
|
||||
ok: true,
|
||||
healthGateId: null,
|
||||
observing: false,
|
||||
newImageId: convergence.imageId,
|
||||
gateWarning: 'Health gate skipped: pre-update container snapshot was unavailable.',
|
||||
};
|
||||
}
|
||||
const begin = this.beginPrepared(prepareToken, convergence.imageId, actor);
|
||||
return {
|
||||
ok: true,
|
||||
healthGateId: begin.runId,
|
||||
observing: begin.observing,
|
||||
newImageId: convergence.imageId,
|
||||
};
|
||||
}
|
||||
|
||||
/** Attach the single converged image id (scale>0) and begin the prepared gate. */
|
||||
private beginPrepared(
|
||||
prepareToken: string,
|
||||
imageId: string | null,
|
||||
actor: string | null,
|
||||
): { runId: string | null; observing: boolean } {
|
||||
if (imageId) {
|
||||
HealthGateService.getInstance().attachExpectedImage(prepareToken, imageId);
|
||||
}
|
||||
return HealthGateService.getInstance().beginPrepared({ prepareToken, actor });
|
||||
}
|
||||
|
||||
/**
|
||||
* Post-mutation replica convergence check. Returns the single shared image id
|
||||
* when every expected primary replica agrees, null for a scale-0 service, or a
|
||||
* typed divergence when replicas are missing or run 2+ distinct images.
|
||||
*/
|
||||
private async resolvePrimaryImage(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
serviceName: string,
|
||||
expectedReplicas: number,
|
||||
): Promise<
|
||||
| { kind: 'converged'; imageId: string | null }
|
||||
| { kind: 'divergent'; error: string }
|
||||
| { kind: 'inspect_failed'; error: string }
|
||||
> {
|
||||
if (expectedReplicas === 0) {
|
||||
return { kind: 'converged', imageId: null };
|
||||
}
|
||||
const inspected = await this.inspectServiceContainers(nodeId, stackName, serviceName);
|
||||
return evaluateServiceReplicaConvergence(
|
||||
serviceName, expectedReplicas, inspected.ids, inspected.inspectErrors,
|
||||
);
|
||||
}
|
||||
|
||||
private async inspectPrimaryImageIds(nodeId: number, stackName: string, serviceName: string): Promise<string[]> {
|
||||
return (await this.inspectServiceContainers(nodeId, stackName, serviceName)).ids;
|
||||
}
|
||||
|
||||
/** Snapshot the current per-replica image ids + repo digests for the recovery row. */
|
||||
private async snapshotServiceReplicas(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
serviceName: string,
|
||||
): Promise<ServiceReplicaSnapshot[]> {
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
const snapshots: ServiceReplicaSnapshot[] = [];
|
||||
for (const imageId of (await this.inspectServiceContainers(nodeId, stackName, serviceName)).ids) {
|
||||
let repoDigest: string | null = null;
|
||||
try {
|
||||
const image = await docker.getImage(imageId).inspect();
|
||||
const digests = (image.RepoDigests ?? []) as string[];
|
||||
repoDigest = digests.length > 0 ? digests[0] : null;
|
||||
} catch {
|
||||
// The digest is optional context; a missing image inspect leaves it null.
|
||||
}
|
||||
snapshots.push({ imageId, repoDigest });
|
||||
}
|
||||
return snapshots;
|
||||
}
|
||||
|
||||
private async inspectServiceContainers(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
serviceName: string,
|
||||
): Promise<{ ids: string[]; inspectErrors: number }> {
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
const listed = await docker.listContainers({
|
||||
all: true,
|
||||
filters: { label: [`com.docker.compose.project=${stackName}`, `com.docker.compose.service=${serviceName}`] },
|
||||
});
|
||||
const ids: string[] = [];
|
||||
let inspectErrors = 0;
|
||||
for (const info of listed) {
|
||||
try {
|
||||
const inspect = await docker.getContainer(info.Id).inspect();
|
||||
// Convergence requires running replicas; exited/created leftovers must
|
||||
// not count toward the expected replica total or the shared image id.
|
||||
if (inspect.State?.Status !== 'running') continue;
|
||||
if (inspect.Image) ids.push(inspect.Image);
|
||||
} catch (error) {
|
||||
if ((error as { statusCode?: number })?.statusCode === 404) continue;
|
||||
inspectErrors += 1;
|
||||
console.warn('[Orchestrator] Replica inspect failed for %s/%s:',
|
||||
sanitizeForLog(stackName), sanitizeForLog(serviceName), getErrorMessage(error, 'unknown'));
|
||||
}
|
||||
}
|
||||
return { ids, inspectErrors };
|
||||
}
|
||||
|
||||
/**
|
||||
* Immutable restore scan target: the captured repo digest when it resolves
|
||||
* locally to the majority image id, otherwise the majority image id directly.
|
||||
* Never the moving declared tag (RESTORE-POLICY-TARGET-2).
|
||||
*/
|
||||
private async resolveRestoreScanTarget(nodeId: number, recovery: ServiceUpdateRecoveryRow): Promise<string> {
|
||||
let replicas: ServiceReplicaSnapshot[] = [];
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(recovery.replicas_json);
|
||||
if (Array.isArray(parsed)) replicas = parsed as ServiceReplicaSnapshot[];
|
||||
} catch {
|
||||
// Corrupt snapshot: fall back to the majority image id below.
|
||||
}
|
||||
const withDigest = replicas.find(r => r.imageId === recovery.majority_image_id && r.repoDigest);
|
||||
if (withDigest?.repoDigest) {
|
||||
const resolvedId = await this.resolveImageId(nodeId, withDigest.repoDigest);
|
||||
if (resolvedId === recovery.majority_image_id) return withDigest.repoDigest;
|
||||
}
|
||||
return recovery.majority_image_id;
|
||||
}
|
||||
|
||||
private async resolveImageId(nodeId: number, ref: string): Promise<string | null> {
|
||||
try {
|
||||
const inspect = await DockerController.getInstance(nodeId).getDocker().getImage(ref).inspect();
|
||||
return inspect.Id ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async imageIsLocal(nodeId: number, imageId: string): Promise<boolean> {
|
||||
try {
|
||||
await DockerController.getInstance(nodeId).getDocker().getImage(imageId).inspect();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,9 @@ import DockerController from './DockerController';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { ComposeDoctorService } from './ComposeDoctorService';
|
||||
import { UpdatePreviewService, isMovingTag } from './UpdatePreviewService';
|
||||
import { UpdatePreviewService, isMovingTag, filterPreviewForService } from './UpdatePreviewService';
|
||||
import { buildEffectiveServiceModel, type EffectiveServiceModelResult } from './effectiveServiceModel';
|
||||
import { filterContainersByComposeService } from '../helpers/composeServiceMatch';
|
||||
import { withTimeout } from '../utils/withTimeout';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
@@ -17,9 +19,12 @@ import {
|
||||
diskSignal,
|
||||
driftSignal,
|
||||
healthchecksSignal,
|
||||
isTroubledContainer,
|
||||
preflightSignal,
|
||||
serviceSignal,
|
||||
updatePreviewSignal,
|
||||
type Errored,
|
||||
type ServiceMembership,
|
||||
} from './updateGuard/readiness';
|
||||
import type { ContainerProbe, RollbackReadinessReport, UpdateReadinessReport } from './updateGuard/types';
|
||||
|
||||
@@ -30,6 +35,19 @@ import type { ContainerProbe, RollbackReadinessReport, UpdateReadinessReport } f
|
||||
// of failing the report.
|
||||
const INPUT_TIMEOUT_MS = 3_000;
|
||||
|
||||
/**
|
||||
* Thrown by `computeUpdateReadiness` when a `serviceName` is given for a
|
||||
* stack that has one (or zero) declared services: service-scoped readiness
|
||||
* requires a real choice among siblings. The route maps this to a 400,
|
||||
* distinct from the 500 a genuine computation failure gets.
|
||||
*/
|
||||
export class SingleServiceUpdateReadinessError extends Error {
|
||||
constructor() {
|
||||
super('Service-scoped readiness requires a stack with more than one service.');
|
||||
this.name = 'SingleServiceUpdateReadinessError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes update readiness and rollback readiness for a stack, on demand,
|
||||
* from existing per-feature stores (preflight runs, drift findings, the atomic
|
||||
@@ -49,16 +67,18 @@ export class UpdateGuardService {
|
||||
/**
|
||||
* Probe the stack's containers via the compose project label, normalized for
|
||||
* the pure scoring functions. Throws on Docker errors; callers map that to
|
||||
* the 'error' sentinel.
|
||||
* the 'error' sentinel. When `serviceName` is given, narrows to that
|
||||
* service's containers before inspecting (labeled containers, per §6).
|
||||
*/
|
||||
async probeContainers(nodeId: number, stackName: string): Promise<ContainerProbe[]> {
|
||||
async probeContainers(nodeId: number, stackName: string, serviceName?: string): Promise<ContainerProbe[]> {
|
||||
const docker = DockerController.getInstance(nodeId).getDocker();
|
||||
const listed = await docker.listContainers({
|
||||
all: true,
|
||||
filters: { label: [`com.docker.compose.project=${stackName}`] },
|
||||
});
|
||||
const scoped = serviceName ? filterContainersByComposeService(listed, serviceName) : listed;
|
||||
const probes = await Promise.all(
|
||||
listed.map(async (info): Promise<ContainerProbe | null> => {
|
||||
scoped.map(async (info): Promise<ContainerProbe | null> => {
|
||||
const name = info.Names?.[0]?.replace(/^\//, '') ?? info.Id.slice(0, 12);
|
||||
let inspect: Awaited<ReturnType<ReturnType<typeof docker.getContainer>['inspect']>>;
|
||||
try {
|
||||
@@ -86,17 +106,40 @@ export class UpdateGuardService {
|
||||
return probes.filter((p): p is ContainerProbe => p !== null);
|
||||
}
|
||||
|
||||
async computeUpdateReadiness(nodeId: number, stackName: string): Promise<UpdateReadinessReport> {
|
||||
/**
|
||||
* When `serviceName` is set, scopes the verdict-affecting inputs (labeled
|
||||
* containers, healthchecks, the update preview, and drift) to that service
|
||||
* only; the stack guardrails (preflight, disk, backup, Docker reachability)
|
||||
* stay unchanged (§6). A render failure or missing service fails closed via
|
||||
* the added `service` signal; a single-service stack throws instead of
|
||||
* scoping (the caller/route maps that to a 400).
|
||||
*/
|
||||
async computeUpdateReadiness(nodeId: number, stackName: string, serviceName?: string): Promise<UpdateReadinessReport> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const now = Date.now();
|
||||
|
||||
const [preflight, drift, containers, preview, backup, disk] = await Promise.all([
|
||||
let model: EffectiveServiceModelResult | null = null;
|
||||
if (serviceName) {
|
||||
model = await buildEffectiveServiceModel(nodeId, stackName);
|
||||
if (model.renderable && model.services.length <= 1) {
|
||||
throw new SingleServiceUpdateReadinessError();
|
||||
}
|
||||
}
|
||||
|
||||
const [preflight, drift, containers, siblings, preview, backup, disk] = await Promise.all([
|
||||
this.collect('preflight', stackName, async () => ComposeDoctorService.getInstance().getLatest(nodeId, stackName)),
|
||||
this.collect('drift', stackName, async () => db.getOpenDriftFindings(nodeId, stackName).length),
|
||||
this.collect('drift', stackName, async () =>
|
||||
db.getOpenDriftFindings(nodeId, stackName).filter(f => !serviceName || f.service === serviceName).length),
|
||||
this.collect('containers', stackName, () =>
|
||||
withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness container probe')),
|
||||
this.collect('update preview', stackName, () =>
|
||||
withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness update preview')),
|
||||
withTimeout(this.probeContainers(nodeId, stackName, serviceName), INPUT_TIMEOUT_MS, 'readiness container probe')),
|
||||
serviceName
|
||||
? this.collect('sibling containers', stackName, () =>
|
||||
withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness sibling probe'))
|
||||
: Promise.resolve<ContainerProbe[] | Errored>([]),
|
||||
this.collect('update preview', stackName, async () => {
|
||||
const full = await withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness update preview');
|
||||
return serviceName ? filterPreviewForService(full, serviceName) : full;
|
||||
}),
|
||||
this.collect('backup info', stackName, () => FileSystemService.getInstance(nodeId).getBackupInfo(stackName)),
|
||||
this.collect('disk', stackName, () => this.readDiskUsage()),
|
||||
]);
|
||||
@@ -114,8 +157,56 @@ export class UpdateGuardService {
|
||||
backupSlotSignal(backup, now),
|
||||
diskSignal(typeof disk === 'number' ? { usePercent: disk, limitPercent } : 'error'),
|
||||
];
|
||||
if (serviceName) {
|
||||
signals.push(serviceSignal(this.resolveServiceMembership(model, serviceName)));
|
||||
}
|
||||
|
||||
return { stack: stackName, computedAt: now, verdict: aggregateVerdict(signals), signals };
|
||||
const advisories = serviceName
|
||||
? [
|
||||
this.siblingHealthAdvisory(containers, siblings),
|
||||
this.dependencyAdvisory(model, serviceName),
|
||||
].filter((note): note is string => note !== null)
|
||||
: [];
|
||||
|
||||
return {
|
||||
stack: stackName,
|
||||
computedAt: now,
|
||||
verdict: aggregateVerdict(signals),
|
||||
signals,
|
||||
serviceName: serviceName ?? null,
|
||||
advisories,
|
||||
};
|
||||
}
|
||||
|
||||
/** Model-membership facts for the selected service; 'error' when the model failed to render. */
|
||||
private resolveServiceMembership(model: EffectiveServiceModelResult | null, serviceName: string): ServiceMembership | Errored {
|
||||
if (!model || !model.renderable) return 'error';
|
||||
const spec = model.services.find(s => s.name === serviceName);
|
||||
if (!spec) return { found: false };
|
||||
return { found: true, hasBuild: spec.hasBuild, declaredImage: spec.declaredImage, expectedReplicas: spec.expectedReplicas };
|
||||
}
|
||||
|
||||
/** Advisory only (§6): a sibling already unhealthy before the update never blocks the selected service's verdict. */
|
||||
private siblingHealthAdvisory(selected: ContainerProbe[] | Errored, all: ContainerProbe[] | Errored): string | null {
|
||||
if (selected === 'error' || all === 'error') return null;
|
||||
const selectedNames = new Set(selected.map(c => c.name));
|
||||
const troubled = all.filter(c => !selectedNames.has(c.name) && isTroubledContainer(c));
|
||||
if (troubled.length === 0) return null;
|
||||
const names = troubled.map(c => c.name).join(', ');
|
||||
return `Other containers in this stack are already unhealthy: ${names}. This does not block the selected service.`;
|
||||
}
|
||||
|
||||
/** Advisory only (§6): dependsOn relationships are informational; this update never recreates siblings. */
|
||||
private dependencyAdvisory(model: EffectiveServiceModelResult | null, serviceName: string): string | null {
|
||||
if (!model || !model.renderable) return null;
|
||||
const spec = model.services.find(s => s.name === serviceName);
|
||||
const dependsOn = spec?.dependsOn ?? [];
|
||||
const dependents = model.services.filter(s => s.dependsOn.includes(serviceName)).map(s => s.name);
|
||||
const parts: string[] = [];
|
||||
if (dependsOn.length > 0) parts.push(`depends on ${dependsOn.join(', ')}`);
|
||||
if (dependents.length > 0) parts.push(`is a dependency of ${dependents.join(', ')}`);
|
||||
if (parts.length === 0) return null;
|
||||
return `This service ${parts.join(' and ')}. Those services are not restarted by this update.`;
|
||||
}
|
||||
|
||||
async computeRollbackReadiness(nodeId: number, stackName: string): Promise<RollbackReadinessReport> {
|
||||
|
||||
@@ -287,6 +287,13 @@ export function buildSummary(
|
||||
};
|
||||
}
|
||||
|
||||
/** Filter a full-stack preview down to one service's images and recompute the summary from that subset. */
|
||||
export function filterPreviewForService(preview: UpdatePreview, serviceName: string): UpdatePreview {
|
||||
const images = preview.images.filter(i => i.service === serviceName);
|
||||
const buildServices = preview.build_services.filter(s => s === serviceName);
|
||||
return buildSummary(preview.stack_name, images, buildServices);
|
||||
}
|
||||
|
||||
export class UpdatePreviewService {
|
||||
private static instance: UpdatePreviewService;
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import crypto from 'crypto';
|
||||
import { ComposeService } from './ComposeService';
|
||||
import { StackUpdateOrchestrator } from './StackUpdateOrchestrator';
|
||||
import { StackOpLockService, stackOpSkipMessage, type StackOpAction } from './StackOpLockService';
|
||||
import { DatabaseService, type Webhook } from './DatabaseService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
@@ -162,7 +163,7 @@ export class WebhookService {
|
||||
atomic,
|
||||
{ source: 'webhook', actor: 'system:webhook' },
|
||||
);
|
||||
HealthGateService.getInstance().begin(nodeId, stackName, 'deploy', 'system:webhook');
|
||||
HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:webhook');
|
||||
break;
|
||||
case 'restart':
|
||||
await compose.runCommand(stackName, 'restart');
|
||||
@@ -179,8 +180,11 @@ export class WebhookService {
|
||||
nodeId,
|
||||
buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }),
|
||||
);
|
||||
await compose.updateStack(stackName, undefined, atomic);
|
||||
HealthGateService.getInstance().begin(nodeId, stackName, 'update', 'system:webhook');
|
||||
await StackUpdateOrchestrator.getInstance().execute(
|
||||
{ nodeId, stackName, target: { scope: 'stack' }, trigger: 'webhook', actor: 'system:webhook' },
|
||||
{ atomic: atomic ?? false, terminalWs: null },
|
||||
);
|
||||
HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook');
|
||||
break;
|
||||
}
|
||||
},
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
/**
|
||||
* Effective Service Model: per-service facts that service-scoped update and
|
||||
* restore key off of (declared image, build presence, expected replica
|
||||
* count, dependencies, healthcheck presence), derived from the
|
||||
* FULLY-MERGED effective model (`docker compose config --format json`)
|
||||
* instead of a single compose file, so a multi-file Git source's overrides
|
||||
* are always reflected.
|
||||
*
|
||||
* Fail closed: unlike the read-only facts readers in this module family
|
||||
* (effectiveAnatomy, storage inventory, etc.), this model backs mutation and
|
||||
* readiness decisions, so a render failure must never fall back to a
|
||||
* root-file parse. A caller that mutates or gates on `renderable: false`
|
||||
* would risk pulling the wrong image or under/over-recreating replicas
|
||||
* against a stale guess.
|
||||
*
|
||||
* Reads only the structural fields a mutation needs; it never reads
|
||||
* `environment`, `command`, `entrypoint`, `labels`, `secrets`, or `configs`.
|
||||
*/
|
||||
import { ComposeService } from './ComposeService';
|
||||
import { parseMissingRequiredVars } from '../helpers/envVarParse';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog';
|
||||
|
||||
const MAX_RENDER_ERROR = 600;
|
||||
|
||||
export interface EffectiveServiceSpec {
|
||||
name: string;
|
||||
declaredImage: string | null;
|
||||
hasBuild: boolean;
|
||||
/** May be 0 (explicit `scale: 0` or `deploy.replicas: 0`); defaults to 1 when neither is set, matching Compose's own default. */
|
||||
expectedReplicas: number;
|
||||
dependsOn: string[];
|
||||
hasHealthcheck: boolean;
|
||||
}
|
||||
|
||||
export type EffectiveServiceModelResult =
|
||||
| { renderable: true; services: EffectiveServiceSpec[] }
|
||||
| { renderable: false; code: 'effective_model_render_failed'; error: string };
|
||||
|
||||
function asString(v: unknown): string | undefined {
|
||||
if (typeof v === 'string') return v;
|
||||
if (typeof v === 'number') return String(v);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function asNumber(v: unknown): number | undefined {
|
||||
if (typeof v === 'number' && Number.isFinite(v)) return v;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* `depends_on` always renders as a map keyed by target service name (Compose
|
||||
* normalizes the short list form into the same map shape), so a plain key
|
||||
* extraction covers both authoring styles; the list branch is a defensive
|
||||
* fallback for a hand-built fixture.
|
||||
*/
|
||||
function dependsOnNames(value: unknown): string[] {
|
||||
if (Array.isArray(value)) {
|
||||
return value.map(asString).filter((s): s is string => s !== undefined);
|
||||
}
|
||||
if (value && typeof value === 'object') return Object.keys(value as Record<string, unknown>);
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Expected replica count mirrors Compose's own `ServiceConfig.GetScale()`:
|
||||
* the top-level `scale` field wins when set (Compose keeps it in sync with
|
||||
* `deploy.replicas`), else `deploy.replicas`, else the Compose default of 1.
|
||||
*/
|
||||
function expectedReplicasOf(svc: Record<string, unknown>, deploy: Record<string, unknown> | undefined): number {
|
||||
return asNumber(svc.scale) ?? asNumber(deploy?.replicas) ?? 1;
|
||||
}
|
||||
|
||||
function parseServiceSpec(name: string, raw: unknown): EffectiveServiceSpec {
|
||||
const svc = (raw ?? {}) as Record<string, unknown>;
|
||||
const deploy = (svc.deploy && typeof svc.deploy === 'object') ? svc.deploy as Record<string, unknown> : undefined;
|
||||
const healthcheck = svc.healthcheck;
|
||||
const hasHealthcheck = !!healthcheck
|
||||
&& typeof healthcheck === 'object'
|
||||
&& (healthcheck as Record<string, unknown>).disable !== true;
|
||||
return {
|
||||
name,
|
||||
declaredImage: asString(svc.image) ?? null,
|
||||
hasBuild: !!svc.build && typeof svc.build === 'object',
|
||||
expectedReplicas: expectedReplicasOf(svc, deploy),
|
||||
dependsOn: dependsOnNames(svc.depends_on),
|
||||
hasHealthcheck,
|
||||
};
|
||||
}
|
||||
|
||||
/** Parse the raw `docker compose config --format json` output into the per-service specs. Tolerant of missing/garbage fields: an empty or malformed `services` map yields an empty list rather than throwing. */
|
||||
function parseEffectiveServices(parsed: unknown): EffectiveServiceSpec[] {
|
||||
const root = (parsed && typeof parsed === 'object') ? parsed as Record<string, unknown> : {};
|
||||
const rawServices = (root.services && typeof root.services === 'object' && !Array.isArray(root.services))
|
||||
? root.services as Record<string, unknown>
|
||||
: {};
|
||||
return Object.entries(rawServices).map(([name, svc]) => parseServiceSpec(name, svc));
|
||||
}
|
||||
|
||||
/**
|
||||
* Render the fully-merged effective Compose model for a stack and extract the
|
||||
* per-service specs service-scoped update/restore key off of. Mirrors the
|
||||
* render-error handling of the other effective-facts readers (redacted,
|
||||
* never raw stderr or an exception), but never falls back to a root-file
|
||||
* parse: a render failure leaves `renderable: false` with no services, so a
|
||||
* caller that skips the `renderable` check cannot silently mutate against a
|
||||
* stale or partial model.
|
||||
*/
|
||||
export async function buildEffectiveServiceModel(nodeId: number, stackName: string): Promise<EffectiveServiceModelResult> {
|
||||
try {
|
||||
const result = await ComposeService.getInstance(nodeId).renderConfig(stackName);
|
||||
if (result.rendered !== null) {
|
||||
try {
|
||||
return { renderable: true, services: parseEffectiveServices(JSON.parse(result.rendered)) };
|
||||
} catch (parseErr) {
|
||||
// JSON.parse errors carry no file content, so the message is safe to log.
|
||||
console.warn('[EffectiveServiceModel] Effective model parse failed for %s:',
|
||||
sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(parseErr, 'unknown')));
|
||||
return {
|
||||
renderable: false,
|
||||
code: 'effective_model_render_failed',
|
||||
error: 'Sencho could not parse the rendered Compose model.',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Raw stderr can echo file content/secrets and is never surfaced; only the
|
||||
// names of any missing required variables, otherwise a generic nudge.
|
||||
const missing = parseMissingRequiredVars(result.stderr);
|
||||
const error = missing.length
|
||||
? `Required variable${missing.length > 1 ? 's' : ''} ${missing.join(', ')} ${missing.length > 1 ? 'have' : 'has'} no value, so the effective model cannot be rendered.`
|
||||
: 'Sencho could not render the effective Compose model. Check the compose and env files for a YAML syntax error, an unresolved include or merge, or a required variable with no value.';
|
||||
return { renderable: false, code: 'effective_model_render_failed', error };
|
||||
} catch (err) {
|
||||
// Spawn failure (docker unavailable), or an unexpected throw before/inside the
|
||||
// render. Leave a sanitized breadcrumb so a non-spawn bug is not invisible, then
|
||||
// redact the surfaced message defensively.
|
||||
console.warn('[EffectiveServiceModel] Render failed for %s:',
|
||||
sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown')));
|
||||
const error = redactSensitiveText(getErrorMessage(err, 'docker compose could not be started.')).slice(0, MAX_RENDER_ERROR).trim()
|
||||
|| 'Sencho could not run docker compose on this node.';
|
||||
return { renderable: false, code: 'effective_model_render_failed', error };
|
||||
}
|
||||
}
|
||||
@@ -67,6 +67,11 @@ export function driftSignal(input: number | Errored): ReadinessSignal {
|
||||
return { ...base, status: 'ok', affectsVerdict: true, detail: 'No open drift findings.' };
|
||||
}
|
||||
|
||||
/** A container already unhealthy, restarting, or crash-exited before any update runs. */
|
||||
export function isTroubledContainer(c: ContainerProbe): boolean {
|
||||
return c.health === 'unhealthy' || c.state === 'restarting' || (c.state === 'exited' && (c.exitCode ?? 0) !== 0);
|
||||
}
|
||||
|
||||
export function containersSignal(input: ContainerProbe[] | Errored): ReadinessSignal {
|
||||
const base = { id: 'containers' as const, title: 'Current containers' };
|
||||
if (input === 'error') {
|
||||
@@ -75,9 +80,7 @@ export function containersSignal(input: ContainerProbe[] | Errored): ReadinessSi
|
||||
if (input.length === 0) {
|
||||
return { ...base, status: 'warning', affectsVerdict: true, detail: 'The stack is not running; updating will start it.' };
|
||||
}
|
||||
const troubled = input.filter(
|
||||
c => c.health === 'unhealthy' || c.state === 'restarting' || (c.state === 'exited' && (c.exitCode ?? 0) !== 0),
|
||||
);
|
||||
const troubled = input.filter(isTroubledContainer);
|
||||
if (troubled.length > 0) {
|
||||
const names = troubled.map(c => c.name).join(', ');
|
||||
return {
|
||||
@@ -162,6 +165,50 @@ export function buildServicesSignal(buildServices: string[] | Errored): Readines
|
||||
};
|
||||
}
|
||||
|
||||
/** Model-membership facts for the selected service, gathered from EffectiveServiceModel. */
|
||||
export type ServiceMembership =
|
||||
| { found: false }
|
||||
| { found: true; hasBuild: boolean; declaredImage: string | null; expectedReplicas: number };
|
||||
|
||||
/** Matches `ServiceUpdateRecoveryService`'s digest-pin check (a recovery snapshot is never eligible for a digest-pinned declared ref). */
|
||||
const DIGEST_PIN_RE = /@sha256:[a-f0-9]{64}$/i;
|
||||
|
||||
/**
|
||||
* Selected-service signal for a service-scoped readiness check: model
|
||||
* membership and whether the update would leave a rollback recovery snapshot
|
||||
* behind. Render failure or a missing service fails closed (blocked), never
|
||||
* unknown, so a service-scoped check cannot silently proceed against a stale
|
||||
* or absent model.
|
||||
*/
|
||||
export function serviceSignal(input: ServiceMembership | Errored): ReadinessSignal {
|
||||
const base = { id: 'service' as const, title: 'Selected service' };
|
||||
if (input === 'error') {
|
||||
return { ...base, status: 'blocked', affectsVerdict: true, detail: 'The compose model could not be rendered, so the selected service cannot be validated. Resolve the compose error before updating.' };
|
||||
}
|
||||
if (!input.found) {
|
||||
return { ...base, status: 'blocked', affectsVerdict: true, detail: 'The selected service is not declared in this stack\u2019s compose model.' };
|
||||
}
|
||||
if (!input.hasBuild && !input.declaredImage) {
|
||||
return { ...base, status: 'blocked', affectsVerdict: true, detail: 'The selected service has neither an image nor a build, so it cannot be updated.' };
|
||||
}
|
||||
const digestPinned = input.declaredImage !== null && DIGEST_PIN_RE.test(input.declaredImage);
|
||||
if (!input.declaredImage || digestPinned) {
|
||||
const reason = !input.declaredImage ? 'it builds from source with no declared image' : 'its declared image is pinned to a digest';
|
||||
return {
|
||||
...base,
|
||||
status: 'warning',
|
||||
affectsVerdict: true,
|
||||
detail: `Declared with ${input.expectedReplicas} expected replica${input.expectedReplicas === 1 ? '' : 's'}. No rollback recovery snapshot will be available for this update because ${reason}.`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
status: 'ok',
|
||||
affectsVerdict: true,
|
||||
detail: `Declared with ${input.expectedReplicas} expected replica${input.expectedReplicas === 1 ? '' : 's'}; a rollback recovery snapshot will be captured before the update.`,
|
||||
};
|
||||
}
|
||||
|
||||
export function backupSlotSignal(
|
||||
input: { exists: boolean; timestamp: number | null } | Errored,
|
||||
now: number,
|
||||
|
||||
@@ -6,7 +6,7 @@ export type SignalStatus = 'ok' | 'warning' | 'attention' | 'blocked' | 'unknown
|
||||
|
||||
/** One input to the readiness verdict (preflight, drift, containers, ...). */
|
||||
export interface ReadinessSignal {
|
||||
id: 'preflight' | 'drift' | 'containers' | 'healthchecks' | 'update_preview' | 'build_services' | 'backup_slot' | 'disk';
|
||||
id: 'preflight' | 'drift' | 'containers' | 'healthchecks' | 'update_preview' | 'build_services' | 'backup_slot' | 'disk' | 'service';
|
||||
status: SignalStatus;
|
||||
/** Short headline ("Compose Doctor", "Running containers"). */
|
||||
title: string;
|
||||
@@ -25,6 +25,13 @@ export interface UpdateReadinessReport {
|
||||
computedAt: number;
|
||||
verdict: ReadinessVerdict;
|
||||
signals: ReadinessSignal[];
|
||||
/** The service this report is scoped to, or null for a full-stack check. */
|
||||
serviceName: string | null;
|
||||
/**
|
||||
* Non-blocking notes that never affect `verdict` (sibling health,
|
||||
* dependsOn relationships). Always empty for a full-stack check.
|
||||
*/
|
||||
advisories: string[];
|
||||
}
|
||||
|
||||
/** State of one rollback readiness item. */
|
||||
@@ -72,6 +79,8 @@ export interface HealthGateContainer {
|
||||
state: string;
|
||||
health: string | null;
|
||||
restarts: number;
|
||||
service?: string | null;
|
||||
role?: 'primary' | 'collateral' | null;
|
||||
}
|
||||
|
||||
/** Payload of GET /:stackName/health-gate ('never-run' when no run exists). */
|
||||
@@ -79,12 +88,15 @@ export interface HealthGateReport {
|
||||
stack: string;
|
||||
id: string | null;
|
||||
status: HealthGateStatus | 'never-run';
|
||||
trigger: 'update' | 'deploy' | null;
|
||||
trigger: 'update' | 'deploy' | 'service_update' | 'service_restore' | null;
|
||||
reason: string | null;
|
||||
windowSeconds: number | null;
|
||||
startedAt: number | null;
|
||||
endedAt: number | null;
|
||||
containers: HealthGateContainer[];
|
||||
targetScope: 'stack' | 'service';
|
||||
serviceName: string | null;
|
||||
failureSource: 'primary' | 'collateral' | null;
|
||||
}
|
||||
|
||||
/** Categories a failed stack deploy or update can be classified into. */
|
||||
|
||||
Reference in New Issue
Block a user