feat: add service-scoped Compose update and restore (#1648)

* feat: add service-scoped Compose update and restore

Allow updating or rebuilding one declared Compose service on multi-service
stacks without recreating siblings, with recovery snapshots, health-gate
observation, and prune holds for rollback images. Full-stack update paths
and single-service UX stay unchanged.

* fix: sanitize service-scoped update log messages for CodeQL

* fix: address service-scoped update audit findings B-01 through B-07

* fix: complete service-scoped update audit metadata and surfaces

* test: wrap Updates readiness tests for deploy-feedback context

* fix: keep service recovery reachable without Deploy Progress

Make failed service-gate recovery discoverable when Deploy Progress is
disabled or dismissed, suppress stale image-scan notification side
effects, normalize ComposeService line endings, and add focused
regression coverage.

* fix: resurface ContainersHealth density and expand on multi-service stacks

Service grouping hid the summary strip and Compact/Detailed/Expand controls that still applied to multi-container stacks.
This commit is contained in:
Anso
2026-07-19 02:42:29 -04:00
committed by GitHub
parent 31d4e4669b
commit 63213c0960
89 changed files with 7608 additions and 331 deletions
@@ -0,0 +1,37 @@
/**
* Ref-counted Auto-Heal suppression for overlapping service-scoped updates.
*/
import { describe, it, expect, beforeEach } from 'vitest';
import { AutoHealService } from '../services/AutoHealService';
describe('AutoHealService suppress refcount', () => {
beforeEach(() => {
AutoHealService.getInstance().stop();
});
it('keeps suppression until every overlapping owner clears', () => {
const svc = AutoHealService.getInstance();
svc.suppress(0, 'web', 'api');
svc.suppress(0, 'web', 'api');
expect(svc.isSuppressed(0, 'web', 'api')).toBe(true);
svc.clearSuppress(0, 'web', 'api');
expect(svc.isSuppressed(0, 'web', 'api')).toBe(true);
svc.clearSuppress(0, 'web', 'api');
expect(svc.isSuppressed(0, 'web', 'api')).toBe(false);
});
it('scopes suppression per service on the same stack', () => {
const svc = AutoHealService.getInstance();
svc.suppress(0, 'web', 'api');
expect(svc.isSuppressed(0, 'web', 'api')).toBe(true);
expect(svc.isSuppressed(0, 'web', 'db')).toBe(false);
svc.clearSuppress(0, 'web', 'api');
expect(svc.isSuppressed(0, 'web', 'api')).toBe(false);
});
it('ignores clearSuppress when nothing is suppressed', () => {
const svc = AutoHealService.getInstance();
svc.clearSuppress(0, 'web', 'api');
expect(svc.isSuppressed(0, 'web', 'api')).toBe(false);
});
});
@@ -660,6 +660,38 @@ describe('ComposeService - deployStack', () => {
expect(mockGetLegacyOrphanContainersByStack).toHaveBeenCalledWith('my-stack');
});
it('blocks a pilot service restore with relative binds before Compose up', async () => {
process.env.SENCHO_MODE = 'pilot';
const composeDir = path.resolve('/test/compose');
mockGetBindMounts.mockResolvedValue([
{ source: '/opt/docker/sencho', destination: composeDir },
]);
const configProc = createMockProcess();
mockSpawn.mockReturnValue(configProc);
const svc = ComposeService.getInstance(1);
const result = svc.recreateServiceFromLocal('my-stack', 'db').then(() => null, (error: Error) => error);
await waitForSpawn();
configProc.stdout.emit('data', Buffer.from(JSON.stringify({
name: 'my-stack',
services: {
db: {
image: 'postgres:17',
volumes: [{
type: 'bind',
source: path.join(composeDir, 'my-stack', 'data'),
target: '/var/lib/postgresql/data',
}],
},
},
})));
configProc.emit('close', 0);
const error = await result;
expect(error?.message).toContain('1:1');
expect(mockSpawn).toHaveBeenCalledTimes(1);
});
it('does not remove compose-managed containers before deploy (#1565)', async () => {
setupAutoCloseSpawn();
mockListContainers.mockResolvedValue([]);
@@ -0,0 +1,251 @@
/**
* Coverage for the service_update_recovery accessors on the real
* DatabaseService (against a temp DB, so the CAS WHERE clauses run against
* actual SQLite semantics rather than a mock):
* - insert / get / list active
* - atomic claim CAS (active -> restoring) and its race/expiry guards
* - renewal CAS never revives a terminal or already-active row
* - mark consumed / reactivate / invalidate transitions
* - sweep of abandoned restoring claims vs a still-live claim
* - held image id projection and stack-delete cleanup
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import type { ServiceUpdateRecoveryRow } from '../services/DatabaseService';
let tmpDir: string;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
});
afterAll(() => cleanupTestDb(tmpDir));
function db() {
return DatabaseService.getInstance();
}
beforeEach(() => {
const raw = (db() as unknown as { db: { prepare: (s: string) => { run: () => void } } }).db;
raw.prepare('DELETE FROM service_update_recovery').run();
});
const NODE = 1;
function makeRow(overrides: Partial<ServiceUpdateRecoveryRow> = {}): ServiceUpdateRecoveryRow {
return {
id: overrides.id ?? 'rec-1',
node_id: NODE,
stack_name: 'web',
service_name: 'api',
replicas_json: JSON.stringify([{ imageId: 'sha256:aaa', repoDigest: 'sha256:digest-aaa' }]),
majority_image_id: 'sha256:aaa',
declared_image_ref: 'ghcr.io/acme/api:v1',
weak_floating_tag: 0,
health_gate_id: null,
status: 'active',
expires_at: 10_000,
claim_expires_at: null,
created_at: 1_000,
created_by: 'tester',
...overrides,
};
}
describe('service_update_recovery accessors', () => {
it('inserts and reads a row back by id', () => {
db().insertServiceUpdateRecovery(makeRow());
const row = db().getServiceUpdateRecovery('rec-1');
expect(row).toMatchObject({ id: 'rec-1', stack_name: 'web', service_name: 'api', status: 'active' });
});
it('lists only active rows for a service, most recent first', () => {
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-old', created_at: 1_000 }));
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-new', created_at: 2_000 }));
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-consumed', status: 'consumed' }));
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-other-service', service_name: 'db' }));
const active = db().listActiveServiceUpdateRecoveries(NODE, 'web', 'api');
expect(active.map(r => r.id)).toEqual(['rec-new', 'rec-old']);
});
it('links a health gate id only while the row is still active', () => {
db().insertServiceUpdateRecovery(makeRow());
db().linkServiceUpdateRecoveryHealthGate('rec-1', 'gate-1');
expect(db().getServiceUpdateRecovery('rec-1')?.health_gate_id).toBe('gate-1');
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-consumed', status: 'consumed' }));
db().linkServiceUpdateRecoveryHealthGate('rec-consumed', 'gate-2');
expect(db().getServiceUpdateRecovery('rec-consumed')?.health_gate_id).toBeNull();
});
describe('claim CAS', () => {
it('claims an active, unexpired row and moves it to restoring', () => {
db().insertServiceUpdateRecovery(makeRow({ expires_at: 50_000 }));
const claimed = db().claimServiceUpdateRecovery('rec-1', 100_000, 20_000);
expect(claimed).toMatchObject({ id: 'rec-1', status: 'restoring', claim_expires_at: 100_000 });
});
it('refuses to claim an already-expired active row', () => {
db().insertServiceUpdateRecovery(makeRow({ expires_at: 10_000 }));
const claimed = db().claimServiceUpdateRecovery('rec-1', 100_000, 20_000);
expect(claimed).toBeUndefined();
expect(db().getServiceUpdateRecovery('rec-1')?.status).toBe('active');
});
it('refuses a second claim on an already-restoring row (no double claim)', () => {
db().insertServiceUpdateRecovery(makeRow({ expires_at: 50_000 }));
const first = db().claimServiceUpdateRecovery('rec-1', 100_000, 20_000);
expect(first).toBeTruthy();
const second = db().claimServiceUpdateRecovery('rec-1', 200_000, 21_000);
expect(second).toBeUndefined();
});
it('refuses to claim a consumed/expired/invalidated row', () => {
for (const status of ['consumed', 'expired', 'invalidated'] as const) {
db().insertServiceUpdateRecovery(makeRow({ id: `rec-${status}`, status, expires_at: 50_000 }));
const claimed = db().claimServiceUpdateRecovery(`rec-${status}`, 100_000, 20_000);
expect(claimed).toBeUndefined();
}
});
});
describe('renewal CAS (no revive of terminal)', () => {
it('extends claim_expires_at while restoring', () => {
db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', claim_expires_at: 100_000 }));
const renewed = db().renewServiceUpdateRecoveryClaim('rec-1', 200_000);
expect(renewed).toBe(true);
expect(db().getServiceUpdateRecovery('rec-1')?.claim_expires_at).toBe(200_000);
});
it.each(['active', 'consumed', 'expired', 'invalidated'] as const)(
'never revives a %s row via renewal',
(status) => {
db().insertServiceUpdateRecovery(makeRow({ status, claim_expires_at: status === 'active' ? null : 100_000 }));
const renewed = db().renewServiceUpdateRecoveryClaim('rec-1', 999_000);
expect(renewed).toBe(false);
expect(db().getServiceUpdateRecovery('rec-1')?.status).toBe(status);
},
);
});
describe('terminal transitions', () => {
it('marks a restoring row consumed and links the restore health gate id', () => {
db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', claim_expires_at: 100_000, health_gate_id: 'update-gate' }));
const ok = db().markServiceUpdateRecoveryConsumed('rec-1', 'restore-gate');
expect(ok).toBe(true);
const row = db().getServiceUpdateRecovery('rec-1');
expect(row).toMatchObject({ status: 'consumed', claim_expires_at: null, health_gate_id: 'restore-gate' });
});
it('keeps the prior health gate id when marking consumed with no runId', () => {
db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', health_gate_id: 'update-gate' }));
db().markServiceUpdateRecoveryConsumed('rec-1', null);
expect(db().getServiceUpdateRecovery('rec-1')?.health_gate_id).toBe('update-gate');
});
it('cannot mark an already-active row consumed', () => {
db().insertServiceUpdateRecovery(makeRow({ status: 'active' }));
expect(db().markServiceUpdateRecoveryConsumed('rec-1')).toBe(false);
});
it('reactivates a restoring row on a mid-flight failure with the image still local', () => {
db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', claim_expires_at: 100_000 }));
const ok = db().reactivateServiceUpdateRecovery('rec-1');
expect(ok).toBe(true);
expect(db().getServiceUpdateRecovery('rec-1')).toMatchObject({ status: 'active', claim_expires_at: null });
});
it('invalidates a restoring row on a mid-flight failure with the image gone', () => {
db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', claim_expires_at: 100_000 }));
const ok = db().invalidateServiceUpdateRecovery('rec-1');
expect(ok).toBe(true);
expect(db().getServiceUpdateRecovery('rec-1')).toMatchObject({ status: 'invalidated', claim_expires_at: null });
});
it('cannot reactivate or invalidate a row that is not restoring', () => {
db().insertServiceUpdateRecovery(makeRow({ status: 'active' }));
expect(db().reactivateServiceUpdateRecovery('rec-1')).toBe(false);
expect(db().invalidateServiceUpdateRecovery('rec-1')).toBe(false);
});
});
describe('sweep', () => {
it('expires an active row whose TTL has lapsed', () => {
db().insertServiceUpdateRecovery(makeRow({ expires_at: 10_000 }));
const count = db().sweepExpiredActiveServiceUpdateRecoveries(20_000);
expect(count).toBe(1);
expect(db().getServiceUpdateRecovery('rec-1')?.status).toBe('expired');
});
it('leaves an unexpired active row alone', () => {
db().insertServiceUpdateRecovery(makeRow({ expires_at: 50_000 }));
expect(db().sweepExpiredActiveServiceUpdateRecoveries(20_000)).toBe(0);
expect(db().getServiceUpdateRecovery('rec-1')?.status).toBe('active');
});
it('expires an abandoned restoring row (dead claim) but never a live one, even past the original expires_at', () => {
db().insertServiceUpdateRecovery(makeRow({
id: 'rec-abandoned', status: 'restoring', expires_at: 10_000, claim_expires_at: 15_000,
}));
db().insertServiceUpdateRecovery(makeRow({
id: 'rec-live', status: 'restoring', expires_at: 10_000, claim_expires_at: 100_000,
}));
const count = db().sweepAbandonedRestoringServiceUpdateRecoveries(20_000);
expect(count).toBe(1);
expect(db().getServiceUpdateRecovery('rec-abandoned')?.status).toBe('expired');
// Original expires_at (10_000) is long past, but the claim is still live: not swept.
expect(db().getServiceUpdateRecovery('rec-live')?.status).toBe('restoring');
});
it('never touches an active row from the restoring sweep, and vice versa', () => {
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-active', status: 'active', expires_at: 10_000 }));
db().insertServiceUpdateRecovery(makeRow({
id: 'rec-restoring', status: 'restoring', expires_at: 999_000, claim_expires_at: 10_000,
}));
expect(db().sweepAbandonedRestoringServiceUpdateRecoveries(20_000)).toBe(1);
expect(db().getServiceUpdateRecovery('rec-active')?.status).toBe('active');
expect(db().sweepExpiredActiveServiceUpdateRecoveries(20_000)).toBe(1);
expect(db().getServiceUpdateRecovery('rec-active')?.status).toBe('expired');
});
});
describe('held image ids', () => {
it('holds the image for an active row and a restoring row with a live claim, not a terminal or abandoned one', () => {
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-active', majority_image_id: 'sha256:held-1', status: 'active', expires_at: 999_000 }));
db().insertServiceUpdateRecovery(makeRow({
id: 'rec-live-claim', majority_image_id: 'sha256:held-2', status: 'restoring', claim_expires_at: 100_000,
}));
db().insertServiceUpdateRecovery(makeRow({
id: 'rec-dead-claim', majority_image_id: 'sha256:not-held-1', status: 'restoring', claim_expires_at: 5_000,
}));
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-consumed', majority_image_id: 'sha256:not-held-2', status: 'consumed' }));
const held = db().listHeldServiceUpdateRecoveryImageIds(NODE, 20_000);
expect(new Set(held)).toEqual(new Set(['sha256:held-1', 'sha256:held-2']));
});
it('scopes held image ids per node', () => {
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-node-1', node_id: 1, majority_image_id: 'sha256:node-1', expires_at: 999_000 }));
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-node-2', node_id: 2, majority_image_id: 'sha256:node-2', expires_at: 999_000 }));
expect(db().listHeldServiceUpdateRecoveryImageIds(1, 0)).toEqual(['sha256:node-1']);
expect(db().listHeldServiceUpdateRecoveryImageIds(2, 0)).toEqual(['sha256:node-2']);
});
});
it('deletes every row for a stack on stack delete, leaving other stacks untouched', () => {
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-web', stack_name: 'web' }));
db().insertServiceUpdateRecovery(makeRow({ id: 'rec-api', stack_name: 'api' }));
db().deleteServiceUpdateRecoveries(NODE, 'web');
expect(db().getServiceUpdateRecovery('rec-web')).toBeUndefined();
expect(db().getServiceUpdateRecovery('rec-api')).toBeTruthy();
});
});
@@ -78,3 +78,86 @@ describe('stack_update_status tri-state accessors', () => {
expect(db().getStackUpdateDetail(2).web.checkStatus).toBe('failed');
});
});
describe('stack_update_status services_json (per-service reduction)', () => {
const SERVICES = [
{ service: 'web', image: 'web:latest', hasUpdate: true, checkStatus: 'ok' as const, lastError: null },
{ service: 'worker', image: 'worker:latest', hasUpdate: false, checkStatus: 'ok' as const, lastError: null },
];
it('persists and returns a per-service breakdown via getStackUpdateDetail', () => {
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null, SERVICES, 3);
const detail = db().getStackUpdateDetail(NODE).stackA;
expect(detail.services).toEqual(SERVICES);
});
it('omits the services field entirely for a stack with no persisted per-service data', () => {
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null);
const detail = db().getStackUpdateDetail(NODE).stackA;
expect(detail.services).toBeUndefined();
expect('services' in detail).toBe(false);
});
it('leaves a prior services_json untouched when a later write omits services (COALESCE)', () => {
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null, SERVICES, 1);
// A legacy-path caller (or the fallback tally) writes without a services array.
db().upsertStackUpdateStatus(NODE, 'stackA', true, 2000, 'ok', null);
expect(db().getStackUpdateDetail(NODE).stackA.services).toEqual(SERVICES);
});
it('getStackServicesJson returns [] for a stack with no persisted row', () => {
expect(db().getStackServicesJson(NODE, 'missing')).toEqual([]);
});
it('getStackServicesJson returns [] for a stack row with no services_json', () => {
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null);
expect(db().getStackServicesJson(NODE, 'stackA')).toEqual([]);
});
it('getStackServicesJson round-trips a persisted per-service breakdown', () => {
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null, SERVICES, 5);
expect(db().getStackServicesJson(NODE, 'stackA')).toEqual(SERVICES);
});
it('recordStackCheckFailure persists a per-service breakdown alongside the failure', () => {
db().recordStackCheckFailure(NODE, 'stackA', 'Registry unreachable', 4000, SERVICES, 2);
const detail = db().getStackUpdateDetail(NODE).stackA;
expect(detail.checkStatus).toBe('failed');
expect(detail.services).toEqual(SERVICES);
});
it('treats corrupt services_json as an empty list rather than throwing (empty model invariant)', () => {
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null);
const raw = (db() as unknown as { db: { prepare: (s: string) => { run: (...args: unknown[]) => void } } }).db;
raw.prepare('UPDATE stack_update_status SET services_json = ? WHERE node_id = ? AND stack_name = ?')
.run('{not valid json', NODE, 'stackA');
expect(db().getStackServicesJson(NODE, 'stackA')).toEqual([]);
const detail = db().getStackUpdateDetail(NODE).stackA;
expect(detail.services).toBeUndefined();
expect(detail.hasUpdate).toBe(true); // aggregate columns are unaffected by corrupt services_json
});
it('treats a services_json shape-version mismatch as an empty list', () => {
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null);
const raw = (db() as unknown as { db: { prepare: (s: string) => { run: (...args: unknown[]) => void } } }).db;
raw.prepare('UPDATE stack_update_status SET services_json = ? WHERE node_id = ? AND stack_name = ?')
.run(JSON.stringify({ version: 999, generation: 1, services: SERVICES }), NODE, 'stackA');
expect(db().getStackServicesJson(NODE, 'stackA')).toEqual([]);
});
it('filters out malformed entries within an otherwise valid services_json array', () => {
db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null);
const raw = (db() as unknown as { db: { prepare: (s: string) => { run: (...args: unknown[]) => void } } }).db;
const malformed = [...SERVICES, { service: 'broken' /* missing required fields */ }];
raw.prepare('UPDATE stack_update_status SET services_json = ? WHERE node_id = ? AND stack_name = ?')
.run(JSON.stringify({ version: 1, generation: 1, services: malformed }), NODE, 'stackA');
expect(db().getStackServicesJson(NODE, 'stackA')).toEqual(SERVICES);
});
});
@@ -428,6 +428,83 @@ describe('DockerController - pruneDanglingImages', () => {
});
});
// ── Prune holds (): images held for a pending service-update
// recovery must never be deleted by any image-prune path. ──────────────
describe('DockerController - prune holds (isImageHeld)', () => {
it('pruneSystem("images") falls back to enumerate-and-delete when isImageHeld is given, skipping the held image', async () => {
mockDocker.listImages.mockResolvedValue([
{ Id: 'img-held', RepoTags: ['app:1'], Containers: 0 },
{ Id: 'img-free', RepoTags: ['app:2'], Containers: 0 },
]);
mockDocker.df
.mockResolvedValueOnce({ LayersSize: 5000 })
.mockResolvedValueOnce({ LayersSize: 3000 });
const removeFn = vi.fn().mockResolvedValue(undefined);
mockDocker.getImage.mockReturnValue({ remove: removeFn });
const dc = DockerController.getInstance(1);
const result = await dc.pruneSystem('images', undefined, (id: string) => id === 'img-held');
expect(mockDocker.pruneImages).not.toHaveBeenCalled();
expect(removeFn).toHaveBeenCalledTimes(1);
expect(mockDocker.getImage).toHaveBeenCalledWith('img-free');
expect(result).toEqual({ success: true, reclaimedBytes: 2000 });
});
it('pruneDanglingImages falls back to enumerate-and-delete when isImageHeld is given, skipping the held image', async () => {
mockDocker.listImages.mockResolvedValue([
{ Id: 'img-held', RepoTags: [], Containers: 0 },
{ Id: 'img-free', RepoTags: ['<none>:<none>'], Containers: 0 },
{ Id: 'img-tagged', RepoTags: ['app:1'], Containers: 0 }, // not dangling; excluded regardless of hold
]);
mockDocker.df
.mockResolvedValueOnce({ LayersSize: 5000 })
.mockResolvedValueOnce({ LayersSize: 4000 });
const removeFn = vi.fn().mockResolvedValue(undefined);
mockDocker.getImage.mockReturnValue({ remove: removeFn });
const dc = DockerController.getInstance(1);
const result = await dc.pruneDanglingImages((id: string) => id === 'img-held');
expect(mockDocker.pruneImages).not.toHaveBeenCalled();
expect(removeFn).toHaveBeenCalledTimes(1);
expect(mockDocker.getImage).toHaveBeenCalledWith('img-free');
expect(result).toEqual({ success: true, reclaimedBytes: 1000 });
});
it('pruneManagedOnly("images") re-checks isImageHeld immediately before each delete', async () => {
mockDocker.df
.mockResolvedValueOnce({ LayersSize: 5000, Images: [] })
.mockResolvedValueOnce({ LayersSize: 4000, Images: [] });
mockDocker.listImages.mockResolvedValue([
{ Id: 'img-held', Containers: 0, Size: 1000, Labels: { 'com.docker.compose.project': 'any-stack' } },
{ Id: 'img-free', Containers: 0, Size: 1000, Labels: { 'com.docker.compose.project': 'any-stack' } },
]);
mockDocker.listContainers.mockResolvedValue([]);
const removeFn = vi.fn().mockResolvedValue(undefined);
mockDocker.getImage.mockReturnValue({ remove: removeFn });
const dc = DockerController.getInstance(1);
const result = await dc.pruneManagedOnly('images', ['any-stack'], (id: string) => id === 'img-held');
expect(removeFn).toHaveBeenCalledTimes(1);
expect(mockDocker.getImage).toHaveBeenCalledWith('img-free');
expect(result.success).toBe(true);
});
it('pruneSystem("images") keeps the bulk Docker prune API when no isImageHeld predicate is given', async () => {
mockDocker.pruneImages.mockResolvedValue({ SpaceReclaimed: 999 });
const dc = DockerController.getInstance(1);
const result = await dc.pruneSystem('images');
expect(mockDocker.pruneImages).toHaveBeenCalled();
expect(mockDocker.listImages).not.toHaveBeenCalled();
expect(result).toEqual({ success: true, reclaimedBytes: 999 });
});
});
// ── getClassifiedResources ─────────────────────────────────────────────
describe('DockerController - getClassifiedResources', () => {
@@ -246,6 +246,74 @@ describe('DriftLedgerService.reconcile', () => {
});
});
describe('DriftLedgerService.reconcileService', () => {
beforeEach(() => clearLedger('svcrec'));
const ledger = () => DriftLedgerService.getInstance();
it('inserts only the selected service\'s findings, ignoring other services in the same report', () => {
const report = reportWith([finding('image-mismatch', 'web'), finding('service-missing', 'db')], { stack: 'svcrec' });
const res = ledger().reconcileService(nodeId, 'svcrec', 'web', report);
expect(res).toEqual({ detected: 1, resolved: 0 });
const open = db().getOpenDriftFindings(nodeId, 'svcrec');
expect(open).toHaveLength(1);
expect(open[0].service).toBe('web');
});
it('resolves only the selected service\'s open finding, leaving other open services untouched', () => {
ledger().reconcile(nodeId, 'svcrec', reportWith([finding('image-mismatch', 'web'), finding('service-missing', 'db')], { stack: 'svcrec' }));
// 'web' cleared, 'db' absent from this service-scoped report too, but only 'web' is in scope.
const res = ledger().reconcileService(nodeId, 'svcrec', 'web', reportWith([], { stack: 'svcrec', status: 'in-sync' }));
expect(res).toEqual({ detected: 0, resolved: 1 });
const open = db().getOpenDriftFindings(nodeId, 'svcrec');
expect(open).toHaveLength(1);
expect(open[0].service).toBe('db');
});
it('preserves an empty-service (network-level) finding when reconciling one named service', () => {
ledger().reconcile(nodeId, 'svcrec', reportWith([finding('image-mismatch', 'web'), finding('network-missing', '')], { stack: 'svcrec' }));
const res = ledger().reconcileService(nodeId, 'svcrec', 'web', reportWith([], { stack: 'svcrec', status: 'in-sync' }));
expect(res).toEqual({ detected: 0, resolved: 1 });
const open = db().getOpenDriftFindings(nodeId, 'svcrec');
expect(open).toHaveLength(1);
expect(open[0].service).toBe('');
});
it('does not touch findings when the same service is idempotently rechecked', () => {
const report = reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' });
ledger().reconcileService(nodeId, 'svcrec', 'web', report);
const res = ledger().reconcileService(nodeId, 'svcrec', 'web', report);
expect(res).toEqual({ detected: 0, resolved: 0 });
expect(db().getOpenDriftFindings(nodeId, 'svcrec')).toHaveLength(1);
});
it('does not reconcile an unreachable report (open findings are not falsely resolved)', () => {
ledger().reconcile(nodeId, 'svcrec', reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' }));
const res = ledger().reconcileService(nodeId, 'svcrec', 'web', { stack: 'svcrec', status: 'unreachable', hasComposeFile: true, hasContainers: false, findings: [] });
expect(res).toEqual({ detected: 0, resolved: 0 });
expect(db().getOpenDriftFindings(nodeId, 'svcrec')).toHaveLength(1);
});
it('does not reconcile a parse-error report', () => {
ledger().reconcile(nodeId, 'svcrec', reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' }));
const res = ledger().reconcileService(nodeId, 'svcrec', 'web', { stack: 'svcrec', status: 'drifted', hasComposeFile: false, hasContainers: false, findings: [], parseError: 'bad yaml' });
expect(res).toEqual({ detected: 0, resolved: 0 });
expect(db().getOpenDriftFindings(nodeId, 'svcrec')).toHaveLength(1);
});
it('names the service in the recorded activity message', () => {
ledger().reconcileService(nodeId, 'svcrec', 'web', reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' }));
const activity = driftActivity('svcrec');
expect(activity).toHaveLength(1);
expect(activity[0].message).toContain('web');
});
it('stamps the dossier last-checked time', () => {
expect(db().getStackDossier(nodeId, 'svcrec')?.last_drift_check_at ?? null).toBeNull();
ledger().reconcileService(nodeId, 'svcrec', 'web', reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' }));
expect(typeof db().getStackDossier(nodeId, 'svcrec')?.last_drift_check_at).toBe('number');
});
});
describe('DriftLedgerService.reconcileStack', () => {
const STACK_A = 'recstacka';
let dirA: string;
@@ -0,0 +1,125 @@
/**
* buildEffectiveServiceModel: turns `docker compose config --format json`
* output into the per-service facts (declared image, build presence,
* expected replicas, dependencies, healthcheck) that service-scoped update
* and restore key off of. Fail-closed is the critical property: a render
* failure must leave `renderable: false` with no services, never a
* root-file fallback.
*/
import { describe, it, expect, afterEach, vi } from 'vitest';
import { ComposeService } from '../services/ComposeService';
import { buildEffectiveServiceModel } from '../services/effectiveServiceModel';
const ENV_SECRET = 'env-secret-9d4a-value';
function stubRender(rendered: string | null, stderr = '') {
vi.spyOn(ComposeService, 'getInstance').mockReturnValue({
renderConfig: vi.fn().mockResolvedValue({ rendered, stderr, code: rendered === null ? 1 : 0, timedOut: false }),
} as unknown as ComposeService);
}
afterEach(() => {
vi.restoreAllMocks();
});
describe('buildEffectiveServiceModel', () => {
it('extracts declared image, dependsOn, and healthcheck for an image-only service', async () => {
stubRender(JSON.stringify({
services: {
web: {
image: 'nginx:latest',
depends_on: { db: { condition: 'service_healthy', required: true } },
healthcheck: { test: ['CMD', 'true'] },
},
db: { image: 'postgres:16' },
},
}));
const result = await buildEffectiveServiceModel(1, 'mystack');
expect(result.renderable).toBe(true);
if (!result.renderable) throw new Error('expected renderable');
expect(result.services).toEqual([
{ name: 'web', declaredImage: 'nginx:latest', hasBuild: false, expectedReplicas: 1, dependsOn: ['db'], hasHealthcheck: true },
{ name: 'db', declaredImage: 'postgres:16', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false },
]);
});
it('flags hasBuild for a build-backed service, including image+build', async () => {
stubRender(JSON.stringify({
services: {
api: { build: { context: '.' } },
worker: { image: 'app:latest', build: { context: '.' } },
},
}));
const result = await buildEffectiveServiceModel(1, 'mystack');
if (!result.renderable) throw new Error('expected renderable');
expect(result.services[0]).toMatchObject({ name: 'api', declaredImage: null, hasBuild: true });
expect(result.services[1]).toMatchObject({ name: 'worker', declaredImage: 'app:latest', hasBuild: true });
});
it('reads expectedReplicas from the top-level scale field, deploy.replicas, or defaults to 1', async () => {
stubRender(JSON.stringify({
services: {
scaled: { image: 'a', scale: 3 },
deployed: { image: 'b', deploy: { replicas: 2 } },
zeroed: { image: 'c', scale: 0 },
unset: { image: 'd' },
},
}));
const result = await buildEffectiveServiceModel(1, 'mystack');
if (!result.renderable) throw new Error('expected renderable');
expect(result.services.map(s => s.expectedReplicas)).toEqual([3, 2, 0, 1]);
});
it('treats a disabled healthcheck as none', async () => {
stubRender(JSON.stringify({
services: { web: { image: 'a', healthcheck: { disable: true } } },
}));
const result = await buildEffectiveServiceModel(1, 'mystack');
if (!result.renderable) throw new Error('expected renderable');
expect(result.services[0].hasHealthcheck).toBe(false);
});
it('parses depends_on given in the short list form', async () => {
stubRender(JSON.stringify({
services: { web: { image: 'a', depends_on: ['db', 'cache'] } },
}));
const result = await buildEffectiveServiceModel(1, 'mystack');
if (!result.renderable) throw new Error('expected renderable');
expect(result.services[0].dependsOn).toEqual(['db', 'cache']);
});
it('fails closed with a redacted error and no services when the render errors', async () => {
stubRender(null, `error: the "${ENV_SECRET}" variable is not set`);
const result = await buildEffectiveServiceModel(1, 'mystack');
expect(result.renderable).toBe(false);
if (result.renderable) throw new Error('expected not renderable');
expect(result.code).toBe('effective_model_render_failed');
expect(result.error).not.toContain(ENV_SECRET);
});
it('fails closed when the rendered output is not valid JSON', async () => {
stubRender('this is not json {');
const result = await buildEffectiveServiceModel(1, 'mystack');
expect(result.renderable).toBe(false);
if (result.renderable) throw new Error('expected not renderable');
expect(result.code).toBe('effective_model_render_failed');
});
it('fails closed and redacts the message when renderConfig throws (docker unavailable)', async () => {
vi.spyOn(ComposeService, 'getInstance').mockReturnValue({
renderConfig: vi.fn().mockRejectedValue(new Error(`spawn failed: password=${ENV_SECRET}`)),
} as unknown as ComposeService);
const result = await buildEffectiveServiceModel(1, 'mystack');
expect(result.renderable).toBe(false);
if (result.renderable) throw new Error('expected not renderable');
expect(result.code).toBe('effective_model_render_failed');
expect(result.error).not.toContain(ENV_SECRET);
});
it('yields an empty service list for a garbage or empty rendered model', async () => {
stubRender(JSON.stringify({ services: {} }));
const result = await buildEffectiveServiceModel(1, 'mystack');
if (!result.renderable) throw new Error('expected renderable');
expect(result.services).toEqual([]);
});
});
@@ -1168,7 +1168,7 @@ describe('GitSourceService.apply', () => {
const { HealthGateService } = await import('../services/HealthGateService');
const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue();
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-git');
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-git');
const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!;
try {
@@ -0,0 +1,66 @@
/**
* Restart-safe rebuild of health_gate_runs when widening the trigger CHECK and
* adding target/failure columns.
*/
import path from 'path';
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import Database from 'better-sqlite3';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
describe('migrateHealthGateTargetSchema', () => {
let tmpDir: string;
beforeEach(async () => {
tmpDir = await setupTestDb();
});
afterEach(async () => {
await cleanupTestDb(tmpDir);
});
it('recovers from a stale health_gate_runs_new and preserves existing rows', async () => {
const dbPath = path.join(tmpDir, 'sencho.db');
const raw = new Database(dbPath);
raw.exec(`
DROP TABLE IF EXISTS health_gate_runs;
DROP TABLE IF EXISTS health_gate_runs_new;
CREATE TABLE health_gate_runs (
id TEXT PRIMARY KEY,
node_id INTEGER NOT NULL,
stack_name TEXT NOT NULL,
trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy')),
status TEXT NOT NULL CHECK (status IN ('observing','passed','failed','unknown')),
reason TEXT,
window_seconds INTEGER NOT NULL,
containers_json TEXT NOT NULL DEFAULT '[]',
started_at INTEGER NOT NULL,
ended_at INTEGER,
created_by TEXT
);
CREATE TABLE health_gate_runs_new (id TEXT PRIMARY KEY);
INSERT INTO health_gate_runs (
id, node_id, stack_name, trigger_action, status, reason, window_seconds, containers_json, started_at, ended_at, created_by
) VALUES ('gate-1', 1, 'web', 'update', 'passed', null, 90, '[]', 1, 2, 'tester');
`);
raw.close();
const { DatabaseService } = await import('../services/DatabaseService');
(DatabaseService as unknown as { instance?: typeof DatabaseService }).instance = undefined;
const db = DatabaseService.getInstance();
const row = db.getDb().prepare('SELECT * FROM health_gate_runs WHERE id = ?').get('gate-1') as {
target_scope: string;
service_name: string | null;
failure_source: string | null;
trigger_action: string;
};
expect(row.trigger_action).toBe('update');
expect(row.target_scope).toBe('stack');
expect(row.service_name).toBeNull();
expect(row.failure_source).toBeNull();
const stale = db.getDb().prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'health_gate_runs_new'",
).get();
expect(stale).toBeUndefined();
});
});
@@ -0,0 +1,293 @@
/**
* Tests for the service-scoped health gate: prepare/attach/beginPrepared
* nullability (disabled, unknown token, concurrency cap, armed), the prepare
* TTL, and primary vs collateral failure attribution (regression-eligible
* siblings can fail the gate; pre-existing unhealthy siblings stay advisory).
*/
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
interface StoredRun {
id: string;
node_id: number;
stack_name: string;
trigger_action: 'update' | 'deploy' | 'service_update' | 'service_restore';
status: 'observing' | 'passed' | 'failed' | 'unknown';
reason: string | null;
window_seconds: number;
containers_json: string;
started_at: number;
ended_at: number | null;
created_by: string | null;
target_scope: 'stack' | 'service';
service_name: string | null;
failure_source: 'primary' | 'collateral' | null;
}
const { state } = vi.hoisted(() => ({
state: {
runs: new Map<string, StoredRun>(),
settings: {} as Record<string, string>,
listContainers: vi.fn(),
inspect: vi.fn(),
},
}));
vi.mock('../services/DatabaseService', () => ({
DatabaseService: {
getInstance: () => ({
getGlobalSettings: () => state.settings,
insertHealthGateRun: (run: StoredRun) => { state.runs.set(run.id, { ...run }); },
finalizeHealthGateRun: (id: string, status: StoredRun['status'], reason: string | null, endedAt: number, containersJson: string, failureSource: StoredRun['failure_source'] = null) => {
const run = state.runs.get(id);
if (run) Object.assign(run, { status, reason, ended_at: endedAt, containers_json: containersJson, failure_source: failureSource });
},
getHealthGateRun: (nodeId: number, stackName: string, id: string) => {
const run = state.runs.get(id);
return run && run.node_id === nodeId && run.stack_name === stackName ? { ...run } : undefined;
},
getLatestHealthGateRun: (nodeId: number, stackName: string) => {
const matches = [...state.runs.values()]
.filter(r => r.node_id === nodeId && r.stack_name === stackName)
.sort((a, b) => b.started_at - a.started_at);
return matches[0] ? { ...matches[0] } : undefined;
},
markInterruptedHealthGateRuns: () => 0,
addNotificationHistory: (_nodeId: number, item: { category?: string; message: string; level: string }) => ({ ...item, id: 1, is_read: false }),
}),
},
}));
vi.mock('../services/DockerController', () => ({
default: {
getInstance: () => ({
getDocker: () => ({
listContainers: state.listContainers,
getContainer: (id: string) => ({ inspect: () => state.inspect(id) }),
}),
}),
},
}));
// AutoHeal suppression is exercised elsewhere; here we only need the calls to
// not throw when the gate finalizes a service run.
vi.mock('../services/AutoHealService', () => ({
AutoHealService: { getInstance: () => ({ clearSuppress: vi.fn() }) },
}));
import { HealthGateService } from '../services/HealthGateService';
type Fixture = {
id: string;
name: string;
service: string;
state?: string;
health?: string | null;
restartCount?: number;
startedAt?: string;
imageId?: string;
};
function setContainers(fixtures: Fixture[]): void {
state.listContainers.mockResolvedValue(fixtures.map(f => ({
Id: f.id,
Names: [`/${f.name}`],
State: f.state ?? 'running',
Labels: { 'com.docker.compose.service': f.service },
})));
state.inspect.mockImplementation((id: string) => {
const f = fixtures.find(c => c.id === id);
if (!f) return Promise.reject(Object.assign(new Error('no such container'), { statusCode: 404 }));
return Promise.resolve({
State: {
Status: f.state ?? 'running',
Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined,
StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z',
},
RestartCount: f.restartCount ?? 0,
Image: f.imageId ?? 'sha256:app',
Config: { Labels: { 'com.docker.compose.service': f.service } },
});
});
}
const svc = () => HealthGateService.getInstance();
async function ticks(n: number): Promise<void> {
for (let i = 0; i < n; i++) await vi.advanceTimersByTimeAsync(5_000);
}
async function prepareService(
fixtures: Fixture[],
opts: { serviceName?: string; expectedReplicas?: number; trigger?: 'service_update' | 'service_restore' } = {},
): Promise<string> {
setContainers(fixtures);
const { prepareToken } = await svc().prepare({
nodeId: 0,
stackName: 'web',
target: { scope: 'service', serviceName: opts.serviceName ?? 'app' },
trigger: opts.trigger ?? 'service_update',
expectedReplicas: opts.expectedReplicas ?? 1,
});
return prepareToken;
}
beforeEach(() => {
vi.useFakeTimers();
state.runs.clear();
state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' };
state.listContainers.mockReset();
state.inspect.mockReset();
svc().start();
});
afterEach(() => {
svc().stop();
expect(vi.getTimerCount()).toBe(0);
vi.useRealTimers();
});
describe('prepare / beginPrepared nullability', () => {
it('prepare returns a token with a TTL that outlives the Compose timeout', async () => {
const before = Date.now();
setContainers([{ id: 'p1', name: 'web-app-1', service: 'app' }]);
const { prepareToken, expiresAt } = await svc().prepare({
nodeId: 0, stackName: 'web', target: { scope: 'service', serviceName: 'app' },
trigger: 'service_update', expectedReplicas: 1,
});
expect(prepareToken).toBeTruthy();
// Default compose timeout is 30m; prepare TTL is max(timeout, 30m) + 5m.
expect(expiresAt).toBe(before + 35 * 60_000);
});
it('returns runId null / observing false when gating is disabled', async () => {
const token = await prepareService([{ id: 'p1', name: 'web-app-1', service: 'app' }]);
state.settings.health_gate_enabled = '0';
expect(svc().beginPrepared({ prepareToken: token, actor: 'tester' })).toEqual({ runId: null, observing: false });
});
it('returns runId null / observing false for an unknown or consumed token', async () => {
const token = await prepareService([{ id: 'p1', name: 'web-app-1', service: 'app' }]);
// First consume arms a gate; the second call sees a consumed token.
svc().attachExpectedImage(token, 'sha256:app');
const first = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
expect(first.observing).toBe(true);
expect(svc().beginPrepared({ prepareToken: token, actor: 'tester' })).toEqual({ runId: null, observing: false });
});
it('persists an immediate unknown past the concurrency cap', async () => {
for (let i = 0; i < 25; i++) svc().beginStack(0, `stack-${i}`, 'update', 'tester');
const token = await prepareService([{ id: 'p1', name: 'web-app-1', service: 'app' }]);
svc().attachExpectedImage(token, 'sha256:app');
const result = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
expect(result.runId).toBeTruthy();
expect(result.observing).toBe(false);
const report = svc().getReport(0, 'web', result.runId!);
expect(report.status).toBe('unknown');
expect(report.reason).toContain('concurrent');
});
});
describe('primary vs collateral attribution', () => {
it('fails with failureSource primary when a replica reports unhealthy', async () => {
const token = await prepareService([
{ id: 'p1', name: 'web-app-1', service: 'app' },
{ id: 's1', name: 'web-db-1', service: 'db' },
]);
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1); // arm expected set
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app', health: 'unhealthy' },
{ id: 's1', name: 'web-db-1', service: 'db' },
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.failureSource).toBe('primary');
});
it('fails with failureSource collateral when a regression-eligible sibling exits', async () => {
const token = await prepareService([
{ id: 'p1', name: 'web-app-1', service: 'app' },
{ id: 's1', name: 'web-db-1', service: 'db' },
]);
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app' },
{ id: 's1', name: 'web-db-1', service: 'db', state: 'exited' },
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.failureSource).toBe('collateral');
});
it('fails with failureSource collateral when a healthy sibling vanishes before the first poll', async () => {
// Sibling is healthy at prepare, then gone before arming. Seeding expected
// from the prepare baseline must still track it so the gate fails.
const token = await prepareService([
{ id: 'p1', name: 'web-app-1', service: 'app' },
{ id: 's1', name: 'web-db-1', service: 'db' },
]);
svc().attachExpectedImage(token, 'sha256:app');
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app' },
]);
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1); // arm expected from primary + prepare collateral baseline
await ticks(1); // first miss on the vanished sibling
await ticks(1); // second miss fails the gate
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.failureSource).toBe('collateral');
expect(report.reason ?? '').toMatch(/disappeared|sibling/i);
});
it('keeps a pre-existing unhealthy sibling advisory: the gate can still pass', async () => {
// The sibling is unhealthy at prepare, so it is not regression-eligible and
// cannot fail the service gate; the healthy primary carries it to a pass.
const token = await prepareService([
{ id: 'p1', name: 'web-app-1', service: 'app' },
{ id: 's1', name: 'web-db-1', service: 'db', health: 'unhealthy' },
]);
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(8); // through the 30s window
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('passed');
expect(report.failureSource).toBeNull();
});
it('lets different-service gates coexist while same-service begin supersedes', async () => {
const appToken = await prepareService(
[{ id: 'p1', name: 'web-app-1', service: 'app' }, { id: 's1', name: 'web-db-1', service: 'db' }],
{ serviceName: 'app' },
);
svc().attachExpectedImage(appToken, 'sha256:app');
const appFirst = svc().beginPrepared({ prepareToken: appToken, actor: 'tester' });
expect(appFirst.observing).toBe(true);
const dbToken = await prepareService(
[{ id: 'p1', name: 'web-app-1', service: 'app' }, { id: 's1', name: 'web-db-1', service: 'db' }],
{ serviceName: 'db' },
);
svc().attachExpectedImage(dbToken, 'sha256:db');
const dbGate = svc().beginPrepared({ prepareToken: dbToken, actor: 'tester' });
expect(dbGate.observing).toBe(true);
expect(svc().getReport(0, 'web', appFirst.runId!).status).toBe('observing');
expect(svc().getReport(0, 'web', dbGate.runId!).status).toBe('observing');
const appToken2 = await prepareService(
[{ id: 'p1', name: 'web-app-1', service: 'app' }, { id: 's1', name: 'web-db-1', service: 'db' }],
{ serviceName: 'app' },
);
svc().attachExpectedImage(appToken2, 'sha256:app2');
const appSecond = svc().beginPrepared({ prepareToken: appToken2, actor: 'tester' });
expect(appSecond.observing).toBe(true);
expect(svc().getReport(0, 'web', appFirst.runId!).status).toBe('unknown');
expect(svc().getReport(0, 'web', dbGate.runId!).status).toBe('observing');
expect(svc().getReport(0, 'web', appSecond.runId!).status).toBe('observing');
});
});
@@ -9,7 +9,7 @@ interface StoredRun {
id: string;
node_id: number;
stack_name: string;
trigger_action: 'update' | 'deploy';
trigger_action: 'update' | 'deploy' | 'service_update' | 'service_restore';
status: 'observing' | 'passed' | 'failed' | 'unknown';
reason: string | null;
window_seconds: number;
@@ -17,6 +17,9 @@ interface StoredRun {
started_at: number;
ended_at: number | null;
created_by: string | null;
target_scope: 'stack' | 'service';
service_name: string | null;
failure_source: 'primary' | 'collateral' | null;
}
const { state } = vi.hoisted(() => ({
@@ -34,9 +37,9 @@ vi.mock('../services/DatabaseService', () => ({
getInstance: () => ({
getGlobalSettings: () => state.settings,
insertHealthGateRun: (run: StoredRun) => { state.runs.set(run.id, { ...run }); },
finalizeHealthGateRun: (id: string, status: StoredRun['status'], reason: string | null, endedAt: number, containersJson: string) => {
finalizeHealthGateRun: (id: string, status: StoredRun['status'], reason: string | null, endedAt: number, containersJson: string, failureSource: StoredRun['failure_source'] = null) => {
const run = state.runs.get(id);
if (run) Object.assign(run, { status, reason, ended_at: endedAt, containers_json: containersJson });
if (run) Object.assign(run, { status, reason, ended_at: endedAt, containers_json: containersJson, failure_source: failureSource });
},
getHealthGateRun: (nodeId: number, stackName: string, id: string) => {
const run = state.runs.get(id);
@@ -134,7 +137,7 @@ afterEach(() => {
describe('HealthGateService verdicts', () => {
it('passes at the window end when containers stay running', async () => {
const id = svc().begin(0, 'web', 'update', 'tester');
const id = svc().beginStack(0, 'web', 'update', 'tester');
expect(id).toBeTruthy();
await ticks(3); // 15s: still observing
expect(latest().status).toBe('observing');
@@ -144,7 +147,7 @@ describe('HealthGateService verdicts', () => {
});
it('fails fast when a container exits', async () => {
svc().begin(0, 'web', 'update', 'tester');
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1); // baseline
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited' }]);
await ticks(1);
@@ -155,7 +158,7 @@ describe('HealthGateService verdicts', () => {
});
it('fails fast when a healthcheck reports unhealthy', async () => {
svc().begin(0, 'web', 'update', 'tester');
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', health: 'unhealthy' }]);
await ticks(1);
@@ -164,7 +167,7 @@ describe('HealthGateService verdicts', () => {
});
it('detects a restart loop via container replacement (new id)', async () => {
svc().begin(0, 'web', 'update', 'tester');
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'bbb', name: 'web-app-1' }]);
await ticks(1); // restart 1 observed; carried as new baseline
@@ -180,7 +183,7 @@ describe('HealthGateService verdicts', () => {
});
it('detects a restart loop via RestartCount and StartedAt movement', async () => {
svc().begin(0, 'web', 'update', 'tester');
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', restartCount: 1 }]);
await ticks(1);
@@ -195,7 +198,7 @@ describe('HealthGateService verdicts', () => {
});
it('tolerates a one-poll disappearance but fails on two consecutive misses', async () => {
svc().begin(0, 'web', 'update', 'tester');
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1); // baseline
setContainers([]); // one missed poll: tolerated
await ticks(1);
@@ -203,7 +206,7 @@ describe('HealthGateService verdicts', () => {
await ticks(5); // through the 30s window
expect(latest().status).toBe('passed');
const second = svc().begin(0, 'web', 'update', 'tester')!;
const second = svc().beginStack(0, 'web', 'update', 'tester')!;
await ticks(1);
setContainers([]);
await ticks(2); // two consecutive misses: disappeared
@@ -213,7 +216,7 @@ describe('HealthGateService verdicts', () => {
});
it('fails when a container is stuck restarting across consecutive polls', async () => {
svc().begin(0, 'web', 'update', 'tester');
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'restarting' }]);
await ticks(2);
@@ -222,7 +225,7 @@ describe('HealthGateService verdicts', () => {
});
it('goes unknown after three consecutive docker errors', async () => {
svc().begin(0, 'web', 'update', 'tester');
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
state.listContainers.mockRejectedValue(new Error('socket gone'));
await ticks(3);
@@ -231,7 +234,7 @@ describe('HealthGateService verdicts', () => {
});
it('resolves unknown when every docker observe hangs', async () => {
svc().begin(0, 'web', 'update', 'tester');
svc().beginStack(0, 'web', 'update', 'tester');
// A wedged socket never settles. The per-observe timeout turns each poll
// into an error, and three in a row finalize the gate unknown instead of
// observing forever on a pending promise.
@@ -244,7 +247,7 @@ describe('HealthGateService verdicts', () => {
});
it('recovers from a transient observe timeout instead of finalizing', async () => {
svc().begin(0, 'web', 'update', 'tester');
svc().beginStack(0, 'web', 'update', 'tester');
// One observe wedges and times out (a single strike), then the socket
// recovers; the gate must keep observing, not give up at one error.
state.listContainers.mockImplementationOnce(() => new Promise<never>(() => {}));
@@ -257,7 +260,7 @@ describe('HealthGateService verdicts', () => {
});
it('runs polls single-flight: no second observe until the first settles', async () => {
svc().begin(0, 'web', 'update', 'tester');
svc().beginStack(0, 'web', 'update', 'tester');
let release: (value: Array<{ Id: string; Names: string[]; State: string }>) => void = () => {};
state.listContainers.mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
// Advance past a second poll interval while the first observe is still
@@ -272,7 +275,7 @@ describe('HealthGateService verdicts', () => {
});
it('ends unknown when a healthcheck is still starting at the window end', async () => {
svc().begin(0, 'web', 'update', 'tester');
svc().beginStack(0, 'web', 'update', 'tester');
setContainers([{ id: 'aaa', name: 'web-app-1', health: 'starting' }]);
await ticks(7);
expect(latest().status).toBe('unknown');
@@ -281,7 +284,7 @@ describe('HealthGateService verdicts', () => {
it('goes unknown when no containers ever appear', async () => {
setContainers([]);
svc().begin(0, 'web', 'update', 'tester');
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(4);
expect(latest().status).toBe('unknown');
expect(latest().reason).toContain('no containers');
@@ -293,7 +296,7 @@ describe('HealthGateService lifecycle', () => {
// A poll is mid-await on Docker when a newer update supersedes the gate;
// when the await resolves with healthy containers, the superseded run
// must keep its terminal unknown verdict.
const first = svc().begin(0, 'web', 'update', 'tester')!;
const first = svc().beginStack(0, 'web', 'update', 'tester')!;
await ticks(2); // baseline established, healthy
let releasePoll: (value: Array<{ Id: string; Names: string[]; State: string }>) => void = () => {};
@@ -302,7 +305,7 @@ describe('HealthGateService lifecycle', () => {
);
const straddlingPoll = vi.advanceTimersByTimeAsync(5_000); // poll now awaiting Docker
const second = svc().begin(0, 'web', 'update', 'tester')!;
const second = svc().beginStack(0, 'web', 'update', 'tester')!;
expect(svc().getReport(0, 'web', first).status).toBe('unknown');
releasePoll([{ Id: 'aaa', Names: ['/web-app-1'], State: 'running' }]);
@@ -317,10 +320,10 @@ describe('HealthGateService lifecycle', () => {
});
it('supersede finalizes the old run as unknown, clears its timer, and getRun still resolves it', async () => {
const first = svc().begin(0, 'web', 'update', 'tester')!;
const first = svc().beginStack(0, 'web', 'update', 'tester')!;
await ticks(1);
const timersBefore = vi.getTimerCount();
const second = svc().begin(0, 'web', 'update', 'tester')!;
const second = svc().beginStack(0, 'web', 'update', 'tester')!;
expect(vi.getTimerCount()).toBe(timersBefore); // old interval cleared, new one added
const superseded = svc().getReport(0, 'web', first);
@@ -337,6 +340,7 @@ describe('HealthGateService lifecycle', () => {
state.runs.set('stale', {
id: 'stale', node_id: 0, stack_name: 'web', trigger_action: 'update', status: 'observing',
reason: null, window_seconds: 30, containers_json: '[]', started_at: 1, ended_at: null, created_by: null,
target_scope: 'stack', service_name: null, failure_source: null,
});
svc().start();
expect(state.runs.get('stale')!.status).toBe('unknown');
@@ -345,7 +349,7 @@ describe('HealthGateService lifecycle', () => {
it('no-ops when disabled but still records the update_started event', () => {
state.settings.health_gate_enabled = '0';
const id = svc().begin(0, 'web', 'update', 'tester');
const id = svc().beginStack(0, 'web', 'update', 'tester');
expect(id).toBeNull();
expect(state.runs.size).toBe(0);
expect(state.activity.some(a => a.category === 'update_started')).toBe(true);
@@ -353,24 +357,24 @@ describe('HealthGateService lifecycle', () => {
});
it('records update_started for update triggers but not deploy triggers', () => {
svc().begin(0, 'web', 'deploy', 'tester');
svc().beginStack(0, 'web', 'deploy', 'tester');
expect(state.activity.some(a => a.category === 'update_started')).toBe(false);
svc().begin(0, 'web', 'update', 'tester');
svc().beginStack(0, 'web', 'update', 'tester');
expect(state.activity.some(a => a.category === 'update_started')).toBe(true);
});
it('refuses to begin before start() so shutdown cannot leak timers', () => {
svc().stop();
expect(svc().begin(0, 'web', 'update', 'tester')).toBeNull();
expect(svc().beginStack(0, 'web', 'update', 'tester')).toBeNull();
expect(vi.getTimerCount()).toBe(0);
svc().start();
});
it('persists an immediate unknown past the concurrency cap', () => {
for (let i = 0; i < 25; i++) {
svc().begin(0, `stack-${i}`, 'update', 'tester');
svc().beginStack(0, `stack-${i}`, 'update', 'tester');
}
const overCap = svc().begin(0, 'one-too-many', 'update', 'tester')!;
const overCap = svc().beginStack(0, 'one-too-many', 'update', 'tester')!;
const report = svc().getReport(0, 'one-too-many', overCap);
expect(report.status).toBe('unknown');
expect(report.reason).toContain('concurrent');
@@ -378,10 +382,10 @@ describe('HealthGateService lifecycle', () => {
it('clamps the configured window into its valid range and falls back on garbage', () => {
state.settings.health_gate_window_seconds = '99999';
const a = svc().begin(0, 'web', 'update', 'tester')!;
const a = svc().beginStack(0, 'web', 'update', 'tester')!;
expect(svc().getReport(0, 'web', a).windowSeconds).toBe(600);
state.settings.health_gate_window_seconds = 'banana';
const b = svc().begin(0, 'web', 'update', 'tester')!;
const b = svc().beginStack(0, 'web', 'update', 'tester')!;
expect(svc().getReport(0, 'web', b).windowSeconds).toBe(90);
});
@@ -392,7 +396,7 @@ describe('HealthGateService lifecycle', () => {
});
it('stop() finalizes in-flight gates as unknown with zero timers left', async () => {
const id = svc().begin(0, 'web', 'update', 'tester')!;
const id = svc().beginStack(0, 'web', 'update', 'tester')!;
await ticks(1);
svc().stop();
expect(vi.getTimerCount()).toBe(0);
@@ -0,0 +1,212 @@
/**
* Unit tests for the §5 per-service reduction pure functions:
* reduceServiceStatus, aggregateServiceCheckStatus, and
* buildAvailabilityNotifyMessage. These operate on plain data (an
* imageUpdateMap and a prior per-service snapshot), so they are tested
* directly without mocking DatabaseService, Docker, or Compose.
*/
import { describe, it, expect } from 'vitest';
import {
reduceServiceStatus,
aggregateServiceCheckStatus,
buildAvailabilityNotifyMessage,
type ImageCheckResult,
} from '../services/ImageUpdateService';
import type { StackServiceStatus } from '../services/DatabaseService';
function ok(hasUpdate: boolean): ImageCheckResult {
return { hasUpdate };
}
function errored(message: string): ImageCheckResult {
return { hasUpdate: false, error: message };
}
function notCheckable(): ImageCheckResult {
return { hasUpdate: false, notCheckable: true };
}
describe('reduceServiceStatus', () => {
it('marks a service not_checkable when it has no checkable ref (STATUS-NOT-CHECKABLE-3)', () => {
const map = new Map([['app:latest', notCheckable()]]);
const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', [], map, undefined);
expect(status).toEqual({ service: 'app', image: 'app:latest', hasUpdate: false, checkStatus: 'not_checkable', lastError: null });
expect(confirmedUpdateThisRun).toBe(false);
});
it('marks a service not_checkable when it declares no image and has no running container', () => {
const map = new Map<string, ImageCheckResult>();
const { status, confirmedUpdateThisRun } = reduceServiceStatus('worker', null, [], map, undefined);
expect(status).toEqual({ service: 'worker', image: null, hasUpdate: false, checkStatus: 'not_checkable', lastError: null });
expect(confirmedUpdateThisRun).toBe(false);
});
it('reports ok + hasUpdate=true when every checkable ref succeeds and one confirms an update', () => {
const map = new Map([['app:latest', ok(true)]]);
const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', [], map, undefined);
expect(status).toEqual({ service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null });
expect(confirmedUpdateThisRun).toBe(true);
});
it('reports ok + hasUpdate=false when every checkable ref succeeds with no update', () => {
const map = new Map([['app:latest', ok(false)]]);
const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', [], map, undefined);
expect(status.checkStatus).toBe('ok');
expect(status.hasUpdate).toBe(false);
expect(confirmedUpdateThisRun).toBe(false);
});
it('STATUS-CONFIRMATION-PARTIAL-5: one newly outdated success + one error -> partial, hasUpdate=true, confirmedUpdateThisRun=true', () => {
// Two refs on the same declared service (declared image + a divergent runtime image).
const map = new Map([
['app:latest', ok(true)],
['app:1.2.3', errored('Registry unreachable for app:1.2.3')],
]);
const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', ['app:1.2.3'], map, undefined);
expect(status.checkStatus).toBe('partial');
expect(status.hasUpdate).toBe(true);
expect(status.lastError).toBe('Registry unreachable for app:1.2.3');
expect(confirmedUpdateThisRun).toBe(true);
});
it('partial with only a preserved prior (no successful outdated check this run) -> confirmedUpdateThisRun=false', () => {
const map = new Map([
['app:latest', ok(false)], // succeeds, but reports no update
['app:1.2.3', errored('timeout')],
]);
const prior: StackServiceStatus = { service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null };
const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', ['app:1.2.3'], map, prior);
expect(status.checkStatus).toBe('partial');
expect(status.hasUpdate).toBe(true); // preserved from prior, not newly confirmed
expect(confirmedUpdateThisRun).toBe(false);
});
it('all checkable refs erroring -> failed, preserves prior hasUpdate, confirmedUpdateThisRun=false', () => {
const map = new Map([['app:latest', errored('Registry unreachable')]]);
const prior: StackServiceStatus = { service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null };
const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', [], map, prior);
expect(status).toEqual({ service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'failed', lastError: 'Registry unreachable' });
expect(confirmedUpdateThisRun).toBe(false);
});
it('all checkable refs erroring with no prior row -> failed, hasUpdate=false', () => {
const map = new Map([['app:latest', errored('boom')]]);
const { status } = reduceServiceStatus('app', 'app:latest', [], map, undefined);
expect(status.hasUpdate).toBe(false);
expect(status.checkStatus).toBe('failed');
});
it('a shared image gets the same check result on every declared service that references it', () => {
const map = new Map([['shared:latest', ok(true)]]);
const web = reduceServiceStatus('web', 'shared:latest', [], map, undefined);
const worker = reduceServiceStatus('worker', 'shared:latest', [], map, undefined);
expect(web.status.hasUpdate).toBe(true);
expect(worker.status.hasUpdate).toBe(true);
expect(web.status.checkStatus).toBe(worker.status.checkStatus);
});
it('records sorted, deduplicated runtimeImages only when a runtime container was observed', () => {
const map = new Map([['app:latest', ok(false)], ['app:1.0', ok(false)]]);
const withRuntime = reduceServiceStatus('app', 'app:latest', ['app:1.0', 'app:1.0'], map, undefined);
expect(withRuntime.status.runtimeImages).toEqual(['app:1.0']);
const noRuntime = reduceServiceStatus('app', 'app:latest', [], map, undefined);
expect(noRuntime.status.runtimeImages).toBeUndefined();
});
});
describe('aggregateServiceCheckStatus', () => {
const svc = (checkStatus: StackServiceStatus['checkStatus'], hasUpdate = false): StackServiceStatus => ({
service: `svc-${checkStatus}-${hasUpdate}`, image: null, hasUpdate, checkStatus, lastError: null,
});
it('is ok for an empty services array (empty effective model)', () => {
expect(aggregateServiceCheckStatus([])).toBe('ok');
});
it('is ok when every service is not_checkable (nothing checkable)', () => {
expect(aggregateServiceCheckStatus([svc('not_checkable'), svc('not_checkable')])).toBe('ok');
});
it('is ok when every checkable service succeeded', () => {
expect(aggregateServiceCheckStatus([svc('ok'), svc('ok'), svc('not_checkable')])).toBe('ok');
});
it('is failed when every checkable service failed, even with a preserved hasUpdate=true', () => {
expect(aggregateServiceCheckStatus([svc('failed', true), svc('failed', true)])).toBe('failed');
});
it('is partial when checkable services mix ok and failed', () => {
expect(aggregateServiceCheckStatus([svc('ok'), svc('failed')])).toBe('partial');
});
it('is partial when a checkable service is itself partial', () => {
expect(aggregateServiceCheckStatus([svc('ok'), svc('partial')])).toBe('partial');
});
it('is not failed when only some (not all) checkable services failed', () => {
expect(aggregateServiceCheckStatus([svc('ok'), svc('failed'), svc('not_checkable')])).not.toBe('failed');
});
});
describe('aggregate invariant: has_update === services.some(hasUpdate)', () => {
it('holds for an empty services array', () => {
const services: StackServiceStatus[] = [];
expect(services.some(s => s.hasUpdate)).toBe(false);
});
it('holds when every checkable service failed but one preserves hasUpdate=true', () => {
const map = new Map([['app:latest', errored('unreachable')]]);
const prior: StackServiceStatus = { service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null };
const { status } = reduceServiceStatus('app', 'app:latest', [], map, prior);
const services = [status];
expect(services.some(s => s.hasUpdate)).toBe(true);
expect(aggregateServiceCheckStatus(services)).toBe('failed');
});
it('holds when one service confirms an update and a sibling service fails outright', () => {
const map = new Map([
['web:latest', ok(true)],
['worker:latest', errored('timeout')],
]);
const web = reduceServiceStatus('web', 'web:latest', [], map, undefined);
const worker = reduceServiceStatus('worker', 'worker:latest', [], map, undefined);
const services = [web.status, worker.status];
// web confirmed an update; worker's sole ref errored, so worker alone is 'failed'.
expect(services.some(s => s.hasUpdate)).toBe(true);
expect(worker.status.checkStatus).toBe('failed');
// At the stack level, one ok service and one failed service is a mix, not a unanimous failure.
expect(aggregateServiceCheckStatus(services)).toBe('partial');
});
});
describe('buildAvailabilityNotifyMessage', () => {
it('uses the generic line for a single-service (or model-unavailable) stack', () => {
expect(buildAvailabilityNotifyMessage('stackA', [])).toBe('Stack "stackA" has image updates available.');
const single: StackServiceStatus[] = [{ service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null }];
expect(buildAvailabilityNotifyMessage('stackA', single)).toBe('Stack "stackA" has image updates available.');
});
it('names sorted services with hasUpdate for a multi-service stack', () => {
const services: StackServiceStatus[] = [
{ service: 'worker', image: 'worker:latest', hasUpdate: true, checkStatus: 'ok', lastError: null },
{ service: 'api', image: 'api:latest', hasUpdate: true, checkStatus: 'ok', lastError: null },
{ service: 'db', image: 'db:latest', hasUpdate: false, checkStatus: 'ok', lastError: null },
];
expect(buildAvailabilityNotifyMessage('stackA', services)).toBe('Stack "stackA" has image updates available for services: api, worker.');
});
it('falls back to the generic line for a multi-service stack with no service currently showing hasUpdate', () => {
const services: StackServiceStatus[] = [
{ service: 'api', image: 'api:latest', hasUpdate: false, checkStatus: 'ok', lastError: null },
{ service: 'db', image: 'db:latest', hasUpdate: false, checkStatus: 'ok', lastError: null },
];
expect(buildAvailabilityNotifyMessage('stackA', services)).toBe('Stack "stackA" has image updates available.');
});
it('uses singular wording for exactly one named service', () => {
const services: StackServiceStatus[] = [
{ service: 'api', image: 'api:latest', hasUpdate: true, checkStatus: 'ok', lastError: null },
{ service: 'db', image: 'db:latest', hasUpdate: false, checkStatus: 'ok', lastError: null },
];
expect(buildAvailabilityNotifyMessage('stackA', services)).toBe('Stack "stackA" has image updates available for service: api.');
});
});
@@ -9,17 +9,19 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
const {
mockGetAuthForRegistry,
mockGetStackUpdateStatus, mockUpsertStackUpdateStatus, mockClearStackUpdateStatus,
mockRecordStackCheckFailure,
mockRecordStackCheckFailure, mockGetStackServicesJson,
mockGetSystemState, mockSetSystemState, mockAddNotificationHistory,
mockDispatchAlert,
mockGetStacks, mockGetStackContent, mockGetEnvContent, mockEnvExists,
mockGetAllContainers, mockGetGlobalSettings, mockInspect,
mockBuildEffectiveServiceModel,
} = vi.hoisted(() => ({
mockGetAuthForRegistry: vi.fn().mockResolvedValue(null),
mockGetStackUpdateStatus: vi.fn().mockReturnValue({}),
mockUpsertStackUpdateStatus: vi.fn(),
mockClearStackUpdateStatus: vi.fn(),
mockRecordStackCheckFailure: vi.fn(),
mockGetStackServicesJson: vi.fn().mockReturnValue([]),
mockGetSystemState: vi.fn().mockReturnValue('1'), // default: backfilled
mockSetSystemState: vi.fn(),
mockAddNotificationHistory: vi.fn(),
@@ -33,6 +35,10 @@ const {
// Backs DockerController.getInstance().getDocker().getImage().inspect() for tests
// that exercise the real checkImage (rather than stubbing it) through checkNode.
mockInspect: vi.fn().mockResolvedValue({ RepoDigests: [] }),
// Defaults to non-renderable so checkNode's per-service reduction is a no-op
// (falls back to the legacy whole-stack tally) unless a test overrides this,
// matching how no real Compose model exists in this unit-test environment.
mockBuildEffectiveServiceModel: vi.fn().mockResolvedValue({ renderable: false, code: 'effective_model_render_failed', error: 'no model in test' }),
}));
vi.mock('../services/RegistryService', () => ({
@@ -54,6 +60,7 @@ vi.mock('../services/DatabaseService', () => ({
getStackUpdateStatus: mockGetStackUpdateStatus,
clearStackUpdateStatus: mockClearStackUpdateStatus,
recordStackCheckFailure: mockRecordStackCheckFailure,
getStackServicesJson: mockGetStackServicesJson,
getSystemState: mockGetSystemState,
setSystemState: mockSetSystemState,
addNotificationHistory: mockAddNotificationHistory,
@@ -61,6 +68,10 @@ vi.mock('../services/DatabaseService', () => ({
},
}));
vi.mock('../services/effectiveServiceModel', () => ({
buildEffectiveServiceModel: mockBuildEffectiveServiceModel,
}));
vi.mock('../services/NotificationService', () => ({
NotificationService: {
getInstance: () => ({
@@ -1437,3 +1448,206 @@ describe('ImageUpdateService cron scheduling', () => {
expect(delay).toBeGreaterThan(0);
});
});
// ── Model-based per-service reduction wiring (§5) ─────────────────
describe('ImageUpdateService - model-based per-service reduction', () => {
const TWO_SERVICE_COMPOSE = `
services:
web:
image: web:latest
worker:
image: worker:latest
`;
const THREE_SERVICE_COMPOSE = `
services:
worker:
image: worker:latest
api:
image: api:latest
db:
image: db:latest
`;
const fakeDb = () => ({
getStackUpdateStatus: mockGetStackUpdateStatus,
upsertStackUpdateStatus: mockUpsertStackUpdateStatus,
clearStackUpdateStatus: mockClearStackUpdateStatus,
recordStackCheckFailure: mockRecordStackCheckFailure,
getSystemState: mockGetSystemState,
setSystemState: mockSetSystemState,
addNotificationHistory: mockAddNotificationHistory,
});
function specFor(name: string, image: string): { name: string; declaredImage: string; hasBuild: boolean; expectedReplicas: number; dependsOn: string[]; hasHealthcheck: boolean } {
return { name, declaredImage: image, hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false };
}
/** Stubs checkImage to report hasUpdate only for the given image refs. */
function stubCheckImagePerRef(service: ImageUpdateService, updatedRefs: string[]) {
(service as any).checkImage = vi.fn().mockImplementation(async (_docker: unknown, ref: string) => ({ hasUpdate: updatedRefs.includes(ref) }));
}
beforeEach(() => {
vi.clearAllMocks();
(ImageUpdateService as any).instance = undefined;
mockGetSystemState.mockReturnValue('1');
mockGetAllContainers.mockResolvedValue([]);
mockEnvExists.mockResolvedValue(false);
});
it('reduces per-service status through the effective model and persists services_json with a generation', async () => {
mockGetStacks.mockResolvedValue(['stackA']);
mockGetStackContent.mockResolvedValue(TWO_SERVICE_COMPOSE);
mockGetStackUpdateStatus.mockReturnValue({});
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
renderable: true,
services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')],
});
const service = ImageUpdateService.getInstance();
stubCheckImagePerRef(service, ['web:latest']);
await (service as any).checkNode(1, fakeDb());
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(
1, 'stackA', true, expect.any(Number), 'ok', null,
[
{ service: 'web', image: 'web:latest', hasUpdate: true, checkStatus: 'ok', lastError: null },
{ service: 'worker', image: 'worker:latest', hasUpdate: false, checkStatus: 'ok', lastError: null },
],
expect.any(Number),
);
});
it('falls back to the legacy whole-stack tally when the effective model is not renderable', async () => {
mockGetStacks.mockResolvedValue(['stackA']);
mockGetStackContent.mockResolvedValue(TWO_SERVICE_COMPOSE);
mockGetStackUpdateStatus.mockReturnValue({});
mockBuildEffectiveServiceModel.mockResolvedValueOnce({ renderable: false, code: 'effective_model_render_failed', error: 'render failed' });
const service = ImageUpdateService.getInstance();
stubCheckImagePerRef(service, ['web:latest']);
await (service as any).checkNode(1, fakeDb());
// Legacy path never passes a services/generation payload.
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(1, 'stackA', true, expect.any(Number), 'ok', null);
});
it('names sorted services with hasUpdate in the availability notification for a multi-service stack', async () => {
mockGetStacks.mockResolvedValue(['stackA']);
mockGetStackContent.mockResolvedValue(THREE_SERVICE_COMPOSE);
mockGetStackUpdateStatus.mockReturnValue({ stackA: false });
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
renderable: true,
services: [specFor('worker', 'worker:latest'), specFor('api', 'api:latest'), specFor('db', 'db:latest')],
});
const service = ImageUpdateService.getInstance();
stubCheckImagePerRef(service, ['worker:latest', 'api:latest']);
await (service as any).checkNode(1, fakeDb());
expect(mockDispatchAlert).toHaveBeenCalledWith(
'info',
'image_update_available',
'Stack "stackA" has image updates available for services: api, worker.',
{ stackName: 'stackA', actor: 'system:image-update' },
);
});
it('does not re-notify a multi-service stack already known to have updates', async () => {
mockGetStacks.mockResolvedValue(['stackA']);
mockGetStackContent.mockResolvedValue(TWO_SERVICE_COMPOSE);
mockGetStackUpdateStatus.mockReturnValue({ stackA: true });
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
renderable: true,
services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')],
});
const service = ImageUpdateService.getInstance();
stubCheckImagePerRef(service, ['web:latest']);
await (service as any).checkNode(1, fakeDb());
expect(mockDispatchAlert).not.toHaveBeenCalled();
});
it('suppresses notification and count side effects when a newer recheck discards a stale full-scan write', async () => {
mockGetStacks.mockResolvedValue(['stackA']);
mockGetStackContent.mockResolvedValue(TWO_SERVICE_COMPOSE);
mockGetStackUpdateStatus.mockReturnValue({});
// Full-scan write path (after registry work) and recheck both need a model.
mockBuildEffectiveServiceModel.mockResolvedValue({
renderable: true,
services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')],
});
const service = ImageUpdateService.getInstance();
let resolveFullScan: ((value: { hasUpdate: boolean }) => void) | undefined;
let fullScanPending = true;
(service as any).checkImage = vi.fn().mockImplementation(async () => {
if (fullScanPending) {
return new Promise<{ hasUpdate: boolean }>((resolve) => {
resolveFullScan = resolve;
});
}
return { hasUpdate: false };
});
const fullScan = (service as any).checkNode(1, fakeDb());
await vi.waitFor(() => expect((service as any).checkImage).toHaveBeenCalled());
// A newer service recheck reserves a higher generation and commits "no update".
fullScanPending = false;
await service.recheckStack(1, 'stackA');
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(
1, 'stackA', false, expect.any(Number), 'ok', null,
expect.any(Array),
expect.any(Number),
);
const upsertsAfterRecheck = mockUpsertStackUpdateStatus.mock.calls.length;
expect(mockDispatchAlert).not.toHaveBeenCalled();
// Stale full scan finishes with hasUpdate=true but must not commit or notify.
resolveFullScan?.({ hasUpdate: true });
await fullScan;
expect(mockUpsertStackUpdateStatus.mock.calls.length).toBe(upsertsAfterRecheck);
expect(mockDispatchAlert).not.toHaveBeenCalled();
});
describe('recheckStack', () => {
it('persists a fresh per-service reduction and returns no warning on success', async () => {
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
renderable: true,
services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')],
});
mockGetAllContainers.mockResolvedValue([
{ Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } },
]);
const service = ImageUpdateService.getInstance();
stubCheckImagePerRef(service, ['web:latest']);
const result = await service.recheckStack(1, 'stackA');
expect(result).toEqual({ warning: null });
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(
1, 'stackA', true, expect.any(Number), 'ok', null,
[
{ service: 'web', image: 'web:latest', runtimeImages: ['web:latest'], hasUpdate: true, checkStatus: 'ok', lastError: null },
{ service: 'worker', image: 'worker:latest', hasUpdate: false, checkStatus: 'ok', lastError: null },
],
expect.any(Number),
);
});
it('returns a warning and leaves the prior row untouched when the model cannot render', async () => {
mockBuildEffectiveServiceModel.mockResolvedValueOnce({ renderable: false, code: 'effective_model_render_failed', error: 'no model in test' });
const service = ImageUpdateService.getInstance();
const result = await service.recheckStack(1, 'stackA');
expect(result).toEqual({ warning: 'no model in test' });
expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled();
});
});
});
@@ -422,7 +422,7 @@ describe('POST /api/auto-update/execute', () => {
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
.mockResolvedValue({ hasUpdate: true } as never);
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue();
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-au');
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-au');
try {
const res = await request(app)
.post('/api/auto-update/execute')
+35
View File
@@ -259,6 +259,18 @@ describe('DockerController.buildPrunePlan', () => {
// Unique bytes per image = Size - SharedSize = 100MB each, not full 1GB each.
expect(plan.reclaimableBytes).toBe(200_000_000);
});
it('excludes an image held for a pending service-update recovery from the plan', async () => {
mockDocker.listImages.mockResolvedValue([
{ Id: 'img-held', RepoTags: ['app:1'], Size: 100, Containers: 0 },
{ Id: 'img-free', RepoTags: ['app:2'], Size: 100, Containers: 0 },
]);
const dc = DockerController.getInstance(1);
const plan = await dc.buildPrunePlan(['images'], 'all', [], 1, (id) => id === 'img-held');
expect(plan.items.map((i) => i.id)).toEqual(['img-free']);
});
});
describe('DockerController.executePrunePlan', () => {
@@ -428,4 +440,27 @@ describe('DockerController.executePrunePlan', () => {
expect(remove).toHaveBeenCalledWith({ force: false });
});
it('re-checks isImageHeld immediately before deleting, skipping a hold that started after the plan was built', async () => {
mockDocker.listImages.mockResolvedValue([
{ Id: 'img1', RepoTags: ['app:latest'], Size: 1000, Containers: 0 },
]);
const imageRemove = vi.fn().mockResolvedValue(undefined);
mockDocker.getImage.mockReturnValue({ remove: imageRemove });
const dc = DockerController.getInstance(1);
const plan = await dc.buildPrunePlan(['images'], 'all', [], 1);
expect(plan.items.map((i) => i.id)).toEqual(['img1']);
// Bypass the rebuild-based staleness check so this isolates the
// immediate per-item recheck (prune holds), simulating a
// recovery snapshot created after the plan was built but before execute.
vi.spyOn(dc, 'assertPlanFresh').mockResolvedValue(plan);
const result = await dc.executePrunePlan(plan, [], (id) => id === 'img1');
expect(imageRemove).not.toHaveBeenCalled();
expect(result.outcomes).toEqual([
expect.objectContaining({ id: 'img1', target: 'images', status: 'skipped', reason: expect.stringMatching(/held/i) }),
]);
});
});
@@ -690,8 +690,8 @@ describe('SchedulerService - executePrune', () => {
await svc.triggerTask(71);
expect(mockPruneSystem).toHaveBeenCalledTimes(2);
expect(mockPruneSystem).toHaveBeenCalledWith('images', undefined);
expect(mockPruneSystem).toHaveBeenCalledWith('volumes', undefined);
expect(mockPruneSystem).toHaveBeenCalledWith('images', undefined, expect.any(Function));
expect(mockPruneSystem).toHaveBeenCalledWith('volumes', undefined, expect.any(Function));
});
it('includes label filter when configured', async () => {
@@ -711,7 +711,7 @@ describe('SchedulerService - executePrune', () => {
const svc = SchedulerService.getInstance();
await svc.triggerTask(72);
expect(mockPruneSystem).toHaveBeenCalledWith('containers', 'env=staging');
expect(mockPruneSystem).toHaveBeenCalledWith('containers', 'env=staging', expect.any(Function));
});
it('marks scheduled prune runs as failed when a target prune fails', async () => {
@@ -828,7 +828,7 @@ describe('SchedulerService - executeUpdate', () => {
it('begins a health gate after a scheduled update succeeds', async () => {
const { HealthGateService } = await import('../services/HealthGateService');
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-1');
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-1');
try {
mockGetScheduledTask.mockReturnValue({
id: 83,
@@ -0,0 +1,217 @@
/**
* Route smoke tests for the nested service update/restore endpoints: auth and
* permission gates, the restore recoveryId requirement, and the mapping from
* OrchestratorResult to HTTP status. The orchestrator itself is mocked so these
* tests exercise only the route wiring.
*/
import fs from 'fs';
import path from 'path';
import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
import request from 'supertest';
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
const { mockExecute } = vi.hoisted(() => ({ mockExecute: vi.fn() }));
vi.mock('../services/StackUpdateOrchestrator', () => ({
StackUpdateOrchestrator: { getInstance: () => ({ execute: mockExecute }) },
}));
let tmpDir: string;
let app: import('express').Express;
let adminCookie: string;
let viewerCookie: string;
function writeStack(name: string) {
const dir = path.join(process.env.COMPOSE_DIR!, name);
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, 'compose.yaml'), 'services:\n app:\n image: nginx\n db:\n image: postgres\n');
}
async function loginAsViewer(): Promise<string> {
const bcrypt = (await import('bcrypt')).default;
const { DatabaseService } = await import('../services/DatabaseService');
const hash = await bcrypt.hash('viewerpass', 1);
DatabaseService.getInstance().addUser({ username: 'viewer1', password_hash: hash, role: 'viewer' });
const res = await request(app).post('/api/auth/login').send({ username: 'viewer1', password: 'viewerpass' });
const cookies = res.headers['set-cookie'] as string | string[];
return Array.isArray(cookies) ? cookies[0] : cookies;
}
beforeAll(async () => {
tmpDir = await setupTestDb();
({ app } = await import('../index'));
adminCookie = await loginAsTestAdmin(app);
viewerCookie = await loginAsViewer();
writeStack('web');
const { NotificationService } = await import('../services/NotificationService');
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
});
afterAll(() => {
vi.restoreAllMocks();
cleanupTestDb(tmpDir);
});
afterEach(async () => {
mockExecute.mockReset();
const { StackOpLockService } = await import('../services/StackOpLockService');
StackOpLockService.resetForTests();
});
describe('nested service route auth and permission gates', () => {
it('rejects an unauthenticated update with 401', async () => {
const res = await request(app).post('/api/stacks/web/services/app/update');
expect(res.status).toBe(401);
expect(mockExecute).not.toHaveBeenCalled();
});
it('rejects an unauthenticated restore with 401', async () => {
const res = await request(app).post('/api/stacks/web/services/app/restore').send({ recoveryId: 'r1' });
expect(res.status).toBe(401);
expect(mockExecute).not.toHaveBeenCalled();
});
it('rejects a viewer without stack:deploy with 403', async () => {
const res = await request(app)
.post('/api/stacks/web/services/app/update')
.set('Cookie', viewerCookie);
expect(res.status).toBe(403);
expect(mockExecute).not.toHaveBeenCalled();
});
});
describe('restore recoveryId requirement', () => {
it('rejects a restore with no recoveryId with 400 recovery_id_required', async () => {
const res = await request(app)
.post('/api/stacks/web/services/app/restore')
.set('Cookie', adminCookie)
.send({});
expect(res.status).toBe(400);
expect(res.body.code).toBe('recovery_id_required');
expect(mockExecute).not.toHaveBeenCalled();
});
});
describe('OrchestratorResult to HTTP mapping', () => {
it('maps service_done to 200 with the result body', async () => {
mockExecute.mockResolvedValue({
kind: 'service_done',
serviceName: 'app',
healthGateId: 'hg-1',
observing: true,
recoveryId: 'rec-1',
recoveryAvailable: true,
});
const res = await request(app)
.post('/api/stacks/web/services/app/update')
.set('Cookie', adminCookie);
expect(res.status).toBe(200);
expect(res.body).toMatchObject({ serviceName: 'app', healthGateId: 'hg-1', recoveryId: 'rec-1', recoveryAvailable: true });
expect(mockExecute).toHaveBeenCalledTimes(1);
});
it('maps service_update_single_service to 400', async () => {
mockExecute.mockResolvedValue({ kind: 'service_failed', code: 'service_update_single_service', error: 'only one service' });
const res = await request(app)
.post('/api/stacks/web/services/app/update')
.set('Cookie', adminCookie);
expect(res.status).toBe(400);
expect(res.body.code).toBe('service_update_single_service');
});
it('maps a policy block to 409', async () => {
mockExecute.mockResolvedValue({ kind: 'service_failed', code: 'policy_blocked', error: 'blocked' });
const res = await request(app)
.post('/api/stacks/web/services/app/update')
.set('Cookie', adminCookie);
expect(res.status).toBe(409);
expect(res.body.code).toBe('policy_blocked');
});
it('maps a service compose failure to 500', async () => {
mockExecute.mockResolvedValue({ kind: 'service_failed', code: 'service_update_compose_failed', error: 'compose exploded' });
const res = await request(app)
.post('/api/stacks/web/services/app/update')
.set('Cookie', adminCookie);
expect(res.status).toBe(500);
expect(res.body.code).toBe('service_update_compose_failed');
});
it('maps a restore service_done to 200', async () => {
mockExecute.mockResolvedValue({
kind: 'service_done',
serviceName: 'app',
healthGateId: 'hg-2',
observing: true,
recoveryId: 'rec-2',
recoveryAvailable: false,
});
const res = await request(app)
.post('/api/stacks/web/services/app/restore')
.set('Cookie', adminCookie)
.send({ recoveryId: 'rec-2' });
expect(res.status).toBe(200);
expect(res.body).toMatchObject({ serviceName: 'app', healthGateId: 'hg-2', recoveryId: 'rec-2' });
expect(mockExecute).toHaveBeenCalledTimes(1);
});
});
describe('GET /api/stacks/:stackName/services/:serviceName/recovery', () => {
it('returns null when no active recovery exists', async () => {
const res = await request(app)
.get('/api/stacks/web/services/app/recovery')
.set('Cookie', adminCookie);
expect(res.status).toBe(200);
expect(res.body).toEqual({ recovery: null });
});
it('returns the newest active recovery row', async () => {
const { DatabaseService } = await import('../services/DatabaseService');
const { NodeRegistry } = await import('../services/NodeRegistry');
const db = DatabaseService.getInstance();
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
const now = Date.now();
db.insertServiceUpdateRecovery({
id: 'rec-old',
node_id: nodeId,
stack_name: 'web',
service_name: 'app',
replicas_json: '[]',
majority_image_id: 'sha256:old',
declared_image_ref: 'nginx:latest',
weak_floating_tag: 0,
health_gate_id: null,
status: 'active',
expires_at: now + 60_000,
claim_expires_at: null,
created_at: now - 1_000,
created_by: 'tester',
});
db.insertServiceUpdateRecovery({
id: 'rec-new',
node_id: nodeId,
stack_name: 'web',
service_name: 'app',
replicas_json: '[]',
majority_image_id: 'sha256:new',
declared_image_ref: 'nginx:latest',
weak_floating_tag: 0,
health_gate_id: 'gate-1',
status: 'active',
expires_at: now + 60_000,
claim_expires_at: null,
created_at: now,
created_by: 'tester',
});
const res = await request(app)
.get('/api/stacks/web/services/app/recovery')
.set('Cookie', adminCookie);
expect(res.status).toBe(200);
expect(res.body.recovery).toMatchObject({
id: 'rec-new',
healthGateId: 'gate-1',
majorityImageId: 'sha256:new',
});
});
});
@@ -0,0 +1,309 @@
/**
* Coverage for ServiceUpdateRecoveryService against a real DatabaseService
* (temp DB) with fake timers: eligibility computation (§8 "Unavailable"
* cases), the claim/renewal lifecycle (RECOVERY-CLAIM-2, no revive of a
* terminal row), the sweep timer, held image ids, and start/stop timer
* hygiene.
*/
import { describe, it, expect, vi, beforeAll, afterAll, beforeEach, afterEach } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import type { ServiceReplicaSnapshot } from '../services/ServiceUpdateRecoveryService';
let tmpDir: string;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let ServiceUpdateRecoveryService: typeof import('../services/ServiceUpdateRecoveryService').ServiceUpdateRecoveryService;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
({ ServiceUpdateRecoveryService } = await import('../services/ServiceUpdateRecoveryService'));
});
afterAll(() => cleanupTestDb(tmpDir));
function db() {
return DatabaseService.getInstance();
}
const svc = () => ServiceUpdateRecoveryService.getInstance();
const BASE_INPUT = {
nodeId: 1,
stackName: 'web',
serviceName: 'api',
declaredImageRef: 'ghcr.io/acme/api:v1',
createdBy: 'tester',
};
beforeEach(() => {
vi.useFakeTimers();
const raw = (db() as unknown as { db: { prepare: (s: string) => { run: () => void } } }).db;
raw.prepare('DELETE FROM service_update_recovery').run();
raw.prepare("DELETE FROM global_settings WHERE key = 'health_gate_window_seconds'").run();
// updateGlobalSetting() invalidates this cache; the raw DELETE above bypasses
// it, so a value set by a previous test would otherwise leak into this one.
(db() as unknown as { cachedGlobalSettings: unknown }).cachedGlobalSettings = null;
svc().start();
});
afterEach(() => {
svc().stop();
expect(vi.getTimerCount()).toBe(0);
vi.useRealTimers();
});
describe('createIfEligible - unavailable cases', () => {
it('rejects a service with no replicas', () => {
const result = svc().createIfEligible({ ...BASE_INPUT, replicas: [] });
expect(result).toEqual({ eligible: false, reason: 'no_replicas' });
});
it('rejects replicas with no local image id', () => {
const replicas: ServiceReplicaSnapshot[] = [{ imageId: '', repoDigest: null }, { imageId: ' ', repoDigest: null }];
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
expect(result).toEqual({ eligible: false, reason: 'missing_local_id' });
});
it('rejects a majority tie between two equally-common images', () => {
const replicas: ServiceReplicaSnapshot[] = [
{ imageId: 'sha256:aaa', repoDigest: null },
{ imageId: 'sha256:bbb', repoDigest: null },
];
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
expect(result).toEqual({ eligible: false, reason: 'majority_tie' });
});
it('rejects a pure-build service with no declared image ref', () => {
const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }];
const result = svc().createIfEligible({ ...BASE_INPUT, declaredImageRef: null, replicas });
expect(result).toEqual({ eligible: false, reason: 'build_only' });
});
it('rejects a digest-pinned declared image ref', () => {
const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }];
const result = svc().createIfEligible({
...BASE_INPUT,
declaredImageRef: 'ghcr.io/acme/api@sha256:' + 'a'.repeat(64),
replicas,
});
expect(result).toEqual({ eligible: false, reason: 'digest_pinned_declared_ref' });
});
});
describe('createIfEligible - eligible cases', () => {
it('persists the majority image and flags a weak floating tag when no replica has a captured digest', () => {
const replicas: ServiceReplicaSnapshot[] = [
{ imageId: 'sha256:aaa', repoDigest: null },
{ imageId: 'sha256:aaa', repoDigest: null },
{ imageId: 'sha256:bbb', repoDigest: null },
];
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
expect(result.eligible).toBe(true);
if (!result.eligible) throw new Error('expected eligible');
expect(result.row).toMatchObject({
node_id: 1, stack_name: 'web', service_name: 'api',
majority_image_id: 'sha256:aaa', declared_image_ref: 'ghcr.io/acme/api:v1',
weak_floating_tag: 1, status: 'active', health_gate_id: null,
});
expect(db().getServiceUpdateRecovery(result.row.id)).toBeTruthy();
});
it('clears the weak floating tag flag when a majority replica has a captured digest', () => {
const replicas: ServiceReplicaSnapshot[] = [
{ imageId: 'sha256:aaa', repoDigest: 'sha256:digest-aaa' },
{ imageId: 'sha256:aaa', repoDigest: 'sha256:digest-aaa' },
{ imageId: 'sha256:bbb', repoDigest: null },
];
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
if (!result.eligible) throw new Error('expected eligible');
expect(result.row.weak_floating_tag).toBe(0);
});
it('sets expires_at from max(compose timeout, health window) plus buffer', () => {
db().updateGlobalSetting('health_gate_window_seconds', '120');
const now = Date.now();
const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }];
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
if (!result.eligible) throw new Error('expected eligible');
// Default compose timeout is 30m; health window 120s is smaller, so compose wins.
expect(result.row.expires_at).toBe(now + 30 * 60_000 + 30 * 60_000);
});
it('uses a raised health window when it exceeds the compose command timeout', () => {
db().updateGlobalSetting('health_gate_window_seconds', String(45 * 60));
const now = Date.now();
const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }];
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
if (!result.eligible) throw new Error('expected eligible');
expect(result.row.expires_at).toBe(now + 45 * 60_000 + 30 * 60_000);
});
it('floors the health window to 90s but still honors the compose timeout floor', () => {
db().updateGlobalSetting('health_gate_window_seconds', '15');
const now = Date.now();
const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }];
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
if (!result.eligible) throw new Error('expected eligible');
expect(result.row.expires_at).toBe(now + 30 * 60_000 + 30 * 60_000);
});
});
describe('claim + mandatory renewal (RECOVERY-CLAIM-2)', () => {
function seedEligible(): string {
const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: 'sha256:digest-aaa' }];
const result = svc().createIfEligible({ ...BASE_INPUT, replicas });
if (!result.eligible) throw new Error('expected eligible');
return result.row.id;
}
it('claims an active row and moves it to restoring with a claim window at least 30m + 5m out', () => {
const id = seedEligible();
const now = Date.now();
const claimed = svc().claim(id);
expect(claimed?.status).toBe('restoring');
expect(claimed?.claim_expires_at).toBeGreaterThanOrEqual(now + 30 * 60_000 + 5 * 60_000);
});
it('renews the claim every 5 minutes while restoring, keeping a long restore held', () => {
const id = seedEligible();
svc().claim(id);
const firstExpiry = db().getServiceUpdateRecovery(id)?.claim_expires_at ?? 0;
svc().startClaimRenewal(id);
vi.advanceTimersByTime(5 * 60_000);
const renewedExpiry = db().getServiceUpdateRecovery(id)?.claim_expires_at ?? 0;
expect(renewedExpiry).toBeGreaterThan(firstExpiry);
expect(db().getServiceUpdateRecovery(id)?.status).toBe('restoring');
svc().stopClaimRenewal(id);
});
it('does not revive a row that left restoring mid-loop (renewal self-stops, never overwrites the terminal status)', () => {
const id = seedEligible();
svc().claim(id);
svc().startClaimRenewal(id);
// Simulate the restore finishing (consumed) between renewal ticks.
db().markServiceUpdateRecoveryConsumed(id, 'restore-gate');
const claimExpiresBefore = db().getServiceUpdateRecovery(id)?.claim_expires_at;
vi.advanceTimersByTime(5 * 60_000);
const row = db().getServiceUpdateRecovery(id);
expect(row?.status).toBe('consumed');
expect(row?.claim_expires_at).toBe(claimExpiresBefore);
svc().stopClaimRenewal(id);
});
it('stops renewing once stopClaimRenewal is called (finally-block contract)', () => {
const id = seedEligible();
svc().claim(id);
svc().startClaimRenewal(id);
svc().stopClaimRenewal(id);
const expiryAtStop = db().getServiceUpdateRecovery(id)?.claim_expires_at;
vi.advanceTimersByTime(15 * 60_000);
expect(db().getServiceUpdateRecovery(id)?.claim_expires_at).toBe(expiryAtStop);
});
it('is idempotent: a second startClaimRenewal call for the same id does not schedule a duplicate timer', () => {
const id = seedEligible();
svc().claim(id);
const before = vi.getTimerCount();
svc().startClaimRenewal(id);
svc().startClaimRenewal(id);
expect(vi.getTimerCount()).toBe(before + 1);
svc().stopClaimRenewal(id);
});
it('refuses to start a renewal loop once the service has been stopped', () => {
const id = seedEligible();
svc().claim(id);
svc().stop();
const before = vi.getTimerCount();
svc().startClaimRenewal(id);
expect(vi.getTimerCount()).toBe(before);
});
});
describe('sweep()', () => {
it('expires an abandoned restoring claim but leaves a live one and an unexpired active one alone', () => {
const now = Date.now();
db().insertServiceUpdateRecovery({
id: 'rec-abandoned', node_id: 1, stack_name: 'web', service_name: 'api',
replicas_json: '[]', majority_image_id: 'sha256:aaa', declared_image_ref: 'x:1',
weak_floating_tag: 0, health_gate_id: null, status: 'restoring',
expires_at: now - 1_000, claim_expires_at: now - 1_000, created_at: now - 60_000, created_by: null,
});
db().insertServiceUpdateRecovery({
id: 'rec-live', node_id: 1, stack_name: 'web', service_name: 'db',
replicas_json: '[]', majority_image_id: 'sha256:bbb', declared_image_ref: 'x:2',
weak_floating_tag: 0, health_gate_id: null, status: 'restoring',
expires_at: now - 1_000, claim_expires_at: now + 60 * 60_000, created_at: now - 60_000, created_by: null,
});
db().insertServiceUpdateRecovery({
id: 'rec-active', node_id: 1, stack_name: 'web', service_name: 'cache',
replicas_json: '[]', majority_image_id: 'sha256:ccc', declared_image_ref: 'x:3',
weak_floating_tag: 0, health_gate_id: null, status: 'active',
expires_at: now + 60 * 60_000, claim_expires_at: null, created_at: now, created_by: null,
});
svc().sweep();
expect(db().getServiceUpdateRecovery('rec-abandoned')?.status).toBe('expired');
expect(db().getServiceUpdateRecovery('rec-live')?.status).toBe('restoring');
expect(db().getServiceUpdateRecovery('rec-active')?.status).toBe('active');
});
it('runs automatically on the interval timer after start()', () => {
const now = Date.now();
db().insertServiceUpdateRecovery({
id: 'rec-ttl', node_id: 1, stack_name: 'web', service_name: 'api',
replicas_json: '[]', majority_image_id: 'sha256:aaa', declared_image_ref: 'x:1',
weak_floating_tag: 0, health_gate_id: null, status: 'active',
expires_at: now + 1_000, claim_expires_at: null, created_at: now, created_by: null,
});
vi.advanceTimersByTime(30_000 + 5 * 60_000 + 1_000);
expect(db().getServiceUpdateRecovery('rec-ttl')?.status).toBe('expired');
});
});
describe('getHeldImageIds', () => {
it('wraps the DB projection for one node', () => {
const now = Date.now();
db().insertServiceUpdateRecovery({
id: 'rec-held', node_id: 1, stack_name: 'web', service_name: 'api',
replicas_json: '[]', majority_image_id: 'sha256:held', declared_image_ref: 'x:1',
weak_floating_tag: 0, health_gate_id: null, status: 'active',
expires_at: now + 60_000, claim_expires_at: null, created_at: now, created_by: null,
});
expect(svc().getHeldImageIds(1)).toEqual(new Set(['sha256:held']));
});
it('fails closed (null) when the DB read throws', () => {
const spy = vi.spyOn(db(), 'listHeldServiceUpdateRecoveryImageIds').mockImplementation(() => {
throw new Error('boom');
});
expect(svc().getHeldImageIds(1)).toBeNull();
spy.mockRestore();
});
});
describe('start() / stop() timer hygiene', () => {
it('clears the sweep timer and stops firing after stop()', () => {
const spy = vi.spyOn(svc(), 'sweep');
svc().stop();
expect(vi.getTimerCount()).toBe(0);
svc().start();
vi.advanceTimersByTime(30_000);
expect(spy).toHaveBeenCalledTimes(1);
svc().stop();
vi.advanceTimersByTime(5 * 60_000);
expect(spy).toHaveBeenCalledTimes(1);
spy.mockRestore();
});
});
@@ -34,6 +34,23 @@ describe('StackOpMetricsService', () => {
successCount: 2,
errorCount: 1,
avgMs: 200,
targetScope: 'stack',
serviceName: null,
});
});
it('records the latest service-scoped metadata on the action bucket', () => {
const svc = StackOpMetricsService.getInstance();
svc.record(1, 'update', 50, true, { targetScope: 'service', serviceName: 'web' });
expect(svc.snapshot()[0]).toMatchObject({
action: 'update',
targetScope: 'service',
serviceName: 'web',
});
svc.record(1, 'update', 40, true, { targetScope: 'stack' });
expect(svc.snapshot()[0]).toMatchObject({
targetScope: 'stack',
serviceName: null,
});
});
@@ -0,0 +1,122 @@
/**
* Unit tests for StackUpdateOrchestrator dispatch: the stack branch runs
* ComposeService.updateStack (side effects stay with callers), the service
* branch hard-rejects any non-manual trigger, and a single-service stack is
* refused with the stable `service_update_single_service` code before any
* mutation.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import type { EffectiveServiceModelResult, EffectiveServiceSpec } from '../services/effectiveServiceModel';
const { state } = vi.hoisted(() => ({
state: {
updateStack: vi.fn(),
model: null as EffectiveServiceModelResult | null,
},
}));
vi.mock('../services/ComposeService', () => ({
ComposeService: {
getInstance: () => ({ updateStack: state.updateStack }),
},
}));
vi.mock('../services/effectiveServiceModel', () => ({
buildEffectiveServiceModel: vi.fn(async () => state.model),
}));
import { StackUpdateOrchestrator, evaluateServiceReplicaConvergence, shortImageId } from '../services/StackUpdateOrchestrator';
const orch = () => StackUpdateOrchestrator.getInstance();
function spec(name: string): EffectiveServiceSpec {
return { name, declaredImage: 'nginx:latest', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false };
}
beforeEach(() => {
state.updateStack.mockReset();
state.updateStack.mockResolvedValue(undefined);
state.model = null;
});
describe('evaluateServiceReplicaConvergence', () => {
it('converges when the exact expected running replicas share one image', () => {
expect(evaluateServiceReplicaConvergence('api', 3, ['sha:a', 'sha:a', 'sha:a'], 0)).toEqual({
kind: 'converged', imageId: 'sha:a',
});
});
it('reports divergent when running replica count is below expected', () => {
const result = evaluateServiceReplicaConvergence('api', 3, ['sha:a'], 0);
expect(result).toMatchObject({ kind: 'divergent' });
expect((result as { error: string }).error).toMatch(/1 running.*expected 3/i);
});
it('reports divergent when no running replicas remain', () => {
const result = evaluateServiceReplicaConvergence('api', 2, [], 0);
expect(result).toMatchObject({ kind: 'divergent' });
expect((result as { error: string }).error).toMatch(/no running replicas/i);
});
it('reports divergent when replicas disagree on image', () => {
const result = evaluateServiceReplicaConvergence('api', 2, ['sha:a', 'sha:b'], 0);
expect(result).toMatchObject({ kind: 'divergent' });
expect((result as { error: string }).error).toMatch(/did not converge/i);
});
it('reports inspect_failed when every inspect failed', () => {
expect(evaluateServiceReplicaConvergence('api', 2, [], 2)).toMatchObject({ kind: 'inspect_failed' });
});
it('treats scale-zero as converged with a null image', () => {
expect(evaluateServiceReplicaConvergence('api', 0, [], 0)).toEqual({ kind: 'converged', imageId: null });
});
});
describe('shortImageId', () => {
it('strips the sha256 prefix and truncates', () => {
expect(shortImageId('sha256:abcdef0123456789')).toBe('abcdef012345');
});
});
describe('StackUpdateOrchestrator stack branch', () => {
it('runs ComposeService.updateStack and returns stack_compose_done', async () => {
const result = await orch().execute(
{ nodeId: 0, stackName: 'web', target: { scope: 'stack' }, trigger: 'manual', actor: 'tester' },
{ atomic: true, terminalWs: null },
);
expect(result).toEqual({ kind: 'stack_compose_done' });
expect(state.updateStack).toHaveBeenCalledWith('web', undefined, true);
});
});
describe('StackUpdateOrchestrator service branch guards', () => {
it('rejects a service-scoped update with a non-manual trigger', async () => {
await expect(
orch().execute(
{ nodeId: 0, stackName: 'web', target: { scope: 'service', serviceName: 'app' }, trigger: 'scheduled', actor: 'system' },
{ policyOptions: { bypass: false, actor: 'system' } },
),
).rejects.toThrow(/manual/i);
expect(state.updateStack).not.toHaveBeenCalled();
});
it('refuses a single-service stack with code service_update_single_service', async () => {
state.model = { renderable: true, services: [spec('only')] };
const result = await orch().execute(
{ nodeId: 0, stackName: 'web', target: { scope: 'service', serviceName: 'only' }, trigger: 'manual', actor: 'tester' },
{ policyOptions: { bypass: false, actor: 'tester' } },
);
expect(result).toMatchObject({ kind: 'service_failed', code: 'service_update_single_service' });
expect(state.updateStack).not.toHaveBeenCalled();
});
it('fails closed when the effective model cannot render', async () => {
state.model = { renderable: false, code: 'effective_model_render_failed', error: 'boom' };
const result = await orch().execute(
{ nodeId: 0, stackName: 'web', target: { scope: 'service', serviceName: 'app' }, trigger: 'manual', actor: 'tester' },
{ policyOptions: { bypass: false, actor: 'tester' } },
);
expect(result).toMatchObject({ kind: 'service_failed', code: 'effective_model_render_failed' });
});
});
@@ -251,7 +251,7 @@ describe('health gate begin call sites', () => {
beforeEach(async () => {
const { HealthGateService } = await import('../services/HealthGateService');
beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-123') as ReturnType<typeof vi.spyOn>;
beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-123') as ReturnType<typeof vi.spyOn>;
});
afterEach(() => {
@@ -12,10 +12,15 @@ import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTes
const {
mockComputeUpdateReadiness,
mockComputeRollbackReadiness,
} = vi.hoisted(() => ({
mockComputeUpdateReadiness: vi.fn(),
mockComputeRollbackReadiness: vi.fn(),
}));
MockSingleServiceUpdateReadinessError,
} = vi.hoisted(() => {
class MockSingleServiceUpdateReadinessError extends Error {}
return {
mockComputeUpdateReadiness: vi.fn(),
mockComputeRollbackReadiness: vi.fn(),
MockSingleServiceUpdateReadinessError,
};
});
vi.mock('../services/UpdateGuardService', () => ({
UpdateGuardService: {
@@ -24,6 +29,7 @@ vi.mock('../services/UpdateGuardService', () => ({
computeRollbackReadiness: mockComputeRollbackReadiness,
}),
},
SingleServiceUpdateReadinessError: MockSingleServiceUpdateReadinessError,
}));
vi.mock('../services/FileSystemService', () => ({
@@ -85,6 +91,28 @@ describe('GET /api/stacks/:stackName/update-readiness', () => {
expect(res.status).toBe(500);
expect(res.body).toEqual({ error: 'Failed to compute update readiness' });
});
it('forwards the service query param to computeUpdateReadiness', async () => {
const report = { stack: 'web', computedAt: 1, verdict: 'ready', signals: [], serviceName: 'api', advisories: [] };
mockComputeUpdateReadiness.mockResolvedValue(report);
const res = await request(app).get('/api/stacks/web/update-readiness?service=api').set('Cookie', authCookie);
expect(res.status).toBe(200);
expect(res.body).toEqual(report);
expect(mockComputeUpdateReadiness).toHaveBeenCalledWith(expect.any(Number), 'web', 'api');
});
it('omits the service argument when no service query param is given', async () => {
mockComputeUpdateReadiness.mockResolvedValue({ stack: 'web', computedAt: 1, verdict: 'ready', signals: [] });
await request(app).get('/api/stacks/web/update-readiness').set('Cookie', authCookie);
expect(mockComputeUpdateReadiness).toHaveBeenCalledWith(expect.any(Number), 'web', undefined);
});
it('returns 400 when the service query targets a single-service stack', async () => {
mockComputeUpdateReadiness.mockRejectedValue(new MockSingleServiceUpdateReadinessError('single service'));
const res = await request(app).get('/api/stacks/web/update-readiness?service=api').set('Cookie', authCookie);
expect(res.status).toBe(400);
expect(res.body).toMatchObject({ code: 'service_update_single_service' });
});
});
describe('GET /api/stacks/:stackName/rollback-readiness', () => {
@@ -120,6 +148,7 @@ describe('GET /api/stacks/:stackName/health-gate', () => {
db.insertHealthGateRun({
id, node_id: defaultNodeId, stack_name: 'web', trigger_action: 'update', status, reason,
window_seconds: 90, containers_json: '[]', started_at: startedAt, ended_at: status === 'observing' ? null : startedAt + 1000, created_by: 'tester',
target_scope: 'stack', service_name: null, failure_source: null,
});
};
@@ -16,6 +16,7 @@ const {
mockGetOpenDriftFindings,
mockGetGlobalSettings,
mockFsSize,
mockBuildEffectiveServiceModel,
} = vi.hoisted(() => ({
mockListContainers: vi.fn(),
mockGetContainer: vi.fn(),
@@ -27,6 +28,7 @@ const {
mockGetOpenDriftFindings: vi.fn(),
mockGetGlobalSettings: vi.fn(),
mockFsSize: vi.fn(),
mockBuildEffectiveServiceModel: vi.fn(),
}));
vi.mock('../services/DockerController', () => ({
@@ -75,7 +77,11 @@ vi.mock('systeminformation', () => ({
default: { fsSize: mockFsSize },
}));
import { UpdateGuardService } from '../services/UpdateGuardService';
vi.mock('../services/effectiveServiceModel', () => ({
buildEffectiveServiceModel: mockBuildEffectiveServiceModel,
}));
import { UpdateGuardService, SingleServiceUpdateReadinessError } from '../services/UpdateGuardService';
const inspectResult = (over: Record<string, unknown> = {}) => ({
State: { Status: 'running', ExitCode: 0 },
@@ -166,6 +172,110 @@ describe('UpdateGuardService.computeUpdateReadiness wiring', () => {
});
});
describe('UpdateGuardService.computeUpdateReadiness with a serviceName', () => {
const twoServiceModel = {
renderable: true as const,
services: [
{ name: 'web', declaredImage: 'nginx:1.27', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: true },
{ name: 'db', declaredImage: 'postgres:16', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false },
],
};
const multiServicePreview = {
stack_name: 'app',
images: [
{ service: 'web', image: 'nginx', current_tag: '1.27.0', next_tag: '1.27.1', has_update: true, semver_bump: 'patch' },
{ service: 'db', image: 'postgres', current_tag: '16.0', next_tag: null, has_update: false, semver_bump: 'none' },
],
build_services: [],
summary: {
has_update: true, primary_image: 'nginx', current_tag: '1.27.0', next_tag: '1.27.1',
semver_bump: 'patch', update_kind: 'tag', blocked: false, blocked_reason: null,
has_build_services: false, rebuild_available: false,
},
rollback_target: 'nginx:1.27.0',
changelog: null,
};
beforeEach(() => {
mockGetLatest.mockReturnValue({ activeStatus: 'pass' });
mockGetOpenDriftFindings.mockReturnValue([]);
mockGetPreview.mockResolvedValue(multiServicePreview);
mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: Date.now() });
mockFsSize.mockResolvedValue([{ mount: '/', use: 42 }]);
mockListContainers.mockResolvedValue([
{ Id: 'web1', Names: ['/web'], State: 'running' },
{ Id: 'db1', Names: ['/db'], State: 'running' },
]);
mockGetContainer.mockReturnValue({ inspect: vi.fn().mockResolvedValue(inspectResult()) });
});
it('scopes the verdict to the selected service, includes a service signal, and filters the preview', async () => {
mockBuildEffectiveServiceModel.mockResolvedValue(twoServiceModel);
const report = await UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'web');
expect(report.serviceName).toBe('web');
expect(report.verdict).toBe('ready');
const serviceSig = report.signals.find(s => s.id === 'service');
expect(serviceSig?.status).toBe('ok');
// update_preview signal is derived from the filtered summary (recomputed for 'web' only).
const previewSig = report.signals.find(s => s.id === 'update_preview');
expect(previewSig?.status).toBe('ok');
});
it('throws SingleServiceUpdateReadinessError for a single-service stack', async () => {
mockBuildEffectiveServiceModel.mockResolvedValue({
renderable: true,
services: [{ name: 'web', declaredImage: 'nginx:1.27', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: true }],
});
await expect(UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'web'))
.rejects.toBeInstanceOf(SingleServiceUpdateReadinessError);
});
it('fails closed (blocked service signal) when the effective model fails to render', async () => {
mockBuildEffectiveServiceModel.mockResolvedValue({
renderable: false,
code: 'effective_model_render_failed',
error: 'bad yaml',
});
const report = await UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'web');
const serviceSig = report.signals.find(s => s.id === 'service');
expect(serviceSig?.status).toBe('blocked');
expect(report.verdict).toBe('blocked');
});
it('fails closed when the selected service is not declared in the model', async () => {
mockBuildEffectiveServiceModel.mockResolvedValue(twoServiceModel);
const report = await UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'cache');
const serviceSig = report.signals.find(s => s.id === 'service');
expect(serviceSig?.status).toBe('blocked');
expect(report.verdict).toBe('blocked');
});
it('reports a sibling advisory without affecting the verdict', async () => {
mockBuildEffectiveServiceModel.mockResolvedValue(twoServiceModel);
mockListContainers.mockResolvedValue([
{ Id: 'web1', Names: ['/web'], State: 'running' },
{ Id: 'db1', Names: ['/db'], State: 'exited', ExitCode: 1 },
]);
mockGetContainer.mockImplementation((id: string) => ({
inspect: vi.fn().mockResolvedValue(id === 'db1'
? inspectResult({ State: { Status: 'exited', ExitCode: 1 } })
: inspectResult()),
}));
const report = await UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'web');
expect(report.verdict).toBe('ready');
expect(report.advisories.some(a => a.includes('db'))).toBe(true);
});
});
describe('UpdateGuardService.computeRollbackReadiness moving-tag wiring', () => {
const preview = (images: Array<{ current_tag: string }>) => ({
stack_name: 'app',
@@ -462,7 +462,7 @@ describe('WebhookService.execute: health gate begin call sites', () => {
vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined);
vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]);
vi.spyOn(compose.ComposeService.prototype, 'deployStack').mockResolvedValue(undefined);
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-hook');
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook');
const result = await WebhookService.getInstance().execute(webhook, 'deploy', 'test', true);
expect(result.success).toBe(true);
@@ -482,7 +482,7 @@ describe('WebhookService.execute: health gate begin call sites', () => {
vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined);
vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]);
vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue(undefined);
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-hook');
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook');
const result = await WebhookService.getInstance().execute(webhook, 'pull', 'test', true);
expect(result.success).toBe(true);
+2
View File
@@ -4,6 +4,7 @@ import { LicenseService } from '../services/LicenseService';
import { MonitorService } from '../services/MonitorService';
import { AutoHealService } from '../services/AutoHealService';
import { HealthGateService } from '../services/HealthGateService';
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
import { FleetSyncRetryService } from '../services/FleetSyncRetryService';
import { DockerEventManager } from '../services/DockerEventManager';
import { ImageUpdateService } from '../services/ImageUpdateService';
@@ -34,6 +35,7 @@ export function installShutdownHandlers(server: Server): void {
}
try { AutoHealService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] AutoHealService cleanup failed:', (e as Error).message); }
try { HealthGateService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] HealthGateService cleanup failed:', (e as Error).message); }
try { ServiceUpdateRecoveryService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] ServiceUpdateRecoveryService cleanup failed:', (e as Error).message); }
try { FleetSyncRetryService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] FleetSyncRetryService cleanup failed:', (e as Error).message); }
try { DockerEventManager.getInstance().stop(); } catch (e) {
console.warn('[Shutdown] DockerEventManager cleanup failed:', (e as Error).message);
+2
View File
@@ -11,6 +11,7 @@ import SelfIdentityService from '../services/SelfIdentityService';
import { MonitorService } from '../services/MonitorService';
import { AutoHealService } from '../services/AutoHealService';
import { HealthGateService } from '../services/HealthGateService';
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
import { FleetSyncRetryService } from '../services/FleetSyncRetryService';
import { DockerEventManager } from '../services/DockerEventManager';
import TrivyService, { sweepStaleTrivyTempDirs } from '../services/TrivyService';
@@ -139,6 +140,7 @@ export async function startServer(server: Server): Promise<void> {
MonitorService.getInstance().start();
AutoHealService.getInstance().start();
HealthGateService.getInstance().start();
ServiceUpdateRecoveryService.getInstance().start();
FleetSyncRetryService.getInstance().start();
ImageUpdateService.getInstance().start();
SchedulerService.getInstance().start();
+2
View File
@@ -26,6 +26,8 @@ const DEPLOY_ALLOWED_PATTERNS: RegExp[] = [
/^\/api\/stacks\/[^/]+\/stop$/,
/^\/api\/stacks\/[^/]+\/start$/,
/^\/api\/stacks\/[^/]+\/update$/,
/^\/api\/stacks\/[^/]+\/services\/[^/]+\/update$/,
/^\/api\/stacks\/[^/]+\/services\/[^/]+\/restore$/,
];
const deny = (res: Response, req: Request, error: string, scope: ApiTokenScope | 'unknown'): void => {
+21 -2
View File
@@ -5,7 +5,7 @@ import { PROXY_TIER_HEADER, PROXY_ROLE_HEADER, PROXY_DEPLOY_SOURCE_HEADER, PROXY
import { LicenseService } from '../services/LicenseService';
import { isProxyExemptPath } from '../helpers/proxyExemptPaths';
import { remoteSupportsCrossNodeRbac, remoteAdvertisesCapability } from '../helpers/remoteCapabilities';
import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY } from '../services/CapabilityRegistry';
import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry';
import { getErrorMessage } from '../utils/errors';
import { DatabaseService } from '../services/DatabaseService';
import { redactSensitiveText } from '../utils/safeLog';
@@ -171,7 +171,7 @@ export function createRemoteProxyMiddleware(): RequestHandler {
finalizeProxyTiming(req, 'error');
console.error('[Proxy] Remote node error:', getErrorMessage(err, 'unknown'));
const path = req.originalUrl || req.url;
if (req.method === 'POST' && /^\/api\/stacks\/[^/]+\/(?:deploy|update)(?:\?|$)/.test(path)) {
if (req.method === 'POST' && /^\/api\/stacks\/[^/]+\/(?:deploy|update|services\/[^/]+\/(?:update|restore))(?:\?|$)/.test(path)) {
try {
DatabaseService.getInstance().insertAuditLog({
timestamp: Date.now(),
@@ -238,6 +238,14 @@ export function createRemoteProxyMiddleware(): RequestHandler {
}
}
if (isServiceScopedUpdateRoute(req)) {
const supported = await remoteAdvertisesCapability(req.nodeId, SERVICE_SCOPED_UPDATE_CAPABILITY);
if (!supported) {
res.status(400).json({ error: 'Service-scoped updates are not supported on this node', code: 'capability_unavailable' });
return;
}
}
// Mixed-version RBAC gate (non-admin only).
if (req.user?.role !== 'admin') {
const rbacSupported = await remoteSupportsCrossNodeRbac(req.nodeId);
@@ -264,3 +272,14 @@ function isStackDownWithRemoveVolumes(req: Request): boolean {
if (!/^\/stacks\/[^/]+\/down$/.test(req.path)) return false;
return req.query.removeVolumes === 'true';
}
/** Nested service update/restore/recovery routes (path is post-/api strip). */
function isServiceScopedUpdateRoute(req: Request): boolean {
if (req.method === 'GET') {
return /^\/stacks\/[^/]+\/services\/[^/]+\/recovery$/.test(req.path);
}
if (req.method === 'POST') {
return /^\/stacks\/[^/]+\/services\/[^/]+\/(?:update|restore)$/.test(req.path);
}
return false;
}
+4 -2
View File
@@ -10,6 +10,7 @@ import { FleetUpdateTrackerService, type UpdateTracker, type TerminalStatus, UPD
import { NodeRegistry } from '../services/NodeRegistry';
import { computeNodeNetworkingSummary, type NodeNetworkingSummary } from '../services/network/networkingSummary';
import DockerController from '../services/DockerController';
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
import { getHostMemory } from '../helpers/hostMemory';
import { FileSystemService } from '../services/FileSystemService';
import { ComposeService } from '../services/ComposeService';
@@ -2115,9 +2116,10 @@ fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res
targetResults.push({ target, success: true, reclaimedBytes: estimate.reclaimableBytes, dryRun: true });
continue;
}
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(node.id);
const result = scope === 'managed'
? await dockerController.pruneManagedOnly(target, knownStacks)
: await dockerController.pruneSystem(target);
? await dockerController.pruneManagedOnly(target, knownStacks, isImageHeld)
: await dockerController.pruneSystem(target, undefined, isImageHeld);
targetResults.push({ target, success: true, reclaimedBytes: result.reclaimedBytes });
if (result.reclaimedBytes > 0 || result.success) anySuccess = true;
} catch (err) {
+6 -4
View File
@@ -7,7 +7,7 @@ import { NodeRegistry } from '../services/NodeRegistry';
import { CacheService } from '../services/CacheService';
import { ImageUpdateService } from '../services/ImageUpdateService';
import { FileSystemService } from '../services/FileSystemService';
import { ComposeService } from '../services/ComposeService';
import { StackUpdateOrchestrator } from '../services/StackUpdateOrchestrator';
import { StackOpLockService, stackOpSkipMessage } from '../services/StackOpLockService';
import { NotificationService } from '../services/NotificationService';
import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement';
@@ -342,7 +342,6 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
const docker = DockerController.getInstance(req.nodeId);
const imageUpdateService = ImageUpdateService.getInstance();
const compose = ComposeService.getInstance(req.nodeId);
const db = DatabaseService.getInstance();
const atomic = true;
const results: string[] = [];
@@ -417,14 +416,17 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
const lock = await StackOpLockService.getInstance().runExclusive(
req.nodeId, stackName, 'update', 'system',
() => compose.updateStack(stackName, undefined, atomic),
() => StackUpdateOrchestrator.getInstance().execute(
{ nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'automatic', actor: `auto-update:${req.user?.username ?? 'scheduler'}` },
{ atomic, terminalWs: null },
),
);
if (!lock.ran) {
results.push(stackOpSkipMessage(stackName, lock.existing.action));
continue;
}
db.clearStackUpdateStatus(req.nodeId, stackName);
HealthGateService.getInstance().begin(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`);
HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`);
NotificationService.getInstance().broadcastEvent({
type: 'state-invalidate',
+265 -8
View File
@@ -13,6 +13,7 @@ import { FileSystemService } from '../services/FileSystemService';
import { StackFileRootsService, STACK_SOURCE_ROOT_ID, stackSourceFileRoot, type StackFileRoot } from '../services/StackFileRootsService';
import { FileRootGateway } from '../services/FileRootGateway';
import { ComposeService, getComposeRollbackInfo } from '../services/ComposeService';
import { StackUpdateOrchestrator, shortImageId, type OrchestratorResult } from '../services/StackUpdateOrchestrator';
import DockerController, { type BulkStackInfo } from '../services/DockerController';
import { DatabaseService, type StackDossierFields } from '../services/DatabaseService';
import { MeshService } from '../services/MeshService';
@@ -30,11 +31,12 @@ import { buildStackNetworkFacts } from '../services/network/composeNetworkInspec
import { buildStorageInventory } from '../services/storage/inventory';
import { probeComposeDiscovery } from '../services/ComposeDiscoveryService';
import { buildEffectiveAnatomy } from '../services/effectiveAnatomy';
import { buildEffectiveServiceModel } from '../services/effectiveServiceModel';
import { buildEnvInventory } from '../services/EnvInventoryService';
import { buildStackLabelInventory } from '../services/LabelInventoryService';
import { labelInventoryOptionsFromRequest, requireRevealAdmin } from '../helpers/labelInventoryRequest';
import { EXPOSURE_INTENTS, type ExposureIntent } from '../services/network/types';
import { UpdateGuardService } from '../services/UpdateGuardService';
import { UpdateGuardService, SingleServiceUpdateReadinessError } from '../services/UpdateGuardService';
import { HealthGateService } from '../services/HealthGateService';
import { classifyFailure } from '../services/updateGuard/failureClassifier';
import { requirePermission, checkPermission } from '../middleware/permissions';
@@ -58,7 +60,8 @@ import { resolveStackEnvSources, discoverStackLocalEnvFiles } from '../helpers/e
import { STACK_STATUSES_CACHE_TTL_MS } from '../helpers/constants';
import { getTerminalWs, DEPLOY_SESSION_HEADER } from '../websocket/generic';
import { isSelfStack, refuseIfSelfStack, selfStackProtectedBulkResult } from '../helpers/selfStackGuard';
import { getActiveCapabilities, STACK_DOWN_REMOVE_VOLUMES_CAPABILITY } from '../services/CapabilityRegistry';
import { getActiveCapabilities, STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry';
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
// Authenticated users with edit permission can write arbitrarily large compose
// files. Refuse to YAML.parse anything beyond this bound so a malformed (or
@@ -471,7 +474,10 @@ async function runStackBulkOp(
};
}
const atomic = true;
await ComposeService.getInstance(req.nodeId).updateStack(stackName, getTerminalWs(req.get(DEPLOY_SESSION_HEADER)), atomic);
await StackUpdateOrchestrator.getInstance().execute(
{ nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'bulk', actor: user },
{ atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) },
);
DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName);
NotificationService.getInstance().broadcastEvent({
type: 'state-invalidate',
@@ -485,7 +491,7 @@ async function runStackBulkOp(
triggerPostDeployScan(stackName, req.nodeId).catch(err =>
console.error('[Security] Post-deploy scan failed for %s:', sanitizeForLog(stackName), err),
);
const healthGateId = HealthGateService.getInstance().begin(req.nodeId, stackName, 'update', req.user?.username ?? null);
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
return { stackName, ok: true, healthGateId };
} else {
const outcome = await containerActionForStack(req.nodeId, stackName, action);
@@ -1143,6 +1149,7 @@ stacksRouter.delete('/:stackName', async (req: Request, res: Response) => {
try {
DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName);
DatabaseService.getInstance().clearStackScanAttempts(req.nodeId, stackName);
DatabaseService.getInstance().deleteServiceUpdateRecoveries(req.nodeId, stackName);
DatabaseService.getInstance().deleteRoleAssignmentsByResource('stack', stackName);
DatabaseService.getInstance().deleteGitSource(stackName);
DatabaseService.getInstance().deleteStackDossier(req.nodeId, stackName);
@@ -1558,6 +1565,24 @@ stacksRouter.get('/:stackName/effective-anatomy', async (req: Request, res: Resp
}
});
// Effective Service Model: per-service facts (declared image, build presence,
// expected replica count, dependencies, healthcheck) that service-scoped
// update/restore key off of, from the fully-merged effective model. Read-only
// and advisory; auto-proxies to the active node. Never returns raw render
// stderr or any environment/label/command value.
stacksRouter.get('/:stackName/effective-services', async (req: Request, res: Response) => {
const stackName = req.params.stackName as string;
if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return;
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
try {
res.json(await buildEffectiveServiceModel(req.nodeId, stackName));
} catch (error) {
console.error('[Stacks] Failed to build effective service model for %s:', sanitizeForLog(stackName),
sanitizeForLog(inspect(error, { depth: 4 })));
res.status(500).json({ error: 'Failed to build effective service model' });
}
});
// Environment inventory: per-stack env vars with their source, scope (Compose
// interpolation vs container injection), and status (present/missing/unused/
// duplicate/unpersisted), plus likely-secret classification. Read-only and
@@ -1656,12 +1681,17 @@ stacksRouter.put('/:stackName/exposure', async (req: Request, res: Response) =>
// that owns it. Read-only, so stack:read is the correct gate.
stacksRouter.get('/:stackName/update-readiness', async (req: Request, res: Response) => {
const stackName = req.params.stackName as string;
const serviceName = typeof req.query.service === 'string' && req.query.service.length > 0 ? req.query.service : undefined;
if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return;
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
try {
const report = await UpdateGuardService.getInstance().computeUpdateReadiness(req.nodeId, stackName);
const report = await UpdateGuardService.getInstance().computeUpdateReadiness(req.nodeId, stackName, serviceName);
res.json(report);
} catch (error) {
if (error instanceof SingleServiceUpdateReadinessError) {
res.status(400).json({ error: error.message, code: 'service_update_single_service' });
return;
}
console.error('[Stacks] Failed to compute update readiness for %s:', sanitizeForLog(stackName),
sanitizeForLog(getErrorMessage(error, 'unknown')));
res.status(500).json({ error: 'Failed to compute update readiness' });
@@ -1724,7 +1754,7 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => {
dlog(`[Stacks] Deploy completed: ${sanitizeForLog(stackName)}`);
if (debug) console.debug(`[Stacks:debug] Deploy finished in ${Date.now() - t0}ms`);
ok = true;
const healthGateId = HealthGateService.getInstance().begin(req.nodeId, stackName, 'deploy', req.user?.username ?? null);
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'deploy', req.user?.username ?? null);
res.json({ message: 'Deployed successfully', healthGateId });
notifyActionSuccess('deploy_success', `${stackName} deployed`, stackName, req.user?.username ?? 'system');
if (!skipScan) {
@@ -1987,6 +2017,230 @@ stacksRouter.post('/:stackName/services/:serviceName/stop', (req, res) =>
stacksRouter.post('/:stackName/services/:serviceName/start', (req, res) =>
handleServiceAction(req, res, 'start'));
/** Map an orchestrator `service_failed` code to an HTTP status. */
function serviceFailureStatus(code: string): number {
switch (code) {
case 'service_update_single_service':
case 'service_not_updatable':
case 'effective_model_render_failed':
case 'recovery_id_required':
return 400;
case 'service_not_found':
case 'recovery_not_found':
return 404;
case 'policy_blocked':
case 'recovery_not_restorable':
case 'recovery_claim_failed':
return 409;
default:
// Compose, retag, inspect, and replica-divergence failures are server-side.
return 500;
}
}
/** Send an orchestrator service result as an HTTP response. Returns success. */
function sendServiceResult(res: Response, result: OrchestratorResult, serviceName: string): boolean {
if (result.kind === 'service_done') {
res.json({
serviceName: result.serviceName,
healthGateId: result.healthGateId,
observing: result.observing,
recoveryId: result.recoveryId,
recoveryAvailable: result.recoveryAvailable,
...(result.previousImageId ? { previousImageId: result.previousImageId } : {}),
...(result.newImageId ? { newImageId: result.newImageId } : {}),
...(result.recheckWarning ? { recheckWarning: result.recheckWarning } : {}),
});
return true;
}
if (result.kind === 'service_failed') {
res.status(serviceFailureStatus(result.code)).json({
error: result.error,
code: result.code,
serviceName: result.serviceName ?? serviceName,
...(result.mutationStage ? { mutationStage: result.mutationStage } : {}),
...(result.recoveryId ? { recoveryId: result.recoveryId } : {}),
});
return false;
}
// Stack results never reach the service routes; treat defensively.
res.status(500).json({ error: 'Unexpected orchestrator result', code: 'unexpected_result' });
return false;
}
function requireServiceScopedUpdateCapability(res: Response): boolean {
if (getActiveCapabilities().includes(SERVICE_SCOPED_UPDATE_CAPABILITY)) return true;
res.status(400).json({ error: 'Service-scoped updates are not supported on this node', code: 'capability_unavailable' });
return false;
}
async function handleServiceScopedMutation(
req: Request,
res: Response,
options: {
lockAction: 'update' | 'rollback';
recoveryId?: string;
notifyFailureAction: 'update' | 'rollback';
failureCode: string;
failureMessage: string;
onSuccess: (
stackName: string,
serviceName: string,
actor: string,
meta: { previousImageId?: string | null; newImageId?: string | null },
) => void;
},
): Promise<void> {
const stackName = req.params.stackName as string;
const serviceName = req.params.serviceName as string;
if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
if (await refuseIfSelfStack(req, res, stackName)) return;
if (!requireServiceScopedUpdateCapability(res)) return;
if (!tryAcquireStackOpLock(req, res, stackName, options.lockAction)) return;
const t0 = Date.now();
let ok = false;
try {
const result = await StackUpdateOrchestrator.getInstance().execute(
{
nodeId: req.nodeId,
stackName,
target: { scope: 'service', serviceName },
trigger: 'manual',
actor: req.user?.username ?? null,
},
{
policyOptions: buildPolicyGateOptions(req),
terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)),
...(options.recoveryId ? { recoveryId: options.recoveryId } : {}),
},
);
invalidateNodeCaches(req.nodeId);
ok = sendServiceResult(res, result, serviceName);
if (ok && result.kind === 'service_done') {
options.onSuccess(stackName, serviceName, req.user?.username ?? 'system', {
previousImageId: result.previousImageId,
newImageId: result.newImageId,
});
NotificationService.getInstance().broadcastEvent({
type: 'state-invalidate',
scope: 'image-updates',
nodeId: req.nodeId,
stackName,
action: 'stack-updated',
ts: Date.now(),
});
} else if (result.kind === 'service_failed') {
notifyActionFailure(
options.notifyFailureAction,
stackName,
new Error(result.error),
req.user?.username ?? 'system',
);
}
} catch (error: unknown) {
console.error(
'[Stacks] Service %s failed: %s/%s: %s',
sanitizeForLog(options.notifyFailureAction),
sanitizeForLog(stackName),
sanitizeForLog(serviceName),
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
notifyActionFailure(options.notifyFailureAction, stackName, error, req.user?.username ?? 'system');
if (!res.headersSent) {
res.status(500).json({ error: getErrorMessage(error, options.failureMessage), code: options.failureCode });
}
} finally {
StackOpMetricsService.getInstance().record(req.nodeId, 'update', Date.now() - t0, ok, {
targetScope: 'service',
serviceName,
});
releaseStackOpLock(req, stackName);
}
}
stacksRouter.post('/:stackName/services/:serviceName/update', async (req: Request, res: Response) => {
await handleServiceScopedMutation(req, res, {
lockAction: 'update',
notifyFailureAction: 'update',
failureCode: 'service_update_failed',
failureMessage: 'Failed to update service',
onSuccess: (stackName, serviceName, actor, meta) => {
const from = shortImageId(meta.previousImageId);
const to = shortImageId(meta.newImageId);
const transition = from && to && from !== to ? ` (${from} -> ${to})` : '';
notifyActionSuccess(
'image_update_applied',
`${stackName}/${serviceName} updated${transition}`,
stackName,
actor,
);
},
});
});
/** Latest restorable service recovery snapshot (active, unexpired), for UI discovery outside Deploy Progress. */
stacksRouter.get('/:stackName/services/:serviceName/recovery', async (req: Request, res: Response) => {
const stackName = req.params.stackName as string;
const serviceName = req.params.serviceName as string;
if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
if (!requireServiceScopedUpdateCapability(res)) return;
try {
const row = ServiceUpdateRecoveryService.getInstance().listActive(req.nodeId, stackName, serviceName)[0];
if (!row) {
res.json({ recovery: null });
return;
}
res.json({
recovery: {
id: row.id,
status: row.status,
healthGateId: row.health_gate_id,
expiresAt: row.expires_at,
createdAt: row.created_at,
majorityImageId: row.majority_image_id,
declaredImageRef: row.declared_image_ref,
},
});
} catch (error: unknown) {
console.error(
'[Stacks] Failed to list service recovery for %s/%s: %s',
sanitizeForLog(stackName),
sanitizeForLog(serviceName),
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
res.status(500).json({ error: 'Failed to load service recovery', code: 'service_recovery_lookup_failed' });
}
});
stacksRouter.post('/:stackName/services/:serviceName/restore', async (req: Request, res: Response) => {
const recoveryId = typeof req.body?.recoveryId === 'string' ? req.body.recoveryId : '';
if (!recoveryId) {
res.status(400).json({ error: 'A recoveryId is required to restore a service.', code: 'recovery_id_required' });
return;
}
await handleServiceScopedMutation(req, res, {
lockAction: 'rollback',
recoveryId,
notifyFailureAction: 'rollback',
failureCode: 'service_restore_failed',
failureMessage: 'Failed to restore service',
onSuccess: (stackName, serviceName, actor, meta) => {
const from = shortImageId(meta.previousImageId);
const to = shortImageId(meta.newImageId);
const transition = from && to && from !== to ? ` (${from} -> ${to})` : '';
notifyActionSuccess(
'deploy_success',
`${stackName}/${serviceName} restored${transition}`,
stackName,
actor,
);
},
});
});
stacksRouter.get('/:stackName/update-preview', async (req: Request, res: Response) => {
const stackName = req.params.stackName as string;
try {
@@ -2013,7 +2267,10 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
const debug = isDebugEnabled();
const atomic = true;
if (debug) console.debug('[Stacks:debug] Update starting', { stackName, atomic, nodeId: req.nodeId });
await ComposeService.getInstance(req.nodeId).updateStack(stackName, getTerminalWs(req.get(DEPLOY_SESSION_HEADER)), atomic);
await StackUpdateOrchestrator.getInstance().execute(
{ nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'manual', actor: req.user?.username ?? null },
{ atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) },
);
DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName);
invalidateNodeCaches(req.nodeId);
NotificationService.getInstance().broadcastEvent({
@@ -2027,7 +2284,7 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
dlog(`[Stacks] Update completed: ${sanitizeForLog(stackName)}`);
if (debug) console.debug(`[Stacks:debug] Update finished in ${Date.now() - t0}ms`);
ok = true;
const healthGateId = HealthGateService.getInstance().begin(req.nodeId, stackName, 'update', req.user?.username ?? null);
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
res.json({ status: 'Update completed', healthGateId });
notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, req.user?.username ?? 'system');
if (!skipScan) {
+11 -7
View File
@@ -8,6 +8,7 @@ import DockerController, {
} from '../services/DockerController';
import { isPruneTarget } from '../services/prunePlan';
import { FileSystemService } from '../services/FileSystemService';
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
import SelfIdentityService from '../services/SelfIdentityService';
import { requireAdmin } from '../middleware/tierGates';
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
@@ -127,8 +128,9 @@ systemMaintenanceRouter.post('/prune/plan', async (req: Request, res: Response)
const pruneScope = parsePruneScope((req.body as { scope?: unknown }).scope);
const dockerController = DockerController.getInstance(req.nodeId);
const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks();
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(req.nodeId);
const plan = await withTimeout(
dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId),
dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId, isImageHeld),
PRUNE_ESTIMATE_TIMEOUT_MS,
'docker prune plan',
);
@@ -171,11 +173,12 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
const planFingerprint = typeof body.planFingerprint === 'string' ? body.planFingerprint : null;
const dockerController = DockerController.getInstance(req.nodeId);
const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks();
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(req.nodeId);
if (isDryRun) {
// Dry-run returns the same itemized plan shape Resources uses for preview.
const plan = await withTimeout(
dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId),
dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId, isImageHeld),
PRUNE_ESTIMATE_TIMEOUT_MS,
'docker prune plan',
);
@@ -201,7 +204,7 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
// fingerprint and keeps the legacy pruneManagedOnly / pruneSystem path.
if (planFingerprint) {
const built = await withTimeout(
dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId),
dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId, isImageHeld),
PRUNE_ESTIMATE_TIMEOUT_MS,
'docker prune plan',
);
@@ -215,7 +218,7 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
`[Resources] System prune (plan): ${built.targets.join(',')} (scope: ${pruneScope}, items: ${built.items.length})`,
);
const pruneStartedAt = Date.now();
const result = await dockerController.executePrunePlan(built, knownStacks);
const result = await dockerController.executePrunePlan(built, knownStacks, isImageHeld);
console.log(
`[Resources] System prune completed: reclaimed ${result.reclaimedBytes} bytes, outcomes=${result.outcomes.length}`,
);
@@ -254,14 +257,15 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
result = await dockerController.pruneManagedOnly(
target as 'images' | 'volumes' | 'networks',
knownStacks,
isImageHeld,
);
} else if (pruneScope === 'managed' && target === 'containers') {
// Managed containers must never fall through to system prune. Build and
// execute an itemized plan for this single target instead.
const plan = await dockerController.buildPrunePlan(['containers'], 'managed', knownStacks, req.nodeId);
result = await dockerController.executePrunePlan(plan, knownStacks);
const plan = await dockerController.buildPrunePlan(['containers'], 'managed', knownStacks, req.nodeId, isImageHeld);
result = await dockerController.executePrunePlan(plan, knownStacks, isImageHeld);
} else {
result = await dockerController.pruneSystem(target);
result = await dockerController.pruneSystem(target, undefined, isImageHeld);
}
console.log(`[Resources] System prune completed: ${target}, reclaimed ${result.reclaimedBytes} bytes`);
+42
View File
@@ -71,6 +71,15 @@ export class AutoHealService {
private observedUnhealthySince = new Map<string, number>();
private historyTimestamps = new Map<string, number>();
private leaseRefreshFailures = new Map<number, number>();
/**
* Ref-counted suppressions for service-scoped updates/restores. While a
* service is being recreated and its post-mutation health gate observes it,
* auto-heal must not restart the churning containers out from under the gate.
* Keyed `${nodeId}:${stackName}:${serviceName}`. Nested owners (overlapping
* same-service ops, or a gate that outlives the orchestrator finally) each
* increment; clearSuppress only unmutes at zero. Process death clears it.
*/
private suppressedServices = new Map<string, number>();
private constructor() {}
@@ -108,6 +117,33 @@ export class AutoHealService {
clearInterval(this.leaseRefreshTimer);
this.leaseRefreshTimer = null;
}
this.suppressedServices.clear();
}
private suppressKey(nodeId: number, stackName: string, serviceName: string): string {
return `${nodeId}:${stackName}:${serviceName}`;
}
/** Suppress auto-heal for one service while a service-scoped update/restore + its health gate run. */
suppress(nodeId: number, stackName: string, serviceName: string): void {
const key = this.suppressKey(nodeId, stackName, serviceName);
this.suppressedServices.set(key, (this.suppressedServices.get(key) ?? 0) + 1);
}
/** Release one suppression owner; Auto-Heal resumes only when the count hits zero. */
clearSuppress(nodeId: number, stackName: string, serviceName: string): void {
const key = this.suppressKey(nodeId, stackName, serviceName);
const current = this.suppressedServices.get(key) ?? 0;
if (current <= 1) {
this.suppressedServices.delete(key);
return;
}
this.suppressedServices.set(key, current - 1);
}
/** True while the given service is suppressed. A stack-wide policy still heals other services. */
isSuppressed(nodeId: number, stackName: string, serviceName: string): boolean {
return (this.suppressedServices.get(this.suppressKey(nodeId, stackName, serviceName)) ?? 0) > 0;
}
/**
@@ -283,6 +319,12 @@ export class AutoHealService {
const containerName =
container.Names?.[0]?.replace(/^\//, '') ?? container.Id.slice(0, 12);
const serviceOverride = container.Labels?.['com.docker.compose.service'] ?? null;
// Skip containers whose service is suppressed by an in-flight
// service-scoped update/restore, so the post-mutation health gate
// owns the churn without auto-heal racing it.
if (serviceOverride && this.isSuppressed(nodeId, policy.stack_name, serviceOverride)) {
continue;
}
const signal = this.getHealSignal(nodeId, container, eventSvc?.getContainerState(container.Id), now);
const decision = this.shouldHeal(signal, policy, this.containerKey(nodeId, container.Id), now);
@@ -56,6 +56,7 @@ export const CAPABILITIES = [
'cross-node-rbac',
'stack-down-remove-volumes',
'guided-external-network-preflight',
'service-scoped-update',
] as const;
/**
@@ -72,6 +73,9 @@ export type Capability = (typeof CAPABILITIES)[number];
/** Capability for optional `?removeVolumes=true` on POST /stacks/:name/down. */
export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes' as const satisfies Capability;
/** Capability for the nested per-service update/restore routes and the `effective-services` model they read. */
export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability;
/** Returns true when the string is a usable semver version. */
export function isValidVersion(v: string | null | undefined): v is string {
return !!v && v !== 'unknown' && v !== '0.0.0-dev' && !!semver.valid(v);
+54 -2
View File
@@ -85,7 +85,8 @@ export function getComposeRollbackInfo(error: unknown): { attempted: boolean; ro
const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000;
function getComposeCommandTimeoutMs(): number {
/** Public so other services (e.g. recovery claim leases) can size their own timers off the same ceiling without depending on a private module-local. */
export function getComposeCommandTimeoutMs(): number {
const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS);
if (Number.isFinite(configured) && configured > 0) {
return configured;
@@ -873,7 +874,11 @@ export class ComposeService {
try {
const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1';
if (pruneOnUpdate) {
const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages();
// Dynamic import: ServiceUpdateRecoveryService imports getComposeCommandTimeoutMs
// from this module, so a static import here would be circular.
const { ServiceUpdateRecoveryService } = await import('./ServiceUpdateRecoveryService');
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(this.nodeId);
const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld);
// The Docker prune API does not report SpaceReclaimed on the containerd
// image store, so only show the figure when the daemon actually returns one.
const reclaimed = result.reclaimedBytes > 0
@@ -906,6 +911,53 @@ export class ComposeService {
}
}
/**
* Service-scoped update: pull (or `build --pull` for a build-backed service)
* and recreate a single service's replicas in place. Always
* `--no-deps --force-recreate`, never `--remove-orphans`, so sibling services
* keep their container ids and StartedAt. No drift re-baseline and no dangling
* prune here; the orchestrator owns per-service post-update reconciliation.
*/
async updateService(stackName: string, serviceName: string, hasBuild: boolean, ws?: WebSocket): Promise<void> {
await this.assertRequiredEnvPresent(stackName);
await this.assertSafePilotBindMapping(stackName);
const stackDir = path.join(this.baseDir, stackName);
const sendOutput = (data: string) => {
if (ws && ws.readyState === WebSocket.OPEN) ws.send(data);
};
await this.withRegistryAuth(async (env) => {
if (hasBuild) {
sendOutput(`=== Building ${serviceName} ===\n`);
await this.execute('docker', await this.authoredComposeArgs(stackName, ['build', '--pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs());
} else {
sendOutput(`=== Pulling ${serviceName} ===\n`);
await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs());
}
sendOutput(`=== Recreating ${serviceName} ===\n`);
await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs());
}, sendOutput);
}
/**
* Recreate a single service from the image already present locally, without
* pulling or building. Used by service restore after the recovery image id has
* been retagged onto the declared ref (`--pull never --no-build` so Compose
* uses the just-retagged local image). Always `--no-deps --force-recreate`,
* never `--remove-orphans`.
*/
async recreateServiceFromLocal(stackName: string, serviceName: string, ws?: WebSocket): Promise<void> {
await this.assertRequiredEnvPresent(stackName);
await this.assertSafePilotBindMapping(stackName);
const stackDir = path.join(this.baseDir, stackName);
const sendOutput = (data: string) => {
if (ws && ws.readyState === WebSocket.OPEN) ws.send(data);
};
await this.withRegistryAuth(async (env) => {
sendOutput(`=== Restoring ${serviceName} ===\n`);
await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', '--pull', 'never', '--no-build', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs());
}, sendOutput);
}
public async downStack(stackName: string): Promise<void> {
const stackPath = path.join(this.baseDir, stackName);
try {
+341 -18
View File
@@ -35,11 +35,58 @@ export interface GlobalSetting {
*/
export type StackCheckStatus = 'ok' | 'partial' | 'failed';
/**
* Per-service check outcome persisted in stack_update_status.services_json.
* Distinct from StackCheckStatus: a service with no checkable image ref is
* `not_checkable`, which never counts as a check failure at the aggregate
* (stack-level) status.
*/
export type ServiceCheckStatus = 'ok' | 'partial' | 'failed' | 'not_checkable';
export interface StackServiceStatus {
service: string;
image: string | null;
runtimeImages?: string[];
hasUpdate: boolean;
checkStatus: ServiceCheckStatus;
lastError: string | null;
}
export interface StackUpdateDetail {
hasUpdate: boolean;
checkStatus: StackCheckStatus;
lastError: string | null;
checkedAt: number;
/** Per-service breakdown; omitted when the stack has no persisted per-service data (no effective model available yet, or corrupt JSON). */
services?: StackServiceStatus[];
}
const SERVICES_JSON_VERSION = 1;
function isStackServiceStatus(value: unknown): value is StackServiceStatus {
if (!value || typeof value !== 'object') return false;
const v = value as Record<string, unknown>;
return typeof v.service === 'string'
&& (v.image === null || typeof v.image === 'string')
&& typeof v.hasUpdate === 'boolean'
&& (v.checkStatus === 'ok' || v.checkStatus === 'partial' || v.checkStatus === 'failed' || v.checkStatus === 'not_checkable')
&& (v.lastError === null || typeof v.lastError === 'string');
}
/** Parses stack_update_status.services_json safely; a missing, corrupt, or version-mismatched value yields an empty list rather than throwing. */
function parseServicesJson(raw: string | null | undefined): StackServiceStatus[] {
if (!raw) return [];
try {
const parsed = JSON.parse(raw) as { version?: unknown; services?: unknown };
if (parsed?.version !== SERVICES_JSON_VERSION || !Array.isArray(parsed.services)) return [];
return parsed.services.filter(isStackServiceStatus);
} catch {
return [];
}
}
function stringifyServicesJson(services: StackServiceStatus[], generation: number): string {
return JSON.stringify({ version: SERVICES_JSON_VERSION, generation, services });
}
export interface StackAlert {
@@ -146,7 +193,7 @@ export interface HealthGateRunRow {
node_id: number;
stack_name: string;
/** Named trigger_action because TRIGGER is reserved in SQLite. */
trigger_action: 'update' | 'deploy';
trigger_action: 'update' | 'deploy' | 'service_update' | 'service_restore';
status: 'observing' | 'passed' | 'failed' | 'unknown';
reason: string | null;
window_seconds: number;
@@ -155,6 +202,30 @@ export interface HealthGateRunRow {
started_at: number;
ended_at: number | null;
created_by: string | null;
/** Additive; legacy rows default to stack. */
target_scope: 'stack' | 'service';
service_name: string | null;
failure_source: 'primary' | 'collateral' | null;
}
/** Pre-update image snapshot enabling a manual per-service restore after a service-scoped update. */
export interface ServiceUpdateRecoveryRow {
id: string;
node_id: number;
stack_name: string;
service_name: string;
/** JSON array of pre-update replica image snapshots (imageId + repoDigest when known). */
replicas_json: string;
majority_image_id: string;
/** Audit/UI only: the tag the majority image is retagged onto during restore, never a policy scan target. */
declared_image_ref: string;
weak_floating_tag: number;
health_gate_id: string | null;
status: 'active' | 'restoring' | 'consumed' | 'expired' | 'invalidated';
expires_at: number;
claim_expires_at: number | null;
created_at: number;
created_by: string | null;
}
/** One finding within a stored preflight run. Never carries an environment value. */
@@ -1005,6 +1076,7 @@ export class DatabaseService {
has_update INTEGER DEFAULT 0,
check_status TEXT NOT NULL DEFAULT 'ok',
last_error TEXT,
services_json TEXT,
checked_at INTEGER NOT NULL,
PRIMARY KEY (node_id, stack_name)
);
@@ -1525,18 +1597,44 @@ export class DatabaseService {
id TEXT PRIMARY KEY,
node_id INTEGER NOT NULL,
stack_name TEXT NOT NULL,
trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy')),
trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy','service_update','service_restore')),
status TEXT NOT NULL CHECK (status IN ('observing','passed','failed','unknown')),
reason TEXT,
window_seconds INTEGER NOT NULL,
containers_json TEXT NOT NULL DEFAULT '[]',
started_at INTEGER NOT NULL,
ended_at INTEGER,
created_by TEXT
created_by TEXT,
target_scope TEXT NOT NULL DEFAULT 'stack' CHECK (target_scope IN ('stack','service')),
service_name TEXT,
failure_source TEXT CHECK (failure_source IS NULL OR failure_source IN ('primary','collateral'))
);
CREATE INDEX IF NOT EXISTS idx_health_gate_runs_node_stack
ON health_gate_runs(node_id, stack_name, started_at);
CREATE TABLE IF NOT EXISTS service_update_recovery (
id TEXT PRIMARY KEY,
node_id INTEGER NOT NULL,
stack_name TEXT NOT NULL,
service_name TEXT NOT NULL,
replicas_json TEXT NOT NULL,
majority_image_id TEXT NOT NULL,
declared_image_ref TEXT NOT NULL,
weak_floating_tag INTEGER NOT NULL DEFAULT 0,
health_gate_id TEXT,
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','restoring','consumed','expired','invalidated')),
expires_at INTEGER NOT NULL,
claim_expires_at INTEGER,
created_at INTEGER NOT NULL,
created_by TEXT
);
CREATE INDEX IF NOT EXISTS idx_service_update_recovery_node_stack_service
ON service_update_recovery(node_id, stack_name, service_name, status);
CREATE INDEX IF NOT EXISTS idx_service_update_recovery_status_expires
ON service_update_recovery(status, expires_at);
CREATE INDEX IF NOT EXISTS idx_service_update_recovery_status_claim_expires
ON service_update_recovery(status, claim_expires_at);
CREATE TABLE IF NOT EXISTS secrets (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
@@ -1664,6 +1762,11 @@ export class DatabaseService {
// failed check is no longer indistinguishable from "up to date".
maybeAddCol('stack_update_status', 'check_status', "TEXT NOT NULL DEFAULT 'ok'");
maybeAddCol('stack_update_status', 'last_error', 'TEXT');
// Per-service image-update snapshot. Must run AFTER the composite-PK
// recreate above so it is never silently dropped on old installs.
maybeAddCol('stack_update_status', 'services_json', 'TEXT');
this.migrateHealthGateTargetSchema();
// Drop legacy SSH/TLS columns from pre-0.7 databases (no longer read or written)
const legacyCols = ['host', 'port', 'ssh_port', 'ssh_user', 'ssh_password', 'ssh_key', 'tls_ca', 'tls_cert', 'tls_key'];
@@ -1791,6 +1894,66 @@ export class DatabaseService {
}
}
/**
* Rebuild health_gate_runs when the installed CHECK still only allows
* update|deploy or target/failure columns are missing. Idempotent and
* restart-safe: drops a stale temporary table, then rebuilds in one
* better-sqlite3 transaction so an interrupted startup cannot leave
* CREATE TABLE health_gate_runs_new blocking the next boot.
*/
private migrateHealthGateTargetSchema(): void {
const tableSql = (this.db.prepare(
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'health_gate_runs'"
).get() as { sql: string } | undefined)?.sql ?? '';
const cols = this.db.pragma('table_info(health_gate_runs)') as Array<{ name: string }>;
const colNames = new Set(cols.map(c => c.name));
const hasTarget = colNames.has('target_scope');
const hasFailure = colNames.has('failure_source');
const hasWideTrigger = tableSql.includes('service_update') && tableSql.includes('service_restore');
if (hasTarget && hasFailure && hasWideTrigger) return;
// A previous crash between CREATE and RENAME leaves this temp table behind.
this.db.exec('DROP TABLE IF EXISTS health_gate_runs_new');
const targetExpr = hasTarget ? 'target_scope' : "'stack'";
const serviceExpr = colNames.has('service_name') ? 'service_name' : 'NULL';
const failureExpr = hasFailure ? 'failure_source' : 'NULL';
this.db.transaction(() => {
this.db.exec(`
CREATE TABLE health_gate_runs_new (
id TEXT PRIMARY KEY,
node_id INTEGER NOT NULL,
stack_name TEXT NOT NULL,
trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy','service_update','service_restore')),
status TEXT NOT NULL CHECK (status IN ('observing','passed','failed','unknown')),
reason TEXT,
window_seconds INTEGER NOT NULL,
containers_json TEXT NOT NULL DEFAULT '[]',
started_at INTEGER NOT NULL,
ended_at INTEGER,
created_by TEXT,
target_scope TEXT NOT NULL DEFAULT 'stack' CHECK (target_scope IN ('stack','service')),
service_name TEXT,
failure_source TEXT CHECK (failure_source IS NULL OR failure_source IN ('primary','collateral'))
);
INSERT INTO health_gate_runs_new (
id, node_id, stack_name, trigger_action, status, reason, window_seconds,
containers_json, started_at, ended_at, created_by, target_scope, service_name, failure_source
)
SELECT
id, node_id, stack_name, trigger_action, status, reason, window_seconds,
containers_json, started_at, ended_at, created_by,
${targetExpr}, ${serviceExpr}, ${failureExpr}
FROM health_gate_runs;
DROP TABLE health_gate_runs;
ALTER TABLE health_gate_runs_new RENAME TO health_gate_runs;
CREATE INDEX IF NOT EXISTS idx_health_gate_runs_node_stack
ON health_gate_runs(node_id, stack_name, started_at);
`);
})();
}
private migrateEncryptNodeTokens(): void {
const crypto = CryptoService.getInstance();
const rows = this.db.prepare("SELECT id, api_token FROM nodes WHERE api_token != '' AND api_token IS NOT NULL").all() as Array<{ id: number; api_token: string }>;
@@ -3128,11 +3291,13 @@ export class DatabaseService {
public insertHealthGateRun(run: HealthGateRunRow): void {
this.db.prepare(
`INSERT INTO health_gate_runs
(id, node_id, stack_name, trigger_action, status, reason, window_seconds, containers_json, started_at, ended_at, created_by)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
(id, node_id, stack_name, trigger_action, status, reason, window_seconds, containers_json,
started_at, ended_at, created_by, target_scope, service_name, failure_source)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
run.id, run.node_id, run.stack_name, run.trigger_action, run.status, run.reason,
run.window_seconds, run.containers_json, run.started_at, run.ended_at, run.created_by,
run.target_scope, run.service_name, run.failure_source,
);
// Bounded history: keep only the 10 most recent runs per stack.
this.db.prepare(
@@ -3152,10 +3317,11 @@ export class DatabaseService {
reason: string | null,
endedAt: number,
containersJson: string,
failureSource: 'primary' | 'collateral' | null = null,
): void {
this.db.prepare(
'UPDATE health_gate_runs SET status = ?, reason = ?, ended_at = ?, containers_json = ? WHERE id = ?'
).run(status, reason, endedAt, containersJson, id);
'UPDATE health_gate_runs SET status = ?, reason = ?, ended_at = ?, containers_json = ?, failure_source = ? WHERE id = ?'
).run(status, reason, endedAt, containersJson, failureSource, id);
}
public getHealthGateRun(nodeId: number, stackName: string, id: string): HealthGateRunRow | undefined {
@@ -3178,6 +3344,128 @@ export class DatabaseService {
return result.changes;
}
// --- Service Update Recovery ---
public insertServiceUpdateRecovery(row: ServiceUpdateRecoveryRow): void {
this.db.prepare(
`INSERT INTO service_update_recovery
(id, node_id, stack_name, service_name, replicas_json, majority_image_id,
declared_image_ref, weak_floating_tag, health_gate_id, status,
expires_at, claim_expires_at, created_at, created_by)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
row.id, row.node_id, row.stack_name, row.service_name, row.replicas_json, row.majority_image_id,
row.declared_image_ref, row.weak_floating_tag, row.health_gate_id, row.status,
row.expires_at, row.claim_expires_at, row.created_at, row.created_by,
);
}
public getServiceUpdateRecovery(id: string): ServiceUpdateRecoveryRow | undefined {
return this.db.prepare(
'SELECT * FROM service_update_recovery WHERE id = ?'
).get(id) as ServiceUpdateRecoveryRow | undefined;
}
/** Active, unexpired rows for one service, most recent first. */
public listActiveServiceUpdateRecoveries(nodeId: number, stackName: string, serviceName: string): ServiceUpdateRecoveryRow[] {
return this.db.prepare(
`SELECT * FROM service_update_recovery
WHERE node_id = ? AND stack_name = ? AND service_name = ? AND status = 'active'
ORDER BY created_at DESC`
).all(nodeId, stackName, serviceName) as ServiceUpdateRecoveryRow[];
}
/** Attach the update flow's own health gate run id while the row is still active. */
public linkServiceUpdateRecoveryHealthGate(id: string, healthGateId: string): void {
this.db.prepare(
`UPDATE service_update_recovery SET health_gate_id = ? WHERE id = ? AND status = 'active'`
).run(healthGateId, id);
}
/**
* Atomic claim CAS for restore: active -> restoring, only when the row has
* not already expired. Returns the updated row, or undefined when another
* claim, a sweep, or a terminal status won the race.
*/
public claimServiceUpdateRecovery(id: string, claimExpiresAt: number, now: number): ServiceUpdateRecoveryRow | undefined {
const result = this.db.prepare(
`UPDATE service_update_recovery
SET status = 'restoring', claim_expires_at = ?
WHERE id = ? AND status = 'active' AND expires_at > ?`
).run(claimExpiresAt, id, now);
if (result.changes === 0) return undefined;
return this.getServiceUpdateRecovery(id);
}
/** Renewal CAS: only while still restoring. Cannot revive a consumed/expired/invalidated/active row. */
public renewServiceUpdateRecoveryClaim(id: string, claimExpiresAt: number): boolean {
const result = this.db.prepare(
`UPDATE service_update_recovery SET claim_expires_at = ? WHERE id = ? AND status = 'restoring'`
).run(claimExpiresAt, id);
return result.changes > 0;
}
/** Restore succeeded: restoring -> consumed, optionally linking the restore's own health gate run. */
public markServiceUpdateRecoveryConsumed(id: string, healthGateId: string | null = null): boolean {
const result = this.db.prepare(
`UPDATE service_update_recovery
SET status = 'consumed', claim_expires_at = NULL, health_gate_id = COALESCE(?, health_gate_id)
WHERE id = ? AND status = 'restoring'`
).run(healthGateId, id);
return result.changes > 0;
}
/** Mid-flight restore failure with the image still local: restoring -> active, claim cleared. */
public reactivateServiceUpdateRecovery(id: string): boolean {
const result = this.db.prepare(
`UPDATE service_update_recovery SET status = 'active', claim_expires_at = NULL WHERE id = ? AND status = 'restoring'`
).run(id);
return result.changes > 0;
}
/** Mid-flight restore failure with the image gone: restoring -> invalidated. */
public invalidateServiceUpdateRecovery(id: string): boolean {
const result = this.db.prepare(
`UPDATE service_update_recovery SET status = 'invalidated', claim_expires_at = NULL WHERE id = ? AND status = 'restoring'`
).run(id);
return result.changes > 0;
}
/** Startup/interval sweep: active rows whose TTL has lapsed. */
public sweepExpiredActiveServiceUpdateRecoveries(now: number): number {
const result = this.db.prepare(
`UPDATE service_update_recovery SET status = 'expired' WHERE status = 'active' AND expires_at <= ?`
).run(now);
return result.changes;
}
/**
* Startup/interval sweep: restoring rows abandoned by a dead claim (process
* died mid-restore). A restoring row with a live claim is never expired here,
* even if its original expires_at has long passed.
*/
public sweepAbandonedRestoringServiceUpdateRecoveries(now: number): number {
const result = this.db.prepare(
`UPDATE service_update_recovery
SET status = 'expired'
WHERE status = 'restoring' AND claim_expires_at IS NOT NULL AND claim_expires_at <= ?`
).run(now);
return result.changes;
}
/** Image IDs currently protected from prune: active rows and restoring rows with a live claim. */
public listHeldServiceUpdateRecoveryImageIds(nodeId: number, now: number): string[] {
const rows = this.db.prepare(
`SELECT DISTINCT majority_image_id FROM service_update_recovery
WHERE node_id = ? AND (status = 'active' OR (status = 'restoring' AND claim_expires_at > ?))`
).all(nodeId, now) as Array<{ majority_image_id: string }>;
return rows.map(r => r.majority_image_id);
}
public deleteServiceUpdateRecoveries(nodeId: number, stackName: string): void {
this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
}
// --- Notification History ---
private mapNotificationRow(row: any): NotificationHistory {
@@ -3569,6 +3857,7 @@ export class DatabaseService {
this.db.prepare('DELETE FROM preflight_acknowledgements WHERE node_id = ?').run(id);
this.db.prepare('DELETE FROM stack_exposure WHERE node_id = ?').run(id);
this.db.prepare('DELETE FROM health_gate_runs WHERE node_id = ?').run(id);
this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ?').run(id);
this.db.prepare('UPDATE blueprints SET pinned_node_id = NULL WHERE pinned_node_id = ?').run(id);
this.deleteRoleAssignmentsByResource('node', String(id));
this.db.prepare('DELETE FROM fleet_sync_status WHERE node_id = ?').run(id);
@@ -3644,6 +3933,12 @@ export class DatabaseService {
// --- Stack Update Status ---
/**
* `services` is omitted by legacy (non-service-aware) callers; omitting it
* leaves the stack's existing services_json untouched (COALESCE against the
* current row) rather than erasing a per-service breakdown that a model-aware
* caller persisted on a previous check.
*/
public upsertStackUpdateStatus(
nodeId: number,
stackName: string,
@@ -3651,16 +3946,20 @@ export class DatabaseService {
checkedAt: number,
checkStatus: StackCheckStatus = 'ok',
lastError: string | null = null,
services?: StackServiceStatus[],
generation = 0,
): void {
const servicesJson = services !== undefined ? stringifyServicesJson(services, generation) : null;
this.db.prepare(
`INSERT INTO stack_update_status (node_id, stack_name, has_update, check_status, last_error, checked_at)
VALUES (?, ?, ?, ?, ?, ?)
`INSERT INTO stack_update_status (node_id, stack_name, has_update, check_status, last_error, checked_at, services_json)
VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(node_id, stack_name) DO UPDATE SET
has_update = excluded.has_update,
check_status = excluded.check_status,
last_error = excluded.last_error,
checked_at = excluded.checked_at`
).run(nodeId, stackName, hasUpdate ? 1 : 0, checkStatus, lastError, checkedAt);
checked_at = excluded.checked_at,
services_json = COALESCE(excluded.services_json, stack_update_status.services_json)`
).run(nodeId, stackName, hasUpdate ? 1 : 0, checkStatus, lastError, checkedAt, servicesJson);
}
/**
@@ -3671,15 +3970,24 @@ export class DatabaseService {
* first-ever failed check inserts a row with has_update = 0 so the stack
* still appears with its failure reason.
*/
public recordStackCheckFailure(nodeId: number, stackName: string, lastError: string, checkedAt: number): void {
public recordStackCheckFailure(
nodeId: number,
stackName: string,
lastError: string,
checkedAt: number,
services?: StackServiceStatus[],
generation = 0,
): void {
const servicesJson = services !== undefined ? stringifyServicesJson(services, generation) : null;
this.db.prepare(
`INSERT INTO stack_update_status (node_id, stack_name, has_update, check_status, last_error, checked_at)
VALUES (?, ?, 0, 'failed', ?, ?)
`INSERT INTO stack_update_status (node_id, stack_name, has_update, check_status, last_error, checked_at, services_json)
VALUES (?, ?, 0, 'failed', ?, ?, ?)
ON CONFLICT(node_id, stack_name) DO UPDATE SET
check_status = 'failed',
last_error = excluded.last_error,
checked_at = excluded.checked_at`
).run(nodeId, stackName, lastError, checkedAt);
checked_at = excluded.checked_at,
services_json = COALESCE(excluded.services_json, stack_update_status.services_json)`
).run(nodeId, stackName, lastError, checkedAt, servicesJson);
}
public getStackUpdateStatus(nodeId?: number): Record<string, boolean> {
@@ -3700,20 +4008,35 @@ export class DatabaseService {
*/
public getStackUpdateDetail(nodeId: number): Record<string, StackUpdateDetail> {
const rows = this.db.prepare(
'SELECT stack_name, has_update, check_status, last_error, checked_at FROM stack_update_status WHERE node_id = ?'
).all(nodeId) as Array<{ stack_name: string; has_update: number; check_status: string | null; last_error: string | null; checked_at: number }>;
'SELECT stack_name, has_update, check_status, last_error, checked_at, services_json FROM stack_update_status WHERE node_id = ?'
).all(nodeId) as Array<{ stack_name: string; has_update: number; check_status: string | null; last_error: string | null; checked_at: number; services_json: string | null }>;
const result: Record<string, StackUpdateDetail> = {};
for (const row of rows) {
const services = parseServicesJson(row.services_json);
result[row.stack_name] = {
hasUpdate: row.has_update === 1,
checkStatus: (row.check_status === 'failed' || row.check_status === 'partial') ? row.check_status : 'ok',
lastError: row.last_error,
checkedAt: row.checked_at,
...(services.length > 0 ? { services } : {}),
};
}
return result;
}
/**
* Prior per-service breakdown for a single stack, used by ImageUpdateService
* to look up each service's last-known hasUpdate before reducing a fresh
* check (so a preserved value survives a partial/failed re-check). Corrupt
* or missing JSON yields an empty list.
*/
public getStackServicesJson(nodeId: number, stackName: string): StackServiceStatus[] {
const row = this.db.prepare(
'SELECT services_json FROM stack_update_status WHERE node_id = ? AND stack_name = ?'
).get(nodeId, stackName) as { services_json: string | null } | undefined;
return parseServicesJson(row?.services_json);
}
public clearStackUpdateStatus(nodeId: number, stackName: string): void {
this.db.prepare('DELETE FROM stack_update_status WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
}
+69 -5
View File
@@ -403,13 +403,56 @@ class DockerController {
};
}
/** True when an image is unused and has no meaningful tag (dangling / untagged). */
private static isDanglingImage(img: { RepoTags?: string[] | null; Containers?: number }): boolean {
if ((img.Containers ?? 0) !== 0) return false;
const tags = img.RepoTags ?? [];
if (tags.length === 0) return true;
return tags.every((t) => t === '<none>:<none>');
}
/**
* Enumerate candidate images and delete individually. Used when a recovery
* hold predicate is supplied so held images are never removed by a bulk
* Docker prune API. Reclaimed bytes come from a LayersSize before/after delta.
*/
private async pruneImagesEnumerated(
eligible: (img: { Id: string; RepoTags?: string[] | null; Containers?: number }) => boolean,
isImageHeld?: (imageId: string) => boolean,
): Promise<{ success: boolean; reclaimedBytes: number }> {
const beforeDf = await this.safeDfSnapshot();
const rawImages = await this.docker.listImages({ all: false });
const candidates = (rawImages as Array<{ Id: string; RepoTags?: string[] | null; Containers?: number }>)
.filter((img) => eligible(img) && !isImageHeld?.(img.Id));
for (const img of candidates) {
if (isImageHeld?.(img.Id)) continue;
try {
await this.docker.getImage(img.Id).remove({ force: true });
} catch (e) {
console.error(`[pruneImagesEnumerated] Failed to remove image ${sanitizeForLog(img.Id)}:`, e);
}
}
const afterDf = await this.safeDfSnapshot();
let reclaimedBytes = 0;
if (typeof beforeDf?.LayersSize === 'number' && typeof afterDf?.LayersSize === 'number') {
reclaimedBytes = Math.max(0, beforeDf.LayersSize - afterDf.LayersSize);
}
return { success: true, reclaimedBytes };
}
// Sencho's own image, networks, and named volumes are always in use by the
// running container, so Docker's server-side prune APIs (pruneContainers,
// pruneImages, pruneNetworks, pruneVolumes) skip them by definition. No
// extra self-guard is needed at this layer; the `managed` scope path goes
// through `pruneManagedOnly`, which adds an explicit self filter for
// defense-in-depth.
public async pruneSystem(target: 'containers' | 'images' | 'networks' | 'volumes', labelFilter?: string) {
public async pruneSystem(
target: 'containers' | 'images' | 'networks' | 'volumes',
labelFilter?: string,
isImageHeld?: (imageId: string) => boolean,
) {
let spaceReclaimed = 0;
if (target === 'containers') {
const filters: Record<string, string[]> = {};
@@ -417,6 +460,9 @@ class DockerController {
const r = await this.docker.pruneContainers({ filters });
spaceReclaimed = r.SpaceReclaimed || 0;
} else if (target === 'images') {
if (isImageHeld) {
return this.pruneImagesEnumerated((img) => (img.Containers ?? 0) === 0, isImageHeld);
}
// Remove all unused images, not just dangling ones
const filters: Record<string, string[] | Record<string, boolean>> = { dangling: { 'false': true } };
if (labelFilter) filters.label = [labelFilter];
@@ -444,7 +490,10 @@ class DockerController {
// which uses { dangling: { 'false': true } } to remove every unused image.
// Used by the prune-on-update flow to reclaim the layers a pull/recreate
// orphans, without touching tagged images for stopped stacks.
public async pruneDanglingImages(): Promise<{ success: boolean; reclaimedBytes: number }> {
public async pruneDanglingImages(isImageHeld?: (imageId: string) => boolean): Promise<{ success: boolean; reclaimedBytes: number }> {
if (isImageHeld) {
return this.pruneImagesEnumerated((img) => DockerController.isDanglingImage(img), isImageHeld);
}
const filters: Record<string, string[] | Record<string, boolean>> = { dangling: { 'true': true } };
const r = await this.docker.pruneImages({ filters });
return { success: true, reclaimedBytes: r.SpaceReclaimed || 0 };
@@ -635,7 +684,8 @@ class DockerController {
public async pruneManagedOnly(
target: 'images' | 'volumes' | 'networks',
knownStackNames: string[]
knownStackNames: string[],
isImageHeld?: (imageId: string) => boolean,
): Promise<{ success: boolean; reclaimedBytes: number }> {
const knownSet = new Set(knownStackNames);
const projectToStack = await DockerController.resolveProjectNameMap(knownStackNames);
@@ -707,6 +757,7 @@ class DockerController {
// once, but the per-image formula subtracts it from every referrer).
const beforeDf = await this.safeDfSnapshot();
await Promise.all(prunable.map(async (img) => {
if (isImageHeld?.(img.Id)) return;
try {
await this.docker.getImage(img.Id).remove({ force: true });
} catch (e) {
@@ -832,6 +883,7 @@ class DockerController {
scope: PruneScope,
knownStackNames: string[],
nodeId: number = this.nodeId,
isImageHeld?: (imageId: string) => boolean,
): Promise<PrunePlan> {
const ordered = normalizePruneTargets(targets);
const knownSet = new Set(knownStackNames);
@@ -980,6 +1032,7 @@ class DockerController {
const freeingImages = ordered.includes('containers');
for (const img of rawImages) {
if (selfIdentity.isOwnImage(img.Id)) continue;
if (isImageHeld?.(img.Id)) continue;
const containers = img.Containers ?? 0;
const refs = imageToContainerIds.get(img.Id) ?? [];
const becomesFree = freeingImages
@@ -1038,8 +1091,9 @@ class DockerController {
public async assertPlanFresh(
plan: PrunePlan,
knownStackNames: string[],
isImageHeld?: (imageId: string) => boolean,
): Promise<PrunePlan | null> {
const rebuilt = await this.buildPrunePlan(plan.targets, plan.scope, knownStackNames, plan.nodeId);
const rebuilt = await this.buildPrunePlan(plan.targets, plan.scope, knownStackNames, plan.nodeId, isImageHeld);
if (rebuilt.fingerprint !== plan.fingerprint) return null;
return rebuilt;
}
@@ -1051,8 +1105,9 @@ class DockerController {
public async executePrunePlan(
plan: PrunePlan,
knownStackNames: string[],
isImageHeld?: (imageId: string) => boolean,
): Promise<{ outcomes: PruneItemOutcome[]; reclaimedBytes: number; success: boolean }> {
const fresh = await this.assertPlanFresh(plan, knownStackNames);
const fresh = await this.assertPlanFresh(plan, knownStackNames, isImageHeld);
if (!fresh) throw new PrunePlanStaleError();
const knownSet = new Set(knownStackNames);
@@ -1123,6 +1178,15 @@ class DockerController {
}
if (target === 'images') {
if (isImageHeld?.(item.id)) {
outcomes.push({
id: item.id,
target: 'images',
status: 'skipped',
reason: 'Image held for pending service-update recovery',
});
continue;
}
if (imageIdBlocked(item.id)) {
const stillReferenced = await this.imageStillReferenced(item.id);
if (stillReferenced) {
@@ -179,6 +179,65 @@ export class DriftLedgerService {
return { detected: toInsert.length, resolved: toResolve.length };
}
/**
* Service-scoped counterpart to `reconcile`: only inserts/resolves findings
* for `serviceName`, leaving every other service's (and the empty/`null`
* service, i.e. network-level) open findings untouched. Used by the service
* update orchestrator so a service-scoped update never resolves drift it
* did not touch. Never pass a report already filtered to one service into
* the stack-wide `reconcile`; that would falsely resolve every other
* service's open findings.
*/
reconcileService(nodeId: number, stackName: string, serviceName: string, report: StackDriftReport): DriftReconcileResult {
if (report.status === 'unreachable' || report.parseError) {
return { detected: 0, resolved: 0 };
}
const db = DatabaseService.getInstance();
const now = Date.now();
const openForService = db.getOpenDriftFindings(nodeId, stackName).filter(r => r.service === serviceName);
const openByKey = new Map(openForService.map(r => [findingKey(r.service, r.finding_type), r]));
const currentForService = report.findings.filter(f => f.service === serviceName);
const currentByKey = new Map(currentForService.map(f => [findingKey(f.service, f.kind), f]));
const toInsert: StackDriftFinding[] = [];
for (const [key, f] of currentByKey) {
if (!openByKey.has(key)) toInsert.push(f);
}
const toResolve: StackDriftFindingRow[] = [];
for (const [key, row] of openByKey) {
if (!currentByKey.has(key)) toResolve.push(row);
}
db.getDb().transaction(() => {
db.setStackDossierDriftCheck(nodeId, stackName, now);
for (const f of toInsert) {
db.insertDriftFinding({
node_id: nodeId,
stack_name: stackName,
service: f.service,
finding_type: f.kind,
severity: 'warning',
message: f.detail,
expected_json: f.expected !== undefined ? JSON.stringify(f.expected) : null,
actual_json: f.actual !== undefined ? JSON.stringify(f.actual) : null,
detected_at: now,
});
}
for (const row of toResolve) {
db.resolveDriftFinding(row.id, now);
}
})();
if (toInsert.length > 0) {
this.recordActivity(nodeId, stackName, 'drift_detected', 'warning',
`Drift detected on ${stackName} (${serviceName}): ${toInsert.length} new finding${toInsert.length === 1 ? '' : 's'}`, now);
}
if (toResolve.length > 0) {
this.recordActivity(nodeId, stackName, 'drift_resolved', 'info',
`Drift resolved on ${stackName} (${serviceName}): ${toResolve.length} finding${toResolve.length === 1 ? '' : 's'} cleared`, now);
}
return { detected: toInsert.length, resolved: toResolve.length };
}
/**
* Build the spatial report for one stack and reconcile it into the ledger.
* Used by the deploy and update success hooks (and the rollback route, which
@@ -196,6 +255,22 @@ export class DriftLedgerService {
}
}
/**
* Build the spatial report for one stack and reconcile only `serviceName`'s
* findings into the ledger. Used by the service update orchestrator's
* success path. Best-effort: a build or reconcile failure is logged and
* swallowed so it never fails the update that triggered it.
*/
async reconcileServiceForStack(nodeId: number, stackName: string, serviceName: string): Promise<DriftReconcileResult> {
try {
const report = await buildStackDriftReport(nodeId, stackName);
return this.reconcileService(nodeId, stackName, serviceName, report);
} catch (error) {
console.error('[DriftLedger] reconcileServiceForStack failed for %s/%s:', sanitizeForLog(stackName), sanitizeForLog(serviceName), sanitizeForLog(getErrorMessage(error, 'unknown')));
return { detected: 0, resolved: 0 };
}
}
/**
* Write a drift transition to the stack activity timeline. History-only (no
* external channel dispatch): a drift signal belongs in the activity feed, not
+1 -1
View File
@@ -1337,7 +1337,7 @@ export class GitSourceService {
console.warn(`[GitSource] ${busy}`);
return { applied: true, deployed: false, deployError: busy };
}
HealthGateService.getInstance().begin(nodeId, stackName, 'deploy', 'system:git-source');
HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:git-source');
console.log(`[GitSource] Applied and deployed ${stackName} at ${commitSha.slice(0, 7)}`);
return { applied: true, deployed: true };
} catch (e) {
+551 -14
View File
@@ -1,12 +1,22 @@
import { randomUUID } from 'crypto';
import DockerController from './DockerController';
import { DatabaseService, type HealthGateRunRow } from './DatabaseService';
import { AutoHealService } from './AutoHealService';
import { sanitizeForLog } from '../utils/safeLog';
import { getErrorMessage } from '../utils/errors';
import { withTimeout } from '../utils/withTimeout';
import type { HealthGateContainer, HealthGateReport } from './updateGuard/types';
import { getComposeCommandTimeoutMs } from './ComposeService';
const POLL_INTERVAL_MS = 5_000;
// A prepared-but-never-begun token (prepare called, mutation then failed before
// beginPrepared) is dropped after this long. Swept lazily on each prepare/
// attach/beginPrepared call rather than on a timer, so the service leaves no
// standing interval (the observation poll timers are the only timers it arms).
// Must outlive the longest Compose mutation (compose timeout + 5m buffer).
function prepareTtlMs(): number {
return Math.max(getComposeCommandTimeoutMs(), 30 * 60_000) + 5 * 60_000;
}
// Per-observe ceiling so a hung Docker socket cannot leave a poll pending
// forever. Above POLL_INTERVAL_MS so a slow-but-live probe is not cut short; a
// timeout counts as a poll error and three in a row resolve the gate unknown.
@@ -20,6 +30,8 @@ const MAX_WINDOW_SECONDS = 600;
// updates). Gates past the cap finalize immediately as unknown.
const MAX_CONCURRENT_GATES = 25;
type GateRole = 'primary' | 'collateral';
interface ObservedContainer {
id: string;
name: string;
@@ -31,6 +43,10 @@ interface ObservedContainer {
restarts: number;
state: string;
health: string | null;
/** `com.docker.compose.service` label, when present (service gates only). */
service: string | null;
/** Image id the container is running (service gates check convergence on it). */
imageId: string;
}
interface ActiveGate {
@@ -53,6 +69,90 @@ interface ActiveGate {
* or stopped can never overwrite the terminal verdict with a stale one.
*/
finalized: boolean;
/** 'stack' for the legacy post-mutation gate, 'service' for prepared gates. */
targetScope: 'stack' | 'service';
/** Named trigger persisted on the row. */
trigger: 'update' | 'deploy' | 'service_update' | 'service_restore';
/** Service gates only. */
serviceName: string | null;
/** Single image id every primary replica must converge on (service gates). */
expectedImageId: string | null;
/** Expected primary replica count from the effective model (service gates; may be 0). */
expectedReplicas: number;
/** Sibling names eligible for regression evaluation (service gates). */
collateralEligibleNames: Set<string>;
/**
* Pre-mutation baselines for regression-eligible collateral siblings.
* Used to seed `expected` at arm time so a sibling that vanishes during the
* mutation gap is still tracked (and can fail) on the first poll.
*/
collateralBaselineByName: Map<string, ObservedContainer>;
/** Role of each expected container (service gates), for failure attribution. */
roleByName: Map<string, GateRole>;
}
/** A pre-mutation baseline container captured at prepare time. */
interface PreparedBaseline {
id: string;
name: string;
service: string | null;
state: string;
health: string | null;
restartCount: number;
startedAt: string | null;
imageId: string;
}
function isRegressionEligibleSibling(baseline: PreparedBaseline): boolean {
return baseline.state === 'running' && (baseline.health === null || baseline.health === 'healthy');
}
function observedFromPreparedBaseline(baseline: PreparedBaseline): ObservedContainer {
return {
id: baseline.id,
name: baseline.name,
startedAt: baseline.startedAt,
restartCount: baseline.restartCount,
restarts: 0,
state: baseline.state,
health: baseline.health,
service: baseline.service,
imageId: baseline.imageId,
};
}
/** An in-memory prepare snapshot awaiting attachExpectedImage + beginPrepared. */
interface PreparedGate {
token: string;
nodeId: number;
stackName: string;
serviceName: string;
trigger: 'service_update' | 'service_restore';
windowSeconds: number;
expiresAt: number;
expectedReplicas: number;
expectedImageId: string | null;
collateralEligibleNames: Set<string>;
primaryBaseline: PreparedBaseline[];
collateralBaseline: PreparedBaseline[];
}
export interface PrepareInput {
nodeId: number;
stackName: string;
target: { scope: 'service'; serviceName: string };
trigger: 'service_update' | 'service_restore';
/**
* Expected primary replica count from the effective model (may be 0). Passed
* by the orchestrator, which already rendered the model, so the gate stays a
* pure observer and does not run a second `docker compose config`.
*/
expectedReplicas: number;
}
export interface BeginPreparedResult {
runId: string | null;
observing: boolean;
}
/**
@@ -71,6 +171,8 @@ interface ActiveGate {
export class HealthGateService {
private static instance: HealthGateService;
private readonly active = new Map<string, ActiveGate>();
/** Prepare tokens awaiting beginPrepared (service update/restore only). */
private readonly prepared = new Map<string, PreparedGate>();
private started = false;
public static getInstance(): HealthGateService {
@@ -101,6 +203,51 @@ export class HealthGateService {
for (const gate of [...this.active.values()]) {
this.finalize(gate, 'unknown', 'shutdown during observation', []);
}
this.prepared.clear();
}
private gateKey(nodeId: number, stackName: string, scope: 'stack' | 'service', serviceName: string | null): string {
return `${nodeId}:${stackName}:${scope}:${serviceName ?? '_'}`;
}
private baselineMapFromPrepared(baselines: PreparedBaseline[]): Map<string, ObservedContainer> {
const map = new Map<string, ObservedContainer>();
for (const baseline of baselines) {
map.set(baseline.name, observedFromPreparedBaseline(baseline));
}
return map;
}
/**
* Finalize in-flight gates that a newer operation must replace:
* - A stack-scoped begin supersedes every gate on the stack.
* - A service-scoped begin supersedes only the same service's gate and any
* stack-scoped gate; other services keep observing independently.
*/
private supersedeGatesForStack(
nodeId: number,
stackName: string,
options?: { serviceName?: string | null; stackOnly?: boolean },
): void {
const serviceName = options?.serviceName;
const stackOnly = options?.stackOnly === true;
for (const gate of [...this.active.values()]) {
if (gate.nodeId !== nodeId || gate.stackName !== stackName) continue;
if (serviceName !== undefined) {
// Service begin: keep other services' gates.
if (gate.targetScope === 'service' && gate.serviceName !== serviceName) continue;
} else if (stackOnly) {
if (gate.targetScope !== 'stack') continue;
}
this.finalize(gate, 'unknown', 'superseded by a newer operation', []);
}
}
/** Drop prepare tokens whose TTL elapsed without a beginPrepared (lazy, no standing timer). */
private sweepExpiredPrepared(now: number): void {
for (const [token, prep] of this.prepared) {
if (now >= prep.expiresAt) this.prepared.delete(token);
}
}
/**
@@ -114,7 +261,7 @@ export class HealthGateService {
* every gated update path gets the timeline marker even when the gate
* itself is disabled.
*/
public begin(
public beginStack(
nodeId: number,
stackName: string,
trigger: 'update' | 'deploy',
@@ -132,12 +279,10 @@ export class HealthGateService {
}
if (!settings.enabled) return null;
// A newer operation supersedes an in-flight gate for the same stack.
const key = `${nodeId}:${stackName}`;
const existing = this.active.get(key);
if (existing) {
this.finalize(existing, 'unknown', 'superseded by a newer update', []);
}
// A newer stack operation supersedes every in-flight gate on this stack
// (including service gates).
const key = this.gateKey(nodeId, stackName, 'stack', null);
this.supersedeGatesForStack(nodeId, stackName);
const runId = randomUUID();
const startedAt = Date.now();
@@ -153,6 +298,9 @@ export class HealthGateService {
started_at: startedAt,
ended_at: null,
created_by: actor,
target_scope: 'stack',
service_name: null,
failure_source: null,
};
if (this.active.size >= MAX_CONCURRENT_GATES) {
@@ -173,6 +321,14 @@ export class HealthGateService {
missingLastPoll: new Set(),
restartingLastPoll: new Set(),
finalized: false,
targetScope: 'stack',
trigger,
serviceName: null,
expectedImageId: null,
expectedReplicas: 0,
collateralEligibleNames: new Set(),
collateralBaselineByName: new Map(),
roleByName: new Map(),
};
this.active.set(key, gate);
this.scheduleNextPoll(gate);
@@ -180,13 +336,169 @@ export class HealthGateService {
} catch (error) {
// The gate is an observer; its failure must never fail the operation.
console.error(
'[HealthGate] begin (%s) failed for %s on node %d:',
'[HealthGate] beginStack (%s) failed for %s on node %d:',
trigger, sanitizeForLog(stackName), nodeId, error,
);
return null;
}
}
/** @deprecated Prefer beginStack; retained as a one-PR alias for callers under migration. */
public begin(
nodeId: number,
stackName: string,
trigger: 'update' | 'deploy',
actor: string | null,
): string | null {
return this.beginStack(nodeId, stackName, trigger, actor);
}
/**
* Pre-mutation snapshot for a service-scoped update/restore. Captures the
* selected service's primary replicas and the sibling collateral set (marking
* only currently-running, healthy or healthcheck-less siblings as
* regression-eligible), and returns an opaque token the orchestrator carries
* through attachExpectedImage and beginPrepared. Never throws: a Docker read
* failure yields snapshotOk=false so the orchestrator can skip observation
* rather than claim a gate with an empty baseline.
*/
public async prepare(input: PrepareInput): Promise<{ prepareToken: string; expiresAt: number; snapshotOk: boolean }> {
const now = Date.now();
this.sweepExpiredPrepared(now);
const { nodeId, stackName, trigger } = input;
const serviceName = input.target.serviceName;
const settings = this.readSettings();
let baselines: PreparedBaseline[] = [];
let snapshotOk = true;
try {
baselines = await this.snapshotBaselines(nodeId, stackName);
} catch (error) {
snapshotOk = false;
console.warn('[HealthGate] prepare snapshot failed for %s/%s:',
sanitizeForLog(stackName), sanitizeForLog(serviceName), getErrorMessage(error, 'unknown'));
}
const primaryBaseline = baselines.filter(b => b.service === serviceName);
const collateralBaseline = baselines.filter(b => b.service !== serviceName);
const collateralEligibleNames = new Set(
collateralBaseline.filter(isRegressionEligibleSibling).map(b => b.name),
);
const token = randomUUID();
const expiresAt = now + prepareTtlMs();
this.prepared.set(token, {
token,
nodeId,
stackName,
serviceName,
trigger,
windowSeconds: settings.windowSeconds,
expiresAt,
expectedReplicas: input.expectedReplicas,
expectedImageId: null,
collateralEligibleNames,
primaryBaseline,
collateralBaseline,
});
return { prepareToken: token, expiresAt, snapshotOk };
}
/** Record the single image id every primary replica must converge on. No-op for an unknown/expired token. */
public attachExpectedImage(prepareToken: string, expectedImageId: string): void {
this.sweepExpiredPrepared(Date.now());
const prep = this.prepared.get(prepareToken);
if (!prep) {
console.warn('[HealthGate] attachExpectedImage: unknown or expired prepare token');
return;
}
prep.expectedImageId = expectedImageId;
}
/**
* Begin observing a prepared service gate after its mutation. Mirrors
* beginStack's nullability: returns a null runId (and observing:false) when
* the service is not started, gating is disabled, or the insert fails; a
* non-null runId with observing:false past the concurrency cap; and a non-null
* runId with observing:true once the poll is armed. The prepare token is
* consumed either way. Never throws.
*/
public beginPrepared(input: { prepareToken: string; actor: string | null }): BeginPreparedResult {
const now = Date.now();
this.sweepExpiredPrepared(now);
const prep = this.prepared.get(input.prepareToken);
// Always consume the token so a caller cannot begin twice off one prepare.
if (prep) this.prepared.delete(input.prepareToken);
if (!this.started || !prep) return { runId: null, observing: false };
try {
const db = DatabaseService.getInstance();
const settings = this.readSettings();
if (!settings.enabled) return { runId: null, observing: false };
const key = this.gateKey(prep.nodeId, prep.stackName, 'service', prep.serviceName);
// Same-service + stack gates only; sibling service observations continue.
this.supersedeGatesForStack(prep.nodeId, prep.stackName, { serviceName: prep.serviceName });
const runId = randomUUID();
const startedAt = Date.now();
const row: HealthGateRunRow = {
id: runId,
node_id: prep.nodeId,
stack_name: prep.stackName,
trigger_action: prep.trigger,
status: 'observing',
reason: null,
window_seconds: prep.windowSeconds,
containers_json: '[]',
started_at: startedAt,
ended_at: null,
created_by: input.actor,
target_scope: 'service',
service_name: prep.serviceName,
failure_source: null,
};
if (this.active.size >= MAX_CONCURRENT_GATES) {
db.insertHealthGateRun({ ...row, status: 'unknown', reason: 'too many concurrent observations', ended_at: startedAt });
return { runId, observing: false };
}
db.insertHealthGateRun(row);
const gate: ActiveGate = {
runId,
nodeId: prep.nodeId,
stackName: prep.stackName,
windowSeconds: prep.windowSeconds,
startedAt,
timer: null,
expected: null,
consecutivePollErrors: 0,
missingLastPoll: new Set(),
restartingLastPoll: new Set(),
finalized: false,
targetScope: 'service',
trigger: prep.trigger,
serviceName: prep.serviceName,
expectedImageId: prep.expectedImageId,
expectedReplicas: prep.expectedReplicas,
collateralEligibleNames: prep.collateralEligibleNames,
collateralBaselineByName: this.baselineMapFromPrepared(
prep.collateralBaseline.filter(b => prep.collateralEligibleNames.has(b.name)),
),
roleByName: new Map(),
};
this.active.set(key, gate);
this.scheduleNextPoll(gate);
return { runId, observing: true };
} catch (error) {
console.error('[HealthGate] beginPrepared failed for %s/%s:',
sanitizeForLog(prep.stackName), sanitizeForLog(prep.serviceName), error);
return { runId: null, observing: false };
}
}
/** A specific run by id, the latest run, or the never-run sentinel. */
public getReport(nodeId: number, stackName: string, gateId?: string): HealthGateReport {
const db = DatabaseService.getInstance();
@@ -197,6 +509,7 @@ export class HealthGateService {
return {
stack: stackName, id: null, status: 'never-run', trigger: null, reason: null,
windowSeconds: null, startedAt: null, endedAt: null, containers: [],
targetScope: 'stack', serviceName: null, failureSource: null,
};
}
let containers: HealthGateContainer[] = [];
@@ -217,6 +530,9 @@ export class HealthGateService {
startedAt: row.started_at,
endedAt: row.ended_at,
containers,
targetScope: row.target_scope ?? 'stack',
serviceName: row.service_name ?? null,
failureSource: row.failure_source ?? null,
};
}
@@ -227,11 +543,15 @@ export class HealthGateService {
* or corrupt the restart/missing accounting that assumes one poll at a time.
*/
private async poll(gate: ActiveGate): Promise<void> {
const key = `${gate.nodeId}:${gate.stackName}`;
const key = this.gateKey(gate.nodeId, gate.stackName, gate.targetScope, gate.serviceName);
// A late timer fire after supersede, stop, or finalize must do nothing.
if (gate.finalized || this.active.get(key) !== gate) return;
try {
await this.runPollCycle(gate, key);
if (gate.targetScope === 'service') {
await this.runServicePollCycle(gate, key);
} else {
await this.runPollCycle(gate, key);
}
} finally {
if (!gate.finalized && this.active.get(key) === gate) {
this.scheduleNextPoll(gate);
@@ -352,16 +672,201 @@ export class HealthGateService {
this.finalize(gate, 'passed', null, summary);
}
/**
* Poll cycle for a prepared service gate. Observes the selected service's
* primary replicas (image convergence, replica count, health/restart) and the
* regression-eligible collateral siblings, attributing a failure to
* 'primary' or 'collateral'. Unrelated containers that appear after prepare do
* not expand the observed set.
*/
private async runServicePollCycle(gate: ActiveGate, key: string): Promise<void> {
let observed: ObservedContainer[];
try {
observed = await withTimeout(
this.observeContainers(gate), OBSERVE_TIMEOUT_MS, 'health gate observe',
);
} catch (error) {
gate.consecutivePollErrors += 1;
console.warn(
'[HealthGate] service poll error %d for %s/%s:',
gate.consecutivePollErrors, sanitizeForLog(gate.stackName),
sanitizeForLog(gate.serviceName ?? ''), getErrorMessage(error, 'unknown'),
);
if (gate.consecutivePollErrors >= 3) {
this.finalize(gate, 'unknown', 'Docker became unreachable during observation', []);
}
return;
}
if (gate.finalized || this.active.get(key) !== gate) return;
gate.consecutivePollErrors = 0;
const elapsedMs = Date.now() - gate.startedAt;
const serviceName = gate.serviceName ?? '';
// Arm the expected set once from post-mutation primary replicas plus the
// pre-mutation regression-eligible collateral baselines. Seeding collateral
// from the prepare baseline (not only from the first post-mutation poll)
// means a healthy sibling that vanished during the mutation gap is still
// tracked and can fail the gate. Scale-0 arms immediately with an empty
// primary set; scale>0 waits (up to the empty grace) for replicas.
if (gate.expected === null) {
const primary = observed.filter(c => c.service === serviceName);
if (gate.expectedReplicas > 0 && primary.length === 0) {
if (elapsedMs >= EMPTY_GRACE_MS) {
this.finalize(gate, 'failed', `service ${serviceName} has no running replicas to observe`, [], 'primary');
}
return;
}
const expected = new Map<string, ObservedContainer>();
for (const c of primary) {
gate.roleByName.set(c.name, 'primary');
expected.set(c.name, c);
}
const observedByName = new Map(observed.map(c => [c.name, c]));
for (const [name, baseline] of gate.collateralBaselineByName) {
if (gate.roleByName.has(name)) continue;
gate.roleByName.set(name, 'collateral');
// Prefer the live observation when the sibling is still present; otherwise
// keep the prepare baseline so a missing sibling enters the miss path.
expected.set(name, observedByName.get(name) ?? baseline);
}
// Also pick up eligible siblings that appeared under a new name after mutation
// (unusual, but keeps prior behavior for containers still in the eligible set).
for (const c of observed) {
if (gate.collateralEligibleNames.has(c.name) && !gate.roleByName.has(c.name)) {
gate.roleByName.set(c.name, 'collateral');
expected.set(c.name, c);
}
}
gate.expected = expected;
return;
}
const byName = new Map(observed.map(c => [c.name, c]));
for (const [name, baseline] of gate.expected) {
const current = byName.get(name);
if (!current) continue;
const restarted =
current.id !== baseline.id ||
current.restartCount > baseline.restartCount ||
(current.startedAt !== null && baseline.startedAt !== null && current.startedAt !== baseline.startedAt);
current.restarts = baseline.restarts + (restarted ? 1 : 0);
}
const summary = this.summarizeService(gate, byName);
// Primary image convergence: a primary replica on any image id other than
// the single attached expected id fails the gate.
if (gate.expectedImageId) {
const wrongImage = observed.find(
c => c.service === serviceName && c.imageId && c.imageId !== gate.expectedImageId,
);
if (wrongImage) {
this.finalize(gate, 'failed', `service ${serviceName} replica ${wrongImage.name} is running an unexpected image`, summary, 'primary');
return;
}
}
for (const [name, baseline] of gate.expected) {
const role = gate.roleByName.get(name) ?? 'collateral';
const noun = role === 'primary' ? 'replica' : 'sibling';
const current = byName.get(name);
if (!current) {
if (gate.missingLastPoll.has(name)) {
this.finalize(gate, 'failed', `${noun} ${name} disappeared during observation`, summary, role);
return;
}
gate.missingLastPoll.add(name);
continue;
}
gate.missingLastPoll.delete(name);
if (current.state === 'exited' && baseline.restarts === current.restarts) {
this.finalize(gate, 'failed', `${noun} ${name} exited during observation`, summary, role);
return;
}
if (current.health === 'unhealthy') {
this.finalize(gate, 'failed', `${noun} ${name} reported unhealthy`, summary, role);
return;
}
if (current.restarts >= 2) {
this.finalize(gate, 'failed', `${noun} ${name} is restart looping`, summary, role);
return;
}
if (current.state === 'restarting') {
if (gate.restartingLastPoll.has(name)) {
this.finalize(gate, 'failed', `${noun} ${name} is stuck restarting`, summary, role);
return;
}
gate.restartingLastPoll.add(name);
} else {
gate.restartingLastPoll.delete(name);
}
gate.expected.set(name, current);
}
if (elapsedMs < gate.windowSeconds * 1000) return;
const stillStarting = observed.filter(
c => c.health === 'starting' && (c.service === serviceName || gate.collateralEligibleNames.has(c.name)),
);
if (stillStarting.length > 0) {
this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary);
return;
}
const runningPrimary = observed.filter(c => c.service === serviceName && c.state === 'running');
if (runningPrimary.length !== gate.expectedReplicas) {
this.finalize(
gate, 'failed',
`service ${serviceName} has ${runningPrimary.length} running replica(s), expected ${gate.expectedReplicas}`,
summary, 'primary',
);
return;
}
const collateralNotRunning = [...gate.expected.keys()]
.filter(name => gate.roleByName.get(name) === 'collateral')
.filter(name => byName.get(name)?.state !== 'running');
if (collateralNotRunning.length > 0) {
this.finalize(gate, 'failed', `sibling(s) not running at the end of the window: ${collateralNotRunning.join(', ')}`, summary, 'collateral');
return;
}
this.finalize(gate, 'passed', null, summary);
}
private summarizeService(
gate: ActiveGate,
current: Map<string, ObservedContainer>,
): HealthGateContainer[] {
if (!gate.expected) return [];
return [...gate.expected.values()].map(baseline => {
const now = current.get(baseline.name);
return {
name: baseline.name,
state: now?.state ?? 'missing',
health: now?.health ?? null,
restarts: now?.restarts ?? baseline.restarts,
service: (now ?? baseline).service ?? gate.serviceName,
role: gate.roleByName.get(baseline.name) ?? null,
};
});
}
private async observeContainers(gate: ActiveGate): Promise<ObservedContainer[]> {
const docker = DockerController.getInstance(gate.nodeId).getDocker();
return this.listStackContainers(gate.nodeId, gate.stackName);
}
/** List and inspect a stack's containers into the gate's observation shape. */
private async listStackContainers(nodeId: number, stackName: string): Promise<ObservedContainer[]> {
const docker = DockerController.getInstance(nodeId).getDocker();
const listed = await docker.listContainers({
all: true,
filters: { label: [`com.docker.compose.project=${gate.stackName}`] },
filters: { label: [`com.docker.compose.project=${stackName}`] },
});
const observed = await Promise.all(
listed.map(async (info): Promise<ObservedContainer | null> => {
try {
const inspect = await docker.getContainer(info.Id).inspect();
const labels = (info.Labels ?? inspect.Config?.Labels ?? {}) as Record<string, string>;
return {
id: info.Id,
name: info.Names?.[0]?.replace(/^\//, '') ?? info.Id.slice(0, 12),
@@ -370,6 +875,8 @@ export class HealthGateService {
restarts: 0,
state: inspect.State?.Status ?? info.State ?? 'unknown',
health: inspect.State?.Health?.Status ?? null,
service: labels['com.docker.compose.service'] ?? null,
imageId: inspect.Image ?? '',
};
} catch (e: unknown) {
// Removed between list and inspect; the missing-container logic will
@@ -382,6 +889,23 @@ export class HealthGateService {
return observed.filter((c): c is ObservedContainer => c !== null);
}
/** Snapshot the current per-container facts for prepare's primary/collateral partition. */
private async snapshotBaselines(nodeId: number, stackName: string): Promise<PreparedBaseline[]> {
const observed = await withTimeout(
this.listStackContainers(nodeId, stackName), OBSERVE_TIMEOUT_MS, 'health gate prepare snapshot',
);
return observed.map(c => ({
id: c.id,
name: c.name,
service: c.service,
state: c.state,
health: c.health,
restartCount: c.restartCount,
startedAt: c.startedAt,
imageId: c.imageId,
}));
}
private summarize(
expected: Map<string, ObservedContainer>,
current: Map<string, ObservedContainer>,
@@ -402,16 +926,29 @@ export class HealthGateService {
status: 'passed' | 'failed' | 'unknown',
reason: string | null,
containers: HealthGateContainer[],
failureSource: 'primary' | 'collateral' | null = null,
): void {
if (gate.finalized) return;
gate.finalized = true;
if (gate.timer) clearTimeout(gate.timer);
const key = `${gate.nodeId}:${gate.stackName}`;
const key = this.gateKey(gate.nodeId, gate.stackName, gate.targetScope, gate.serviceName);
if (this.active.get(key) === gate) this.active.delete(key);
// A service gate owns its service's Auto-Heal suppression while observing;
// release it here so a failed/superseded/stopped gate never leaves auto-heal
// muted for that service.
if (gate.targetScope === 'service' && gate.serviceName) {
try {
AutoHealService.getInstance().clearSuppress(gate.nodeId, gate.stackName, gate.serviceName);
} catch (error) {
console.warn('[HealthGate] Failed to clear auto-heal suppression for %s/%s:',
sanitizeForLog(gate.stackName), sanitizeForLog(gate.serviceName), getErrorMessage(error, 'unknown'));
}
}
try {
DatabaseService.getInstance().finalizeHealthGateRun(
gate.runId, status, reason, Date.now(), JSON.stringify(containers),
gate.runId, status, reason, Date.now(), JSON.stringify(containers), failureSource,
);
} catch (error) {
// The verdict is lost from the DB (the startup sweep will later rewrite
+349 -38
View File
@@ -2,7 +2,7 @@ import path from 'path';
import YAML from 'yaml';
import { CronExpressionParser } from 'cron-parser';
import DockerController from './DockerController';
import { DatabaseService } from './DatabaseService';
import { DatabaseService, type StackCheckStatus, type StackServiceStatus } from './DatabaseService';
import { FileSystemService } from './FileSystemService';
import { RegistryService } from './RegistryService';
import { NodeRegistry } from './NodeRegistry';
@@ -12,6 +12,7 @@ import { parseImageRef, selectLocalRepoDigest, compareLocalToRemoteTag } from '.
import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
import { sanitizeForLog } from '../utils/safeLog';
import { buildEffectiveServiceModel } from './effectiveServiceModel';
const BACKFILL_KEY = 'image_update_notifications_backfilled';
@@ -243,11 +244,124 @@ export async function loadStackBuildServices(nodeId: number, stackName: string):
return extractBuildServicesFromCompose(composeContent);
}
// ─── Per-service reduction (model-based status) ─────────────────────────────
export interface ServiceReduction {
status: StackServiceStatus;
confirmedUpdateThisRun: boolean;
}
export function reduceServiceStatus(
service: string,
declaredImage: string | null,
runtimeImages: string[],
imageUpdateMap: Map<string, ImageCheckResult>,
prior: StackServiceStatus | undefined,
): ServiceReduction {
const dedupedRuntime = [...new Set(runtimeImages)].sort();
const refs = new Set<string>();
if (declaredImage) refs.add(declaredImage);
for (const ref of dedupedRuntime) refs.add(ref);
if (refs.size === 0) {
return {
status: { service, image: declaredImage, hasUpdate: false, checkStatus: 'not_checkable', lastError: null },
confirmedUpdateThisRun: false,
};
}
const checkableResults: ImageCheckResult[] = [];
for (const ref of refs) {
const result = imageUpdateMap.get(ref);
if (!result || result.notCheckable) continue;
checkableResults.push(result);
}
if (checkableResults.length === 0) {
return {
status: { service, image: declaredImage, hasUpdate: false, checkStatus: 'not_checkable', lastError: null },
confirmedUpdateThisRun: false,
};
}
const errored = checkableResults.filter((r) => r.error !== undefined);
const confirmedUpdateThisRun = checkableResults.some(
(r) => r.error === undefined && r.hasUpdate === true,
);
if (errored.length === checkableResults.length) {
const priorHasUpdate = prior?.hasUpdate ?? false;
return {
status: {
service,
image: declaredImage,
...(dedupedRuntime.length > 0 ? { runtimeImages: dedupedRuntime } : {}),
hasUpdate: priorHasUpdate,
checkStatus: 'failed',
lastError: errored[0].error ?? 'Update check failed',
},
confirmedUpdateThisRun: false,
};
}
const checkStatus: StackServiceStatus['checkStatus'] = errored.length > 0 ? 'partial' : 'ok';
const lastError = errored.length > 0 ? (errored[0].error ?? null) : null;
const hasUpdate = checkStatus === 'partial'
? (confirmedUpdateThisRun || (prior?.hasUpdate ?? false))
: confirmedUpdateThisRun;
return {
status: {
service,
image: declaredImage,
...(dedupedRuntime.length > 0 ? { runtimeImages: dedupedRuntime } : {}),
hasUpdate,
checkStatus,
lastError,
},
confirmedUpdateThisRun,
};
}
export function aggregateServiceCheckStatus(services: StackServiceStatus[]): StackCheckStatus {
if (services.length === 0) return 'ok';
const checkable = services.filter((s) => s.checkStatus !== 'not_checkable');
if (checkable.length === 0) return 'ok';
const failedCount = checkable.filter((s) => s.checkStatus === 'failed').length;
if (failedCount === checkable.length) return 'failed';
if (checkable.some((s) => s.checkStatus === 'partial')
|| checkable.some((s) => s.checkStatus === 'failed')) {
return 'partial';
}
return 'ok';
}
export function buildAvailabilityNotifyMessage(stackName: string, services: StackServiceStatus[]): string {
const named = services.filter((s) => s.hasUpdate).map((s) => s.service).sort();
if (named.length === 0 || services.length <= 1) {
return `Stack "${stackName}" has image updates available.`;
}
if (named.length === 1) {
return `Stack "${stackName}" has image updates available for service: ${named[0]}.`;
}
return `Stack "${stackName}" has image updates available for services: ${named.join(', ')}.`;
}
function stackStatusLastError(services: StackServiceStatus[]): string | null {
for (const svc of services) {
if (svc.checkStatus !== 'not_checkable' && svc.lastError) return svc.lastError;
}
return null;
}
// ─── Service ──────────────────────────────────────────────────────────────────
export class ImageUpdateService {
private static instance: ImageUpdateService;
/** Per-stack write serialization and generation for stale-writer discard. */
private stackWriteState = new Map<string, { chain: Promise<void>; generation: number }>();
private static readonly MIN_INTERVAL_MINUTES = 15;
private static readonly MAX_INTERVAL_MINUTES = 1440; // 24 hours
private static readonly DEFAULT_INTERVAL_MINUTES = 120; // 2 hours
@@ -594,9 +708,10 @@ export class ImageUpdateService {
}
// Phase 3: Container augmentation (captures actual deployed image tags)
let allContainers: Awaited<ReturnType<DockerController['getAllContainers']>> = [];
try {
const containers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers');
for (const c of containers) {
allContainers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers');
for (const c of allContainers) {
// Prefer the pinned project label (== stackName for Sencho-deployed
// stacks, including multi-file / context-dir ones where
// --project-directory would otherwise change the working-dir
@@ -635,6 +750,14 @@ export class ImageUpdateService {
}
// Phase 4: Deduplicate and check all unique images
// Reserve write generations before the slow registry checks so a concurrent
// recheckStack that finishes first keeps its write (stale full-scan commits
// are dropped in withStackWriteLock).
const writeGenerations = new Map<string, number>();
for (const stackName of stackImages.keys()) {
writeGenerations.set(stackName, this.reserveStackWriteGeneration(nodeId, stackName));
}
const allImages = new Set<string>();
for (const imgs of stackImages.values()) for (const img of imgs) allImages.add(img);
@@ -662,54 +785,36 @@ export class ImageUpdateService {
// Write status for ALL stacks (including those with no pullable images)
const now = Date.now();
let updatesFound = 0;
const newlyUpdated: string[] = [];
for (const [stackName, images] of stackImages) {
// Tally only checkable images: a not-checkable image (locally built,
// or a bare digest ref) is neither a pass nor a failure.
const checkable = Array.from(images)
.map(img => imageUpdateMap.get(img))
.filter((r): r is ImageCheckResult => !!r && !r.notCheckable);
const errored = checkable.filter(r => r.error !== undefined);
const confirmedHasUpdate = checkable.some(r => r.error === undefined && r.hasUpdate === true);
// Every checkable image failed: status is undeterminable. Preserve the
// last-known has_update so a transient registry outage neither erases a
// real update nor flaps the notification state.
if (checkable.length > 0 && errored.length === checkable.length) {
db.recordStackCheckFailure(nodeId, stackName, errored[0].error ?? 'Update check failed', now);
continue;
}
const checkStatus = errored.length > 0 ? 'partial' : 'ok';
const lastError = errored.length > 0 ? (errored[0].error ?? null) : null;
// Only a fully-ok check is authoritative enough to lower has_update to
// false. On a partial check some image could not be reached, so a
// previously confirmed update is preserved rather than erased (which
// would also re-fire the notification when that image recovers).
const hasUpdate = checkStatus === 'partial'
? (confirmedHasUpdate || previousState[stackName] === true)
: confirmedHasUpdate;
if (hasUpdate) {
const newlyUpdated: Array<{ stackName: string; message: string }> = [];
for (const [stackName] of stackImages) {
const outcome = await this.writeStackUpdateStatus(
nodeId,
stackName,
stackImages.get(stackName) ?? new Set(),
imageUpdateMap,
previousState,
now,
allContainers,
db,
writeGenerations.get(stackName) ?? this.reserveStackWriteGeneration(nodeId, stackName),
);
if (outcome.committed && outcome.hasUpdate) {
updatesFound++;
// Notify only on state transition: was false/absent, now true
if (!isBackfilled || !previousState[stackName]) {
newlyUpdated.push(stackName);
newlyUpdated.push({ stackName, message: outcome.notifyMessage });
}
}
db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now, checkStatus, lastError);
}
// Dispatch notifications for stacks that newly have updates
if (newlyUpdated.length > 0) {
const notifier = NotificationService.getInstance();
for (const stackName of newlyUpdated) {
for (const { stackName, message } of newlyUpdated) {
try {
await notifier.dispatchAlert(
'info',
'image_update_available',
// Node-neutral body: the hub bell badge attributes remotes.
`Stack "${stackName}" has image updates available.`,
message,
{ stackName, actor: 'system:image-update' },
);
} catch (e) {
@@ -751,6 +856,212 @@ export class ImageUpdateService {
}
}
/**
* Re-check a single stack after a service-scoped update or restore. On a
* render failure the prior row is left untouched and a warning is returned.
*/
public async recheckStack(nodeId: number, stackName: string): Promise<{ warning: string | null }> {
const generation = this.reserveStackWriteGeneration(nodeId, stackName);
const db = DatabaseService.getInstance();
const docker = DockerController.getInstance(nodeId);
const model = await buildEffectiveServiceModel(nodeId, stackName);
if (!model.renderable) {
return { warning: model.error };
}
let containers: Array<{ Image?: string; Labels?: Record<string, string> }> = [];
try {
containers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers');
} catch (e) {
console.warn(
'[ImageUpdateService] recheckStack container read failed for %s: %s',
sanitizeForLog(stackName),
sanitizeForLog(getErrorMessage(e, 'unknown')),
);
}
const refs = new Set<string>();
const runtimeByService = this.runtimeImagesByService(stackName, containers);
for (const spec of model.services) {
if (spec.declaredImage) refs.add(spec.declaredImage);
for (const ref of runtimeByService.get(spec.name) ?? []) refs.add(ref);
}
const imageUpdateMap = new Map<string, ImageCheckResult>();
for (const imageRef of refs) {
try {
imageUpdateMap.set(imageRef, await this.checkImage(docker, imageRef));
} catch (e) {
imageUpdateMap.set(imageRef, { hasUpdate: false, error: getErrorMessage(e, 'Update check failed') });
}
await sleep(ImageUpdateService.INTER_IMAGE_DELAY_MS);
}
const priorByService = new Map(db.getStackServicesJson(nodeId, stackName).map((s) => [s.service, s]));
const reductions = model.services.map((spec) => reduceServiceStatus(
spec.name,
spec.declaredImage,
runtimeByService.get(spec.name) ?? [],
imageUpdateMap,
priorByService.get(spec.name),
));
const services = reductions.map((r) => r.status);
const checkStatus = aggregateServiceCheckStatus(services);
const hasUpdate = services.some((s) => s.hasUpdate);
const lastError = stackStatusLastError(services);
const now = Date.now();
await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => {
if (checkStatus === 'failed') {
db.recordStackCheckFailure(nodeId, stackName, lastError ?? 'Update check failed', now, services, gen);
} else {
db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now, checkStatus, lastError, services, gen);
}
});
return { warning: null };
}
private stackWriteKey(nodeId: number, stackName: string): string {
return `${nodeId}:${stackName}`;
}
/** Bump the per-stack write generation before async registry work so a later
* slower scan cannot commit after a newer recheck reserved a higher gen. */
private reserveStackWriteGeneration(nodeId: number, stackName: string): number {
const key = this.stackWriteKey(nodeId, stackName);
let state = this.stackWriteState.get(key);
if (!state) {
state = { chain: Promise.resolve(), generation: 0 };
this.stackWriteState.set(key, state);
}
state.generation += 1;
return state.generation;
}
private async withStackWriteLock(
nodeId: number,
stackName: string,
generation: number,
write: (generation: number) => void | Promise<void>,
): Promise<boolean> {
const key = this.stackWriteKey(nodeId, stackName);
let state = this.stackWriteState.get(key);
if (!state) {
state = { chain: Promise.resolve(), generation };
this.stackWriteState.set(key, state);
}
let committed = false;
state.chain = state.chain.then(async () => {
const current = this.stackWriteState.get(key);
// A newer reservation supersedes this writer; drop the stale commit.
if (!current || generation < current.generation) return;
await write(generation);
committed = true;
});
await state.chain;
return committed;
}
private runtimeImagesByService(
stackName: string,
containers: Array<{ Image?: string; Labels?: Record<string, string> }>,
): Map<string, string[]> {
const out = new Map<string, string[]>();
for (const c of containers) {
if (c.Labels?.['com.docker.compose.project'] !== stackName) continue;
const service = c.Labels?.['com.docker.compose.service'];
if (!service) continue;
const imageRef = c.Image ?? '';
if (!imageRef || imageRef.startsWith('sha256:')) continue;
const list = out.get(service) ?? [];
list.push(imageRef);
out.set(service, list);
}
return out;
}
private async writeStackUpdateStatus(
nodeId: number,
stackName: string,
images: Set<string>,
imageUpdateMap: Map<string, ImageCheckResult>,
previousState: Record<string, boolean>,
checkedAt: number,
containers: Array<{ Image?: string; Labels?: Record<string, string> }>,
db: DatabaseService,
generation: number,
): Promise<{ hasUpdate: boolean; notifyMessage: string; committed: boolean }> {
const model = await buildEffectiveServiceModel(nodeId, stackName);
if (model.renderable) {
const priorByService = new Map(
DatabaseService.getInstance().getStackServicesJson(nodeId, stackName).map((s) => [s.service, s]),
);
const runtimeByService = this.runtimeImagesByService(stackName, containers);
const reductions = model.services.map((spec) => reduceServiceStatus(
spec.name,
spec.declaredImage,
runtimeByService.get(spec.name) ?? [],
imageUpdateMap,
priorByService.get(spec.name),
));
const services = reductions.map((r) => r.status);
const checkStatus = aggregateServiceCheckStatus(services);
const hasUpdate = services.some((s) => s.hasUpdate);
const lastError = stackStatusLastError(services);
const notifyMessage = buildAvailabilityNotifyMessage(stackName, services);
const committed = await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => {
if (checkStatus === 'failed') {
db.recordStackCheckFailure(
nodeId, stackName, lastError ?? 'Update check failed', checkedAt, services, gen,
);
} else {
db.upsertStackUpdateStatus(
nodeId, stackName, hasUpdate, checkedAt, checkStatus, lastError, services, gen,
);
}
});
return { hasUpdate, notifyMessage, committed };
}
const checkable = Array.from(images)
.map((img) => imageUpdateMap.get(img))
.filter((r): r is ImageCheckResult => !!r && !r.notCheckable);
const errored = checkable.filter((r) => r.error !== undefined);
const confirmedHasUpdate = checkable.some((r) => r.error === undefined && r.hasUpdate === true);
if (checkable.length > 0 && errored.length === checkable.length) {
const committed = await this.withStackWriteLock(nodeId, stackName, generation, async () => {
db.recordStackCheckFailure(
nodeId, stackName, errored[0].error ?? 'Update check failed', checkedAt,
);
});
return {
hasUpdate: previousState[stackName] === true,
notifyMessage: buildAvailabilityNotifyMessage(stackName, []),
committed,
};
}
const checkStatus: StackCheckStatus = errored.length > 0 ? 'partial' : 'ok';
const lastError = errored.length > 0 ? (errored[0].error ?? null) : null;
const hasUpdate = checkStatus === 'partial'
? (confirmedHasUpdate || previousState[stackName] === true)
: confirmedHasUpdate;
const committed = await this.withStackWriteLock(nodeId, stackName, generation, async () => {
db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, checkedAt, checkStatus, lastError);
});
return {
hasUpdate,
notifyMessage: buildAvailabilityNotifyMessage(stackName, []),
committed,
};
}
public async checkImage(docker: DockerController, imageRef: string): Promise<ImageCheckResult> {
const parsed = parseImageRef(imageRef);
// A bare digest ref (sha256:...) has no tag to track upstream; not applicable.
+10 -6
View File
@@ -5,9 +5,11 @@ import { LicenseService } from './LicenseService';
import { PROXY_TIER_HEADER, deployProvenanceHeaders } from './license-headers';
import DockerController from './DockerController';
import { ComposeService } from './ComposeService';
import { StackUpdateOrchestrator } from './StackUpdateOrchestrator';
import { StackOpLockService, stackOpSkipMessage as skipMessage } from './StackOpLockService';
import { FileSystemService } from './FileSystemService';
import { HealthGateService } from './HealthGateService';
import { ServiceUpdateRecoveryService } from './ServiceUpdateRecoveryService';
import { ImageUpdateService } from './ImageUpdateService';
import type { ImageCheckResult } from './ImageUpdateService';
import { isDebugEnabled } from '../utils/debug';
@@ -706,12 +708,13 @@ export class SchedulerService {
? (JSON.parse(task.prune_targets) as string[]).filter((t): t is PruneTarget => allTargets.includes(t as PruneTarget))
: [...allTargets];
const labelFilter = task.prune_label_filter || undefined;
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(nodeId);
const results: string[] = [];
const failures: string[] = [];
for (const target of targets) {
try {
const result = await docker.pruneSystem(target, labelFilter);
const result = await docker.pruneSystem(target, labelFilter, isImageHeld);
results.push(`${target}: ${result.reclaimedBytes ?? 0} bytes reclaimed`);
} catch (error: unknown) {
const msg = error instanceof Error ? error.message : String(error);
@@ -766,12 +769,11 @@ export class SchedulerService {
const db = DatabaseService.getInstance();
const docker = DockerController.getInstance(task.node_id);
const imageUpdateService = ImageUpdateService.getInstance();
const compose = ComposeService.getInstance(task.node_id);
const results: string[] = [];
for (const stackName of stackNames) {
try {
const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, compose, db, isFleet || isWildcard);
const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, db, isFleet || isWildcard);
results.push(output);
} catch (e) {
const msg = getErrorMessage(e, String(e));
@@ -1028,7 +1030,6 @@ export class SchedulerService {
nodeId: number,
docker: DockerController,
imageUpdateService: ImageUpdateService,
compose: ComposeService,
db: DatabaseService,
isWildcard = false
): Promise<string> {
@@ -1096,11 +1097,14 @@ export class SchedulerService {
const atomic = true;
const lock = await StackOpLockService.getInstance().runExclusive(
nodeId, stackName, 'update', 'system',
() => compose.updateStack(stackName, undefined, atomic),
() => StackUpdateOrchestrator.getInstance().execute(
{ nodeId, stackName, target: { scope: 'stack' }, trigger: 'scheduled', actor: 'system:scheduler' },
{ atomic, terminalWs: null },
),
);
if (!lock.ran) return skipMessage(stackName, lock.existing.action);
db.clearStackUpdateStatus(nodeId, stackName);
HealthGateService.getInstance().begin(nodeId, stackName, 'update', 'system:scheduler');
HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler');
this.safeDispatch(
'info',
@@ -0,0 +1,280 @@
import { randomUUID } from 'crypto';
import { DatabaseService, type ServiceUpdateRecoveryRow } from './DatabaseService';
import { getComposeCommandTimeoutMs } from './ComposeService';
import { getErrorMessage } from '../utils/errors';
const SWEEP_INTERVAL_MS = 5 * 60_000;
const INITIAL_SWEEP_DELAY_MS = 30_000;
const CLAIM_RENEWAL_INTERVAL_MS = 5 * 60_000;
const MIN_CLAIM_WINDOW_MS = 30 * 60_000;
const CLAIM_RENEWAL_BUFFER_MS = 5 * 60_000;
const RECOVERY_TTL_BUFFER_MS = 30 * 60_000;
const MIN_RECOVERY_WINDOW_SECONDS = 90;
const DIGEST_PIN_PATTERN = /@sha256:[a-f0-9]{64}$/i;
/** A single pre-update replica snapshot for one service. */
export interface ServiceReplicaSnapshot {
imageId: string;
/** Repo digest captured from the running container, when known. */
repoDigest: string | null;
}
export interface CreateServiceUpdateRecoveryInput {
nodeId: number;
stackName: string;
serviceName: string;
replicas: ServiceReplicaSnapshot[];
/** EffectiveServiceSpec.declaredImage; null for a pure-build service with no image field. */
declaredImageRef: string | null;
createdBy: string | null;
}
export type ServiceUpdateRecoveryUnavailableReason =
| 'no_replicas'
| 'majority_tie'
| 'build_only'
| 'digest_pinned_declared_ref'
| 'missing_local_id';
export type CreateServiceUpdateRecoveryResult =
| { eligible: true; row: ServiceUpdateRecoveryRow }
| { eligible: false; reason: ServiceUpdateRecoveryUnavailableReason };
/**
* Pre-update image snapshots that let an operator manually restore a single
* service after a service-scoped update, without touching its siblings.
*
* Purely a data/lifecycle layer: it decides whether a snapshot is eligible to
* persist, holds the claimed row's image from prune while a restore is in
* flight, and sweeps stale rows. It never runs Compose or touches Docker; the
* update/restore orchestration (StackUpdateOrchestrator) calls into this
* service at the right points and is responsible for policy, health gating,
* and Auto-Heal suppression around those calls.
*/
export class ServiceUpdateRecoveryService {
private static instance: ServiceUpdateRecoveryService;
private started = false;
private intervalId: NodeJS.Timeout | null = null;
private initialTimer: NodeJS.Timeout | null = null;
private readonly renewalTimers = new Map<string, NodeJS.Timeout>();
private constructor() {}
public static getInstance(): ServiceUpdateRecoveryService {
if (!ServiceUpdateRecoveryService.instance) {
ServiceUpdateRecoveryService.instance = new ServiceUpdateRecoveryService();
}
return ServiceUpdateRecoveryService.instance;
}
public start(): void {
this.started = true;
if (this.initialTimer || this.intervalId) return;
this.initialTimer = setTimeout(() => {
this.sweep();
this.intervalId = setInterval(() => this.sweep(), SWEEP_INTERVAL_MS);
}, INITIAL_SWEEP_DELAY_MS);
}
/** Clear the sweep timer and every in-flight claim renewal loop. No callbacks fire after this returns. */
public stop(): void {
this.started = false;
if (this.initialTimer) {
clearTimeout(this.initialTimer);
this.initialTimer = null;
}
if (this.intervalId) {
clearInterval(this.intervalId);
this.intervalId = null;
}
for (const timer of this.renewalTimers.values()) clearInterval(timer);
this.renewalTimers.clear();
}
/** Expire active rows past their TTL and restoring rows abandoned by a dead claim. Never throws. */
public sweep(): void {
try {
const now = Date.now();
const db = DatabaseService.getInstance();
const expiredActive = db.sweepExpiredActiveServiceUpdateRecoveries(now);
const expiredAbandoned = db.sweepAbandonedRestoringServiceUpdateRecoveries(now);
if (expiredActive > 0 || expiredAbandoned > 0) {
console.log(
`[ServiceUpdateRecovery] Swept ${expiredActive} expired active and ${expiredAbandoned} abandoned restoring record(s)`,
);
}
} catch (error) {
console.error('[ServiceUpdateRecovery] Sweep failed:', getErrorMessage(error, 'unknown'));
}
}
/**
* Decide whether a pre-update snapshot is eligible to persist and, if so,
* insert it. Unavailable per §8: no replicas, a majority tie, a pure-build
* service (no declared image ref), a digest-pinned declared ref, or a
* replica missing its local image id.
*/
public createIfEligible(input: CreateServiceUpdateRecoveryInput): CreateServiceUpdateRecoveryResult {
const validReplicas = input.replicas.filter(r => r.imageId.trim().length > 0);
if (validReplicas.length === 0) {
return { eligible: false, reason: input.replicas.length === 0 ? 'no_replicas' : 'missing_local_id' };
}
const majority = this.computeMajorityImage(validReplicas);
if (!majority) return { eligible: false, reason: 'majority_tie' };
if (!input.declaredImageRef) return { eligible: false, reason: 'build_only' };
if (DIGEST_PIN_PATTERN.test(input.declaredImageRef)) return { eligible: false, reason: 'digest_pinned_declared_ref' };
const now = Date.now();
const row: ServiceUpdateRecoveryRow = {
id: randomUUID(),
node_id: input.nodeId,
stack_name: input.stackName,
service_name: input.serviceName,
replicas_json: JSON.stringify(input.replicas),
majority_image_id: majority.imageId,
declared_image_ref: input.declaredImageRef,
weak_floating_tag: majority.weakFloatingTag ? 1 : 0,
health_gate_id: null,
status: 'active',
expires_at: now + this.activeRecoveryTtlMs() + RECOVERY_TTL_BUFFER_MS,
claim_expires_at: null,
created_at: now,
created_by: input.createdBy,
};
DatabaseService.getInstance().insertServiceUpdateRecovery(row);
return { eligible: true, row };
}
/** Active, unexpired recovery rows for one service (drives `recoveryAvailable` in the update/restore responses). */
public listActive(nodeId: number, stackName: string, serviceName: string): ServiceUpdateRecoveryRow[] {
return DatabaseService.getInstance().listActiveServiceUpdateRecoveries(nodeId, stackName, serviceName);
}
public get(id: string): ServiceUpdateRecoveryRow | undefined {
return DatabaseService.getInstance().getServiceUpdateRecovery(id);
}
/** Attach the update flow's own health gate run id while the row is still active. */
public linkHealthGate(id: string, healthGateId: string): void {
DatabaseService.getInstance().linkServiceUpdateRecoveryHealthGate(id, healthGateId);
}
/** Atomic claim: active -> restoring. Returns the claimed row, or undefined if another actor won the race. */
public claim(id: string): ServiceUpdateRecoveryRow | undefined {
const now = Date.now();
return DatabaseService.getInstance().claimServiceUpdateRecovery(id, this.nextClaimExpiry(now), now);
}
/**
* Start the mandatory 5-minute renewal loop for a claimed row (RECOVERY-CLAIM-2).
* Idempotent per id. Each tick CAS-extends claim_expires_at only while the row
* is still restoring; a renewal that finds the row no longer restoring (superseded,
* consumed, or force-expired) stops itself rather than reviving a terminal row.
*/
public startClaimRenewal(id: string): void {
if (!this.started || this.renewalTimers.has(id)) return;
const timer = setInterval(() => {
try {
const renewed = DatabaseService.getInstance().renewServiceUpdateRecoveryClaim(id, this.nextClaimExpiry(Date.now()));
if (!renewed) this.stopClaimRenewal(id);
} catch (error) {
console.error('[ServiceUpdateRecovery] Claim renewal failed for %s:', id, getErrorMessage(error, 'unknown'));
}
}, CLAIM_RENEWAL_INTERVAL_MS);
this.renewalTimers.set(id, timer);
}
/** Stop the renewal loop for a row. Callers must call this in the restore operation's `finally` regardless of outcome. */
public stopClaimRenewal(id: string): void {
const timer = this.renewalTimers.get(id);
if (timer) {
clearInterval(timer);
this.renewalTimers.delete(id);
}
}
/** Restore succeeded: restoring -> consumed, optionally linking the restore's own health gate run. */
public markConsumed(id: string, healthGateId: string | null = null): boolean {
return DatabaseService.getInstance().markServiceUpdateRecoveryConsumed(id, healthGateId);
}
/** Mid-flight restore failure with the image still local: restoring -> active, claim cleared. */
public reactivate(id: string): boolean {
return DatabaseService.getInstance().reactivateServiceUpdateRecovery(id);
}
/** Mid-flight restore failure with the image gone: restoring -> invalidated. */
public invalidate(id: string): boolean {
return DatabaseService.getInstance().invalidateServiceUpdateRecovery(id);
}
/**
* Image IDs currently held for this node (active rows and restoring rows
* with a live claim). Returns null when the held set cannot be read so
* callers can fail closed (skip prune) instead of treating the miss as empty.
*/
public getHeldImageIds(nodeId: number): Set<string> | null {
try {
return new Set(DatabaseService.getInstance().listHeldServiceUpdateRecoveryImageIds(nodeId, Date.now()));
} catch (error) {
console.warn(
'[ServiceUpdateRecovery] Failed to compute held image ids for node %d:', nodeId, getErrorMessage(error, 'unknown'),
);
return null;
}
}
/**
* A predicate a pruner can call immediately before deleting each candidate
* image. Re-reads the held set on every call (rather than snapshotting it
* once) so a snapshot that becomes eligible between plan and delete is
* still honored. When the held set cannot be read, returns true for every
* id so prune skips deletes (fail closed).
*/
public buildHeldImagePredicate(nodeId: number): (imageId: string) => boolean {
return (imageId: string) => {
const held = this.getHeldImageIds(nodeId);
if (held === null) return true;
return held.has(imageId);
};
}
private nextClaimExpiry(now: number): number {
return now + Math.max(getComposeCommandTimeoutMs(), MIN_CLAIM_WINDOW_MS) + CLAIM_RENEWAL_BUFFER_MS;
}
/**
* Active recovery must outlive both the Compose mutation and the post-update
* health window. A raised SENCHO_COMPOSE_COMMAND_TIMEOUT_MS must not leave the
* snapshot eligible for sweep while Compose is still running.
*/
private activeRecoveryTtlMs(): number {
return Math.max(getComposeCommandTimeoutMs(), this.readRecoveryWindowSeconds() * 1000);
}
private readRecoveryWindowSeconds(): number {
try {
const raw = parseInt(DatabaseService.getInstance().getGlobalSettings()['health_gate_window_seconds'] ?? '', 10);
return Number.isFinite(raw) ? Math.max(raw, MIN_RECOVERY_WINDOW_SECONDS) : MIN_RECOVERY_WINDOW_SECONDS;
} catch (error) {
console.warn('[ServiceUpdateRecovery] Settings read failed; using default window:', getErrorMessage(error, 'unknown'));
return MIN_RECOVERY_WINDOW_SECONDS;
}
}
/** Majority image id among replicas, or null on a tie (no single winner). */
private computeMajorityImage(replicas: ServiceReplicaSnapshot[]): { imageId: string; weakFloatingTag: boolean } | null {
const counts = new Map<string, number>();
for (const replica of replicas) {
counts.set(replica.imageId, (counts.get(replica.imageId) ?? 0) + 1);
}
const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1]);
if (ranked.length === 0) return null;
if (ranked.length > 1 && ranked[0][1] === ranked[1][1]) return null;
const winner = ranked[0][0];
const weakFloatingTag = !replicas.some(r => r.imageId === winner && r.repoDigest);
return { imageId: winner, weakFloatingTag };
}
}
+35 -2
View File
@@ -11,6 +11,11 @@
export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update';
export interface StackOpRecordMeta {
targetScope?: 'stack' | 'service';
serviceName?: string | null;
}
interface StackOpStats {
count: number;
successCount: number;
@@ -22,6 +27,10 @@ interface StackOpStats {
* computed from this window on demand.
*/
recentSamples: number[];
/** Most recent target scope recorded for this bucket (additive; stack default). */
lastTargetScope: 'stack' | 'service';
/** Most recent service name when lastTargetScope is service; otherwise null. */
lastServiceName: string | null;
}
export interface StackOpSnapshotEntry {
@@ -33,6 +42,8 @@ export interface StackOpSnapshotEntry {
avgMs: number;
p50Ms: number;
p95Ms: number;
targetScope: 'stack' | 'service';
serviceName: string | null;
}
const MAX_SAMPLES = 1000;
@@ -66,12 +77,26 @@ export class StackOpMetricsService {
* Record one completed op. Call from the route layer after the lifecycle
* call resolves or rejects; `ok=false` for the rejection path.
*/
public record(nodeId: number, action: StackOpAction, durationMs: number, ok: boolean): void {
public record(
nodeId: number,
action: StackOpAction,
durationMs: number,
ok: boolean,
meta?: StackOpRecordMeta,
): void {
if (!Number.isFinite(durationMs) || durationMs < 0) return;
const k = this.key(nodeId, action);
let s = this.stats.get(k);
if (!s) {
s = { count: 0, successCount: 0, errorCount: 0, totalMs: 0, recentSamples: [] };
s = {
count: 0,
successCount: 0,
errorCount: 0,
totalMs: 0,
recentSamples: [],
lastTargetScope: 'stack',
lastServiceName: null,
};
this.stats.set(k, s);
}
s.count += 1;
@@ -84,6 +109,12 @@ export class StackOpMetricsService {
// and this path is once-per-stack-op (low cadence).
s.recentSamples.shift();
}
if (meta?.targetScope) s.lastTargetScope = meta.targetScope;
if (meta?.targetScope === 'service') {
s.lastServiceName = meta.serviceName ?? null;
} else if (meta?.targetScope === 'stack') {
s.lastServiceName = null;
}
}
public snapshot(): StackOpSnapshotEntry[] {
@@ -102,6 +133,8 @@ export class StackOpMetricsService {
avgMs: s.count === 0 ? 0 : Math.round(s.totalMs / s.count),
p50Ms: percentile(sorted, 0.5),
p95Ms: percentile(sorted, 0.95),
targetScope: s.lastTargetScope,
serviceName: s.lastServiceName,
});
}
// Stable ordering: nodeId asc, then action asc.
@@ -0,0 +1,684 @@
import WebSocket from 'ws';
import DockerController from './DockerController';
import { ComposeService } from './ComposeService';
import { HealthGateService } from './HealthGateService';
import { AutoHealService } from './AutoHealService';
import { ImageUpdateService } from './ImageUpdateService';
import {
ServiceUpdateRecoveryService,
type ServiceReplicaSnapshot,
} from './ServiceUpdateRecoveryService';
import type { ServiceUpdateRecoveryRow } from './DatabaseService';
import { buildEffectiveServiceModel } from './effectiveServiceModel';
import { DriftLedgerService } from './DriftLedgerService';
import { MeshService } from './MeshService';
import { DatabaseService } from './DatabaseService';
import { enforcePolicyForImageRefs, type PolicyEnforcementOptions } from './PolicyEnforcement';
import { describePolicyBlock } from '../helpers/policyGate';
import { getErrorMessage } from '../utils/errors';
import { sanitizeForLog } from '../utils/safeLog';
import type { EffectiveServiceSpec } from './effectiveServiceModel';
export type UpdateTarget =
| { scope: 'stack' }
| { scope: 'service'; serviceName: string };
export type UpdateTrigger =
| 'manual' | 'scheduled' | 'automatic' | 'blueprint' | 'webhook' | 'bulk';
export interface UpdateOperationContext {
nodeId: number;
stackName: string;
target: UpdateTarget;
trigger: UpdateTrigger;
actor: string | null;
}
export interface StackComposeOptions {
atomic: boolean;
terminalWs?: WebSocket | null;
}
export interface ServiceUpdateOptions {
policyOptions: PolicyEnforcementOptions;
terminalWs?: WebSocket | null;
/** Restore only: binds the operation to a recovery snapshot. */
recoveryId?: string;
}
export type OrchestratorResult =
| { kind: 'stack_compose_done' }
| {
kind: 'service_done';
serviceName: string;
healthGateId: string | null;
observing: boolean;
recoveryId: string | null;
recoveryAvailable: boolean;
previousImageId?: string | null;
newImageId?: string | null;
recheckWarning?: string;
}
| {
kind: 'service_failed';
code: string;
error: string;
serviceName?: string;
mutationStage?: string;
recoveryId?: string | null;
};
function serviceFailed(
code: string,
error: string,
extra?: { serviceName?: string; mutationStage?: string; recoveryId?: string | null },
): Extract<OrchestratorResult, { kind: 'service_failed' }> {
return { kind: 'service_failed', code, error, ...extra };
}
/** Shorten a local image id for activity copy (sha256:abcdef… → abcdef…). */
export function shortImageId(imageId: string | null | undefined): string | null {
if (!imageId) return null;
const bare = imageId.startsWith('sha256:') ? imageId.slice(7) : imageId;
return bare.slice(0, 12);
}
/**
* Post-mutation replica convergence rules shared by update and restore.
* Requires the exact expected count of inspected running image ids and a
* single shared image before success (independent of optional health gating).
*/
export function evaluateServiceReplicaConvergence(
serviceName: string,
expectedReplicas: number,
ids: string[],
inspectErrors: number,
):
| { kind: 'converged'; imageId: string | null }
| { kind: 'divergent'; error: string }
| { kind: 'inspect_failed'; error: string } {
if (expectedReplicas === 0) {
return { kind: 'converged', imageId: null };
}
if (inspectErrors > 0 && ids.length === 0) {
return {
kind: 'inspect_failed',
error: `Could not inspect replicas for service "${serviceName}" after the update.`,
};
}
const distinct = new Set(ids);
if (distinct.size === 0) {
return { kind: 'divergent', error: `Service "${serviceName}" has no running replicas after the update.` };
}
if (ids.length !== expectedReplicas) {
return {
kind: 'divergent',
error: `Service "${serviceName}" has ${ids.length} running replica(s); expected ${expectedReplicas}.`,
};
}
if (distinct.size > 1) {
return { kind: 'divergent', error: `Service "${serviceName}" replicas did not converge on a single image.` };
}
return { kind: 'converged', imageId: [...distinct][0] };
}
/** Parse a tag image ref into a Docker tag() repo + tag pair. */
function splitImageRef(ref: string): { repo: string; tag: string } {
const lastSlash = ref.lastIndexOf('/');
const lastColon = ref.lastIndexOf(':');
if (lastColon > lastSlash) {
return { repo: ref.slice(0, lastColon), tag: ref.slice(lastColon + 1) };
}
return { repo: ref, tag: 'latest' };
}
/**
* Single entry point for stack-scope Compose updates and the manual
* service-scoped update/restore flows.
*
* Ownership boundaries (§3): the five stack update callers keep every
* side effect (policy, lock, clear status, invalidate, broadcast, notify,
* post-scan, and the post-success `beginStack`); this orchestrator's stack
* branch only runs `ComposeService.updateStack`. The service branch (nested
* manual routes only) owns policy, the recovery snapshot, the pre-mutation
* health gate (prepare/attach/beginPrepared), Auto-Heal suppression, and the
* per-service recheck; the route holds the stack lock and maps the result to
* HTTP. Expected failures never throw past `execute`; they return a
* `service_failed` result.
*/
export class StackUpdateOrchestrator {
private static instance: StackUpdateOrchestrator;
public static getInstance(): StackUpdateOrchestrator {
if (!StackUpdateOrchestrator.instance) {
StackUpdateOrchestrator.instance = new StackUpdateOrchestrator();
}
return StackUpdateOrchestrator.instance;
}
public async execute(
ctx: UpdateOperationContext,
options: StackComposeOptions | ServiceUpdateOptions,
): Promise<OrchestratorResult> {
if (ctx.target.scope === 'stack') {
return this.executeStack(ctx, options as StackComposeOptions);
}
// Service scope: automation never selects this branch.
if (ctx.trigger !== 'manual') {
throw new Error('Service-scoped updates require a manual trigger');
}
const serviceOptions = options as ServiceUpdateOptions;
if (serviceOptions.recoveryId) {
return this.executeRestore(ctx, ctx.target.serviceName, serviceOptions);
}
return this.executeServiceUpdate(ctx, ctx.target.serviceName, serviceOptions);
}
/**
* Stack branch: run the full-stack Compose update only. Side effects and the
* post-success `beginStack(..., 'update')` stay with the caller.
*/
private async executeStack(
ctx: UpdateOperationContext,
options: StackComposeOptions,
): Promise<OrchestratorResult> {
await ComposeService.getInstance(ctx.nodeId).updateStack(
ctx.stackName, options.terminalWs ?? undefined, options.atomic,
);
return { kind: 'stack_compose_done' };
}
private async executeServiceUpdate(
ctx: UpdateOperationContext,
serviceName: string,
options: ServiceUpdateOptions,
): Promise<OrchestratorResult> {
const { nodeId, stackName } = ctx;
const loaded = await this.loadServiceSpec(nodeId, stackName, serviceName);
if (!loaded.ok) return loaded.result;
const { spec, services } = loaded;
if (services.length <= 1) {
return serviceFailed(
'service_update_single_service',
'Service-scoped updates require a stack with more than one service.',
);
}
if (!spec.hasBuild && !spec.declaredImage) {
return serviceFailed(
'service_not_updatable',
`Service "${serviceName}" has neither an image nor a build, so it cannot be updated.`,
);
}
AutoHealService.getInstance().suppress(nodeId, stackName, serviceName);
let observing = false;
let recoveryId: string | null = null;
let recoveryAvailable = false;
try {
const policyFailure = await this.checkPolicyOrFail(
stackName, nodeId, serviceName, spec.declaredImage ? [spec.declaredImage] : [], options.policyOptions, 'update',
);
if (policyFailure) return policyFailure;
const prepared = await HealthGateService.getInstance().prepare({
nodeId, stackName,
target: { scope: 'service', serviceName },
trigger: 'service_update',
expectedReplicas: spec.expectedReplicas,
});
const prepareToken = prepared.prepareToken;
const prepareSnapshotOk = prepared.snapshotOk;
const replicas = await this.snapshotServiceReplicas(nodeId, stackName, serviceName);
const recovery = ServiceUpdateRecoveryService.getInstance().createIfEligible({
nodeId, stackName, serviceName,
replicas,
declaredImageRef: spec.declaredImage,
createdBy: ctx.actor,
});
const previousImageId = recovery.eligible
? recovery.row.majority_image_id
: (replicas[0]?.imageId ?? null);
if (recovery.eligible) {
recoveryId = recovery.row.id;
recoveryAvailable = true;
}
try {
await ComposeService.getInstance(nodeId).updateService(
stackName, serviceName, spec.hasBuild, options.terminalWs ?? undefined,
);
} catch (composeError) {
return serviceFailed(
'service_update_compose_failed',
getErrorMessage(composeError, 'Service update failed'),
{ serviceName, mutationStage: 'compose', recoveryId },
);
}
const observed = await this.observePreparedService(
prepareToken, nodeId, stackName, serviceName, spec.expectedReplicas, ctx.actor,
'service_update_replica_divergence', recoveryId, prepareSnapshotOk,
);
if (!observed.ok) return observed.result;
observing = observed.observing;
const healthGateId = observed.healthGateId;
if (recoveryId && healthGateId) {
try {
ServiceUpdateRecoveryService.getInstance().linkHealthGate(recoveryId, healthGateId);
} catch (linkError) {
console.warn(
'[StackUpdateOrchestrator] Failed to link recovery %s to gate %s: %s',
sanitizeForLog(recoveryId), sanitizeForLog(healthGateId),
sanitizeForLog(getErrorMessage(linkError, 'unknown')),
);
}
}
const recheck = await ImageUpdateService.getInstance().recheckStack(nodeId, stackName);
await DriftLedgerService.getInstance().reconcileServiceForStack(nodeId, stackName, serviceName);
await this.refreshMeshIfEnabled(nodeId, stackName);
const warnings = [observed.gateWarning, recheck.warning].filter((w): w is string => !!w);
return {
kind: 'service_done',
serviceName,
healthGateId,
observing,
recoveryId,
recoveryAvailable,
previousImageId,
newImageId: observed.newImageId ?? null,
recheckWarning: warnings.length > 0 ? warnings.join(' ') : undefined,
};
} finally {
if (!observing) {
AutoHealService.getInstance().clearSuppress(nodeId, stackName, serviceName);
}
}
}
private async executeRestore(
ctx: UpdateOperationContext,
serviceName: string,
options: ServiceUpdateOptions,
): Promise<OrchestratorResult> {
const { nodeId, stackName } = ctx;
const recoveryId = options.recoveryId as string;
const recovery = ServiceUpdateRecoveryService.getInstance().get(recoveryId);
if (
!recovery ||
recovery.node_id !== nodeId ||
recovery.stack_name !== stackName ||
recovery.service_name !== serviceName
) {
return serviceFailed('recovery_not_found', 'No matching recovery snapshot for this service.');
}
if (recovery.status !== 'active') {
return serviceFailed(
'recovery_not_restorable',
`This recovery snapshot is ${recovery.status} and can no longer be restored.`,
{ recoveryId },
);
}
AutoHealService.getInstance().suppress(nodeId, stackName, serviceName);
let observing = false;
let claimed = false;
let consumed = false;
try {
const loaded = await this.loadServiceSpec(nodeId, stackName, serviceName, recoveryId);
if (!loaded.ok) return loaded.result;
const { spec, services } = loaded;
if (services.length <= 1) {
return serviceFailed(
'service_update_single_service',
'Service-scoped restore requires a stack with more than one service.',
{ recoveryId },
);
}
const scanTarget = await this.resolveRestoreScanTarget(nodeId, recovery);
const policyFailure = await this.checkPolicyOrFail(
stackName, nodeId, serviceName, [scanTarget], options.policyOptions, 'rollback', recoveryId,
);
if (policyFailure) return policyFailure;
const prepared = await HealthGateService.getInstance().prepare({
nodeId, stackName,
target: { scope: 'service', serviceName },
trigger: 'service_restore',
expectedReplicas: spec.expectedReplicas,
});
const prepareToken = prepared.prepareToken;
const prepareSnapshotOk = prepared.snapshotOk;
const claimedRow = ServiceUpdateRecoveryService.getInstance().claim(recoveryId);
if (!claimedRow) {
return serviceFailed(
'recovery_claim_failed',
'This recovery snapshot is being restored by another operation.',
{ recoveryId },
);
}
claimed = true;
ServiceUpdateRecoveryService.getInstance().startClaimRenewal(recoveryId);
const previousImageId = (await this.snapshotServiceReplicas(nodeId, stackName, serviceName))[0]?.imageId ?? null;
try {
const { repo, tag } = splitImageRef(recovery.declared_image_ref);
await DockerController.getInstance(nodeId).getDocker()
.getImage(recovery.majority_image_id).tag({ repo, tag });
} catch (retagError) {
return serviceFailed(
'restore_retag_failed',
getErrorMessage(retagError, 'Could not retag the recovery image'),
{ serviceName, mutationStage: 'retag', recoveryId },
);
}
try {
await ComposeService.getInstance(nodeId).recreateServiceFromLocal(
stackName, serviceName, options.terminalWs ?? undefined,
);
} catch (composeError) {
return serviceFailed(
'restore_compose_failed',
getErrorMessage(composeError, 'Service restore failed'),
{ serviceName, mutationStage: 'compose', recoveryId },
);
}
const observed = await this.observePreparedService(
prepareToken, nodeId, stackName, serviceName, spec.expectedReplicas, ctx.actor,
'restore_replica_divergence', recoveryId, prepareSnapshotOk,
);
if (!observed.ok) return observed.result;
observing = observed.observing;
const healthGateId = observed.healthGateId;
try {
ServiceUpdateRecoveryService.getInstance().markConsumed(recoveryId, healthGateId);
consumed = true;
} catch (consumeError) {
console.warn(
'[StackUpdateOrchestrator] Failed to mark recovery %s consumed: %s',
sanitizeForLog(recoveryId), sanitizeForLog(getErrorMessage(consumeError, 'unknown')),
);
// Compose already succeeded; treat as consumed for lease/cleanup purposes so
// we do not reactivate a snapshot that may no longer match the running image.
consumed = true;
}
const recheck = await ImageUpdateService.getInstance().recheckStack(nodeId, stackName);
await this.refreshMeshIfEnabled(nodeId, stackName);
const warnings = [observed.gateWarning, recheck.warning].filter((w): w is string => !!w);
return {
kind: 'service_done',
serviceName,
healthGateId,
observing,
recoveryId,
recoveryAvailable: false,
previousImageId,
newImageId: observed.newImageId ?? null,
recheckWarning: warnings.length > 0 ? warnings.join(' ') : undefined,
};
} finally {
ServiceUpdateRecoveryService.getInstance().stopClaimRenewal(recoveryId);
if (claimed && !consumed) {
const stillLocal = await this.imageIsLocal(nodeId, recovery.majority_image_id);
if (stillLocal) {
ServiceUpdateRecoveryService.getInstance().reactivate(recoveryId);
} else {
ServiceUpdateRecoveryService.getInstance().invalidate(recoveryId);
}
}
if (!observing) {
AutoHealService.getInstance().clearSuppress(nodeId, stackName, serviceName);
}
}
}
/** Best-effort mesh alias refresh when this stack is opted into Mesh. */
private async refreshMeshIfEnabled(nodeId: number, stackName: string): Promise<void> {
try {
if (!DatabaseService.getInstance().isMeshStackEnabled(nodeId, stackName)) return;
await MeshService.getInstance().refreshAliasCache();
} catch (error) {
console.warn(
'[StackUpdateOrchestrator] Mesh alias refresh failed for %s on node %d: %s',
sanitizeForLog(stackName), nodeId, sanitizeForLog(getErrorMessage(error, 'unknown')),
);
}
}
private async loadServiceSpec(
nodeId: number,
stackName: string,
serviceName: string,
recoveryId?: string | null,
): Promise<
| { ok: true; spec: EffectiveServiceSpec; services: EffectiveServiceSpec[] }
| { ok: false; result: Extract<OrchestratorResult, { kind: 'service_failed' }> }
> {
const model = await buildEffectiveServiceModel(nodeId, stackName);
if (!model.renderable) {
return { ok: false, result: serviceFailed(model.code, model.error, { recoveryId }) };
}
const spec = model.services.find(s => s.name === serviceName);
if (!spec) {
return {
ok: false,
result: serviceFailed(
'service_not_found',
`Service "${serviceName}" is not declared in this stack.`,
{ recoveryId },
),
};
}
return { ok: true, spec, services: model.services };
}
private async checkPolicyOrFail(
stackName: string,
nodeId: number,
serviceName: string,
refs: string[],
policyOptions: PolicyEnforcementOptions,
action: 'update' | 'rollback',
recoveryId?: string | null,
): Promise<Extract<OrchestratorResult, { kind: 'service_failed' }> | null> {
if (refs.length === 0) return null;
const gate = await enforcePolicyForImageRefs(stackName, nodeId, refs, policyOptions);
if (gate.ok || gate.bypassed) return null;
return serviceFailed(
'policy_blocked',
`Service "${serviceName}": ${describePolicyBlock(gate.policy, gate.violations, action)}`,
{ serviceName, mutationStage: 'policy', recoveryId },
);
}
private async observePreparedService(
prepareToken: string,
nodeId: number,
stackName: string,
serviceName: string,
expectedReplicas: number,
actor: string | null,
divergenceCode: string,
recoveryId: string | null,
prepareSnapshotOk: boolean,
): Promise<
| { ok: true; healthGateId: string | null; observing: boolean; newImageId: string | null; gateWarning?: string }
| { ok: false; result: Extract<OrchestratorResult, { kind: 'service_failed' }> }
> {
const convergence = await this.resolvePrimaryImage(nodeId, stackName, serviceName, expectedReplicas);
if (convergence.kind === 'inspect_failed') {
return {
ok: false,
result: serviceFailed('replica_inspect_failed', convergence.error, {
serviceName, mutationStage: 'inspect', recoveryId,
}),
};
}
if (convergence.kind === 'divergent') {
return {
ok: false,
result: serviceFailed(divergenceCode, convergence.error, {
serviceName, mutationStage: 'inspect', recoveryId,
}),
};
}
if (!prepareSnapshotOk) {
return {
ok: true,
healthGateId: null,
observing: false,
newImageId: convergence.imageId,
gateWarning: 'Health gate skipped: pre-update container snapshot was unavailable.',
};
}
const begin = this.beginPrepared(prepareToken, convergence.imageId, actor);
return {
ok: true,
healthGateId: begin.runId,
observing: begin.observing,
newImageId: convergence.imageId,
};
}
/** Attach the single converged image id (scale>0) and begin the prepared gate. */
private beginPrepared(
prepareToken: string,
imageId: string | null,
actor: string | null,
): { runId: string | null; observing: boolean } {
if (imageId) {
HealthGateService.getInstance().attachExpectedImage(prepareToken, imageId);
}
return HealthGateService.getInstance().beginPrepared({ prepareToken, actor });
}
/**
* Post-mutation replica convergence check. Returns the single shared image id
* when every expected primary replica agrees, null for a scale-0 service, or a
* typed divergence when replicas are missing or run 2+ distinct images.
*/
private async resolvePrimaryImage(
nodeId: number,
stackName: string,
serviceName: string,
expectedReplicas: number,
): Promise<
| { kind: 'converged'; imageId: string | null }
| { kind: 'divergent'; error: string }
| { kind: 'inspect_failed'; error: string }
> {
if (expectedReplicas === 0) {
return { kind: 'converged', imageId: null };
}
const inspected = await this.inspectServiceContainers(nodeId, stackName, serviceName);
return evaluateServiceReplicaConvergence(
serviceName, expectedReplicas, inspected.ids, inspected.inspectErrors,
);
}
private async inspectPrimaryImageIds(nodeId: number, stackName: string, serviceName: string): Promise<string[]> {
return (await this.inspectServiceContainers(nodeId, stackName, serviceName)).ids;
}
/** Snapshot the current per-replica image ids + repo digests for the recovery row. */
private async snapshotServiceReplicas(
nodeId: number,
stackName: string,
serviceName: string,
): Promise<ServiceReplicaSnapshot[]> {
const docker = DockerController.getInstance(nodeId).getDocker();
const snapshots: ServiceReplicaSnapshot[] = [];
for (const imageId of (await this.inspectServiceContainers(nodeId, stackName, serviceName)).ids) {
let repoDigest: string | null = null;
try {
const image = await docker.getImage(imageId).inspect();
const digests = (image.RepoDigests ?? []) as string[];
repoDigest = digests.length > 0 ? digests[0] : null;
} catch {
// The digest is optional context; a missing image inspect leaves it null.
}
snapshots.push({ imageId, repoDigest });
}
return snapshots;
}
private async inspectServiceContainers(
nodeId: number,
stackName: string,
serviceName: string,
): Promise<{ ids: string[]; inspectErrors: number }> {
const docker = DockerController.getInstance(nodeId).getDocker();
const listed = await docker.listContainers({
all: true,
filters: { label: [`com.docker.compose.project=${stackName}`, `com.docker.compose.service=${serviceName}`] },
});
const ids: string[] = [];
let inspectErrors = 0;
for (const info of listed) {
try {
const inspect = await docker.getContainer(info.Id).inspect();
// Convergence requires running replicas; exited/created leftovers must
// not count toward the expected replica total or the shared image id.
if (inspect.State?.Status !== 'running') continue;
if (inspect.Image) ids.push(inspect.Image);
} catch (error) {
if ((error as { statusCode?: number })?.statusCode === 404) continue;
inspectErrors += 1;
console.warn('[Orchestrator] Replica inspect failed for %s/%s:',
sanitizeForLog(stackName), sanitizeForLog(serviceName), getErrorMessage(error, 'unknown'));
}
}
return { ids, inspectErrors };
}
/**
* Immutable restore scan target: the captured repo digest when it resolves
* locally to the majority image id, otherwise the majority image id directly.
* Never the moving declared tag (RESTORE-POLICY-TARGET-2).
*/
private async resolveRestoreScanTarget(nodeId: number, recovery: ServiceUpdateRecoveryRow): Promise<string> {
let replicas: ServiceReplicaSnapshot[] = [];
try {
const parsed: unknown = JSON.parse(recovery.replicas_json);
if (Array.isArray(parsed)) replicas = parsed as ServiceReplicaSnapshot[];
} catch {
// Corrupt snapshot: fall back to the majority image id below.
}
const withDigest = replicas.find(r => r.imageId === recovery.majority_image_id && r.repoDigest);
if (withDigest?.repoDigest) {
const resolvedId = await this.resolveImageId(nodeId, withDigest.repoDigest);
if (resolvedId === recovery.majority_image_id) return withDigest.repoDigest;
}
return recovery.majority_image_id;
}
private async resolveImageId(nodeId: number, ref: string): Promise<string | null> {
try {
const inspect = await DockerController.getInstance(nodeId).getDocker().getImage(ref).inspect();
return inspect.Id ?? null;
} catch {
return null;
}
}
private async imageIsLocal(nodeId: number, imageId: string): Promise<boolean> {
try {
await DockerController.getInstance(nodeId).getDocker().getImage(imageId).inspect();
return true;
} catch {
return false;
}
}
}
+102 -11
View File
@@ -3,7 +3,9 @@ import DockerController from './DockerController';
import { DatabaseService } from './DatabaseService';
import { FileSystemService } from './FileSystemService';
import { ComposeDoctorService } from './ComposeDoctorService';
import { UpdatePreviewService, isMovingTag } from './UpdatePreviewService';
import { UpdatePreviewService, isMovingTag, filterPreviewForService } from './UpdatePreviewService';
import { buildEffectiveServiceModel, type EffectiveServiceModelResult } from './effectiveServiceModel';
import { filterContainersByComposeService } from '../helpers/composeServiceMatch';
import { withTimeout } from '../utils/withTimeout';
import { getErrorMessage } from '../utils/errors';
import { sanitizeForLog } from '../utils/safeLog';
@@ -17,9 +19,12 @@ import {
diskSignal,
driftSignal,
healthchecksSignal,
isTroubledContainer,
preflightSignal,
serviceSignal,
updatePreviewSignal,
type Errored,
type ServiceMembership,
} from './updateGuard/readiness';
import type { ContainerProbe, RollbackReadinessReport, UpdateReadinessReport } from './updateGuard/types';
@@ -30,6 +35,19 @@ import type { ContainerProbe, RollbackReadinessReport, UpdateReadinessReport } f
// of failing the report.
const INPUT_TIMEOUT_MS = 3_000;
/**
* Thrown by `computeUpdateReadiness` when a `serviceName` is given for a
* stack that has one (or zero) declared services: service-scoped readiness
* requires a real choice among siblings. The route maps this to a 400,
* distinct from the 500 a genuine computation failure gets.
*/
export class SingleServiceUpdateReadinessError extends Error {
constructor() {
super('Service-scoped readiness requires a stack with more than one service.');
this.name = 'SingleServiceUpdateReadinessError';
}
}
/**
* Computes update readiness and rollback readiness for a stack, on demand,
* from existing per-feature stores (preflight runs, drift findings, the atomic
@@ -49,16 +67,18 @@ export class UpdateGuardService {
/**
* Probe the stack's containers via the compose project label, normalized for
* the pure scoring functions. Throws on Docker errors; callers map that to
* the 'error' sentinel.
* the 'error' sentinel. When `serviceName` is given, narrows to that
* service's containers before inspecting (labeled containers, per §6).
*/
async probeContainers(nodeId: number, stackName: string): Promise<ContainerProbe[]> {
async probeContainers(nodeId: number, stackName: string, serviceName?: string): Promise<ContainerProbe[]> {
const docker = DockerController.getInstance(nodeId).getDocker();
const listed = await docker.listContainers({
all: true,
filters: { label: [`com.docker.compose.project=${stackName}`] },
});
const scoped = serviceName ? filterContainersByComposeService(listed, serviceName) : listed;
const probes = await Promise.all(
listed.map(async (info): Promise<ContainerProbe | null> => {
scoped.map(async (info): Promise<ContainerProbe | null> => {
const name = info.Names?.[0]?.replace(/^\//, '') ?? info.Id.slice(0, 12);
let inspect: Awaited<ReturnType<ReturnType<typeof docker.getContainer>['inspect']>>;
try {
@@ -86,17 +106,40 @@ export class UpdateGuardService {
return probes.filter((p): p is ContainerProbe => p !== null);
}
async computeUpdateReadiness(nodeId: number, stackName: string): Promise<UpdateReadinessReport> {
/**
* When `serviceName` is set, scopes the verdict-affecting inputs (labeled
* containers, healthchecks, the update preview, and drift) to that service
* only; the stack guardrails (preflight, disk, backup, Docker reachability)
* stay unchanged (§6). A render failure or missing service fails closed via
* the added `service` signal; a single-service stack throws instead of
* scoping (the caller/route maps that to a 400).
*/
async computeUpdateReadiness(nodeId: number, stackName: string, serviceName?: string): Promise<UpdateReadinessReport> {
const db = DatabaseService.getInstance();
const now = Date.now();
const [preflight, drift, containers, preview, backup, disk] = await Promise.all([
let model: EffectiveServiceModelResult | null = null;
if (serviceName) {
model = await buildEffectiveServiceModel(nodeId, stackName);
if (model.renderable && model.services.length <= 1) {
throw new SingleServiceUpdateReadinessError();
}
}
const [preflight, drift, containers, siblings, preview, backup, disk] = await Promise.all([
this.collect('preflight', stackName, async () => ComposeDoctorService.getInstance().getLatest(nodeId, stackName)),
this.collect('drift', stackName, async () => db.getOpenDriftFindings(nodeId, stackName).length),
this.collect('drift', stackName, async () =>
db.getOpenDriftFindings(nodeId, stackName).filter(f => !serviceName || f.service === serviceName).length),
this.collect('containers', stackName, () =>
withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness container probe')),
this.collect('update preview', stackName, () =>
withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness update preview')),
withTimeout(this.probeContainers(nodeId, stackName, serviceName), INPUT_TIMEOUT_MS, 'readiness container probe')),
serviceName
? this.collect('sibling containers', stackName, () =>
withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness sibling probe'))
: Promise.resolve<ContainerProbe[] | Errored>([]),
this.collect('update preview', stackName, async () => {
const full = await withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness update preview');
return serviceName ? filterPreviewForService(full, serviceName) : full;
}),
this.collect('backup info', stackName, () => FileSystemService.getInstance(nodeId).getBackupInfo(stackName)),
this.collect('disk', stackName, () => this.readDiskUsage()),
]);
@@ -114,8 +157,56 @@ export class UpdateGuardService {
backupSlotSignal(backup, now),
diskSignal(typeof disk === 'number' ? { usePercent: disk, limitPercent } : 'error'),
];
if (serviceName) {
signals.push(serviceSignal(this.resolveServiceMembership(model, serviceName)));
}
return { stack: stackName, computedAt: now, verdict: aggregateVerdict(signals), signals };
const advisories = serviceName
? [
this.siblingHealthAdvisory(containers, siblings),
this.dependencyAdvisory(model, serviceName),
].filter((note): note is string => note !== null)
: [];
return {
stack: stackName,
computedAt: now,
verdict: aggregateVerdict(signals),
signals,
serviceName: serviceName ?? null,
advisories,
};
}
/** Model-membership facts for the selected service; 'error' when the model failed to render. */
private resolveServiceMembership(model: EffectiveServiceModelResult | null, serviceName: string): ServiceMembership | Errored {
if (!model || !model.renderable) return 'error';
const spec = model.services.find(s => s.name === serviceName);
if (!spec) return { found: false };
return { found: true, hasBuild: spec.hasBuild, declaredImage: spec.declaredImage, expectedReplicas: spec.expectedReplicas };
}
/** Advisory only (§6): a sibling already unhealthy before the update never blocks the selected service's verdict. */
private siblingHealthAdvisory(selected: ContainerProbe[] | Errored, all: ContainerProbe[] | Errored): string | null {
if (selected === 'error' || all === 'error') return null;
const selectedNames = new Set(selected.map(c => c.name));
const troubled = all.filter(c => !selectedNames.has(c.name) && isTroubledContainer(c));
if (troubled.length === 0) return null;
const names = troubled.map(c => c.name).join(', ');
return `Other containers in this stack are already unhealthy: ${names}. This does not block the selected service.`;
}
/** Advisory only (§6): dependsOn relationships are informational; this update never recreates siblings. */
private dependencyAdvisory(model: EffectiveServiceModelResult | null, serviceName: string): string | null {
if (!model || !model.renderable) return null;
const spec = model.services.find(s => s.name === serviceName);
const dependsOn = spec?.dependsOn ?? [];
const dependents = model.services.filter(s => s.dependsOn.includes(serviceName)).map(s => s.name);
const parts: string[] = [];
if (dependsOn.length > 0) parts.push(`depends on ${dependsOn.join(', ')}`);
if (dependents.length > 0) parts.push(`is a dependency of ${dependents.join(', ')}`);
if (parts.length === 0) return null;
return `This service ${parts.join(' and ')}. Those services are not restarted by this update.`;
}
async computeRollbackReadiness(nodeId: number, stackName: string): Promise<RollbackReadinessReport> {
@@ -287,6 +287,13 @@ export function buildSummary(
};
}
/** Filter a full-stack preview down to one service's images and recompute the summary from that subset. */
export function filterPreviewForService(preview: UpdatePreview, serviceName: string): UpdatePreview {
const images = preview.images.filter(i => i.service === serviceName);
const buildServices = preview.build_services.filter(s => s === serviceName);
return buildSummary(preview.stack_name, images, buildServices);
}
export class UpdatePreviewService {
private static instance: UpdatePreviewService;
+7 -3
View File
@@ -1,5 +1,6 @@
import crypto from 'crypto';
import { ComposeService } from './ComposeService';
import { StackUpdateOrchestrator } from './StackUpdateOrchestrator';
import { StackOpLockService, stackOpSkipMessage, type StackOpAction } from './StackOpLockService';
import { DatabaseService, type Webhook } from './DatabaseService';
import { FileSystemService } from './FileSystemService';
@@ -162,7 +163,7 @@ export class WebhookService {
atomic,
{ source: 'webhook', actor: 'system:webhook' },
);
HealthGateService.getInstance().begin(nodeId, stackName, 'deploy', 'system:webhook');
HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:webhook');
break;
case 'restart':
await compose.runCommand(stackName, 'restart');
@@ -179,8 +180,11 @@ export class WebhookService {
nodeId,
buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }),
);
await compose.updateStack(stackName, undefined, atomic);
HealthGateService.getInstance().begin(nodeId, stackName, 'update', 'system:webhook');
await StackUpdateOrchestrator.getInstance().execute(
{ nodeId, stackName, target: { scope: 'stack' }, trigger: 'webhook', actor: 'system:webhook' },
{ atomic: atomic ?? false, terminalWs: null },
);
HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook');
break;
}
},
@@ -0,0 +1,144 @@
/**
* Effective Service Model: per-service facts that service-scoped update and
* restore key off of (declared image, build presence, expected replica
* count, dependencies, healthcheck presence), derived from the
* FULLY-MERGED effective model (`docker compose config --format json`)
* instead of a single compose file, so a multi-file Git source's overrides
* are always reflected.
*
* Fail closed: unlike the read-only facts readers in this module family
* (effectiveAnatomy, storage inventory, etc.), this model backs mutation and
* readiness decisions, so a render failure must never fall back to a
* root-file parse. A caller that mutates or gates on `renderable: false`
* would risk pulling the wrong image or under/over-recreating replicas
* against a stale guess.
*
* Reads only the structural fields a mutation needs; it never reads
* `environment`, `command`, `entrypoint`, `labels`, `secrets`, or `configs`.
*/
import { ComposeService } from './ComposeService';
import { parseMissingRequiredVars } from '../helpers/envVarParse';
import { getErrorMessage } from '../utils/errors';
import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog';
const MAX_RENDER_ERROR = 600;
export interface EffectiveServiceSpec {
name: string;
declaredImage: string | null;
hasBuild: boolean;
/** May be 0 (explicit `scale: 0` or `deploy.replicas: 0`); defaults to 1 when neither is set, matching Compose's own default. */
expectedReplicas: number;
dependsOn: string[];
hasHealthcheck: boolean;
}
export type EffectiveServiceModelResult =
| { renderable: true; services: EffectiveServiceSpec[] }
| { renderable: false; code: 'effective_model_render_failed'; error: string };
function asString(v: unknown): string | undefined {
if (typeof v === 'string') return v;
if (typeof v === 'number') return String(v);
return undefined;
}
function asNumber(v: unknown): number | undefined {
if (typeof v === 'number' && Number.isFinite(v)) return v;
return undefined;
}
/**
* `depends_on` always renders as a map keyed by target service name (Compose
* normalizes the short list form into the same map shape), so a plain key
* extraction covers both authoring styles; the list branch is a defensive
* fallback for a hand-built fixture.
*/
function dependsOnNames(value: unknown): string[] {
if (Array.isArray(value)) {
return value.map(asString).filter((s): s is string => s !== undefined);
}
if (value && typeof value === 'object') return Object.keys(value as Record<string, unknown>);
return [];
}
/**
* Expected replica count mirrors Compose's own `ServiceConfig.GetScale()`:
* the top-level `scale` field wins when set (Compose keeps it in sync with
* `deploy.replicas`), else `deploy.replicas`, else the Compose default of 1.
*/
function expectedReplicasOf(svc: Record<string, unknown>, deploy: Record<string, unknown> | undefined): number {
return asNumber(svc.scale) ?? asNumber(deploy?.replicas) ?? 1;
}
function parseServiceSpec(name: string, raw: unknown): EffectiveServiceSpec {
const svc = (raw ?? {}) as Record<string, unknown>;
const deploy = (svc.deploy && typeof svc.deploy === 'object') ? svc.deploy as Record<string, unknown> : undefined;
const healthcheck = svc.healthcheck;
const hasHealthcheck = !!healthcheck
&& typeof healthcheck === 'object'
&& (healthcheck as Record<string, unknown>).disable !== true;
return {
name,
declaredImage: asString(svc.image) ?? null,
hasBuild: !!svc.build && typeof svc.build === 'object',
expectedReplicas: expectedReplicasOf(svc, deploy),
dependsOn: dependsOnNames(svc.depends_on),
hasHealthcheck,
};
}
/** Parse the raw `docker compose config --format json` output into the per-service specs. Tolerant of missing/garbage fields: an empty or malformed `services` map yields an empty list rather than throwing. */
function parseEffectiveServices(parsed: unknown): EffectiveServiceSpec[] {
const root = (parsed && typeof parsed === 'object') ? parsed as Record<string, unknown> : {};
const rawServices = (root.services && typeof root.services === 'object' && !Array.isArray(root.services))
? root.services as Record<string, unknown>
: {};
return Object.entries(rawServices).map(([name, svc]) => parseServiceSpec(name, svc));
}
/**
* Render the fully-merged effective Compose model for a stack and extract the
* per-service specs service-scoped update/restore key off of. Mirrors the
* render-error handling of the other effective-facts readers (redacted,
* never raw stderr or an exception), but never falls back to a root-file
* parse: a render failure leaves `renderable: false` with no services, so a
* caller that skips the `renderable` check cannot silently mutate against a
* stale or partial model.
*/
export async function buildEffectiveServiceModel(nodeId: number, stackName: string): Promise<EffectiveServiceModelResult> {
try {
const result = await ComposeService.getInstance(nodeId).renderConfig(stackName);
if (result.rendered !== null) {
try {
return { renderable: true, services: parseEffectiveServices(JSON.parse(result.rendered)) };
} catch (parseErr) {
// JSON.parse errors carry no file content, so the message is safe to log.
console.warn('[EffectiveServiceModel] Effective model parse failed for %s:',
sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(parseErr, 'unknown')));
return {
renderable: false,
code: 'effective_model_render_failed',
error: 'Sencho could not parse the rendered Compose model.',
};
}
}
// Raw stderr can echo file content/secrets and is never surfaced; only the
// names of any missing required variables, otherwise a generic nudge.
const missing = parseMissingRequiredVars(result.stderr);
const error = missing.length
? `Required variable${missing.length > 1 ? 's' : ''} ${missing.join(', ')} ${missing.length > 1 ? 'have' : 'has'} no value, so the effective model cannot be rendered.`
: 'Sencho could not render the effective Compose model. Check the compose and env files for a YAML syntax error, an unresolved include or merge, or a required variable with no value.';
return { renderable: false, code: 'effective_model_render_failed', error };
} catch (err) {
// Spawn failure (docker unavailable), or an unexpected throw before/inside the
// render. Leave a sanitized breadcrumb so a non-spawn bug is not invisible, then
// redact the surfaced message defensively.
console.warn('[EffectiveServiceModel] Render failed for %s:',
sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown')));
const error = redactSensitiveText(getErrorMessage(err, 'docker compose could not be started.')).slice(0, MAX_RENDER_ERROR).trim()
|| 'Sencho could not run docker compose on this node.';
return { renderable: false, code: 'effective_model_render_failed', error };
}
}
+50 -3
View File
@@ -67,6 +67,11 @@ export function driftSignal(input: number | Errored): ReadinessSignal {
return { ...base, status: 'ok', affectsVerdict: true, detail: 'No open drift findings.' };
}
/** A container already unhealthy, restarting, or crash-exited before any update runs. */
export function isTroubledContainer(c: ContainerProbe): boolean {
return c.health === 'unhealthy' || c.state === 'restarting' || (c.state === 'exited' && (c.exitCode ?? 0) !== 0);
}
export function containersSignal(input: ContainerProbe[] | Errored): ReadinessSignal {
const base = { id: 'containers' as const, title: 'Current containers' };
if (input === 'error') {
@@ -75,9 +80,7 @@ export function containersSignal(input: ContainerProbe[] | Errored): ReadinessSi
if (input.length === 0) {
return { ...base, status: 'warning', affectsVerdict: true, detail: 'The stack is not running; updating will start it.' };
}
const troubled = input.filter(
c => c.health === 'unhealthy' || c.state === 'restarting' || (c.state === 'exited' && (c.exitCode ?? 0) !== 0),
);
const troubled = input.filter(isTroubledContainer);
if (troubled.length > 0) {
const names = troubled.map(c => c.name).join(', ');
return {
@@ -162,6 +165,50 @@ export function buildServicesSignal(buildServices: string[] | Errored): Readines
};
}
/** Model-membership facts for the selected service, gathered from EffectiveServiceModel. */
export type ServiceMembership =
| { found: false }
| { found: true; hasBuild: boolean; declaredImage: string | null; expectedReplicas: number };
/** Matches `ServiceUpdateRecoveryService`'s digest-pin check (a recovery snapshot is never eligible for a digest-pinned declared ref). */
const DIGEST_PIN_RE = /@sha256:[a-f0-9]{64}$/i;
/**
* Selected-service signal for a service-scoped readiness check: model
* membership and whether the update would leave a rollback recovery snapshot
* behind. Render failure or a missing service fails closed (blocked), never
* unknown, so a service-scoped check cannot silently proceed against a stale
* or absent model.
*/
export function serviceSignal(input: ServiceMembership | Errored): ReadinessSignal {
const base = { id: 'service' as const, title: 'Selected service' };
if (input === 'error') {
return { ...base, status: 'blocked', affectsVerdict: true, detail: 'The compose model could not be rendered, so the selected service cannot be validated. Resolve the compose error before updating.' };
}
if (!input.found) {
return { ...base, status: 'blocked', affectsVerdict: true, detail: 'The selected service is not declared in this stack\u2019s compose model.' };
}
if (!input.hasBuild && !input.declaredImage) {
return { ...base, status: 'blocked', affectsVerdict: true, detail: 'The selected service has neither an image nor a build, so it cannot be updated.' };
}
const digestPinned = input.declaredImage !== null && DIGEST_PIN_RE.test(input.declaredImage);
if (!input.declaredImage || digestPinned) {
const reason = !input.declaredImage ? 'it builds from source with no declared image' : 'its declared image is pinned to a digest';
return {
...base,
status: 'warning',
affectsVerdict: true,
detail: `Declared with ${input.expectedReplicas} expected replica${input.expectedReplicas === 1 ? '' : 's'}. No rollback recovery snapshot will be available for this update because ${reason}.`,
};
}
return {
...base,
status: 'ok',
affectsVerdict: true,
detail: `Declared with ${input.expectedReplicas} expected replica${input.expectedReplicas === 1 ? '' : 's'}; a rollback recovery snapshot will be captured before the update.`,
};
}
export function backupSlotSignal(
input: { exists: boolean; timestamp: number | null } | Errored,
now: number,
+14 -2
View File
@@ -6,7 +6,7 @@ export type SignalStatus = 'ok' | 'warning' | 'attention' | 'blocked' | 'unknown
/** One input to the readiness verdict (preflight, drift, containers, ...). */
export interface ReadinessSignal {
id: 'preflight' | 'drift' | 'containers' | 'healthchecks' | 'update_preview' | 'build_services' | 'backup_slot' | 'disk';
id: 'preflight' | 'drift' | 'containers' | 'healthchecks' | 'update_preview' | 'build_services' | 'backup_slot' | 'disk' | 'service';
status: SignalStatus;
/** Short headline ("Compose Doctor", "Running containers"). */
title: string;
@@ -25,6 +25,13 @@ export interface UpdateReadinessReport {
computedAt: number;
verdict: ReadinessVerdict;
signals: ReadinessSignal[];
/** The service this report is scoped to, or null for a full-stack check. */
serviceName: string | null;
/**
* Non-blocking notes that never affect `verdict` (sibling health,
* dependsOn relationships). Always empty for a full-stack check.
*/
advisories: string[];
}
/** State of one rollback readiness item. */
@@ -72,6 +79,8 @@ export interface HealthGateContainer {
state: string;
health: string | null;
restarts: number;
service?: string | null;
role?: 'primary' | 'collateral' | null;
}
/** Payload of GET /:stackName/health-gate ('never-run' when no run exists). */
@@ -79,12 +88,15 @@ export interface HealthGateReport {
stack: string;
id: string | null;
status: HealthGateStatus | 'never-run';
trigger: 'update' | 'deploy' | null;
trigger: 'update' | 'deploy' | 'service_update' | 'service_restore' | null;
reason: string | null;
windowSeconds: number | null;
startedAt: number | null;
endedAt: number | null;
containers: HealthGateContainer[];
targetScope: 'stack' | 'service';
serviceName: string | null;
failureSource: 'primary' | 'collateral' | null;
}
/** Categories a failed stack deploy or update can be classified into. */