diff --git a/backend/src/__tests__/auto-heal-suppress.test.ts b/backend/src/__tests__/auto-heal-suppress.test.ts new file mode 100644 index 00000000..7f4e087d --- /dev/null +++ b/backend/src/__tests__/auto-heal-suppress.test.ts @@ -0,0 +1,37 @@ +/** + * Ref-counted Auto-Heal suppression for overlapping service-scoped updates. + */ +import { describe, it, expect, beforeEach } from 'vitest'; +import { AutoHealService } from '../services/AutoHealService'; + +describe('AutoHealService suppress refcount', () => { + beforeEach(() => { + AutoHealService.getInstance().stop(); + }); + + it('keeps suppression until every overlapping owner clears', () => { + const svc = AutoHealService.getInstance(); + svc.suppress(0, 'web', 'api'); + svc.suppress(0, 'web', 'api'); + expect(svc.isSuppressed(0, 'web', 'api')).toBe(true); + svc.clearSuppress(0, 'web', 'api'); + expect(svc.isSuppressed(0, 'web', 'api')).toBe(true); + svc.clearSuppress(0, 'web', 'api'); + expect(svc.isSuppressed(0, 'web', 'api')).toBe(false); + }); + + it('scopes suppression per service on the same stack', () => { + const svc = AutoHealService.getInstance(); + svc.suppress(0, 'web', 'api'); + expect(svc.isSuppressed(0, 'web', 'api')).toBe(true); + expect(svc.isSuppressed(0, 'web', 'db')).toBe(false); + svc.clearSuppress(0, 'web', 'api'); + expect(svc.isSuppressed(0, 'web', 'api')).toBe(false); + }); + + it('ignores clearSuppress when nothing is suppressed', () => { + const svc = AutoHealService.getInstance(); + svc.clearSuppress(0, 'web', 'api'); + expect(svc.isSuppressed(0, 'web', 'api')).toBe(false); + }); +}); diff --git a/backend/src/__tests__/compose-service.test.ts b/backend/src/__tests__/compose-service.test.ts index d603b79b..8ca97de8 100644 --- a/backend/src/__tests__/compose-service.test.ts +++ b/backend/src/__tests__/compose-service.test.ts @@ -660,6 +660,38 @@ describe('ComposeService - deployStack', () => { expect(mockGetLegacyOrphanContainersByStack).toHaveBeenCalledWith('my-stack'); }); + it('blocks a pilot service restore with relative binds before Compose up', async () => { + process.env.SENCHO_MODE = 'pilot'; + const composeDir = path.resolve('/test/compose'); + mockGetBindMounts.mockResolvedValue([ + { source: '/opt/docker/sencho', destination: composeDir }, + ]); + const configProc = createMockProcess(); + mockSpawn.mockReturnValue(configProc); + + const svc = ComposeService.getInstance(1); + const result = svc.recreateServiceFromLocal('my-stack', 'db').then(() => null, (error: Error) => error); + await waitForSpawn(); + configProc.stdout.emit('data', Buffer.from(JSON.stringify({ + name: 'my-stack', + services: { + db: { + image: 'postgres:17', + volumes: [{ + type: 'bind', + source: path.join(composeDir, 'my-stack', 'data'), + target: '/var/lib/postgresql/data', + }], + }, + }, + }))); + configProc.emit('close', 0); + + const error = await result; + expect(error?.message).toContain('1:1'); + expect(mockSpawn).toHaveBeenCalledTimes(1); + }); + it('does not remove compose-managed containers before deploy (#1565)', async () => { setupAutoCloseSpawn(); mockListContainers.mockResolvedValue([]); diff --git a/backend/src/__tests__/database-service-update-recovery.test.ts b/backend/src/__tests__/database-service-update-recovery.test.ts new file mode 100644 index 00000000..a8e06950 --- /dev/null +++ b/backend/src/__tests__/database-service-update-recovery.test.ts @@ -0,0 +1,251 @@ +/** + * Coverage for the service_update_recovery accessors on the real + * DatabaseService (against a temp DB, so the CAS WHERE clauses run against + * actual SQLite semantics rather than a mock): + * - insert / get / list active + * - atomic claim CAS (active -> restoring) and its race/expiry guards + * - renewal CAS never revives a terminal or already-active row + * - mark consumed / reactivate / invalidate transitions + * - sweep of abandoned restoring claims vs a still-live claim + * - held image id projection and stack-delete cleanup + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import type { ServiceUpdateRecoveryRow } from '../services/DatabaseService'; + +let tmpDir: string; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +function db() { + return DatabaseService.getInstance(); +} + +beforeEach(() => { + const raw = (db() as unknown as { db: { prepare: (s: string) => { run: () => void } } }).db; + raw.prepare('DELETE FROM service_update_recovery').run(); +}); + +const NODE = 1; + +function makeRow(overrides: Partial = {}): ServiceUpdateRecoveryRow { + return { + id: overrides.id ?? 'rec-1', + node_id: NODE, + stack_name: 'web', + service_name: 'api', + replicas_json: JSON.stringify([{ imageId: 'sha256:aaa', repoDigest: 'sha256:digest-aaa' }]), + majority_image_id: 'sha256:aaa', + declared_image_ref: 'ghcr.io/acme/api:v1', + weak_floating_tag: 0, + health_gate_id: null, + status: 'active', + expires_at: 10_000, + claim_expires_at: null, + created_at: 1_000, + created_by: 'tester', + ...overrides, + }; +} + +describe('service_update_recovery accessors', () => { + it('inserts and reads a row back by id', () => { + db().insertServiceUpdateRecovery(makeRow()); + const row = db().getServiceUpdateRecovery('rec-1'); + expect(row).toMatchObject({ id: 'rec-1', stack_name: 'web', service_name: 'api', status: 'active' }); + }); + + it('lists only active rows for a service, most recent first', () => { + db().insertServiceUpdateRecovery(makeRow({ id: 'rec-old', created_at: 1_000 })); + db().insertServiceUpdateRecovery(makeRow({ id: 'rec-new', created_at: 2_000 })); + db().insertServiceUpdateRecovery(makeRow({ id: 'rec-consumed', status: 'consumed' })); + db().insertServiceUpdateRecovery(makeRow({ id: 'rec-other-service', service_name: 'db' })); + + const active = db().listActiveServiceUpdateRecoveries(NODE, 'web', 'api'); + expect(active.map(r => r.id)).toEqual(['rec-new', 'rec-old']); + }); + + it('links a health gate id only while the row is still active', () => { + db().insertServiceUpdateRecovery(makeRow()); + db().linkServiceUpdateRecoveryHealthGate('rec-1', 'gate-1'); + expect(db().getServiceUpdateRecovery('rec-1')?.health_gate_id).toBe('gate-1'); + + db().insertServiceUpdateRecovery(makeRow({ id: 'rec-consumed', status: 'consumed' })); + db().linkServiceUpdateRecoveryHealthGate('rec-consumed', 'gate-2'); + expect(db().getServiceUpdateRecovery('rec-consumed')?.health_gate_id).toBeNull(); + }); + + describe('claim CAS', () => { + it('claims an active, unexpired row and moves it to restoring', () => { + db().insertServiceUpdateRecovery(makeRow({ expires_at: 50_000 })); + const claimed = db().claimServiceUpdateRecovery('rec-1', 100_000, 20_000); + expect(claimed).toMatchObject({ id: 'rec-1', status: 'restoring', claim_expires_at: 100_000 }); + }); + + it('refuses to claim an already-expired active row', () => { + db().insertServiceUpdateRecovery(makeRow({ expires_at: 10_000 })); + const claimed = db().claimServiceUpdateRecovery('rec-1', 100_000, 20_000); + expect(claimed).toBeUndefined(); + expect(db().getServiceUpdateRecovery('rec-1')?.status).toBe('active'); + }); + + it('refuses a second claim on an already-restoring row (no double claim)', () => { + db().insertServiceUpdateRecovery(makeRow({ expires_at: 50_000 })); + const first = db().claimServiceUpdateRecovery('rec-1', 100_000, 20_000); + expect(first).toBeTruthy(); + const second = db().claimServiceUpdateRecovery('rec-1', 200_000, 21_000); + expect(second).toBeUndefined(); + }); + + it('refuses to claim a consumed/expired/invalidated row', () => { + for (const status of ['consumed', 'expired', 'invalidated'] as const) { + db().insertServiceUpdateRecovery(makeRow({ id: `rec-${status}`, status, expires_at: 50_000 })); + const claimed = db().claimServiceUpdateRecovery(`rec-${status}`, 100_000, 20_000); + expect(claimed).toBeUndefined(); + } + }); + }); + + describe('renewal CAS (no revive of terminal)', () => { + it('extends claim_expires_at while restoring', () => { + db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', claim_expires_at: 100_000 })); + const renewed = db().renewServiceUpdateRecoveryClaim('rec-1', 200_000); + expect(renewed).toBe(true); + expect(db().getServiceUpdateRecovery('rec-1')?.claim_expires_at).toBe(200_000); + }); + + it.each(['active', 'consumed', 'expired', 'invalidated'] as const)( + 'never revives a %s row via renewal', + (status) => { + db().insertServiceUpdateRecovery(makeRow({ status, claim_expires_at: status === 'active' ? null : 100_000 })); + const renewed = db().renewServiceUpdateRecoveryClaim('rec-1', 999_000); + expect(renewed).toBe(false); + expect(db().getServiceUpdateRecovery('rec-1')?.status).toBe(status); + }, + ); + }); + + describe('terminal transitions', () => { + it('marks a restoring row consumed and links the restore health gate id', () => { + db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', claim_expires_at: 100_000, health_gate_id: 'update-gate' })); + const ok = db().markServiceUpdateRecoveryConsumed('rec-1', 'restore-gate'); + expect(ok).toBe(true); + const row = db().getServiceUpdateRecovery('rec-1'); + expect(row).toMatchObject({ status: 'consumed', claim_expires_at: null, health_gate_id: 'restore-gate' }); + }); + + it('keeps the prior health gate id when marking consumed with no runId', () => { + db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', health_gate_id: 'update-gate' })); + db().markServiceUpdateRecoveryConsumed('rec-1', null); + expect(db().getServiceUpdateRecovery('rec-1')?.health_gate_id).toBe('update-gate'); + }); + + it('cannot mark an already-active row consumed', () => { + db().insertServiceUpdateRecovery(makeRow({ status: 'active' })); + expect(db().markServiceUpdateRecoveryConsumed('rec-1')).toBe(false); + }); + + it('reactivates a restoring row on a mid-flight failure with the image still local', () => { + db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', claim_expires_at: 100_000 })); + const ok = db().reactivateServiceUpdateRecovery('rec-1'); + expect(ok).toBe(true); + expect(db().getServiceUpdateRecovery('rec-1')).toMatchObject({ status: 'active', claim_expires_at: null }); + }); + + it('invalidates a restoring row on a mid-flight failure with the image gone', () => { + db().insertServiceUpdateRecovery(makeRow({ status: 'restoring', claim_expires_at: 100_000 })); + const ok = db().invalidateServiceUpdateRecovery('rec-1'); + expect(ok).toBe(true); + expect(db().getServiceUpdateRecovery('rec-1')).toMatchObject({ status: 'invalidated', claim_expires_at: null }); + }); + + it('cannot reactivate or invalidate a row that is not restoring', () => { + db().insertServiceUpdateRecovery(makeRow({ status: 'active' })); + expect(db().reactivateServiceUpdateRecovery('rec-1')).toBe(false); + expect(db().invalidateServiceUpdateRecovery('rec-1')).toBe(false); + }); + }); + + describe('sweep', () => { + it('expires an active row whose TTL has lapsed', () => { + db().insertServiceUpdateRecovery(makeRow({ expires_at: 10_000 })); + const count = db().sweepExpiredActiveServiceUpdateRecoveries(20_000); + expect(count).toBe(1); + expect(db().getServiceUpdateRecovery('rec-1')?.status).toBe('expired'); + }); + + it('leaves an unexpired active row alone', () => { + db().insertServiceUpdateRecovery(makeRow({ expires_at: 50_000 })); + expect(db().sweepExpiredActiveServiceUpdateRecoveries(20_000)).toBe(0); + expect(db().getServiceUpdateRecovery('rec-1')?.status).toBe('active'); + }); + + it('expires an abandoned restoring row (dead claim) but never a live one, even past the original expires_at', () => { + db().insertServiceUpdateRecovery(makeRow({ + id: 'rec-abandoned', status: 'restoring', expires_at: 10_000, claim_expires_at: 15_000, + })); + db().insertServiceUpdateRecovery(makeRow({ + id: 'rec-live', status: 'restoring', expires_at: 10_000, claim_expires_at: 100_000, + })); + + const count = db().sweepAbandonedRestoringServiceUpdateRecoveries(20_000); + expect(count).toBe(1); + expect(db().getServiceUpdateRecovery('rec-abandoned')?.status).toBe('expired'); + // Original expires_at (10_000) is long past, but the claim is still live: not swept. + expect(db().getServiceUpdateRecovery('rec-live')?.status).toBe('restoring'); + }); + + it('never touches an active row from the restoring sweep, and vice versa', () => { + db().insertServiceUpdateRecovery(makeRow({ id: 'rec-active', status: 'active', expires_at: 10_000 })); + db().insertServiceUpdateRecovery(makeRow({ + id: 'rec-restoring', status: 'restoring', expires_at: 999_000, claim_expires_at: 10_000, + })); + + expect(db().sweepAbandonedRestoringServiceUpdateRecoveries(20_000)).toBe(1); + expect(db().getServiceUpdateRecovery('rec-active')?.status).toBe('active'); + + expect(db().sweepExpiredActiveServiceUpdateRecoveries(20_000)).toBe(1); + expect(db().getServiceUpdateRecovery('rec-active')?.status).toBe('expired'); + }); + }); + + describe('held image ids', () => { + it('holds the image for an active row and a restoring row with a live claim, not a terminal or abandoned one', () => { + db().insertServiceUpdateRecovery(makeRow({ id: 'rec-active', majority_image_id: 'sha256:held-1', status: 'active', expires_at: 999_000 })); + db().insertServiceUpdateRecovery(makeRow({ + id: 'rec-live-claim', majority_image_id: 'sha256:held-2', status: 'restoring', claim_expires_at: 100_000, + })); + db().insertServiceUpdateRecovery(makeRow({ + id: 'rec-dead-claim', majority_image_id: 'sha256:not-held-1', status: 'restoring', claim_expires_at: 5_000, + })); + db().insertServiceUpdateRecovery(makeRow({ id: 'rec-consumed', majority_image_id: 'sha256:not-held-2', status: 'consumed' })); + + const held = db().listHeldServiceUpdateRecoveryImageIds(NODE, 20_000); + expect(new Set(held)).toEqual(new Set(['sha256:held-1', 'sha256:held-2'])); + }); + + it('scopes held image ids per node', () => { + db().insertServiceUpdateRecovery(makeRow({ id: 'rec-node-1', node_id: 1, majority_image_id: 'sha256:node-1', expires_at: 999_000 })); + db().insertServiceUpdateRecovery(makeRow({ id: 'rec-node-2', node_id: 2, majority_image_id: 'sha256:node-2', expires_at: 999_000 })); + + expect(db().listHeldServiceUpdateRecoveryImageIds(1, 0)).toEqual(['sha256:node-1']); + expect(db().listHeldServiceUpdateRecoveryImageIds(2, 0)).toEqual(['sha256:node-2']); + }); + }); + + it('deletes every row for a stack on stack delete, leaving other stacks untouched', () => { + db().insertServiceUpdateRecovery(makeRow({ id: 'rec-web', stack_name: 'web' })); + db().insertServiceUpdateRecovery(makeRow({ id: 'rec-api', stack_name: 'api' })); + + db().deleteServiceUpdateRecoveries(NODE, 'web'); + + expect(db().getServiceUpdateRecovery('rec-web')).toBeUndefined(); + expect(db().getServiceUpdateRecovery('rec-api')).toBeTruthy(); + }); +}); diff --git a/backend/src/__tests__/database-stack-update-status.test.ts b/backend/src/__tests__/database-stack-update-status.test.ts index fd397dc9..ce74e2bb 100644 --- a/backend/src/__tests__/database-stack-update-status.test.ts +++ b/backend/src/__tests__/database-stack-update-status.test.ts @@ -78,3 +78,86 @@ describe('stack_update_status tri-state accessors', () => { expect(db().getStackUpdateDetail(2).web.checkStatus).toBe('failed'); }); }); + +describe('stack_update_status services_json (per-service reduction)', () => { + const SERVICES = [ + { service: 'web', image: 'web:latest', hasUpdate: true, checkStatus: 'ok' as const, lastError: null }, + { service: 'worker', image: 'worker:latest', hasUpdate: false, checkStatus: 'ok' as const, lastError: null }, + ]; + + it('persists and returns a per-service breakdown via getStackUpdateDetail', () => { + db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null, SERVICES, 3); + + const detail = db().getStackUpdateDetail(NODE).stackA; + expect(detail.services).toEqual(SERVICES); + }); + + it('omits the services field entirely for a stack with no persisted per-service data', () => { + db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null); + + const detail = db().getStackUpdateDetail(NODE).stackA; + expect(detail.services).toBeUndefined(); + expect('services' in detail).toBe(false); + }); + + it('leaves a prior services_json untouched when a later write omits services (COALESCE)', () => { + db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null, SERVICES, 1); + // A legacy-path caller (or the fallback tally) writes without a services array. + db().upsertStackUpdateStatus(NODE, 'stackA', true, 2000, 'ok', null); + + expect(db().getStackUpdateDetail(NODE).stackA.services).toEqual(SERVICES); + }); + + it('getStackServicesJson returns [] for a stack with no persisted row', () => { + expect(db().getStackServicesJson(NODE, 'missing')).toEqual([]); + }); + + it('getStackServicesJson returns [] for a stack row with no services_json', () => { + db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null); + expect(db().getStackServicesJson(NODE, 'stackA')).toEqual([]); + }); + + it('getStackServicesJson round-trips a persisted per-service breakdown', () => { + db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null, SERVICES, 5); + expect(db().getStackServicesJson(NODE, 'stackA')).toEqual(SERVICES); + }); + + it('recordStackCheckFailure persists a per-service breakdown alongside the failure', () => { + db().recordStackCheckFailure(NODE, 'stackA', 'Registry unreachable', 4000, SERVICES, 2); + + const detail = db().getStackUpdateDetail(NODE).stackA; + expect(detail.checkStatus).toBe('failed'); + expect(detail.services).toEqual(SERVICES); + }); + + it('treats corrupt services_json as an empty list rather than throwing (empty model invariant)', () => { + db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null); + const raw = (db() as unknown as { db: { prepare: (s: string) => { run: (...args: unknown[]) => void } } }).db; + raw.prepare('UPDATE stack_update_status SET services_json = ? WHERE node_id = ? AND stack_name = ?') + .run('{not valid json', NODE, 'stackA'); + + expect(db().getStackServicesJson(NODE, 'stackA')).toEqual([]); + const detail = db().getStackUpdateDetail(NODE).stackA; + expect(detail.services).toBeUndefined(); + expect(detail.hasUpdate).toBe(true); // aggregate columns are unaffected by corrupt services_json + }); + + it('treats a services_json shape-version mismatch as an empty list', () => { + db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null); + const raw = (db() as unknown as { db: { prepare: (s: string) => { run: (...args: unknown[]) => void } } }).db; + raw.prepare('UPDATE stack_update_status SET services_json = ? WHERE node_id = ? AND stack_name = ?') + .run(JSON.stringify({ version: 999, generation: 1, services: SERVICES }), NODE, 'stackA'); + + expect(db().getStackServicesJson(NODE, 'stackA')).toEqual([]); + }); + + it('filters out malformed entries within an otherwise valid services_json array', () => { + db().upsertStackUpdateStatus(NODE, 'stackA', true, 1000, 'ok', null); + const raw = (db() as unknown as { db: { prepare: (s: string) => { run: (...args: unknown[]) => void } } }).db; + const malformed = [...SERVICES, { service: 'broken' /* missing required fields */ }]; + raw.prepare('UPDATE stack_update_status SET services_json = ? WHERE node_id = ? AND stack_name = ?') + .run(JSON.stringify({ version: 1, generation: 1, services: malformed }), NODE, 'stackA'); + + expect(db().getStackServicesJson(NODE, 'stackA')).toEqual(SERVICES); + }); +}); diff --git a/backend/src/__tests__/docker-controller.test.ts b/backend/src/__tests__/docker-controller.test.ts index 18b23984..5afd827d 100644 --- a/backend/src/__tests__/docker-controller.test.ts +++ b/backend/src/__tests__/docker-controller.test.ts @@ -428,6 +428,83 @@ describe('DockerController - pruneDanglingImages', () => { }); }); +// ── Prune holds (): images held for a pending service-update +// recovery must never be deleted by any image-prune path. ────────────── + +describe('DockerController - prune holds (isImageHeld)', () => { + it('pruneSystem("images") falls back to enumerate-and-delete when isImageHeld is given, skipping the held image', async () => { + mockDocker.listImages.mockResolvedValue([ + { Id: 'img-held', RepoTags: ['app:1'], Containers: 0 }, + { Id: 'img-free', RepoTags: ['app:2'], Containers: 0 }, + ]); + mockDocker.df + .mockResolvedValueOnce({ LayersSize: 5000 }) + .mockResolvedValueOnce({ LayersSize: 3000 }); + const removeFn = vi.fn().mockResolvedValue(undefined); + mockDocker.getImage.mockReturnValue({ remove: removeFn }); + + const dc = DockerController.getInstance(1); + const result = await dc.pruneSystem('images', undefined, (id: string) => id === 'img-held'); + + expect(mockDocker.pruneImages).not.toHaveBeenCalled(); + expect(removeFn).toHaveBeenCalledTimes(1); + expect(mockDocker.getImage).toHaveBeenCalledWith('img-free'); + expect(result).toEqual({ success: true, reclaimedBytes: 2000 }); + }); + + it('pruneDanglingImages falls back to enumerate-and-delete when isImageHeld is given, skipping the held image', async () => { + mockDocker.listImages.mockResolvedValue([ + { Id: 'img-held', RepoTags: [], Containers: 0 }, + { Id: 'img-free', RepoTags: [':'], Containers: 0 }, + { Id: 'img-tagged', RepoTags: ['app:1'], Containers: 0 }, // not dangling; excluded regardless of hold + ]); + mockDocker.df + .mockResolvedValueOnce({ LayersSize: 5000 }) + .mockResolvedValueOnce({ LayersSize: 4000 }); + const removeFn = vi.fn().mockResolvedValue(undefined); + mockDocker.getImage.mockReturnValue({ remove: removeFn }); + + const dc = DockerController.getInstance(1); + const result = await dc.pruneDanglingImages((id: string) => id === 'img-held'); + + expect(mockDocker.pruneImages).not.toHaveBeenCalled(); + expect(removeFn).toHaveBeenCalledTimes(1); + expect(mockDocker.getImage).toHaveBeenCalledWith('img-free'); + expect(result).toEqual({ success: true, reclaimedBytes: 1000 }); + }); + + it('pruneManagedOnly("images") re-checks isImageHeld immediately before each delete', async () => { + mockDocker.df + .mockResolvedValueOnce({ LayersSize: 5000, Images: [] }) + .mockResolvedValueOnce({ LayersSize: 4000, Images: [] }); + mockDocker.listImages.mockResolvedValue([ + { Id: 'img-held', Containers: 0, Size: 1000, Labels: { 'com.docker.compose.project': 'any-stack' } }, + { Id: 'img-free', Containers: 0, Size: 1000, Labels: { 'com.docker.compose.project': 'any-stack' } }, + ]); + mockDocker.listContainers.mockResolvedValue([]); + const removeFn = vi.fn().mockResolvedValue(undefined); + mockDocker.getImage.mockReturnValue({ remove: removeFn }); + + const dc = DockerController.getInstance(1); + const result = await dc.pruneManagedOnly('images', ['any-stack'], (id: string) => id === 'img-held'); + + expect(removeFn).toHaveBeenCalledTimes(1); + expect(mockDocker.getImage).toHaveBeenCalledWith('img-free'); + expect(result.success).toBe(true); + }); + + it('pruneSystem("images") keeps the bulk Docker prune API when no isImageHeld predicate is given', async () => { + mockDocker.pruneImages.mockResolvedValue({ SpaceReclaimed: 999 }); + + const dc = DockerController.getInstance(1); + const result = await dc.pruneSystem('images'); + + expect(mockDocker.pruneImages).toHaveBeenCalled(); + expect(mockDocker.listImages).not.toHaveBeenCalled(); + expect(result).toEqual({ success: true, reclaimedBytes: 999 }); + }); +}); + // ── getClassifiedResources ───────────────────────────────────────────── describe('DockerController - getClassifiedResources', () => { diff --git a/backend/src/__tests__/drift-ledger.test.ts b/backend/src/__tests__/drift-ledger.test.ts index 7a14ff52..f81fbcd2 100644 --- a/backend/src/__tests__/drift-ledger.test.ts +++ b/backend/src/__tests__/drift-ledger.test.ts @@ -246,6 +246,74 @@ describe('DriftLedgerService.reconcile', () => { }); }); +describe('DriftLedgerService.reconcileService', () => { + beforeEach(() => clearLedger('svcrec')); + const ledger = () => DriftLedgerService.getInstance(); + + it('inserts only the selected service\'s findings, ignoring other services in the same report', () => { + const report = reportWith([finding('image-mismatch', 'web'), finding('service-missing', 'db')], { stack: 'svcrec' }); + const res = ledger().reconcileService(nodeId, 'svcrec', 'web', report); + expect(res).toEqual({ detected: 1, resolved: 0 }); + const open = db().getOpenDriftFindings(nodeId, 'svcrec'); + expect(open).toHaveLength(1); + expect(open[0].service).toBe('web'); + }); + + it('resolves only the selected service\'s open finding, leaving other open services untouched', () => { + ledger().reconcile(nodeId, 'svcrec', reportWith([finding('image-mismatch', 'web'), finding('service-missing', 'db')], { stack: 'svcrec' })); + // 'web' cleared, 'db' absent from this service-scoped report too, but only 'web' is in scope. + const res = ledger().reconcileService(nodeId, 'svcrec', 'web', reportWith([], { stack: 'svcrec', status: 'in-sync' })); + expect(res).toEqual({ detected: 0, resolved: 1 }); + const open = db().getOpenDriftFindings(nodeId, 'svcrec'); + expect(open).toHaveLength(1); + expect(open[0].service).toBe('db'); + }); + + it('preserves an empty-service (network-level) finding when reconciling one named service', () => { + ledger().reconcile(nodeId, 'svcrec', reportWith([finding('image-mismatch', 'web'), finding('network-missing', '')], { stack: 'svcrec' })); + const res = ledger().reconcileService(nodeId, 'svcrec', 'web', reportWith([], { stack: 'svcrec', status: 'in-sync' })); + expect(res).toEqual({ detected: 0, resolved: 1 }); + const open = db().getOpenDriftFindings(nodeId, 'svcrec'); + expect(open).toHaveLength(1); + expect(open[0].service).toBe(''); + }); + + it('does not touch findings when the same service is idempotently rechecked', () => { + const report = reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' }); + ledger().reconcileService(nodeId, 'svcrec', 'web', report); + const res = ledger().reconcileService(nodeId, 'svcrec', 'web', report); + expect(res).toEqual({ detected: 0, resolved: 0 }); + expect(db().getOpenDriftFindings(nodeId, 'svcrec')).toHaveLength(1); + }); + + it('does not reconcile an unreachable report (open findings are not falsely resolved)', () => { + ledger().reconcile(nodeId, 'svcrec', reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' })); + const res = ledger().reconcileService(nodeId, 'svcrec', 'web', { stack: 'svcrec', status: 'unreachable', hasComposeFile: true, hasContainers: false, findings: [] }); + expect(res).toEqual({ detected: 0, resolved: 0 }); + expect(db().getOpenDriftFindings(nodeId, 'svcrec')).toHaveLength(1); + }); + + it('does not reconcile a parse-error report', () => { + ledger().reconcile(nodeId, 'svcrec', reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' })); + const res = ledger().reconcileService(nodeId, 'svcrec', 'web', { stack: 'svcrec', status: 'drifted', hasComposeFile: false, hasContainers: false, findings: [], parseError: 'bad yaml' }); + expect(res).toEqual({ detected: 0, resolved: 0 }); + expect(db().getOpenDriftFindings(nodeId, 'svcrec')).toHaveLength(1); + }); + + it('names the service in the recorded activity message', () => { + ledger().reconcileService(nodeId, 'svcrec', 'web', reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' })); + const activity = driftActivity('svcrec'); + expect(activity).toHaveLength(1); + expect(activity[0].message).toContain('web'); + }); + + it('stamps the dossier last-checked time', () => { + expect(db().getStackDossier(nodeId, 'svcrec')?.last_drift_check_at ?? null).toBeNull(); + ledger().reconcileService(nodeId, 'svcrec', 'web', reportWith([finding('image-mismatch', 'web')], { stack: 'svcrec' })); + expect(typeof db().getStackDossier(nodeId, 'svcrec')?.last_drift_check_at).toBe('number'); + }); +}); + describe('DriftLedgerService.reconcileStack', () => { const STACK_A = 'recstacka'; let dirA: string; diff --git a/backend/src/__tests__/effective-service-model.test.ts b/backend/src/__tests__/effective-service-model.test.ts new file mode 100644 index 00000000..e7ffccfb --- /dev/null +++ b/backend/src/__tests__/effective-service-model.test.ts @@ -0,0 +1,125 @@ +/** + * buildEffectiveServiceModel: turns `docker compose config --format json` + * output into the per-service facts (declared image, build presence, + * expected replicas, dependencies, healthcheck) that service-scoped update + * and restore key off of. Fail-closed is the critical property: a render + * failure must leave `renderable: false` with no services, never a + * root-file fallback. + */ +import { describe, it, expect, afterEach, vi } from 'vitest'; +import { ComposeService } from '../services/ComposeService'; +import { buildEffectiveServiceModel } from '../services/effectiveServiceModel'; + +const ENV_SECRET = 'env-secret-9d4a-value'; + +function stubRender(rendered: string | null, stderr = '') { + vi.spyOn(ComposeService, 'getInstance').mockReturnValue({ + renderConfig: vi.fn().mockResolvedValue({ rendered, stderr, code: rendered === null ? 1 : 0, timedOut: false }), + } as unknown as ComposeService); +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('buildEffectiveServiceModel', () => { + it('extracts declared image, dependsOn, and healthcheck for an image-only service', async () => { + stubRender(JSON.stringify({ + services: { + web: { + image: 'nginx:latest', + depends_on: { db: { condition: 'service_healthy', required: true } }, + healthcheck: { test: ['CMD', 'true'] }, + }, + db: { image: 'postgres:16' }, + }, + })); + const result = await buildEffectiveServiceModel(1, 'mystack'); + expect(result.renderable).toBe(true); + if (!result.renderable) throw new Error('expected renderable'); + expect(result.services).toEqual([ + { name: 'web', declaredImage: 'nginx:latest', hasBuild: false, expectedReplicas: 1, dependsOn: ['db'], hasHealthcheck: true }, + { name: 'db', declaredImage: 'postgres:16', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false }, + ]); + }); + + it('flags hasBuild for a build-backed service, including image+build', async () => { + stubRender(JSON.stringify({ + services: { + api: { build: { context: '.' } }, + worker: { image: 'app:latest', build: { context: '.' } }, + }, + })); + const result = await buildEffectiveServiceModel(1, 'mystack'); + if (!result.renderable) throw new Error('expected renderable'); + expect(result.services[0]).toMatchObject({ name: 'api', declaredImage: null, hasBuild: true }); + expect(result.services[1]).toMatchObject({ name: 'worker', declaredImage: 'app:latest', hasBuild: true }); + }); + + it('reads expectedReplicas from the top-level scale field, deploy.replicas, or defaults to 1', async () => { + stubRender(JSON.stringify({ + services: { + scaled: { image: 'a', scale: 3 }, + deployed: { image: 'b', deploy: { replicas: 2 } }, + zeroed: { image: 'c', scale: 0 }, + unset: { image: 'd' }, + }, + })); + const result = await buildEffectiveServiceModel(1, 'mystack'); + if (!result.renderable) throw new Error('expected renderable'); + expect(result.services.map(s => s.expectedReplicas)).toEqual([3, 2, 0, 1]); + }); + + it('treats a disabled healthcheck as none', async () => { + stubRender(JSON.stringify({ + services: { web: { image: 'a', healthcheck: { disable: true } } }, + })); + const result = await buildEffectiveServiceModel(1, 'mystack'); + if (!result.renderable) throw new Error('expected renderable'); + expect(result.services[0].hasHealthcheck).toBe(false); + }); + + it('parses depends_on given in the short list form', async () => { + stubRender(JSON.stringify({ + services: { web: { image: 'a', depends_on: ['db', 'cache'] } }, + })); + const result = await buildEffectiveServiceModel(1, 'mystack'); + if (!result.renderable) throw new Error('expected renderable'); + expect(result.services[0].dependsOn).toEqual(['db', 'cache']); + }); + + it('fails closed with a redacted error and no services when the render errors', async () => { + stubRender(null, `error: the "${ENV_SECRET}" variable is not set`); + const result = await buildEffectiveServiceModel(1, 'mystack'); + expect(result.renderable).toBe(false); + if (result.renderable) throw new Error('expected not renderable'); + expect(result.code).toBe('effective_model_render_failed'); + expect(result.error).not.toContain(ENV_SECRET); + }); + + it('fails closed when the rendered output is not valid JSON', async () => { + stubRender('this is not json {'); + const result = await buildEffectiveServiceModel(1, 'mystack'); + expect(result.renderable).toBe(false); + if (result.renderable) throw new Error('expected not renderable'); + expect(result.code).toBe('effective_model_render_failed'); + }); + + it('fails closed and redacts the message when renderConfig throws (docker unavailable)', async () => { + vi.spyOn(ComposeService, 'getInstance').mockReturnValue({ + renderConfig: vi.fn().mockRejectedValue(new Error(`spawn failed: password=${ENV_SECRET}`)), + } as unknown as ComposeService); + const result = await buildEffectiveServiceModel(1, 'mystack'); + expect(result.renderable).toBe(false); + if (result.renderable) throw new Error('expected not renderable'); + expect(result.code).toBe('effective_model_render_failed'); + expect(result.error).not.toContain(ENV_SECRET); + }); + + it('yields an empty service list for a garbage or empty rendered model', async () => { + stubRender(JSON.stringify({ services: {} })); + const result = await buildEffectiveServiceModel(1, 'mystack'); + if (!result.renderable) throw new Error('expected renderable'); + expect(result.services).toEqual([]); + }); +}); diff --git a/backend/src/__tests__/git-source-service.test.ts b/backend/src/__tests__/git-source-service.test.ts index c52194c3..c7a3775b 100644 --- a/backend/src/__tests__/git-source-service.test.ts +++ b/backend/src/__tests__/git-source-service.test.ts @@ -1168,7 +1168,7 @@ describe('GitSourceService.apply', () => { const { HealthGateService } = await import('../services/HealthGateService'); const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); - const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-git'); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-git'); const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; try { diff --git a/backend/src/__tests__/health-gate-migration.test.ts b/backend/src/__tests__/health-gate-migration.test.ts new file mode 100644 index 00000000..2953bc43 --- /dev/null +++ b/backend/src/__tests__/health-gate-migration.test.ts @@ -0,0 +1,66 @@ +/** + * Restart-safe rebuild of health_gate_runs when widening the trigger CHECK and + * adding target/failure columns. + */ +import path from 'path'; +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import Database from 'better-sqlite3'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +describe('migrateHealthGateTargetSchema', () => { + let tmpDir: string; + + beforeEach(async () => { + tmpDir = await setupTestDb(); + }); + + afterEach(async () => { + await cleanupTestDb(tmpDir); + }); + + it('recovers from a stale health_gate_runs_new and preserves existing rows', async () => { + const dbPath = path.join(tmpDir, 'sencho.db'); + const raw = new Database(dbPath); + raw.exec(` + DROP TABLE IF EXISTS health_gate_runs; + DROP TABLE IF EXISTS health_gate_runs_new; + CREATE TABLE health_gate_runs ( + id TEXT PRIMARY KEY, + node_id INTEGER NOT NULL, + stack_name TEXT NOT NULL, + trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy')), + status TEXT NOT NULL CHECK (status IN ('observing','passed','failed','unknown')), + reason TEXT, + window_seconds INTEGER NOT NULL, + containers_json TEXT NOT NULL DEFAULT '[]', + started_at INTEGER NOT NULL, + ended_at INTEGER, + created_by TEXT + ); + CREATE TABLE health_gate_runs_new (id TEXT PRIMARY KEY); + INSERT INTO health_gate_runs ( + id, node_id, stack_name, trigger_action, status, reason, window_seconds, containers_json, started_at, ended_at, created_by + ) VALUES ('gate-1', 1, 'web', 'update', 'passed', null, 90, '[]', 1, 2, 'tester'); + `); + raw.close(); + + const { DatabaseService } = await import('../services/DatabaseService'); + (DatabaseService as unknown as { instance?: typeof DatabaseService }).instance = undefined; + const db = DatabaseService.getInstance(); + const row = db.getDb().prepare('SELECT * FROM health_gate_runs WHERE id = ?').get('gate-1') as { + target_scope: string; + service_name: string | null; + failure_source: string | null; + trigger_action: string; + }; + expect(row.trigger_action).toBe('update'); + expect(row.target_scope).toBe('stack'); + expect(row.service_name).toBeNull(); + expect(row.failure_source).toBeNull(); + + const stale = db.getDb().prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'health_gate_runs_new'", + ).get(); + expect(stale).toBeUndefined(); + }); +}); diff --git a/backend/src/__tests__/health-gate-prepare.test.ts b/backend/src/__tests__/health-gate-prepare.test.ts new file mode 100644 index 00000000..258be3e2 --- /dev/null +++ b/backend/src/__tests__/health-gate-prepare.test.ts @@ -0,0 +1,293 @@ +/** + * Tests for the service-scoped health gate: prepare/attach/beginPrepared + * nullability (disabled, unknown token, concurrency cap, armed), the prepare + * TTL, and primary vs collateral failure attribution (regression-eligible + * siblings can fail the gate; pre-existing unhealthy siblings stay advisory). + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +interface StoredRun { + id: string; + node_id: number; + stack_name: string; + trigger_action: 'update' | 'deploy' | 'service_update' | 'service_restore'; + status: 'observing' | 'passed' | 'failed' | 'unknown'; + reason: string | null; + window_seconds: number; + containers_json: string; + started_at: number; + ended_at: number | null; + created_by: string | null; + target_scope: 'stack' | 'service'; + service_name: string | null; + failure_source: 'primary' | 'collateral' | null; +} + +const { state } = vi.hoisted(() => ({ + state: { + runs: new Map(), + settings: {} as Record, + listContainers: vi.fn(), + inspect: vi.fn(), + }, +})); + +vi.mock('../services/DatabaseService', () => ({ + DatabaseService: { + getInstance: () => ({ + getGlobalSettings: () => state.settings, + insertHealthGateRun: (run: StoredRun) => { state.runs.set(run.id, { ...run }); }, + finalizeHealthGateRun: (id: string, status: StoredRun['status'], reason: string | null, endedAt: number, containersJson: string, failureSource: StoredRun['failure_source'] = null) => { + const run = state.runs.get(id); + if (run) Object.assign(run, { status, reason, ended_at: endedAt, containers_json: containersJson, failure_source: failureSource }); + }, + getHealthGateRun: (nodeId: number, stackName: string, id: string) => { + const run = state.runs.get(id); + return run && run.node_id === nodeId && run.stack_name === stackName ? { ...run } : undefined; + }, + getLatestHealthGateRun: (nodeId: number, stackName: string) => { + const matches = [...state.runs.values()] + .filter(r => r.node_id === nodeId && r.stack_name === stackName) + .sort((a, b) => b.started_at - a.started_at); + return matches[0] ? { ...matches[0] } : undefined; + }, + markInterruptedHealthGateRuns: () => 0, + addNotificationHistory: (_nodeId: number, item: { category?: string; message: string; level: string }) => ({ ...item, id: 1, is_read: false }), + }), + }, +})); + +vi.mock('../services/DockerController', () => ({ + default: { + getInstance: () => ({ + getDocker: () => ({ + listContainers: state.listContainers, + getContainer: (id: string) => ({ inspect: () => state.inspect(id) }), + }), + }), + }, +})); + +// AutoHeal suppression is exercised elsewhere; here we only need the calls to +// not throw when the gate finalizes a service run. +vi.mock('../services/AutoHealService', () => ({ + AutoHealService: { getInstance: () => ({ clearSuppress: vi.fn() }) }, +})); + +import { HealthGateService } from '../services/HealthGateService'; + +type Fixture = { + id: string; + name: string; + service: string; + state?: string; + health?: string | null; + restartCount?: number; + startedAt?: string; + imageId?: string; +}; + +function setContainers(fixtures: Fixture[]): void { + state.listContainers.mockResolvedValue(fixtures.map(f => ({ + Id: f.id, + Names: [`/${f.name}`], + State: f.state ?? 'running', + Labels: { 'com.docker.compose.service': f.service }, + }))); + state.inspect.mockImplementation((id: string) => { + const f = fixtures.find(c => c.id === id); + if (!f) return Promise.reject(Object.assign(new Error('no such container'), { statusCode: 404 })); + return Promise.resolve({ + State: { + Status: f.state ?? 'running', + Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined, + StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z', + }, + RestartCount: f.restartCount ?? 0, + Image: f.imageId ?? 'sha256:app', + Config: { Labels: { 'com.docker.compose.service': f.service } }, + }); + }); +} + +const svc = () => HealthGateService.getInstance(); + +async function ticks(n: number): Promise { + for (let i = 0; i < n; i++) await vi.advanceTimersByTimeAsync(5_000); +} + +async function prepareService( + fixtures: Fixture[], + opts: { serviceName?: string; expectedReplicas?: number; trigger?: 'service_update' | 'service_restore' } = {}, +): Promise { + setContainers(fixtures); + const { prepareToken } = await svc().prepare({ + nodeId: 0, + stackName: 'web', + target: { scope: 'service', serviceName: opts.serviceName ?? 'app' }, + trigger: opts.trigger ?? 'service_update', + expectedReplicas: opts.expectedReplicas ?? 1, + }); + return prepareToken; +} + +beforeEach(() => { + vi.useFakeTimers(); + state.runs.clear(); + state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' }; + state.listContainers.mockReset(); + state.inspect.mockReset(); + svc().start(); +}); + +afterEach(() => { + svc().stop(); + expect(vi.getTimerCount()).toBe(0); + vi.useRealTimers(); +}); + +describe('prepare / beginPrepared nullability', () => { + it('prepare returns a token with a TTL that outlives the Compose timeout', async () => { + const before = Date.now(); + setContainers([{ id: 'p1', name: 'web-app-1', service: 'app' }]); + const { prepareToken, expiresAt } = await svc().prepare({ + nodeId: 0, stackName: 'web', target: { scope: 'service', serviceName: 'app' }, + trigger: 'service_update', expectedReplicas: 1, + }); + expect(prepareToken).toBeTruthy(); + // Default compose timeout is 30m; prepare TTL is max(timeout, 30m) + 5m. + expect(expiresAt).toBe(before + 35 * 60_000); + }); + + it('returns runId null / observing false when gating is disabled', async () => { + const token = await prepareService([{ id: 'p1', name: 'web-app-1', service: 'app' }]); + state.settings.health_gate_enabled = '0'; + expect(svc().beginPrepared({ prepareToken: token, actor: 'tester' })).toEqual({ runId: null, observing: false }); + }); + + it('returns runId null / observing false for an unknown or consumed token', async () => { + const token = await prepareService([{ id: 'p1', name: 'web-app-1', service: 'app' }]); + // First consume arms a gate; the second call sees a consumed token. + svc().attachExpectedImage(token, 'sha256:app'); + const first = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + expect(first.observing).toBe(true); + expect(svc().beginPrepared({ prepareToken: token, actor: 'tester' })).toEqual({ runId: null, observing: false }); + }); + + it('persists an immediate unknown past the concurrency cap', async () => { + for (let i = 0; i < 25; i++) svc().beginStack(0, `stack-${i}`, 'update', 'tester'); + const token = await prepareService([{ id: 'p1', name: 'web-app-1', service: 'app' }]); + svc().attachExpectedImage(token, 'sha256:app'); + const result = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + expect(result.runId).toBeTruthy(); + expect(result.observing).toBe(false); + const report = svc().getReport(0, 'web', result.runId!); + expect(report.status).toBe('unknown'); + expect(report.reason).toContain('concurrent'); + }); +}); + +describe('primary vs collateral attribution', () => { + it('fails with failureSource primary when a replica reports unhealthy', async () => { + const token = await prepareService([ + { id: 'p1', name: 'web-app-1', service: 'app' }, + { id: 's1', name: 'web-db-1', service: 'db' }, + ]); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); // arm expected set + setContainers([ + { id: 'p1', name: 'web-app-1', service: 'app', health: 'unhealthy' }, + { id: 's1', name: 'web-db-1', service: 'db' }, + ]); + await ticks(1); + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.failureSource).toBe('primary'); + }); + + it('fails with failureSource collateral when a regression-eligible sibling exits', async () => { + const token = await prepareService([ + { id: 'p1', name: 'web-app-1', service: 'app' }, + { id: 's1', name: 'web-db-1', service: 'db' }, + ]); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-app-1', service: 'app' }, + { id: 's1', name: 'web-db-1', service: 'db', state: 'exited' }, + ]); + await ticks(1); + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.failureSource).toBe('collateral'); + }); + + it('fails with failureSource collateral when a healthy sibling vanishes before the first poll', async () => { + // Sibling is healthy at prepare, then gone before arming. Seeding expected + // from the prepare baseline must still track it so the gate fails. + const token = await prepareService([ + { id: 'p1', name: 'web-app-1', service: 'app' }, + { id: 's1', name: 'web-db-1', service: 'db' }, + ]); + svc().attachExpectedImage(token, 'sha256:app'); + setContainers([ + { id: 'p1', name: 'web-app-1', service: 'app' }, + ]); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); // arm expected from primary + prepare collateral baseline + await ticks(1); // first miss on the vanished sibling + await ticks(1); // second miss fails the gate + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.failureSource).toBe('collateral'); + expect(report.reason ?? '').toMatch(/disappeared|sibling/i); + }); + + it('keeps a pre-existing unhealthy sibling advisory: the gate can still pass', async () => { + // The sibling is unhealthy at prepare, so it is not regression-eligible and + // cannot fail the service gate; the healthy primary carries it to a pass. + const token = await prepareService([ + { id: 'p1', name: 'web-app-1', service: 'app' }, + { id: 's1', name: 'web-db-1', service: 'db', health: 'unhealthy' }, + ]); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(8); // through the 30s window + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('passed'); + expect(report.failureSource).toBeNull(); + }); + + it('lets different-service gates coexist while same-service begin supersedes', async () => { + const appToken = await prepareService( + [{ id: 'p1', name: 'web-app-1', service: 'app' }, { id: 's1', name: 'web-db-1', service: 'db' }], + { serviceName: 'app' }, + ); + svc().attachExpectedImage(appToken, 'sha256:app'); + const appFirst = svc().beginPrepared({ prepareToken: appToken, actor: 'tester' }); + expect(appFirst.observing).toBe(true); + + const dbToken = await prepareService( + [{ id: 'p1', name: 'web-app-1', service: 'app' }, { id: 's1', name: 'web-db-1', service: 'db' }], + { serviceName: 'db' }, + ); + svc().attachExpectedImage(dbToken, 'sha256:db'); + const dbGate = svc().beginPrepared({ prepareToken: dbToken, actor: 'tester' }); + expect(dbGate.observing).toBe(true); + expect(svc().getReport(0, 'web', appFirst.runId!).status).toBe('observing'); + expect(svc().getReport(0, 'web', dbGate.runId!).status).toBe('observing'); + + const appToken2 = await prepareService( + [{ id: 'p1', name: 'web-app-1', service: 'app' }, { id: 's1', name: 'web-db-1', service: 'db' }], + { serviceName: 'app' }, + ); + svc().attachExpectedImage(appToken2, 'sha256:app2'); + const appSecond = svc().beginPrepared({ prepareToken: appToken2, actor: 'tester' }); + expect(appSecond.observing).toBe(true); + expect(svc().getReport(0, 'web', appFirst.runId!).status).toBe('unknown'); + expect(svc().getReport(0, 'web', dbGate.runId!).status).toBe('observing'); + expect(svc().getReport(0, 'web', appSecond.runId!).status).toBe('observing'); + }); +}); diff --git a/backend/src/__tests__/health-gate-service.test.ts b/backend/src/__tests__/health-gate-service.test.ts index fdd0ac23..161bc570 100644 --- a/backend/src/__tests__/health-gate-service.test.ts +++ b/backend/src/__tests__/health-gate-service.test.ts @@ -9,7 +9,7 @@ interface StoredRun { id: string; node_id: number; stack_name: string; - trigger_action: 'update' | 'deploy'; + trigger_action: 'update' | 'deploy' | 'service_update' | 'service_restore'; status: 'observing' | 'passed' | 'failed' | 'unknown'; reason: string | null; window_seconds: number; @@ -17,6 +17,9 @@ interface StoredRun { started_at: number; ended_at: number | null; created_by: string | null; + target_scope: 'stack' | 'service'; + service_name: string | null; + failure_source: 'primary' | 'collateral' | null; } const { state } = vi.hoisted(() => ({ @@ -34,9 +37,9 @@ vi.mock('../services/DatabaseService', () => ({ getInstance: () => ({ getGlobalSettings: () => state.settings, insertHealthGateRun: (run: StoredRun) => { state.runs.set(run.id, { ...run }); }, - finalizeHealthGateRun: (id: string, status: StoredRun['status'], reason: string | null, endedAt: number, containersJson: string) => { + finalizeHealthGateRun: (id: string, status: StoredRun['status'], reason: string | null, endedAt: number, containersJson: string, failureSource: StoredRun['failure_source'] = null) => { const run = state.runs.get(id); - if (run) Object.assign(run, { status, reason, ended_at: endedAt, containers_json: containersJson }); + if (run) Object.assign(run, { status, reason, ended_at: endedAt, containers_json: containersJson, failure_source: failureSource }); }, getHealthGateRun: (nodeId: number, stackName: string, id: string) => { const run = state.runs.get(id); @@ -134,7 +137,7 @@ afterEach(() => { describe('HealthGateService verdicts', () => { it('passes at the window end when containers stay running', async () => { - const id = svc().begin(0, 'web', 'update', 'tester'); + const id = svc().beginStack(0, 'web', 'update', 'tester'); expect(id).toBeTruthy(); await ticks(3); // 15s: still observing expect(latest().status).toBe('observing'); @@ -144,7 +147,7 @@ describe('HealthGateService verdicts', () => { }); it('fails fast when a container exits', async () => { - svc().begin(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester'); await ticks(1); // baseline setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited' }]); await ticks(1); @@ -155,7 +158,7 @@ describe('HealthGateService verdicts', () => { }); it('fails fast when a healthcheck reports unhealthy', async () => { - svc().begin(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester'); await ticks(1); setContainers([{ id: 'aaa', name: 'web-app-1', health: 'unhealthy' }]); await ticks(1); @@ -164,7 +167,7 @@ describe('HealthGateService verdicts', () => { }); it('detects a restart loop via container replacement (new id)', async () => { - svc().begin(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester'); await ticks(1); setContainers([{ id: 'bbb', name: 'web-app-1' }]); await ticks(1); // restart 1 observed; carried as new baseline @@ -180,7 +183,7 @@ describe('HealthGateService verdicts', () => { }); it('detects a restart loop via RestartCount and StartedAt movement', async () => { - svc().begin(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester'); await ticks(1); setContainers([{ id: 'aaa', name: 'web-app-1', restartCount: 1 }]); await ticks(1); @@ -195,7 +198,7 @@ describe('HealthGateService verdicts', () => { }); it('tolerates a one-poll disappearance but fails on two consecutive misses', async () => { - svc().begin(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester'); await ticks(1); // baseline setContainers([]); // one missed poll: tolerated await ticks(1); @@ -203,7 +206,7 @@ describe('HealthGateService verdicts', () => { await ticks(5); // through the 30s window expect(latest().status).toBe('passed'); - const second = svc().begin(0, 'web', 'update', 'tester')!; + const second = svc().beginStack(0, 'web', 'update', 'tester')!; await ticks(1); setContainers([]); await ticks(2); // two consecutive misses: disappeared @@ -213,7 +216,7 @@ describe('HealthGateService verdicts', () => { }); it('fails when a container is stuck restarting across consecutive polls', async () => { - svc().begin(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester'); await ticks(1); setContainers([{ id: 'aaa', name: 'web-app-1', state: 'restarting' }]); await ticks(2); @@ -222,7 +225,7 @@ describe('HealthGateService verdicts', () => { }); it('goes unknown after three consecutive docker errors', async () => { - svc().begin(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester'); await ticks(1); state.listContainers.mockRejectedValue(new Error('socket gone')); await ticks(3); @@ -231,7 +234,7 @@ describe('HealthGateService verdicts', () => { }); it('resolves unknown when every docker observe hangs', async () => { - svc().begin(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester'); // A wedged socket never settles. The per-observe timeout turns each poll // into an error, and three in a row finalize the gate unknown instead of // observing forever on a pending promise. @@ -244,7 +247,7 @@ describe('HealthGateService verdicts', () => { }); it('recovers from a transient observe timeout instead of finalizing', async () => { - svc().begin(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester'); // One observe wedges and times out (a single strike), then the socket // recovers; the gate must keep observing, not give up at one error. state.listContainers.mockImplementationOnce(() => new Promise(() => {})); @@ -257,7 +260,7 @@ describe('HealthGateService verdicts', () => { }); it('runs polls single-flight: no second observe until the first settles', async () => { - svc().begin(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester'); let release: (value: Array<{ Id: string; Names: string[]; State: string }>) => void = () => {}; state.listContainers.mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); // Advance past a second poll interval while the first observe is still @@ -272,7 +275,7 @@ describe('HealthGateService verdicts', () => { }); it('ends unknown when a healthcheck is still starting at the window end', async () => { - svc().begin(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester'); setContainers([{ id: 'aaa', name: 'web-app-1', health: 'starting' }]); await ticks(7); expect(latest().status).toBe('unknown'); @@ -281,7 +284,7 @@ describe('HealthGateService verdicts', () => { it('goes unknown when no containers ever appear', async () => { setContainers([]); - svc().begin(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester'); await ticks(4); expect(latest().status).toBe('unknown'); expect(latest().reason).toContain('no containers'); @@ -293,7 +296,7 @@ describe('HealthGateService lifecycle', () => { // A poll is mid-await on Docker when a newer update supersedes the gate; // when the await resolves with healthy containers, the superseded run // must keep its terminal unknown verdict. - const first = svc().begin(0, 'web', 'update', 'tester')!; + const first = svc().beginStack(0, 'web', 'update', 'tester')!; await ticks(2); // baseline established, healthy let releasePoll: (value: Array<{ Id: string; Names: string[]; State: string }>) => void = () => {}; @@ -302,7 +305,7 @@ describe('HealthGateService lifecycle', () => { ); const straddlingPoll = vi.advanceTimersByTimeAsync(5_000); // poll now awaiting Docker - const second = svc().begin(0, 'web', 'update', 'tester')!; + const second = svc().beginStack(0, 'web', 'update', 'tester')!; expect(svc().getReport(0, 'web', first).status).toBe('unknown'); releasePoll([{ Id: 'aaa', Names: ['/web-app-1'], State: 'running' }]); @@ -317,10 +320,10 @@ describe('HealthGateService lifecycle', () => { }); it('supersede finalizes the old run as unknown, clears its timer, and getRun still resolves it', async () => { - const first = svc().begin(0, 'web', 'update', 'tester')!; + const first = svc().beginStack(0, 'web', 'update', 'tester')!; await ticks(1); const timersBefore = vi.getTimerCount(); - const second = svc().begin(0, 'web', 'update', 'tester')!; + const second = svc().beginStack(0, 'web', 'update', 'tester')!; expect(vi.getTimerCount()).toBe(timersBefore); // old interval cleared, new one added const superseded = svc().getReport(0, 'web', first); @@ -337,6 +340,7 @@ describe('HealthGateService lifecycle', () => { state.runs.set('stale', { id: 'stale', node_id: 0, stack_name: 'web', trigger_action: 'update', status: 'observing', reason: null, window_seconds: 30, containers_json: '[]', started_at: 1, ended_at: null, created_by: null, + target_scope: 'stack', service_name: null, failure_source: null, }); svc().start(); expect(state.runs.get('stale')!.status).toBe('unknown'); @@ -345,7 +349,7 @@ describe('HealthGateService lifecycle', () => { it('no-ops when disabled but still records the update_started event', () => { state.settings.health_gate_enabled = '0'; - const id = svc().begin(0, 'web', 'update', 'tester'); + const id = svc().beginStack(0, 'web', 'update', 'tester'); expect(id).toBeNull(); expect(state.runs.size).toBe(0); expect(state.activity.some(a => a.category === 'update_started')).toBe(true); @@ -353,24 +357,24 @@ describe('HealthGateService lifecycle', () => { }); it('records update_started for update triggers but not deploy triggers', () => { - svc().begin(0, 'web', 'deploy', 'tester'); + svc().beginStack(0, 'web', 'deploy', 'tester'); expect(state.activity.some(a => a.category === 'update_started')).toBe(false); - svc().begin(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester'); expect(state.activity.some(a => a.category === 'update_started')).toBe(true); }); it('refuses to begin before start() so shutdown cannot leak timers', () => { svc().stop(); - expect(svc().begin(0, 'web', 'update', 'tester')).toBeNull(); + expect(svc().beginStack(0, 'web', 'update', 'tester')).toBeNull(); expect(vi.getTimerCount()).toBe(0); svc().start(); }); it('persists an immediate unknown past the concurrency cap', () => { for (let i = 0; i < 25; i++) { - svc().begin(0, `stack-${i}`, 'update', 'tester'); + svc().beginStack(0, `stack-${i}`, 'update', 'tester'); } - const overCap = svc().begin(0, 'one-too-many', 'update', 'tester')!; + const overCap = svc().beginStack(0, 'one-too-many', 'update', 'tester')!; const report = svc().getReport(0, 'one-too-many', overCap); expect(report.status).toBe('unknown'); expect(report.reason).toContain('concurrent'); @@ -378,10 +382,10 @@ describe('HealthGateService lifecycle', () => { it('clamps the configured window into its valid range and falls back on garbage', () => { state.settings.health_gate_window_seconds = '99999'; - const a = svc().begin(0, 'web', 'update', 'tester')!; + const a = svc().beginStack(0, 'web', 'update', 'tester')!; expect(svc().getReport(0, 'web', a).windowSeconds).toBe(600); state.settings.health_gate_window_seconds = 'banana'; - const b = svc().begin(0, 'web', 'update', 'tester')!; + const b = svc().beginStack(0, 'web', 'update', 'tester')!; expect(svc().getReport(0, 'web', b).windowSeconds).toBe(90); }); @@ -392,7 +396,7 @@ describe('HealthGateService lifecycle', () => { }); it('stop() finalizes in-flight gates as unknown with zero timers left', async () => { - const id = svc().begin(0, 'web', 'update', 'tester')!; + const id = svc().beginStack(0, 'web', 'update', 'tester')!; await ticks(1); svc().stop(); expect(vi.getTimerCount()).toBe(0); diff --git a/backend/src/__tests__/image-update-service-reduction.test.ts b/backend/src/__tests__/image-update-service-reduction.test.ts new file mode 100644 index 00000000..75287f94 --- /dev/null +++ b/backend/src/__tests__/image-update-service-reduction.test.ts @@ -0,0 +1,212 @@ +/** + * Unit tests for the §5 per-service reduction pure functions: + * reduceServiceStatus, aggregateServiceCheckStatus, and + * buildAvailabilityNotifyMessage. These operate on plain data (an + * imageUpdateMap and a prior per-service snapshot), so they are tested + * directly without mocking DatabaseService, Docker, or Compose. + */ +import { describe, it, expect } from 'vitest'; +import { + reduceServiceStatus, + aggregateServiceCheckStatus, + buildAvailabilityNotifyMessage, + type ImageCheckResult, +} from '../services/ImageUpdateService'; +import type { StackServiceStatus } from '../services/DatabaseService'; + +function ok(hasUpdate: boolean): ImageCheckResult { + return { hasUpdate }; +} +function errored(message: string): ImageCheckResult { + return { hasUpdate: false, error: message }; +} +function notCheckable(): ImageCheckResult { + return { hasUpdate: false, notCheckable: true }; +} + +describe('reduceServiceStatus', () => { + it('marks a service not_checkable when it has no checkable ref (STATUS-NOT-CHECKABLE-3)', () => { + const map = new Map([['app:latest', notCheckable()]]); + const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', [], map, undefined); + expect(status).toEqual({ service: 'app', image: 'app:latest', hasUpdate: false, checkStatus: 'not_checkable', lastError: null }); + expect(confirmedUpdateThisRun).toBe(false); + }); + + it('marks a service not_checkable when it declares no image and has no running container', () => { + const map = new Map(); + const { status, confirmedUpdateThisRun } = reduceServiceStatus('worker', null, [], map, undefined); + expect(status).toEqual({ service: 'worker', image: null, hasUpdate: false, checkStatus: 'not_checkable', lastError: null }); + expect(confirmedUpdateThisRun).toBe(false); + }); + + it('reports ok + hasUpdate=true when every checkable ref succeeds and one confirms an update', () => { + const map = new Map([['app:latest', ok(true)]]); + const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', [], map, undefined); + expect(status).toEqual({ service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null }); + expect(confirmedUpdateThisRun).toBe(true); + }); + + it('reports ok + hasUpdate=false when every checkable ref succeeds with no update', () => { + const map = new Map([['app:latest', ok(false)]]); + const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', [], map, undefined); + expect(status.checkStatus).toBe('ok'); + expect(status.hasUpdate).toBe(false); + expect(confirmedUpdateThisRun).toBe(false); + }); + + it('STATUS-CONFIRMATION-PARTIAL-5: one newly outdated success + one error -> partial, hasUpdate=true, confirmedUpdateThisRun=true', () => { + // Two refs on the same declared service (declared image + a divergent runtime image). + const map = new Map([ + ['app:latest', ok(true)], + ['app:1.2.3', errored('Registry unreachable for app:1.2.3')], + ]); + const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', ['app:1.2.3'], map, undefined); + expect(status.checkStatus).toBe('partial'); + expect(status.hasUpdate).toBe(true); + expect(status.lastError).toBe('Registry unreachable for app:1.2.3'); + expect(confirmedUpdateThisRun).toBe(true); + }); + + it('partial with only a preserved prior (no successful outdated check this run) -> confirmedUpdateThisRun=false', () => { + const map = new Map([ + ['app:latest', ok(false)], // succeeds, but reports no update + ['app:1.2.3', errored('timeout')], + ]); + const prior: StackServiceStatus = { service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null }; + const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', ['app:1.2.3'], map, prior); + expect(status.checkStatus).toBe('partial'); + expect(status.hasUpdate).toBe(true); // preserved from prior, not newly confirmed + expect(confirmedUpdateThisRun).toBe(false); + }); + + it('all checkable refs erroring -> failed, preserves prior hasUpdate, confirmedUpdateThisRun=false', () => { + const map = new Map([['app:latest', errored('Registry unreachable')]]); + const prior: StackServiceStatus = { service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null }; + const { status, confirmedUpdateThisRun } = reduceServiceStatus('app', 'app:latest', [], map, prior); + expect(status).toEqual({ service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'failed', lastError: 'Registry unreachable' }); + expect(confirmedUpdateThisRun).toBe(false); + }); + + it('all checkable refs erroring with no prior row -> failed, hasUpdate=false', () => { + const map = new Map([['app:latest', errored('boom')]]); + const { status } = reduceServiceStatus('app', 'app:latest', [], map, undefined); + expect(status.hasUpdate).toBe(false); + expect(status.checkStatus).toBe('failed'); + }); + + it('a shared image gets the same check result on every declared service that references it', () => { + const map = new Map([['shared:latest', ok(true)]]); + const web = reduceServiceStatus('web', 'shared:latest', [], map, undefined); + const worker = reduceServiceStatus('worker', 'shared:latest', [], map, undefined); + expect(web.status.hasUpdate).toBe(true); + expect(worker.status.hasUpdate).toBe(true); + expect(web.status.checkStatus).toBe(worker.status.checkStatus); + }); + + it('records sorted, deduplicated runtimeImages only when a runtime container was observed', () => { + const map = new Map([['app:latest', ok(false)], ['app:1.0', ok(false)]]); + const withRuntime = reduceServiceStatus('app', 'app:latest', ['app:1.0', 'app:1.0'], map, undefined); + expect(withRuntime.status.runtimeImages).toEqual(['app:1.0']); + + const noRuntime = reduceServiceStatus('app', 'app:latest', [], map, undefined); + expect(noRuntime.status.runtimeImages).toBeUndefined(); + }); +}); + +describe('aggregateServiceCheckStatus', () => { + const svc = (checkStatus: StackServiceStatus['checkStatus'], hasUpdate = false): StackServiceStatus => ({ + service: `svc-${checkStatus}-${hasUpdate}`, image: null, hasUpdate, checkStatus, lastError: null, + }); + + it('is ok for an empty services array (empty effective model)', () => { + expect(aggregateServiceCheckStatus([])).toBe('ok'); + }); + + it('is ok when every service is not_checkable (nothing checkable)', () => { + expect(aggregateServiceCheckStatus([svc('not_checkable'), svc('not_checkable')])).toBe('ok'); + }); + + it('is ok when every checkable service succeeded', () => { + expect(aggregateServiceCheckStatus([svc('ok'), svc('ok'), svc('not_checkable')])).toBe('ok'); + }); + + it('is failed when every checkable service failed, even with a preserved hasUpdate=true', () => { + expect(aggregateServiceCheckStatus([svc('failed', true), svc('failed', true)])).toBe('failed'); + }); + + it('is partial when checkable services mix ok and failed', () => { + expect(aggregateServiceCheckStatus([svc('ok'), svc('failed')])).toBe('partial'); + }); + + it('is partial when a checkable service is itself partial', () => { + expect(aggregateServiceCheckStatus([svc('ok'), svc('partial')])).toBe('partial'); + }); + + it('is not failed when only some (not all) checkable services failed', () => { + expect(aggregateServiceCheckStatus([svc('ok'), svc('failed'), svc('not_checkable')])).not.toBe('failed'); + }); +}); + +describe('aggregate invariant: has_update === services.some(hasUpdate)', () => { + it('holds for an empty services array', () => { + const services: StackServiceStatus[] = []; + expect(services.some(s => s.hasUpdate)).toBe(false); + }); + + it('holds when every checkable service failed but one preserves hasUpdate=true', () => { + const map = new Map([['app:latest', errored('unreachable')]]); + const prior: StackServiceStatus = { service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null }; + const { status } = reduceServiceStatus('app', 'app:latest', [], map, prior); + const services = [status]; + expect(services.some(s => s.hasUpdate)).toBe(true); + expect(aggregateServiceCheckStatus(services)).toBe('failed'); + }); + + it('holds when one service confirms an update and a sibling service fails outright', () => { + const map = new Map([ + ['web:latest', ok(true)], + ['worker:latest', errored('timeout')], + ]); + const web = reduceServiceStatus('web', 'web:latest', [], map, undefined); + const worker = reduceServiceStatus('worker', 'worker:latest', [], map, undefined); + const services = [web.status, worker.status]; + // web confirmed an update; worker's sole ref errored, so worker alone is 'failed'. + expect(services.some(s => s.hasUpdate)).toBe(true); + expect(worker.status.checkStatus).toBe('failed'); + // At the stack level, one ok service and one failed service is a mix, not a unanimous failure. + expect(aggregateServiceCheckStatus(services)).toBe('partial'); + }); +}); + +describe('buildAvailabilityNotifyMessage', () => { + it('uses the generic line for a single-service (or model-unavailable) stack', () => { + expect(buildAvailabilityNotifyMessage('stackA', [])).toBe('Stack "stackA" has image updates available.'); + const single: StackServiceStatus[] = [{ service: 'app', image: 'app:latest', hasUpdate: true, checkStatus: 'ok', lastError: null }]; + expect(buildAvailabilityNotifyMessage('stackA', single)).toBe('Stack "stackA" has image updates available.'); + }); + + it('names sorted services with hasUpdate for a multi-service stack', () => { + const services: StackServiceStatus[] = [ + { service: 'worker', image: 'worker:latest', hasUpdate: true, checkStatus: 'ok', lastError: null }, + { service: 'api', image: 'api:latest', hasUpdate: true, checkStatus: 'ok', lastError: null }, + { service: 'db', image: 'db:latest', hasUpdate: false, checkStatus: 'ok', lastError: null }, + ]; + expect(buildAvailabilityNotifyMessage('stackA', services)).toBe('Stack "stackA" has image updates available for services: api, worker.'); + }); + + it('falls back to the generic line for a multi-service stack with no service currently showing hasUpdate', () => { + const services: StackServiceStatus[] = [ + { service: 'api', image: 'api:latest', hasUpdate: false, checkStatus: 'ok', lastError: null }, + { service: 'db', image: 'db:latest', hasUpdate: false, checkStatus: 'ok', lastError: null }, + ]; + expect(buildAvailabilityNotifyMessage('stackA', services)).toBe('Stack "stackA" has image updates available.'); + }); + + it('uses singular wording for exactly one named service', () => { + const services: StackServiceStatus[] = [ + { service: 'api', image: 'api:latest', hasUpdate: true, checkStatus: 'ok', lastError: null }, + { service: 'db', image: 'db:latest', hasUpdate: false, checkStatus: 'ok', lastError: null }, + ]; + expect(buildAvailabilityNotifyMessage('stackA', services)).toBe('Stack "stackA" has image updates available for service: api.'); + }); +}); diff --git a/backend/src/__tests__/image-update-service.test.ts b/backend/src/__tests__/image-update-service.test.ts index 60f63521..65a0b828 100644 --- a/backend/src/__tests__/image-update-service.test.ts +++ b/backend/src/__tests__/image-update-service.test.ts @@ -9,17 +9,19 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; const { mockGetAuthForRegistry, mockGetStackUpdateStatus, mockUpsertStackUpdateStatus, mockClearStackUpdateStatus, - mockRecordStackCheckFailure, + mockRecordStackCheckFailure, mockGetStackServicesJson, mockGetSystemState, mockSetSystemState, mockAddNotificationHistory, mockDispatchAlert, mockGetStacks, mockGetStackContent, mockGetEnvContent, mockEnvExists, mockGetAllContainers, mockGetGlobalSettings, mockInspect, + mockBuildEffectiveServiceModel, } = vi.hoisted(() => ({ mockGetAuthForRegistry: vi.fn().mockResolvedValue(null), mockGetStackUpdateStatus: vi.fn().mockReturnValue({}), mockUpsertStackUpdateStatus: vi.fn(), mockClearStackUpdateStatus: vi.fn(), mockRecordStackCheckFailure: vi.fn(), + mockGetStackServicesJson: vi.fn().mockReturnValue([]), mockGetSystemState: vi.fn().mockReturnValue('1'), // default: backfilled mockSetSystemState: vi.fn(), mockAddNotificationHistory: vi.fn(), @@ -33,6 +35,10 @@ const { // Backs DockerController.getInstance().getDocker().getImage().inspect() for tests // that exercise the real checkImage (rather than stubbing it) through checkNode. mockInspect: vi.fn().mockResolvedValue({ RepoDigests: [] }), + // Defaults to non-renderable so checkNode's per-service reduction is a no-op + // (falls back to the legacy whole-stack tally) unless a test overrides this, + // matching how no real Compose model exists in this unit-test environment. + mockBuildEffectiveServiceModel: vi.fn().mockResolvedValue({ renderable: false, code: 'effective_model_render_failed', error: 'no model in test' }), })); vi.mock('../services/RegistryService', () => ({ @@ -54,6 +60,7 @@ vi.mock('../services/DatabaseService', () => ({ getStackUpdateStatus: mockGetStackUpdateStatus, clearStackUpdateStatus: mockClearStackUpdateStatus, recordStackCheckFailure: mockRecordStackCheckFailure, + getStackServicesJson: mockGetStackServicesJson, getSystemState: mockGetSystemState, setSystemState: mockSetSystemState, addNotificationHistory: mockAddNotificationHistory, @@ -61,6 +68,10 @@ vi.mock('../services/DatabaseService', () => ({ }, })); +vi.mock('../services/effectiveServiceModel', () => ({ + buildEffectiveServiceModel: mockBuildEffectiveServiceModel, +})); + vi.mock('../services/NotificationService', () => ({ NotificationService: { getInstance: () => ({ @@ -1437,3 +1448,206 @@ describe('ImageUpdateService cron scheduling', () => { expect(delay).toBeGreaterThan(0); }); }); + +// ── Model-based per-service reduction wiring (§5) ───────────────── + +describe('ImageUpdateService - model-based per-service reduction', () => { + const TWO_SERVICE_COMPOSE = ` +services: + web: + image: web:latest + worker: + image: worker:latest +`; + + const THREE_SERVICE_COMPOSE = ` +services: + worker: + image: worker:latest + api: + image: api:latest + db: + image: db:latest +`; + + const fakeDb = () => ({ + getStackUpdateStatus: mockGetStackUpdateStatus, + upsertStackUpdateStatus: mockUpsertStackUpdateStatus, + clearStackUpdateStatus: mockClearStackUpdateStatus, + recordStackCheckFailure: mockRecordStackCheckFailure, + getSystemState: mockGetSystemState, + setSystemState: mockSetSystemState, + addNotificationHistory: mockAddNotificationHistory, + }); + + function specFor(name: string, image: string): { name: string; declaredImage: string; hasBuild: boolean; expectedReplicas: number; dependsOn: string[]; hasHealthcheck: boolean } { + return { name, declaredImage: image, hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false }; + } + + /** Stubs checkImage to report hasUpdate only for the given image refs. */ + function stubCheckImagePerRef(service: ImageUpdateService, updatedRefs: string[]) { + (service as any).checkImage = vi.fn().mockImplementation(async (_docker: unknown, ref: string) => ({ hasUpdate: updatedRefs.includes(ref) })); + } + + beforeEach(() => { + vi.clearAllMocks(); + (ImageUpdateService as any).instance = undefined; + mockGetSystemState.mockReturnValue('1'); + mockGetAllContainers.mockResolvedValue([]); + mockEnvExists.mockResolvedValue(false); + }); + + it('reduces per-service status through the effective model and persists services_json with a generation', async () => { + mockGetStacks.mockResolvedValue(['stackA']); + mockGetStackContent.mockResolvedValue(TWO_SERVICE_COMPOSE); + mockGetStackUpdateStatus.mockReturnValue({}); + mockBuildEffectiveServiceModel.mockResolvedValueOnce({ + renderable: true, + services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')], + }); + const service = ImageUpdateService.getInstance(); + stubCheckImagePerRef(service, ['web:latest']); + + await (service as any).checkNode(1, fakeDb()); + + expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith( + 1, 'stackA', true, expect.any(Number), 'ok', null, + [ + { service: 'web', image: 'web:latest', hasUpdate: true, checkStatus: 'ok', lastError: null }, + { service: 'worker', image: 'worker:latest', hasUpdate: false, checkStatus: 'ok', lastError: null }, + ], + expect.any(Number), + ); + }); + + it('falls back to the legacy whole-stack tally when the effective model is not renderable', async () => { + mockGetStacks.mockResolvedValue(['stackA']); + mockGetStackContent.mockResolvedValue(TWO_SERVICE_COMPOSE); + mockGetStackUpdateStatus.mockReturnValue({}); + mockBuildEffectiveServiceModel.mockResolvedValueOnce({ renderable: false, code: 'effective_model_render_failed', error: 'render failed' }); + const service = ImageUpdateService.getInstance(); + stubCheckImagePerRef(service, ['web:latest']); + + await (service as any).checkNode(1, fakeDb()); + + // Legacy path never passes a services/generation payload. + expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(1, 'stackA', true, expect.any(Number), 'ok', null); + }); + + it('names sorted services with hasUpdate in the availability notification for a multi-service stack', async () => { + mockGetStacks.mockResolvedValue(['stackA']); + mockGetStackContent.mockResolvedValue(THREE_SERVICE_COMPOSE); + mockGetStackUpdateStatus.mockReturnValue({ stackA: false }); + mockBuildEffectiveServiceModel.mockResolvedValueOnce({ + renderable: true, + services: [specFor('worker', 'worker:latest'), specFor('api', 'api:latest'), specFor('db', 'db:latest')], + }); + const service = ImageUpdateService.getInstance(); + stubCheckImagePerRef(service, ['worker:latest', 'api:latest']); + + await (service as any).checkNode(1, fakeDb()); + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'info', + 'image_update_available', + 'Stack "stackA" has image updates available for services: api, worker.', + { stackName: 'stackA', actor: 'system:image-update' }, + ); + }); + + it('does not re-notify a multi-service stack already known to have updates', async () => { + mockGetStacks.mockResolvedValue(['stackA']); + mockGetStackContent.mockResolvedValue(TWO_SERVICE_COMPOSE); + mockGetStackUpdateStatus.mockReturnValue({ stackA: true }); + mockBuildEffectiveServiceModel.mockResolvedValueOnce({ + renderable: true, + services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')], + }); + const service = ImageUpdateService.getInstance(); + stubCheckImagePerRef(service, ['web:latest']); + + await (service as any).checkNode(1, fakeDb()); + + expect(mockDispatchAlert).not.toHaveBeenCalled(); + }); + + it('suppresses notification and count side effects when a newer recheck discards a stale full-scan write', async () => { + mockGetStacks.mockResolvedValue(['stackA']); + mockGetStackContent.mockResolvedValue(TWO_SERVICE_COMPOSE); + mockGetStackUpdateStatus.mockReturnValue({}); + // Full-scan write path (after registry work) and recheck both need a model. + mockBuildEffectiveServiceModel.mockResolvedValue({ + renderable: true, + services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')], + }); + + const service = ImageUpdateService.getInstance(); + let resolveFullScan: ((value: { hasUpdate: boolean }) => void) | undefined; + let fullScanPending = true; + (service as any).checkImage = vi.fn().mockImplementation(async () => { + if (fullScanPending) { + return new Promise<{ hasUpdate: boolean }>((resolve) => { + resolveFullScan = resolve; + }); + } + return { hasUpdate: false }; + }); + + const fullScan = (service as any).checkNode(1, fakeDb()); + await vi.waitFor(() => expect((service as any).checkImage).toHaveBeenCalled()); + + // A newer service recheck reserves a higher generation and commits "no update". + fullScanPending = false; + await service.recheckStack(1, 'stackA'); + expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith( + 1, 'stackA', false, expect.any(Number), 'ok', null, + expect.any(Array), + expect.any(Number), + ); + const upsertsAfterRecheck = mockUpsertStackUpdateStatus.mock.calls.length; + expect(mockDispatchAlert).not.toHaveBeenCalled(); + + // Stale full scan finishes with hasUpdate=true but must not commit or notify. + resolveFullScan?.({ hasUpdate: true }); + await fullScan; + + expect(mockUpsertStackUpdateStatus.mock.calls.length).toBe(upsertsAfterRecheck); + expect(mockDispatchAlert).not.toHaveBeenCalled(); + }); + + describe('recheckStack', () => { + it('persists a fresh per-service reduction and returns no warning on success', async () => { + mockBuildEffectiveServiceModel.mockResolvedValueOnce({ + renderable: true, + services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')], + }); + mockGetAllContainers.mockResolvedValue([ + { Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } }, + ]); + const service = ImageUpdateService.getInstance(); + stubCheckImagePerRef(service, ['web:latest']); + + const result = await service.recheckStack(1, 'stackA'); + + expect(result).toEqual({ warning: null }); + expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith( + 1, 'stackA', true, expect.any(Number), 'ok', null, + [ + { service: 'web', image: 'web:latest', runtimeImages: ['web:latest'], hasUpdate: true, checkStatus: 'ok', lastError: null }, + { service: 'worker', image: 'worker:latest', hasUpdate: false, checkStatus: 'ok', lastError: null }, + ], + expect.any(Number), + ); + }); + + it('returns a warning and leaves the prior row untouched when the model cannot render', async () => { + mockBuildEffectiveServiceModel.mockResolvedValueOnce({ renderable: false, code: 'effective_model_render_failed', error: 'no model in test' }); + const service = ImageUpdateService.getInstance(); + + const result = await service.recheckStack(1, 'stackA'); + + expect(result).toEqual({ warning: 'no model in test' }); + expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/backend/src/__tests__/image-updates-routes.test.ts b/backend/src/__tests__/image-updates-routes.test.ts index 0743ed4d..40157d54 100644 --- a/backend/src/__tests__/image-updates-routes.test.ts +++ b/backend/src/__tests__/image-updates-routes.test.ts @@ -422,7 +422,7 @@ describe('POST /api/auto-update/execute', () => { const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') .mockResolvedValue({ hasUpdate: true } as never); const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue(); - const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-au'); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-au'); try { const res = await request(app) .post('/api/auto-update/execute') diff --git a/backend/src/__tests__/prune-plan.test.ts b/backend/src/__tests__/prune-plan.test.ts index 2f722f84..5c4fc81d 100644 --- a/backend/src/__tests__/prune-plan.test.ts +++ b/backend/src/__tests__/prune-plan.test.ts @@ -259,6 +259,18 @@ describe('DockerController.buildPrunePlan', () => { // Unique bytes per image = Size - SharedSize = 100MB each, not full 1GB each. expect(plan.reclaimableBytes).toBe(200_000_000); }); + + it('excludes an image held for a pending service-update recovery from the plan', async () => { + mockDocker.listImages.mockResolvedValue([ + { Id: 'img-held', RepoTags: ['app:1'], Size: 100, Containers: 0 }, + { Id: 'img-free', RepoTags: ['app:2'], Size: 100, Containers: 0 }, + ]); + + const dc = DockerController.getInstance(1); + const plan = await dc.buildPrunePlan(['images'], 'all', [], 1, (id) => id === 'img-held'); + + expect(plan.items.map((i) => i.id)).toEqual(['img-free']); + }); }); describe('DockerController.executePrunePlan', () => { @@ -428,4 +440,27 @@ describe('DockerController.executePrunePlan', () => { expect(remove).toHaveBeenCalledWith({ force: false }); }); + + it('re-checks isImageHeld immediately before deleting, skipping a hold that started after the plan was built', async () => { + mockDocker.listImages.mockResolvedValue([ + { Id: 'img1', RepoTags: ['app:latest'], Size: 1000, Containers: 0 }, + ]); + const imageRemove = vi.fn().mockResolvedValue(undefined); + mockDocker.getImage.mockReturnValue({ remove: imageRemove }); + + const dc = DockerController.getInstance(1); + const plan = await dc.buildPrunePlan(['images'], 'all', [], 1); + expect(plan.items.map((i) => i.id)).toEqual(['img1']); + + // Bypass the rebuild-based staleness check so this isolates the + // immediate per-item recheck (prune holds), simulating a + // recovery snapshot created after the plan was built but before execute. + vi.spyOn(dc, 'assertPlanFresh').mockResolvedValue(plan); + const result = await dc.executePrunePlan(plan, [], (id) => id === 'img1'); + + expect(imageRemove).not.toHaveBeenCalled(); + expect(result.outcomes).toEqual([ + expect.objectContaining({ id: 'img1', target: 'images', status: 'skipped', reason: expect.stringMatching(/held/i) }), + ]); + }); }); diff --git a/backend/src/__tests__/scheduler-service.test.ts b/backend/src/__tests__/scheduler-service.test.ts index cf3eca2b..3b07ed5f 100644 --- a/backend/src/__tests__/scheduler-service.test.ts +++ b/backend/src/__tests__/scheduler-service.test.ts @@ -690,8 +690,8 @@ describe('SchedulerService - executePrune', () => { await svc.triggerTask(71); expect(mockPruneSystem).toHaveBeenCalledTimes(2); - expect(mockPruneSystem).toHaveBeenCalledWith('images', undefined); - expect(mockPruneSystem).toHaveBeenCalledWith('volumes', undefined); + expect(mockPruneSystem).toHaveBeenCalledWith('images', undefined, expect.any(Function)); + expect(mockPruneSystem).toHaveBeenCalledWith('volumes', undefined, expect.any(Function)); }); it('includes label filter when configured', async () => { @@ -711,7 +711,7 @@ describe('SchedulerService - executePrune', () => { const svc = SchedulerService.getInstance(); await svc.triggerTask(72); - expect(mockPruneSystem).toHaveBeenCalledWith('containers', 'env=staging'); + expect(mockPruneSystem).toHaveBeenCalledWith('containers', 'env=staging', expect.any(Function)); }); it('marks scheduled prune runs as failed when a target prune fails', async () => { @@ -828,7 +828,7 @@ describe('SchedulerService - executeUpdate', () => { it('begins a health gate after a scheduled update succeeds', async () => { const { HealthGateService } = await import('../services/HealthGateService'); - const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-1'); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-1'); try { mockGetScheduledTask.mockReturnValue({ id: 83, diff --git a/backend/src/__tests__/service-scoped-update-routes.test.ts b/backend/src/__tests__/service-scoped-update-routes.test.ts new file mode 100644 index 00000000..f2564257 --- /dev/null +++ b/backend/src/__tests__/service-scoped-update-routes.test.ts @@ -0,0 +1,217 @@ +/** + * Route smoke tests for the nested service update/restore endpoints: auth and + * permission gates, the restore recoveryId requirement, and the mapping from + * OrchestratorResult to HTTP status. The orchestrator itself is mocked so these + * tests exercise only the route wiring. + */ +import fs from 'fs'; +import path from 'path'; +import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest'; +import request from 'supertest'; +import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; + +const { mockExecute } = vi.hoisted(() => ({ mockExecute: vi.fn() })); + +vi.mock('../services/StackUpdateOrchestrator', () => ({ + StackUpdateOrchestrator: { getInstance: () => ({ execute: mockExecute }) }, +})); + +let tmpDir: string; +let app: import('express').Express; +let adminCookie: string; +let viewerCookie: string; + +function writeStack(name: string) { + const dir = path.join(process.env.COMPOSE_DIR!, name); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'compose.yaml'), 'services:\n app:\n image: nginx\n db:\n image: postgres\n'); +} + +async function loginAsViewer(): Promise { + const bcrypt = (await import('bcrypt')).default; + const { DatabaseService } = await import('../services/DatabaseService'); + const hash = await bcrypt.hash('viewerpass', 1); + DatabaseService.getInstance().addUser({ username: 'viewer1', password_hash: hash, role: 'viewer' }); + const res = await request(app).post('/api/auth/login').send({ username: 'viewer1', password: 'viewerpass' }); + const cookies = res.headers['set-cookie'] as string | string[]; + return Array.isArray(cookies) ? cookies[0] : cookies; +} + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + adminCookie = await loginAsTestAdmin(app); + viewerCookie = await loginAsViewer(); + writeStack('web'); + + const { NotificationService } = await import('../services/NotificationService'); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); +}); + +afterAll(() => { + vi.restoreAllMocks(); + cleanupTestDb(tmpDir); +}); + +afterEach(async () => { + mockExecute.mockReset(); + const { StackOpLockService } = await import('../services/StackOpLockService'); + StackOpLockService.resetForTests(); +}); + +describe('nested service route auth and permission gates', () => { + it('rejects an unauthenticated update with 401', async () => { + const res = await request(app).post('/api/stacks/web/services/app/update'); + expect(res.status).toBe(401); + expect(mockExecute).not.toHaveBeenCalled(); + }); + + it('rejects an unauthenticated restore with 401', async () => { + const res = await request(app).post('/api/stacks/web/services/app/restore').send({ recoveryId: 'r1' }); + expect(res.status).toBe(401); + expect(mockExecute).not.toHaveBeenCalled(); + }); + + it('rejects a viewer without stack:deploy with 403', async () => { + const res = await request(app) + .post('/api/stacks/web/services/app/update') + .set('Cookie', viewerCookie); + expect(res.status).toBe(403); + expect(mockExecute).not.toHaveBeenCalled(); + }); +}); + +describe('restore recoveryId requirement', () => { + it('rejects a restore with no recoveryId with 400 recovery_id_required', async () => { + const res = await request(app) + .post('/api/stacks/web/services/app/restore') + .set('Cookie', adminCookie) + .send({}); + expect(res.status).toBe(400); + expect(res.body.code).toBe('recovery_id_required'); + expect(mockExecute).not.toHaveBeenCalled(); + }); +}); + +describe('OrchestratorResult to HTTP mapping', () => { + it('maps service_done to 200 with the result body', async () => { + mockExecute.mockResolvedValue({ + kind: 'service_done', + serviceName: 'app', + healthGateId: 'hg-1', + observing: true, + recoveryId: 'rec-1', + recoveryAvailable: true, + }); + const res = await request(app) + .post('/api/stacks/web/services/app/update') + .set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ serviceName: 'app', healthGateId: 'hg-1', recoveryId: 'rec-1', recoveryAvailable: true }); + expect(mockExecute).toHaveBeenCalledTimes(1); + }); + + it('maps service_update_single_service to 400', async () => { + mockExecute.mockResolvedValue({ kind: 'service_failed', code: 'service_update_single_service', error: 'only one service' }); + const res = await request(app) + .post('/api/stacks/web/services/app/update') + .set('Cookie', adminCookie); + expect(res.status).toBe(400); + expect(res.body.code).toBe('service_update_single_service'); + }); + + it('maps a policy block to 409', async () => { + mockExecute.mockResolvedValue({ kind: 'service_failed', code: 'policy_blocked', error: 'blocked' }); + const res = await request(app) + .post('/api/stacks/web/services/app/update') + .set('Cookie', adminCookie); + expect(res.status).toBe(409); + expect(res.body.code).toBe('policy_blocked'); + }); + + it('maps a service compose failure to 500', async () => { + mockExecute.mockResolvedValue({ kind: 'service_failed', code: 'service_update_compose_failed', error: 'compose exploded' }); + const res = await request(app) + .post('/api/stacks/web/services/app/update') + .set('Cookie', adminCookie); + expect(res.status).toBe(500); + expect(res.body.code).toBe('service_update_compose_failed'); + }); + + it('maps a restore service_done to 200', async () => { + mockExecute.mockResolvedValue({ + kind: 'service_done', + serviceName: 'app', + healthGateId: 'hg-2', + observing: true, + recoveryId: 'rec-2', + recoveryAvailable: false, + }); + const res = await request(app) + .post('/api/stacks/web/services/app/restore') + .set('Cookie', adminCookie) + .send({ recoveryId: 'rec-2' }); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ serviceName: 'app', healthGateId: 'hg-2', recoveryId: 'rec-2' }); + expect(mockExecute).toHaveBeenCalledTimes(1); + }); +}); + +describe('GET /api/stacks/:stackName/services/:serviceName/recovery', () => { + it('returns null when no active recovery exists', async () => { + const res = await request(app) + .get('/api/stacks/web/services/app/recovery') + .set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body).toEqual({ recovery: null }); + }); + + it('returns the newest active recovery row', async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const { NodeRegistry } = await import('../services/NodeRegistry'); + const db = DatabaseService.getInstance(); + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const now = Date.now(); + db.insertServiceUpdateRecovery({ + id: 'rec-old', + node_id: nodeId, + stack_name: 'web', + service_name: 'app', + replicas_json: '[]', + majority_image_id: 'sha256:old', + declared_image_ref: 'nginx:latest', + weak_floating_tag: 0, + health_gate_id: null, + status: 'active', + expires_at: now + 60_000, + claim_expires_at: null, + created_at: now - 1_000, + created_by: 'tester', + }); + db.insertServiceUpdateRecovery({ + id: 'rec-new', + node_id: nodeId, + stack_name: 'web', + service_name: 'app', + replicas_json: '[]', + majority_image_id: 'sha256:new', + declared_image_ref: 'nginx:latest', + weak_floating_tag: 0, + health_gate_id: 'gate-1', + status: 'active', + expires_at: now + 60_000, + claim_expires_at: null, + created_at: now, + created_by: 'tester', + }); + const res = await request(app) + .get('/api/stacks/web/services/app/recovery') + .set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body.recovery).toMatchObject({ + id: 'rec-new', + healthGateId: 'gate-1', + majorityImageId: 'sha256:new', + }); + }); +}); diff --git a/backend/src/__tests__/service-update-recovery-service.test.ts b/backend/src/__tests__/service-update-recovery-service.test.ts new file mode 100644 index 00000000..aed4dcc6 --- /dev/null +++ b/backend/src/__tests__/service-update-recovery-service.test.ts @@ -0,0 +1,309 @@ +/** + * Coverage for ServiceUpdateRecoveryService against a real DatabaseService + * (temp DB) with fake timers: eligibility computation (§8 "Unavailable" + * cases), the claim/renewal lifecycle (RECOVERY-CLAIM-2, no revive of a + * terminal row), the sweep timer, held image ids, and start/stop timer + * hygiene. + */ +import { describe, it, expect, vi, beforeAll, afterAll, beforeEach, afterEach } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import type { ServiceReplicaSnapshot } from '../services/ServiceUpdateRecoveryService'; + +let tmpDir: string; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let ServiceUpdateRecoveryService: typeof import('../services/ServiceUpdateRecoveryService').ServiceUpdateRecoveryService; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ ServiceUpdateRecoveryService } = await import('../services/ServiceUpdateRecoveryService')); +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +function db() { + return DatabaseService.getInstance(); +} + +const svc = () => ServiceUpdateRecoveryService.getInstance(); + +const BASE_INPUT = { + nodeId: 1, + stackName: 'web', + serviceName: 'api', + declaredImageRef: 'ghcr.io/acme/api:v1', + createdBy: 'tester', +}; + +beforeEach(() => { + vi.useFakeTimers(); + const raw = (db() as unknown as { db: { prepare: (s: string) => { run: () => void } } }).db; + raw.prepare('DELETE FROM service_update_recovery').run(); + raw.prepare("DELETE FROM global_settings WHERE key = 'health_gate_window_seconds'").run(); + // updateGlobalSetting() invalidates this cache; the raw DELETE above bypasses + // it, so a value set by a previous test would otherwise leak into this one. + (db() as unknown as { cachedGlobalSettings: unknown }).cachedGlobalSettings = null; + svc().start(); +}); + +afterEach(() => { + svc().stop(); + expect(vi.getTimerCount()).toBe(0); + vi.useRealTimers(); +}); + +describe('createIfEligible - unavailable cases', () => { + it('rejects a service with no replicas', () => { + const result = svc().createIfEligible({ ...BASE_INPUT, replicas: [] }); + expect(result).toEqual({ eligible: false, reason: 'no_replicas' }); + }); + + it('rejects replicas with no local image id', () => { + const replicas: ServiceReplicaSnapshot[] = [{ imageId: '', repoDigest: null }, { imageId: ' ', repoDigest: null }]; + const result = svc().createIfEligible({ ...BASE_INPUT, replicas }); + expect(result).toEqual({ eligible: false, reason: 'missing_local_id' }); + }); + + it('rejects a majority tie between two equally-common images', () => { + const replicas: ServiceReplicaSnapshot[] = [ + { imageId: 'sha256:aaa', repoDigest: null }, + { imageId: 'sha256:bbb', repoDigest: null }, + ]; + const result = svc().createIfEligible({ ...BASE_INPUT, replicas }); + expect(result).toEqual({ eligible: false, reason: 'majority_tie' }); + }); + + it('rejects a pure-build service with no declared image ref', () => { + const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }]; + const result = svc().createIfEligible({ ...BASE_INPUT, declaredImageRef: null, replicas }); + expect(result).toEqual({ eligible: false, reason: 'build_only' }); + }); + + it('rejects a digest-pinned declared image ref', () => { + const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }]; + const result = svc().createIfEligible({ + ...BASE_INPUT, + declaredImageRef: 'ghcr.io/acme/api@sha256:' + 'a'.repeat(64), + replicas, + }); + expect(result).toEqual({ eligible: false, reason: 'digest_pinned_declared_ref' }); + }); +}); + +describe('createIfEligible - eligible cases', () => { + it('persists the majority image and flags a weak floating tag when no replica has a captured digest', () => { + const replicas: ServiceReplicaSnapshot[] = [ + { imageId: 'sha256:aaa', repoDigest: null }, + { imageId: 'sha256:aaa', repoDigest: null }, + { imageId: 'sha256:bbb', repoDigest: null }, + ]; + const result = svc().createIfEligible({ ...BASE_INPUT, replicas }); + expect(result.eligible).toBe(true); + if (!result.eligible) throw new Error('expected eligible'); + expect(result.row).toMatchObject({ + node_id: 1, stack_name: 'web', service_name: 'api', + majority_image_id: 'sha256:aaa', declared_image_ref: 'ghcr.io/acme/api:v1', + weak_floating_tag: 1, status: 'active', health_gate_id: null, + }); + expect(db().getServiceUpdateRecovery(result.row.id)).toBeTruthy(); + }); + + it('clears the weak floating tag flag when a majority replica has a captured digest', () => { + const replicas: ServiceReplicaSnapshot[] = [ + { imageId: 'sha256:aaa', repoDigest: 'sha256:digest-aaa' }, + { imageId: 'sha256:aaa', repoDigest: 'sha256:digest-aaa' }, + { imageId: 'sha256:bbb', repoDigest: null }, + ]; + const result = svc().createIfEligible({ ...BASE_INPUT, replicas }); + if (!result.eligible) throw new Error('expected eligible'); + expect(result.row.weak_floating_tag).toBe(0); + }); + + it('sets expires_at from max(compose timeout, health window) plus buffer', () => { + db().updateGlobalSetting('health_gate_window_seconds', '120'); + const now = Date.now(); + const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }]; + const result = svc().createIfEligible({ ...BASE_INPUT, replicas }); + if (!result.eligible) throw new Error('expected eligible'); + // Default compose timeout is 30m; health window 120s is smaller, so compose wins. + expect(result.row.expires_at).toBe(now + 30 * 60_000 + 30 * 60_000); + }); + + it('uses a raised health window when it exceeds the compose command timeout', () => { + db().updateGlobalSetting('health_gate_window_seconds', String(45 * 60)); + const now = Date.now(); + const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }]; + const result = svc().createIfEligible({ ...BASE_INPUT, replicas }); + if (!result.eligible) throw new Error('expected eligible'); + expect(result.row.expires_at).toBe(now + 45 * 60_000 + 30 * 60_000); + }); + + it('floors the health window to 90s but still honors the compose timeout floor', () => { + db().updateGlobalSetting('health_gate_window_seconds', '15'); + const now = Date.now(); + const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: null }]; + const result = svc().createIfEligible({ ...BASE_INPUT, replicas }); + if (!result.eligible) throw new Error('expected eligible'); + expect(result.row.expires_at).toBe(now + 30 * 60_000 + 30 * 60_000); + }); +}); + +describe('claim + mandatory renewal (RECOVERY-CLAIM-2)', () => { + function seedEligible(): string { + const replicas: ServiceReplicaSnapshot[] = [{ imageId: 'sha256:aaa', repoDigest: 'sha256:digest-aaa' }]; + const result = svc().createIfEligible({ ...BASE_INPUT, replicas }); + if (!result.eligible) throw new Error('expected eligible'); + return result.row.id; + } + + it('claims an active row and moves it to restoring with a claim window at least 30m + 5m out', () => { + const id = seedEligible(); + const now = Date.now(); + const claimed = svc().claim(id); + expect(claimed?.status).toBe('restoring'); + expect(claimed?.claim_expires_at).toBeGreaterThanOrEqual(now + 30 * 60_000 + 5 * 60_000); + }); + + it('renews the claim every 5 minutes while restoring, keeping a long restore held', () => { + const id = seedEligible(); + svc().claim(id); + const firstExpiry = db().getServiceUpdateRecovery(id)?.claim_expires_at ?? 0; + + svc().startClaimRenewal(id); + vi.advanceTimersByTime(5 * 60_000); + + const renewedExpiry = db().getServiceUpdateRecovery(id)?.claim_expires_at ?? 0; + expect(renewedExpiry).toBeGreaterThan(firstExpiry); + expect(db().getServiceUpdateRecovery(id)?.status).toBe('restoring'); + + svc().stopClaimRenewal(id); + }); + + it('does not revive a row that left restoring mid-loop (renewal self-stops, never overwrites the terminal status)', () => { + const id = seedEligible(); + svc().claim(id); + svc().startClaimRenewal(id); + + // Simulate the restore finishing (consumed) between renewal ticks. + db().markServiceUpdateRecoveryConsumed(id, 'restore-gate'); + const claimExpiresBefore = db().getServiceUpdateRecovery(id)?.claim_expires_at; + + vi.advanceTimersByTime(5 * 60_000); + + const row = db().getServiceUpdateRecovery(id); + expect(row?.status).toBe('consumed'); + expect(row?.claim_expires_at).toBe(claimExpiresBefore); + + svc().stopClaimRenewal(id); + }); + + it('stops renewing once stopClaimRenewal is called (finally-block contract)', () => { + const id = seedEligible(); + svc().claim(id); + svc().startClaimRenewal(id); + svc().stopClaimRenewal(id); + + const expiryAtStop = db().getServiceUpdateRecovery(id)?.claim_expires_at; + vi.advanceTimersByTime(15 * 60_000); + expect(db().getServiceUpdateRecovery(id)?.claim_expires_at).toBe(expiryAtStop); + }); + + it('is idempotent: a second startClaimRenewal call for the same id does not schedule a duplicate timer', () => { + const id = seedEligible(); + svc().claim(id); + const before = vi.getTimerCount(); + svc().startClaimRenewal(id); + svc().startClaimRenewal(id); + expect(vi.getTimerCount()).toBe(before + 1); + svc().stopClaimRenewal(id); + }); + + it('refuses to start a renewal loop once the service has been stopped', () => { + const id = seedEligible(); + svc().claim(id); + svc().stop(); + const before = vi.getTimerCount(); + svc().startClaimRenewal(id); + expect(vi.getTimerCount()).toBe(before); + }); +}); + +describe('sweep()', () => { + it('expires an abandoned restoring claim but leaves a live one and an unexpired active one alone', () => { + const now = Date.now(); + db().insertServiceUpdateRecovery({ + id: 'rec-abandoned', node_id: 1, stack_name: 'web', service_name: 'api', + replicas_json: '[]', majority_image_id: 'sha256:aaa', declared_image_ref: 'x:1', + weak_floating_tag: 0, health_gate_id: null, status: 'restoring', + expires_at: now - 1_000, claim_expires_at: now - 1_000, created_at: now - 60_000, created_by: null, + }); + db().insertServiceUpdateRecovery({ + id: 'rec-live', node_id: 1, stack_name: 'web', service_name: 'db', + replicas_json: '[]', majority_image_id: 'sha256:bbb', declared_image_ref: 'x:2', + weak_floating_tag: 0, health_gate_id: null, status: 'restoring', + expires_at: now - 1_000, claim_expires_at: now + 60 * 60_000, created_at: now - 60_000, created_by: null, + }); + db().insertServiceUpdateRecovery({ + id: 'rec-active', node_id: 1, stack_name: 'web', service_name: 'cache', + replicas_json: '[]', majority_image_id: 'sha256:ccc', declared_image_ref: 'x:3', + weak_floating_tag: 0, health_gate_id: null, status: 'active', + expires_at: now + 60 * 60_000, claim_expires_at: null, created_at: now, created_by: null, + }); + + svc().sweep(); + + expect(db().getServiceUpdateRecovery('rec-abandoned')?.status).toBe('expired'); + expect(db().getServiceUpdateRecovery('rec-live')?.status).toBe('restoring'); + expect(db().getServiceUpdateRecovery('rec-active')?.status).toBe('active'); + }); + + it('runs automatically on the interval timer after start()', () => { + const now = Date.now(); + db().insertServiceUpdateRecovery({ + id: 'rec-ttl', node_id: 1, stack_name: 'web', service_name: 'api', + replicas_json: '[]', majority_image_id: 'sha256:aaa', declared_image_ref: 'x:1', + weak_floating_tag: 0, health_gate_id: null, status: 'active', + expires_at: now + 1_000, claim_expires_at: null, created_at: now, created_by: null, + }); + + vi.advanceTimersByTime(30_000 + 5 * 60_000 + 1_000); + + expect(db().getServiceUpdateRecovery('rec-ttl')?.status).toBe('expired'); + }); +}); + +describe('getHeldImageIds', () => { + it('wraps the DB projection for one node', () => { + const now = Date.now(); + db().insertServiceUpdateRecovery({ + id: 'rec-held', node_id: 1, stack_name: 'web', service_name: 'api', + replicas_json: '[]', majority_image_id: 'sha256:held', declared_image_ref: 'x:1', + weak_floating_tag: 0, health_gate_id: null, status: 'active', + expires_at: now + 60_000, claim_expires_at: null, created_at: now, created_by: null, + }); + expect(svc().getHeldImageIds(1)).toEqual(new Set(['sha256:held'])); + }); + + it('fails closed (null) when the DB read throws', () => { + const spy = vi.spyOn(db(), 'listHeldServiceUpdateRecoveryImageIds').mockImplementation(() => { + throw new Error('boom'); + }); + expect(svc().getHeldImageIds(1)).toBeNull(); + spy.mockRestore(); + }); +}); + +describe('start() / stop() timer hygiene', () => { + it('clears the sweep timer and stops firing after stop()', () => { + const spy = vi.spyOn(svc(), 'sweep'); + svc().stop(); + expect(vi.getTimerCount()).toBe(0); + svc().start(); + vi.advanceTimersByTime(30_000); + expect(spy).toHaveBeenCalledTimes(1); + svc().stop(); + vi.advanceTimersByTime(5 * 60_000); + expect(spy).toHaveBeenCalledTimes(1); + spy.mockRestore(); + }); +}); diff --git a/backend/src/__tests__/stack-op-metrics-service.test.ts b/backend/src/__tests__/stack-op-metrics-service.test.ts index 6c12768b..77bbbebd 100644 --- a/backend/src/__tests__/stack-op-metrics-service.test.ts +++ b/backend/src/__tests__/stack-op-metrics-service.test.ts @@ -34,6 +34,23 @@ describe('StackOpMetricsService', () => { successCount: 2, errorCount: 1, avgMs: 200, + targetScope: 'stack', + serviceName: null, + }); + }); + + it('records the latest service-scoped metadata on the action bucket', () => { + const svc = StackOpMetricsService.getInstance(); + svc.record(1, 'update', 50, true, { targetScope: 'service', serviceName: 'web' }); + expect(svc.snapshot()[0]).toMatchObject({ + action: 'update', + targetScope: 'service', + serviceName: 'web', + }); + svc.record(1, 'update', 40, true, { targetScope: 'stack' }); + expect(svc.snapshot()[0]).toMatchObject({ + targetScope: 'stack', + serviceName: null, }); }); diff --git a/backend/src/__tests__/stack-update-orchestrator.test.ts b/backend/src/__tests__/stack-update-orchestrator.test.ts new file mode 100644 index 00000000..103ad017 --- /dev/null +++ b/backend/src/__tests__/stack-update-orchestrator.test.ts @@ -0,0 +1,122 @@ +/** + * Unit tests for StackUpdateOrchestrator dispatch: the stack branch runs + * ComposeService.updateStack (side effects stay with callers), the service + * branch hard-rejects any non-manual trigger, and a single-service stack is + * refused with the stable `service_update_single_service` code before any + * mutation. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import type { EffectiveServiceModelResult, EffectiveServiceSpec } from '../services/effectiveServiceModel'; + +const { state } = vi.hoisted(() => ({ + state: { + updateStack: vi.fn(), + model: null as EffectiveServiceModelResult | null, + }, +})); + +vi.mock('../services/ComposeService', () => ({ + ComposeService: { + getInstance: () => ({ updateStack: state.updateStack }), + }, +})); + +vi.mock('../services/effectiveServiceModel', () => ({ + buildEffectiveServiceModel: vi.fn(async () => state.model), +})); + +import { StackUpdateOrchestrator, evaluateServiceReplicaConvergence, shortImageId } from '../services/StackUpdateOrchestrator'; + +const orch = () => StackUpdateOrchestrator.getInstance(); + +function spec(name: string): EffectiveServiceSpec { + return { name, declaredImage: 'nginx:latest', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false }; +} + +beforeEach(() => { + state.updateStack.mockReset(); + state.updateStack.mockResolvedValue(undefined); + state.model = null; +}); + +describe('evaluateServiceReplicaConvergence', () => { + it('converges when the exact expected running replicas share one image', () => { + expect(evaluateServiceReplicaConvergence('api', 3, ['sha:a', 'sha:a', 'sha:a'], 0)).toEqual({ + kind: 'converged', imageId: 'sha:a', + }); + }); + + it('reports divergent when running replica count is below expected', () => { + const result = evaluateServiceReplicaConvergence('api', 3, ['sha:a'], 0); + expect(result).toMatchObject({ kind: 'divergent' }); + expect((result as { error: string }).error).toMatch(/1 running.*expected 3/i); + }); + + it('reports divergent when no running replicas remain', () => { + const result = evaluateServiceReplicaConvergence('api', 2, [], 0); + expect(result).toMatchObject({ kind: 'divergent' }); + expect((result as { error: string }).error).toMatch(/no running replicas/i); + }); + + it('reports divergent when replicas disagree on image', () => { + const result = evaluateServiceReplicaConvergence('api', 2, ['sha:a', 'sha:b'], 0); + expect(result).toMatchObject({ kind: 'divergent' }); + expect((result as { error: string }).error).toMatch(/did not converge/i); + }); + + it('reports inspect_failed when every inspect failed', () => { + expect(evaluateServiceReplicaConvergence('api', 2, [], 2)).toMatchObject({ kind: 'inspect_failed' }); + }); + + it('treats scale-zero as converged with a null image', () => { + expect(evaluateServiceReplicaConvergence('api', 0, [], 0)).toEqual({ kind: 'converged', imageId: null }); + }); +}); + +describe('shortImageId', () => { + it('strips the sha256 prefix and truncates', () => { + expect(shortImageId('sha256:abcdef0123456789')).toBe('abcdef012345'); + }); +}); + +describe('StackUpdateOrchestrator stack branch', () => { + it('runs ComposeService.updateStack and returns stack_compose_done', async () => { + const result = await orch().execute( + { nodeId: 0, stackName: 'web', target: { scope: 'stack' }, trigger: 'manual', actor: 'tester' }, + { atomic: true, terminalWs: null }, + ); + expect(result).toEqual({ kind: 'stack_compose_done' }); + expect(state.updateStack).toHaveBeenCalledWith('web', undefined, true); + }); +}); + +describe('StackUpdateOrchestrator service branch guards', () => { + it('rejects a service-scoped update with a non-manual trigger', async () => { + await expect( + orch().execute( + { nodeId: 0, stackName: 'web', target: { scope: 'service', serviceName: 'app' }, trigger: 'scheduled', actor: 'system' }, + { policyOptions: { bypass: false, actor: 'system' } }, + ), + ).rejects.toThrow(/manual/i); + expect(state.updateStack).not.toHaveBeenCalled(); + }); + + it('refuses a single-service stack with code service_update_single_service', async () => { + state.model = { renderable: true, services: [spec('only')] }; + const result = await orch().execute( + { nodeId: 0, stackName: 'web', target: { scope: 'service', serviceName: 'only' }, trigger: 'manual', actor: 'tester' }, + { policyOptions: { bypass: false, actor: 'tester' } }, + ); + expect(result).toMatchObject({ kind: 'service_failed', code: 'service_update_single_service' }); + expect(state.updateStack).not.toHaveBeenCalled(); + }); + + it('fails closed when the effective model cannot render', async () => { + state.model = { renderable: false, code: 'effective_model_render_failed', error: 'boom' }; + const result = await orch().execute( + { nodeId: 0, stackName: 'web', target: { scope: 'service', serviceName: 'app' }, trigger: 'manual', actor: 'tester' }, + { policyOptions: { bypass: false, actor: 'tester' } }, + ); + expect(result).toMatchObject({ kind: 'service_failed', code: 'effective_model_render_failed' }); + }); +}); diff --git a/backend/src/__tests__/stacks-failure-notifications.test.ts b/backend/src/__tests__/stacks-failure-notifications.test.ts index b742b2ec..8adaae5f 100644 --- a/backend/src/__tests__/stacks-failure-notifications.test.ts +++ b/backend/src/__tests__/stacks-failure-notifications.test.ts @@ -251,7 +251,7 @@ describe('health gate begin call sites', () => { beforeEach(async () => { const { HealthGateService } = await import('../services/HealthGateService'); - beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-123') as ReturnType; + beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-123') as ReturnType; }); afterEach(() => { diff --git a/backend/src/__tests__/update-guard-routes.test.ts b/backend/src/__tests__/update-guard-routes.test.ts index d2493527..96f70ab0 100644 --- a/backend/src/__tests__/update-guard-routes.test.ts +++ b/backend/src/__tests__/update-guard-routes.test.ts @@ -12,10 +12,15 @@ import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTes const { mockComputeUpdateReadiness, mockComputeRollbackReadiness, -} = vi.hoisted(() => ({ - mockComputeUpdateReadiness: vi.fn(), - mockComputeRollbackReadiness: vi.fn(), -})); + MockSingleServiceUpdateReadinessError, +} = vi.hoisted(() => { + class MockSingleServiceUpdateReadinessError extends Error {} + return { + mockComputeUpdateReadiness: vi.fn(), + mockComputeRollbackReadiness: vi.fn(), + MockSingleServiceUpdateReadinessError, + }; +}); vi.mock('../services/UpdateGuardService', () => ({ UpdateGuardService: { @@ -24,6 +29,7 @@ vi.mock('../services/UpdateGuardService', () => ({ computeRollbackReadiness: mockComputeRollbackReadiness, }), }, + SingleServiceUpdateReadinessError: MockSingleServiceUpdateReadinessError, })); vi.mock('../services/FileSystemService', () => ({ @@ -85,6 +91,28 @@ describe('GET /api/stacks/:stackName/update-readiness', () => { expect(res.status).toBe(500); expect(res.body).toEqual({ error: 'Failed to compute update readiness' }); }); + + it('forwards the service query param to computeUpdateReadiness', async () => { + const report = { stack: 'web', computedAt: 1, verdict: 'ready', signals: [], serviceName: 'api', advisories: [] }; + mockComputeUpdateReadiness.mockResolvedValue(report); + const res = await request(app).get('/api/stacks/web/update-readiness?service=api').set('Cookie', authCookie); + expect(res.status).toBe(200); + expect(res.body).toEqual(report); + expect(mockComputeUpdateReadiness).toHaveBeenCalledWith(expect.any(Number), 'web', 'api'); + }); + + it('omits the service argument when no service query param is given', async () => { + mockComputeUpdateReadiness.mockResolvedValue({ stack: 'web', computedAt: 1, verdict: 'ready', signals: [] }); + await request(app).get('/api/stacks/web/update-readiness').set('Cookie', authCookie); + expect(mockComputeUpdateReadiness).toHaveBeenCalledWith(expect.any(Number), 'web', undefined); + }); + + it('returns 400 when the service query targets a single-service stack', async () => { + mockComputeUpdateReadiness.mockRejectedValue(new MockSingleServiceUpdateReadinessError('single service')); + const res = await request(app).get('/api/stacks/web/update-readiness?service=api').set('Cookie', authCookie); + expect(res.status).toBe(400); + expect(res.body).toMatchObject({ code: 'service_update_single_service' }); + }); }); describe('GET /api/stacks/:stackName/rollback-readiness', () => { @@ -120,6 +148,7 @@ describe('GET /api/stacks/:stackName/health-gate', () => { db.insertHealthGateRun({ id, node_id: defaultNodeId, stack_name: 'web', trigger_action: 'update', status, reason, window_seconds: 90, containers_json: '[]', started_at: startedAt, ended_at: status === 'observing' ? null : startedAt + 1000, created_by: 'tester', + target_scope: 'stack', service_name: null, failure_source: null, }); }; diff --git a/backend/src/__tests__/update-guard-service.test.ts b/backend/src/__tests__/update-guard-service.test.ts index 2a725c39..ad6c055a 100644 --- a/backend/src/__tests__/update-guard-service.test.ts +++ b/backend/src/__tests__/update-guard-service.test.ts @@ -16,6 +16,7 @@ const { mockGetOpenDriftFindings, mockGetGlobalSettings, mockFsSize, + mockBuildEffectiveServiceModel, } = vi.hoisted(() => ({ mockListContainers: vi.fn(), mockGetContainer: vi.fn(), @@ -27,6 +28,7 @@ const { mockGetOpenDriftFindings: vi.fn(), mockGetGlobalSettings: vi.fn(), mockFsSize: vi.fn(), + mockBuildEffectiveServiceModel: vi.fn(), })); vi.mock('../services/DockerController', () => ({ @@ -75,7 +77,11 @@ vi.mock('systeminformation', () => ({ default: { fsSize: mockFsSize }, })); -import { UpdateGuardService } from '../services/UpdateGuardService'; +vi.mock('../services/effectiveServiceModel', () => ({ + buildEffectiveServiceModel: mockBuildEffectiveServiceModel, +})); + +import { UpdateGuardService, SingleServiceUpdateReadinessError } from '../services/UpdateGuardService'; const inspectResult = (over: Record = {}) => ({ State: { Status: 'running', ExitCode: 0 }, @@ -166,6 +172,110 @@ describe('UpdateGuardService.computeUpdateReadiness wiring', () => { }); }); +describe('UpdateGuardService.computeUpdateReadiness with a serviceName', () => { + const twoServiceModel = { + renderable: true as const, + services: [ + { name: 'web', declaredImage: 'nginx:1.27', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: true }, + { name: 'db', declaredImage: 'postgres:16', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: false }, + ], + }; + const multiServicePreview = { + stack_name: 'app', + images: [ + { service: 'web', image: 'nginx', current_tag: '1.27.0', next_tag: '1.27.1', has_update: true, semver_bump: 'patch' }, + { service: 'db', image: 'postgres', current_tag: '16.0', next_tag: null, has_update: false, semver_bump: 'none' }, + ], + build_services: [], + summary: { + has_update: true, primary_image: 'nginx', current_tag: '1.27.0', next_tag: '1.27.1', + semver_bump: 'patch', update_kind: 'tag', blocked: false, blocked_reason: null, + has_build_services: false, rebuild_available: false, + }, + rollback_target: 'nginx:1.27.0', + changelog: null, + }; + + beforeEach(() => { + mockGetLatest.mockReturnValue({ activeStatus: 'pass' }); + mockGetOpenDriftFindings.mockReturnValue([]); + mockGetPreview.mockResolvedValue(multiServicePreview); + mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: Date.now() }); + mockFsSize.mockResolvedValue([{ mount: '/', use: 42 }]); + mockListContainers.mockResolvedValue([ + { Id: 'web1', Names: ['/web'], State: 'running' }, + { Id: 'db1', Names: ['/db'], State: 'running' }, + ]); + mockGetContainer.mockReturnValue({ inspect: vi.fn().mockResolvedValue(inspectResult()) }); + }); + + it('scopes the verdict to the selected service, includes a service signal, and filters the preview', async () => { + mockBuildEffectiveServiceModel.mockResolvedValue(twoServiceModel); + + const report = await UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'web'); + + expect(report.serviceName).toBe('web'); + expect(report.verdict).toBe('ready'); + const serviceSig = report.signals.find(s => s.id === 'service'); + expect(serviceSig?.status).toBe('ok'); + // update_preview signal is derived from the filtered summary (recomputed for 'web' only). + const previewSig = report.signals.find(s => s.id === 'update_preview'); + expect(previewSig?.status).toBe('ok'); + }); + + it('throws SingleServiceUpdateReadinessError for a single-service stack', async () => { + mockBuildEffectiveServiceModel.mockResolvedValue({ + renderable: true, + services: [{ name: 'web', declaredImage: 'nginx:1.27', hasBuild: false, expectedReplicas: 1, dependsOn: [], hasHealthcheck: true }], + }); + + await expect(UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'web')) + .rejects.toBeInstanceOf(SingleServiceUpdateReadinessError); + }); + + it('fails closed (blocked service signal) when the effective model fails to render', async () => { + mockBuildEffectiveServiceModel.mockResolvedValue({ + renderable: false, + code: 'effective_model_render_failed', + error: 'bad yaml', + }); + + const report = await UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'web'); + + const serviceSig = report.signals.find(s => s.id === 'service'); + expect(serviceSig?.status).toBe('blocked'); + expect(report.verdict).toBe('blocked'); + }); + + it('fails closed when the selected service is not declared in the model', async () => { + mockBuildEffectiveServiceModel.mockResolvedValue(twoServiceModel); + + const report = await UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'cache'); + + const serviceSig = report.signals.find(s => s.id === 'service'); + expect(serviceSig?.status).toBe('blocked'); + expect(report.verdict).toBe('blocked'); + }); + + it('reports a sibling advisory without affecting the verdict', async () => { + mockBuildEffectiveServiceModel.mockResolvedValue(twoServiceModel); + mockListContainers.mockResolvedValue([ + { Id: 'web1', Names: ['/web'], State: 'running' }, + { Id: 'db1', Names: ['/db'], State: 'exited', ExitCode: 1 }, + ]); + mockGetContainer.mockImplementation((id: string) => ({ + inspect: vi.fn().mockResolvedValue(id === 'db1' + ? inspectResult({ State: { Status: 'exited', ExitCode: 1 } }) + : inspectResult()), + })); + + const report = await UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app', 'web'); + + expect(report.verdict).toBe('ready'); + expect(report.advisories.some(a => a.includes('db'))).toBe(true); + }); +}); + describe('UpdateGuardService.computeRollbackReadiness moving-tag wiring', () => { const preview = (images: Array<{ current_tag: string }>) => ({ stack_name: 'app', diff --git a/backend/src/__tests__/webhooks-trigger.test.ts b/backend/src/__tests__/webhooks-trigger.test.ts index a6b953cc..ad264d2f 100644 --- a/backend/src/__tests__/webhooks-trigger.test.ts +++ b/backend/src/__tests__/webhooks-trigger.test.ts @@ -462,7 +462,7 @@ describe('WebhookService.execute: health gate begin call sites', () => { vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined); vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]); vi.spyOn(compose.ComposeService.prototype, 'deployStack').mockResolvedValue(undefined); - const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-hook'); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook'); const result = await WebhookService.getInstance().execute(webhook, 'deploy', 'test', true); expect(result.success).toBe(true); @@ -482,7 +482,7 @@ describe('WebhookService.execute: health gate begin call sites', () => { vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined); vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]); vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue(undefined); - const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-hook'); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook'); const result = await WebhookService.getInstance().execute(webhook, 'pull', 'test', true); expect(result.success).toBe(true); diff --git a/backend/src/bootstrap/shutdown.ts b/backend/src/bootstrap/shutdown.ts index a50f1ffa..10533ffa 100644 --- a/backend/src/bootstrap/shutdown.ts +++ b/backend/src/bootstrap/shutdown.ts @@ -4,6 +4,7 @@ import { LicenseService } from '../services/LicenseService'; import { MonitorService } from '../services/MonitorService'; import { AutoHealService } from '../services/AutoHealService'; import { HealthGateService } from '../services/HealthGateService'; +import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService'; import { FleetSyncRetryService } from '../services/FleetSyncRetryService'; import { DockerEventManager } from '../services/DockerEventManager'; import { ImageUpdateService } from '../services/ImageUpdateService'; @@ -34,6 +35,7 @@ export function installShutdownHandlers(server: Server): void { } try { AutoHealService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] AutoHealService cleanup failed:', (e as Error).message); } try { HealthGateService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] HealthGateService cleanup failed:', (e as Error).message); } + try { ServiceUpdateRecoveryService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] ServiceUpdateRecoveryService cleanup failed:', (e as Error).message); } try { FleetSyncRetryService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] FleetSyncRetryService cleanup failed:', (e as Error).message); } try { DockerEventManager.getInstance().stop(); } catch (e) { console.warn('[Shutdown] DockerEventManager cleanup failed:', (e as Error).message); diff --git a/backend/src/bootstrap/startup.ts b/backend/src/bootstrap/startup.ts index 13463e06..e46401d7 100644 --- a/backend/src/bootstrap/startup.ts +++ b/backend/src/bootstrap/startup.ts @@ -11,6 +11,7 @@ import SelfIdentityService from '../services/SelfIdentityService'; import { MonitorService } from '../services/MonitorService'; import { AutoHealService } from '../services/AutoHealService'; import { HealthGateService } from '../services/HealthGateService'; +import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService'; import { FleetSyncRetryService } from '../services/FleetSyncRetryService'; import { DockerEventManager } from '../services/DockerEventManager'; import TrivyService, { sweepStaleTrivyTempDirs } from '../services/TrivyService'; @@ -139,6 +140,7 @@ export async function startServer(server: Server): Promise { MonitorService.getInstance().start(); AutoHealService.getInstance().start(); HealthGateService.getInstance().start(); + ServiceUpdateRecoveryService.getInstance().start(); FleetSyncRetryService.getInstance().start(); ImageUpdateService.getInstance().start(); SchedulerService.getInstance().start(); diff --git a/backend/src/middleware/apiTokenScope.ts b/backend/src/middleware/apiTokenScope.ts index e84e6154..a69023c5 100644 --- a/backend/src/middleware/apiTokenScope.ts +++ b/backend/src/middleware/apiTokenScope.ts @@ -26,6 +26,8 @@ const DEPLOY_ALLOWED_PATTERNS: RegExp[] = [ /^\/api\/stacks\/[^/]+\/stop$/, /^\/api\/stacks\/[^/]+\/start$/, /^\/api\/stacks\/[^/]+\/update$/, + /^\/api\/stacks\/[^/]+\/services\/[^/]+\/update$/, + /^\/api\/stacks\/[^/]+\/services\/[^/]+\/restore$/, ]; const deny = (res: Response, req: Request, error: string, scope: ApiTokenScope | 'unknown'): void => { diff --git a/backend/src/proxy/remoteNodeProxy.ts b/backend/src/proxy/remoteNodeProxy.ts index e0cf6c12..b359dd0c 100644 --- a/backend/src/proxy/remoteNodeProxy.ts +++ b/backend/src/proxy/remoteNodeProxy.ts @@ -5,7 +5,7 @@ import { PROXY_TIER_HEADER, PROXY_ROLE_HEADER, PROXY_DEPLOY_SOURCE_HEADER, PROXY import { LicenseService } from '../services/LicenseService'; import { isProxyExemptPath } from '../helpers/proxyExemptPaths'; import { remoteSupportsCrossNodeRbac, remoteAdvertisesCapability } from '../helpers/remoteCapabilities'; -import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY } from '../services/CapabilityRegistry'; +import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry'; import { getErrorMessage } from '../utils/errors'; import { DatabaseService } from '../services/DatabaseService'; import { redactSensitiveText } from '../utils/safeLog'; @@ -171,7 +171,7 @@ export function createRemoteProxyMiddleware(): RequestHandler { finalizeProxyTiming(req, 'error'); console.error('[Proxy] Remote node error:', getErrorMessage(err, 'unknown')); const path = req.originalUrl || req.url; - if (req.method === 'POST' && /^\/api\/stacks\/[^/]+\/(?:deploy|update)(?:\?|$)/.test(path)) { + if (req.method === 'POST' && /^\/api\/stacks\/[^/]+\/(?:deploy|update|services\/[^/]+\/(?:update|restore))(?:\?|$)/.test(path)) { try { DatabaseService.getInstance().insertAuditLog({ timestamp: Date.now(), @@ -238,6 +238,14 @@ export function createRemoteProxyMiddleware(): RequestHandler { } } + if (isServiceScopedUpdateRoute(req)) { + const supported = await remoteAdvertisesCapability(req.nodeId, SERVICE_SCOPED_UPDATE_CAPABILITY); + if (!supported) { + res.status(400).json({ error: 'Service-scoped updates are not supported on this node', code: 'capability_unavailable' }); + return; + } + } + // Mixed-version RBAC gate (non-admin only). if (req.user?.role !== 'admin') { const rbacSupported = await remoteSupportsCrossNodeRbac(req.nodeId); @@ -264,3 +272,14 @@ function isStackDownWithRemoveVolumes(req: Request): boolean { if (!/^\/stacks\/[^/]+\/down$/.test(req.path)) return false; return req.query.removeVolumes === 'true'; } + +/** Nested service update/restore/recovery routes (path is post-/api strip). */ +function isServiceScopedUpdateRoute(req: Request): boolean { + if (req.method === 'GET') { + return /^\/stacks\/[^/]+\/services\/[^/]+\/recovery$/.test(req.path); + } + if (req.method === 'POST') { + return /^\/stacks\/[^/]+\/services\/[^/]+\/(?:update|restore)$/.test(req.path); + } + return false; +} diff --git a/backend/src/routes/fleet.ts b/backend/src/routes/fleet.ts index 794c3cd4..b78ca283 100644 --- a/backend/src/routes/fleet.ts +++ b/backend/src/routes/fleet.ts @@ -10,6 +10,7 @@ import { FleetUpdateTrackerService, type UpdateTracker, type TerminalStatus, UPD import { NodeRegistry } from '../services/NodeRegistry'; import { computeNodeNetworkingSummary, type NodeNetworkingSummary } from '../services/network/networkingSummary'; import DockerController from '../services/DockerController'; +import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService'; import { getHostMemory } from '../helpers/hostMemory'; import { FileSystemService } from '../services/FileSystemService'; import { ComposeService } from '../services/ComposeService'; @@ -2115,9 +2116,10 @@ fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res targetResults.push({ target, success: true, reclaimedBytes: estimate.reclaimableBytes, dryRun: true }); continue; } + const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(node.id); const result = scope === 'managed' - ? await dockerController.pruneManagedOnly(target, knownStacks) - : await dockerController.pruneSystem(target); + ? await dockerController.pruneManagedOnly(target, knownStacks, isImageHeld) + : await dockerController.pruneSystem(target, undefined, isImageHeld); targetResults.push({ target, success: true, reclaimedBytes: result.reclaimedBytes }); if (result.reclaimedBytes > 0 || result.success) anySuccess = true; } catch (err) { diff --git a/backend/src/routes/imageUpdates.ts b/backend/src/routes/imageUpdates.ts index b8806776..213a8380 100644 --- a/backend/src/routes/imageUpdates.ts +++ b/backend/src/routes/imageUpdates.ts @@ -7,7 +7,7 @@ import { NodeRegistry } from '../services/NodeRegistry'; import { CacheService } from '../services/CacheService'; import { ImageUpdateService } from '../services/ImageUpdateService'; import { FileSystemService } from '../services/FileSystemService'; -import { ComposeService } from '../services/ComposeService'; +import { StackUpdateOrchestrator } from '../services/StackUpdateOrchestrator'; import { StackOpLockService, stackOpSkipMessage } from '../services/StackOpLockService'; import { NotificationService } from '../services/NotificationService'; import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement'; @@ -342,7 +342,6 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp const docker = DockerController.getInstance(req.nodeId); const imageUpdateService = ImageUpdateService.getInstance(); - const compose = ComposeService.getInstance(req.nodeId); const db = DatabaseService.getInstance(); const atomic = true; const results: string[] = []; @@ -417,14 +416,17 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp const lock = await StackOpLockService.getInstance().runExclusive( req.nodeId, stackName, 'update', 'system', - () => compose.updateStack(stackName, undefined, atomic), + () => StackUpdateOrchestrator.getInstance().execute( + { nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'automatic', actor: `auto-update:${req.user?.username ?? 'scheduler'}` }, + { atomic, terminalWs: null }, + ), ); if (!lock.ran) { results.push(stackOpSkipMessage(stackName, lock.existing.action)); continue; } db.clearStackUpdateStatus(req.nodeId, stackName); - HealthGateService.getInstance().begin(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`); + HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`); NotificationService.getInstance().broadcastEvent({ type: 'state-invalidate', diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index fabfe6b7..0d753c4c 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -13,6 +13,7 @@ import { FileSystemService } from '../services/FileSystemService'; import { StackFileRootsService, STACK_SOURCE_ROOT_ID, stackSourceFileRoot, type StackFileRoot } from '../services/StackFileRootsService'; import { FileRootGateway } from '../services/FileRootGateway'; import { ComposeService, getComposeRollbackInfo } from '../services/ComposeService'; +import { StackUpdateOrchestrator, shortImageId, type OrchestratorResult } from '../services/StackUpdateOrchestrator'; import DockerController, { type BulkStackInfo } from '../services/DockerController'; import { DatabaseService, type StackDossierFields } from '../services/DatabaseService'; import { MeshService } from '../services/MeshService'; @@ -30,11 +31,12 @@ import { buildStackNetworkFacts } from '../services/network/composeNetworkInspec import { buildStorageInventory } from '../services/storage/inventory'; import { probeComposeDiscovery } from '../services/ComposeDiscoveryService'; import { buildEffectiveAnatomy } from '../services/effectiveAnatomy'; +import { buildEffectiveServiceModel } from '../services/effectiveServiceModel'; import { buildEnvInventory } from '../services/EnvInventoryService'; import { buildStackLabelInventory } from '../services/LabelInventoryService'; import { labelInventoryOptionsFromRequest, requireRevealAdmin } from '../helpers/labelInventoryRequest'; import { EXPOSURE_INTENTS, type ExposureIntent } from '../services/network/types'; -import { UpdateGuardService } from '../services/UpdateGuardService'; +import { UpdateGuardService, SingleServiceUpdateReadinessError } from '../services/UpdateGuardService'; import { HealthGateService } from '../services/HealthGateService'; import { classifyFailure } from '../services/updateGuard/failureClassifier'; import { requirePermission, checkPermission } from '../middleware/permissions'; @@ -58,7 +60,8 @@ import { resolveStackEnvSources, discoverStackLocalEnvFiles } from '../helpers/e import { STACK_STATUSES_CACHE_TTL_MS } from '../helpers/constants'; import { getTerminalWs, DEPLOY_SESSION_HEADER } from '../websocket/generic'; import { isSelfStack, refuseIfSelfStack, selfStackProtectedBulkResult } from '../helpers/selfStackGuard'; -import { getActiveCapabilities, STACK_DOWN_REMOVE_VOLUMES_CAPABILITY } from '../services/CapabilityRegistry'; +import { getActiveCapabilities, STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry'; +import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService'; // Authenticated users with edit permission can write arbitrarily large compose // files. Refuse to YAML.parse anything beyond this bound so a malformed (or @@ -471,7 +474,10 @@ async function runStackBulkOp( }; } const atomic = true; - await ComposeService.getInstance(req.nodeId).updateStack(stackName, getTerminalWs(req.get(DEPLOY_SESSION_HEADER)), atomic); + await StackUpdateOrchestrator.getInstance().execute( + { nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'bulk', actor: user }, + { atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) }, + ); DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName); NotificationService.getInstance().broadcastEvent({ type: 'state-invalidate', @@ -485,7 +491,7 @@ async function runStackBulkOp( triggerPostDeployScan(stackName, req.nodeId).catch(err => console.error('[Security] Post-deploy scan failed for %s:', sanitizeForLog(stackName), err), ); - const healthGateId = HealthGateService.getInstance().begin(req.nodeId, stackName, 'update', req.user?.username ?? null); + const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null); return { stackName, ok: true, healthGateId }; } else { const outcome = await containerActionForStack(req.nodeId, stackName, action); @@ -1143,6 +1149,7 @@ stacksRouter.delete('/:stackName', async (req: Request, res: Response) => { try { DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName); DatabaseService.getInstance().clearStackScanAttempts(req.nodeId, stackName); + DatabaseService.getInstance().deleteServiceUpdateRecoveries(req.nodeId, stackName); DatabaseService.getInstance().deleteRoleAssignmentsByResource('stack', stackName); DatabaseService.getInstance().deleteGitSource(stackName); DatabaseService.getInstance().deleteStackDossier(req.nodeId, stackName); @@ -1558,6 +1565,24 @@ stacksRouter.get('/:stackName/effective-anatomy', async (req: Request, res: Resp } }); +// Effective Service Model: per-service facts (declared image, build presence, +// expected replica count, dependencies, healthcheck) that service-scoped +// update/restore key off of, from the fully-merged effective model. Read-only +// and advisory; auto-proxies to the active node. Never returns raw render +// stderr or any environment/label/command value. +stacksRouter.get('/:stackName/effective-services', async (req: Request, res: Response) => { + const stackName = req.params.stackName as string; + if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return; + if (!(await requireStackExists(req.nodeId, stackName, res))) return; + try { + res.json(await buildEffectiveServiceModel(req.nodeId, stackName)); + } catch (error) { + console.error('[Stacks] Failed to build effective service model for %s:', sanitizeForLog(stackName), + sanitizeForLog(inspect(error, { depth: 4 }))); + res.status(500).json({ error: 'Failed to build effective service model' }); + } +}); + // Environment inventory: per-stack env vars with their source, scope (Compose // interpolation vs container injection), and status (present/missing/unused/ // duplicate/unpersisted), plus likely-secret classification. Read-only and @@ -1656,12 +1681,17 @@ stacksRouter.put('/:stackName/exposure', async (req: Request, res: Response) => // that owns it. Read-only, so stack:read is the correct gate. stacksRouter.get('/:stackName/update-readiness', async (req: Request, res: Response) => { const stackName = req.params.stackName as string; + const serviceName = typeof req.query.service === 'string' && req.query.service.length > 0 ? req.query.service : undefined; if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return; if (!(await requireStackExists(req.nodeId, stackName, res))) return; try { - const report = await UpdateGuardService.getInstance().computeUpdateReadiness(req.nodeId, stackName); + const report = await UpdateGuardService.getInstance().computeUpdateReadiness(req.nodeId, stackName, serviceName); res.json(report); } catch (error) { + if (error instanceof SingleServiceUpdateReadinessError) { + res.status(400).json({ error: error.message, code: 'service_update_single_service' }); + return; + } console.error('[Stacks] Failed to compute update readiness for %s:', sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(error, 'unknown'))); res.status(500).json({ error: 'Failed to compute update readiness' }); @@ -1724,7 +1754,7 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => { dlog(`[Stacks] Deploy completed: ${sanitizeForLog(stackName)}`); if (debug) console.debug(`[Stacks:debug] Deploy finished in ${Date.now() - t0}ms`); ok = true; - const healthGateId = HealthGateService.getInstance().begin(req.nodeId, stackName, 'deploy', req.user?.username ?? null); + const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'deploy', req.user?.username ?? null); res.json({ message: 'Deployed successfully', healthGateId }); notifyActionSuccess('deploy_success', `${stackName} deployed`, stackName, req.user?.username ?? 'system'); if (!skipScan) { @@ -1987,6 +2017,230 @@ stacksRouter.post('/:stackName/services/:serviceName/stop', (req, res) => stacksRouter.post('/:stackName/services/:serviceName/start', (req, res) => handleServiceAction(req, res, 'start')); +/** Map an orchestrator `service_failed` code to an HTTP status. */ +function serviceFailureStatus(code: string): number { + switch (code) { + case 'service_update_single_service': + case 'service_not_updatable': + case 'effective_model_render_failed': + case 'recovery_id_required': + return 400; + case 'service_not_found': + case 'recovery_not_found': + return 404; + case 'policy_blocked': + case 'recovery_not_restorable': + case 'recovery_claim_failed': + return 409; + default: + // Compose, retag, inspect, and replica-divergence failures are server-side. + return 500; + } +} + +/** Send an orchestrator service result as an HTTP response. Returns success. */ +function sendServiceResult(res: Response, result: OrchestratorResult, serviceName: string): boolean { + if (result.kind === 'service_done') { + res.json({ + serviceName: result.serviceName, + healthGateId: result.healthGateId, + observing: result.observing, + recoveryId: result.recoveryId, + recoveryAvailable: result.recoveryAvailable, + ...(result.previousImageId ? { previousImageId: result.previousImageId } : {}), + ...(result.newImageId ? { newImageId: result.newImageId } : {}), + ...(result.recheckWarning ? { recheckWarning: result.recheckWarning } : {}), + }); + return true; + } + if (result.kind === 'service_failed') { + res.status(serviceFailureStatus(result.code)).json({ + error: result.error, + code: result.code, + serviceName: result.serviceName ?? serviceName, + ...(result.mutationStage ? { mutationStage: result.mutationStage } : {}), + ...(result.recoveryId ? { recoveryId: result.recoveryId } : {}), + }); + return false; + } + // Stack results never reach the service routes; treat defensively. + res.status(500).json({ error: 'Unexpected orchestrator result', code: 'unexpected_result' }); + return false; +} + +function requireServiceScopedUpdateCapability(res: Response): boolean { + if (getActiveCapabilities().includes(SERVICE_SCOPED_UPDATE_CAPABILITY)) return true; + res.status(400).json({ error: 'Service-scoped updates are not supported on this node', code: 'capability_unavailable' }); + return false; +} + +async function handleServiceScopedMutation( + req: Request, + res: Response, + options: { + lockAction: 'update' | 'rollback'; + recoveryId?: string; + notifyFailureAction: 'update' | 'rollback'; + failureCode: string; + failureMessage: string; + onSuccess: ( + stackName: string, + serviceName: string, + actor: string, + meta: { previousImageId?: string | null; newImageId?: string | null }, + ) => void; + }, +): Promise { + const stackName = req.params.stackName as string; + const serviceName = req.params.serviceName as string; + if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return; + if (!(await requireStackExists(req.nodeId, stackName, res))) return; + if (await refuseIfSelfStack(req, res, stackName)) return; + if (!requireServiceScopedUpdateCapability(res)) return; + if (!tryAcquireStackOpLock(req, res, stackName, options.lockAction)) return; + + const t0 = Date.now(); + let ok = false; + try { + const result = await StackUpdateOrchestrator.getInstance().execute( + { + nodeId: req.nodeId, + stackName, + target: { scope: 'service', serviceName }, + trigger: 'manual', + actor: req.user?.username ?? null, + }, + { + policyOptions: buildPolicyGateOptions(req), + terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)), + ...(options.recoveryId ? { recoveryId: options.recoveryId } : {}), + }, + ); + invalidateNodeCaches(req.nodeId); + ok = sendServiceResult(res, result, serviceName); + if (ok && result.kind === 'service_done') { + options.onSuccess(stackName, serviceName, req.user?.username ?? 'system', { + previousImageId: result.previousImageId, + newImageId: result.newImageId, + }); + NotificationService.getInstance().broadcastEvent({ + type: 'state-invalidate', + scope: 'image-updates', + nodeId: req.nodeId, + stackName, + action: 'stack-updated', + ts: Date.now(), + }); + } else if (result.kind === 'service_failed') { + notifyActionFailure( + options.notifyFailureAction, + stackName, + new Error(result.error), + req.user?.username ?? 'system', + ); + } + } catch (error: unknown) { + console.error( + '[Stacks] Service %s failed: %s/%s: %s', + sanitizeForLog(options.notifyFailureAction), + sanitizeForLog(stackName), + sanitizeForLog(serviceName), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + notifyActionFailure(options.notifyFailureAction, stackName, error, req.user?.username ?? 'system'); + if (!res.headersSent) { + res.status(500).json({ error: getErrorMessage(error, options.failureMessage), code: options.failureCode }); + } + } finally { + StackOpMetricsService.getInstance().record(req.nodeId, 'update', Date.now() - t0, ok, { + targetScope: 'service', + serviceName, + }); + releaseStackOpLock(req, stackName); + } +} + +stacksRouter.post('/:stackName/services/:serviceName/update', async (req: Request, res: Response) => { + await handleServiceScopedMutation(req, res, { + lockAction: 'update', + notifyFailureAction: 'update', + failureCode: 'service_update_failed', + failureMessage: 'Failed to update service', + onSuccess: (stackName, serviceName, actor, meta) => { + const from = shortImageId(meta.previousImageId); + const to = shortImageId(meta.newImageId); + const transition = from && to && from !== to ? ` (${from} -> ${to})` : ''; + notifyActionSuccess( + 'image_update_applied', + `${stackName}/${serviceName} updated${transition}`, + stackName, + actor, + ); + }, + }); +}); + +/** Latest restorable service recovery snapshot (active, unexpired), for UI discovery outside Deploy Progress. */ +stacksRouter.get('/:stackName/services/:serviceName/recovery', async (req: Request, res: Response) => { + const stackName = req.params.stackName as string; + const serviceName = req.params.serviceName as string; + if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return; + if (!(await requireStackExists(req.nodeId, stackName, res))) return; + if (!requireServiceScopedUpdateCapability(res)) return; + try { + const row = ServiceUpdateRecoveryService.getInstance().listActive(req.nodeId, stackName, serviceName)[0]; + if (!row) { + res.json({ recovery: null }); + return; + } + res.json({ + recovery: { + id: row.id, + status: row.status, + healthGateId: row.health_gate_id, + expiresAt: row.expires_at, + createdAt: row.created_at, + majorityImageId: row.majority_image_id, + declaredImageRef: row.declared_image_ref, + }, + }); + } catch (error: unknown) { + console.error( + '[Stacks] Failed to list service recovery for %s/%s: %s', + sanitizeForLog(stackName), + sanitizeForLog(serviceName), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + res.status(500).json({ error: 'Failed to load service recovery', code: 'service_recovery_lookup_failed' }); + } +}); + +stacksRouter.post('/:stackName/services/:serviceName/restore', async (req: Request, res: Response) => { + const recoveryId = typeof req.body?.recoveryId === 'string' ? req.body.recoveryId : ''; + if (!recoveryId) { + res.status(400).json({ error: 'A recoveryId is required to restore a service.', code: 'recovery_id_required' }); + return; + } + await handleServiceScopedMutation(req, res, { + lockAction: 'rollback', + recoveryId, + notifyFailureAction: 'rollback', + failureCode: 'service_restore_failed', + failureMessage: 'Failed to restore service', + onSuccess: (stackName, serviceName, actor, meta) => { + const from = shortImageId(meta.previousImageId); + const to = shortImageId(meta.newImageId); + const transition = from && to && from !== to ? ` (${from} -> ${to})` : ''; + notifyActionSuccess( + 'deploy_success', + `${stackName}/${serviceName} restored${transition}`, + stackName, + actor, + ); + }, + }); +}); + stacksRouter.get('/:stackName/update-preview', async (req: Request, res: Response) => { const stackName = req.params.stackName as string; try { @@ -2013,7 +2267,10 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { const debug = isDebugEnabled(); const atomic = true; if (debug) console.debug('[Stacks:debug] Update starting', { stackName, atomic, nodeId: req.nodeId }); - await ComposeService.getInstance(req.nodeId).updateStack(stackName, getTerminalWs(req.get(DEPLOY_SESSION_HEADER)), atomic); + await StackUpdateOrchestrator.getInstance().execute( + { nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'manual', actor: req.user?.username ?? null }, + { atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) }, + ); DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName); invalidateNodeCaches(req.nodeId); NotificationService.getInstance().broadcastEvent({ @@ -2027,7 +2284,7 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { dlog(`[Stacks] Update completed: ${sanitizeForLog(stackName)}`); if (debug) console.debug(`[Stacks:debug] Update finished in ${Date.now() - t0}ms`); ok = true; - const healthGateId = HealthGateService.getInstance().begin(req.nodeId, stackName, 'update', req.user?.username ?? null); + const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null); res.json({ status: 'Update completed', healthGateId }); notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, req.user?.username ?? 'system'); if (!skipScan) { diff --git a/backend/src/routes/systemMaintenance.ts b/backend/src/routes/systemMaintenance.ts index 38dba944..e86afda0 100644 --- a/backend/src/routes/systemMaintenance.ts +++ b/backend/src/routes/systemMaintenance.ts @@ -8,6 +8,7 @@ import DockerController, { } from '../services/DockerController'; import { isPruneTarget } from '../services/prunePlan'; import { FileSystemService } from '../services/FileSystemService'; +import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService'; import SelfIdentityService from '../services/SelfIdentityService'; import { requireAdmin } from '../middleware/tierGates'; import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; @@ -127,8 +128,9 @@ systemMaintenanceRouter.post('/prune/plan', async (req: Request, res: Response) const pruneScope = parsePruneScope((req.body as { scope?: unknown }).scope); const dockerController = DockerController.getInstance(req.nodeId); const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks(); + const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(req.nodeId); const plan = await withTimeout( - dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId), + dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId, isImageHeld), PRUNE_ESTIMATE_TIMEOUT_MS, 'docker prune plan', ); @@ -171,11 +173,12 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response const planFingerprint = typeof body.planFingerprint === 'string' ? body.planFingerprint : null; const dockerController = DockerController.getInstance(req.nodeId); const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks(); + const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(req.nodeId); if (isDryRun) { // Dry-run returns the same itemized plan shape Resources uses for preview. const plan = await withTimeout( - dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId), + dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId, isImageHeld), PRUNE_ESTIMATE_TIMEOUT_MS, 'docker prune plan', ); @@ -201,7 +204,7 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response // fingerprint and keeps the legacy pruneManagedOnly / pruneSystem path. if (planFingerprint) { const built = await withTimeout( - dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId), + dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId, isImageHeld), PRUNE_ESTIMATE_TIMEOUT_MS, 'docker prune plan', ); @@ -215,7 +218,7 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response `[Resources] System prune (plan): ${built.targets.join(',')} (scope: ${pruneScope}, items: ${built.items.length})`, ); const pruneStartedAt = Date.now(); - const result = await dockerController.executePrunePlan(built, knownStacks); + const result = await dockerController.executePrunePlan(built, knownStacks, isImageHeld); console.log( `[Resources] System prune completed: reclaimed ${result.reclaimedBytes} bytes, outcomes=${result.outcomes.length}`, ); @@ -254,14 +257,15 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response result = await dockerController.pruneManagedOnly( target as 'images' | 'volumes' | 'networks', knownStacks, + isImageHeld, ); } else if (pruneScope === 'managed' && target === 'containers') { // Managed containers must never fall through to system prune. Build and // execute an itemized plan for this single target instead. - const plan = await dockerController.buildPrunePlan(['containers'], 'managed', knownStacks, req.nodeId); - result = await dockerController.executePrunePlan(plan, knownStacks); + const plan = await dockerController.buildPrunePlan(['containers'], 'managed', knownStacks, req.nodeId, isImageHeld); + result = await dockerController.executePrunePlan(plan, knownStacks, isImageHeld); } else { - result = await dockerController.pruneSystem(target); + result = await dockerController.pruneSystem(target, undefined, isImageHeld); } console.log(`[Resources] System prune completed: ${target}, reclaimed ${result.reclaimedBytes} bytes`); diff --git a/backend/src/services/AutoHealService.ts b/backend/src/services/AutoHealService.ts index a3d2d4a2..fe7f1abb 100644 --- a/backend/src/services/AutoHealService.ts +++ b/backend/src/services/AutoHealService.ts @@ -71,6 +71,15 @@ export class AutoHealService { private observedUnhealthySince = new Map(); private historyTimestamps = new Map(); private leaseRefreshFailures = new Map(); + /** + * Ref-counted suppressions for service-scoped updates/restores. While a + * service is being recreated and its post-mutation health gate observes it, + * auto-heal must not restart the churning containers out from under the gate. + * Keyed `${nodeId}:${stackName}:${serviceName}`. Nested owners (overlapping + * same-service ops, or a gate that outlives the orchestrator finally) each + * increment; clearSuppress only unmutes at zero. Process death clears it. + */ + private suppressedServices = new Map(); private constructor() {} @@ -108,6 +117,33 @@ export class AutoHealService { clearInterval(this.leaseRefreshTimer); this.leaseRefreshTimer = null; } + this.suppressedServices.clear(); + } + + private suppressKey(nodeId: number, stackName: string, serviceName: string): string { + return `${nodeId}:${stackName}:${serviceName}`; + } + + /** Suppress auto-heal for one service while a service-scoped update/restore + its health gate run. */ + suppress(nodeId: number, stackName: string, serviceName: string): void { + const key = this.suppressKey(nodeId, stackName, serviceName); + this.suppressedServices.set(key, (this.suppressedServices.get(key) ?? 0) + 1); + } + + /** Release one suppression owner; Auto-Heal resumes only when the count hits zero. */ + clearSuppress(nodeId: number, stackName: string, serviceName: string): void { + const key = this.suppressKey(nodeId, stackName, serviceName); + const current = this.suppressedServices.get(key) ?? 0; + if (current <= 1) { + this.suppressedServices.delete(key); + return; + } + this.suppressedServices.set(key, current - 1); + } + + /** True while the given service is suppressed. A stack-wide policy still heals other services. */ + isSuppressed(nodeId: number, stackName: string, serviceName: string): boolean { + return (this.suppressedServices.get(this.suppressKey(nodeId, stackName, serviceName)) ?? 0) > 0; } /** @@ -283,6 +319,12 @@ export class AutoHealService { const containerName = container.Names?.[0]?.replace(/^\//, '') ?? container.Id.slice(0, 12); const serviceOverride = container.Labels?.['com.docker.compose.service'] ?? null; + // Skip containers whose service is suppressed by an in-flight + // service-scoped update/restore, so the post-mutation health gate + // owns the churn without auto-heal racing it. + if (serviceOverride && this.isSuppressed(nodeId, policy.stack_name, serviceOverride)) { + continue; + } const signal = this.getHealSignal(nodeId, container, eventSvc?.getContainerState(container.Id), now); const decision = this.shouldHeal(signal, policy, this.containerKey(nodeId, container.Id), now); diff --git a/backend/src/services/CapabilityRegistry.ts b/backend/src/services/CapabilityRegistry.ts index 17f5d31b..3b9461fa 100644 --- a/backend/src/services/CapabilityRegistry.ts +++ b/backend/src/services/CapabilityRegistry.ts @@ -56,6 +56,7 @@ export const CAPABILITIES = [ 'cross-node-rbac', 'stack-down-remove-volumes', 'guided-external-network-preflight', + 'service-scoped-update', ] as const; /** @@ -72,6 +73,9 @@ export type Capability = (typeof CAPABILITIES)[number]; /** Capability for optional `?removeVolumes=true` on POST /stacks/:name/down. */ export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes' as const satisfies Capability; +/** Capability for the nested per-service update/restore routes and the `effective-services` model they read. */ +export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability; + /** Returns true when the string is a usable semver version. */ export function isValidVersion(v: string | null | undefined): v is string { return !!v && v !== 'unknown' && v !== '0.0.0-dev' && !!semver.valid(v); diff --git a/backend/src/services/ComposeService.ts b/backend/src/services/ComposeService.ts index 6d7caf6a..f6aaf99d 100644 --- a/backend/src/services/ComposeService.ts +++ b/backend/src/services/ComposeService.ts @@ -85,7 +85,8 @@ export function getComposeRollbackInfo(error: unknown): { attempted: boolean; ro const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000; -function getComposeCommandTimeoutMs(): number { +/** Public so other services (e.g. recovery claim leases) can size their own timers off the same ceiling without depending on a private module-local. */ +export function getComposeCommandTimeoutMs(): number { const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS); if (Number.isFinite(configured) && configured > 0) { return configured; @@ -873,7 +874,11 @@ export class ComposeService { try { const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1'; if (pruneOnUpdate) { - const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(); + // Dynamic import: ServiceUpdateRecoveryService imports getComposeCommandTimeoutMs + // from this module, so a static import here would be circular. + const { ServiceUpdateRecoveryService } = await import('./ServiceUpdateRecoveryService'); + const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(this.nodeId); + const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld); // The Docker prune API does not report SpaceReclaimed on the containerd // image store, so only show the figure when the daemon actually returns one. const reclaimed = result.reclaimedBytes > 0 @@ -906,6 +911,53 @@ export class ComposeService { } } + /** + * Service-scoped update: pull (or `build --pull` for a build-backed service) + * and recreate a single service's replicas in place. Always + * `--no-deps --force-recreate`, never `--remove-orphans`, so sibling services + * keep their container ids and StartedAt. No drift re-baseline and no dangling + * prune here; the orchestrator owns per-service post-update reconciliation. + */ + async updateService(stackName: string, serviceName: string, hasBuild: boolean, ws?: WebSocket): Promise { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + const stackDir = path.join(this.baseDir, stackName); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + await this.withRegistryAuth(async (env) => { + if (hasBuild) { + sendOutput(`=== Building ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['build', '--pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + } else { + sendOutput(`=== Pulling ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + } + sendOutput(`=== Recreating ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + }, sendOutput); + } + + /** + * Recreate a single service from the image already present locally, without + * pulling or building. Used by service restore after the recovery image id has + * been retagged onto the declared ref (`--pull never --no-build` so Compose + * uses the just-retagged local image). Always `--no-deps --force-recreate`, + * never `--remove-orphans`. + */ + async recreateServiceFromLocal(stackName: string, serviceName: string, ws?: WebSocket): Promise { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + const stackDir = path.join(this.baseDir, stackName); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + await this.withRegistryAuth(async (env) => { + sendOutput(`=== Restoring ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', '--pull', 'never', '--no-build', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + }, sendOutput); + } + public async downStack(stackName: string): Promise { const stackPath = path.join(this.baseDir, stackName); try { diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 997b81f0..afa394b3 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -35,11 +35,58 @@ export interface GlobalSetting { */ export type StackCheckStatus = 'ok' | 'partial' | 'failed'; +/** + * Per-service check outcome persisted in stack_update_status.services_json. + * Distinct from StackCheckStatus: a service with no checkable image ref is + * `not_checkable`, which never counts as a check failure at the aggregate + * (stack-level) status. + */ +export type ServiceCheckStatus = 'ok' | 'partial' | 'failed' | 'not_checkable'; + +export interface StackServiceStatus { + service: string; + image: string | null; + runtimeImages?: string[]; + hasUpdate: boolean; + checkStatus: ServiceCheckStatus; + lastError: string | null; +} + export interface StackUpdateDetail { hasUpdate: boolean; checkStatus: StackCheckStatus; lastError: string | null; checkedAt: number; + /** Per-service breakdown; omitted when the stack has no persisted per-service data (no effective model available yet, or corrupt JSON). */ + services?: StackServiceStatus[]; +} + +const SERVICES_JSON_VERSION = 1; + +function isStackServiceStatus(value: unknown): value is StackServiceStatus { + if (!value || typeof value !== 'object') return false; + const v = value as Record; + return typeof v.service === 'string' + && (v.image === null || typeof v.image === 'string') + && typeof v.hasUpdate === 'boolean' + && (v.checkStatus === 'ok' || v.checkStatus === 'partial' || v.checkStatus === 'failed' || v.checkStatus === 'not_checkable') + && (v.lastError === null || typeof v.lastError === 'string'); +} + +/** Parses stack_update_status.services_json safely; a missing, corrupt, or version-mismatched value yields an empty list rather than throwing. */ +function parseServicesJson(raw: string | null | undefined): StackServiceStatus[] { + if (!raw) return []; + try { + const parsed = JSON.parse(raw) as { version?: unknown; services?: unknown }; + if (parsed?.version !== SERVICES_JSON_VERSION || !Array.isArray(parsed.services)) return []; + return parsed.services.filter(isStackServiceStatus); + } catch { + return []; + } +} + +function stringifyServicesJson(services: StackServiceStatus[], generation: number): string { + return JSON.stringify({ version: SERVICES_JSON_VERSION, generation, services }); } export interface StackAlert { @@ -146,7 +193,7 @@ export interface HealthGateRunRow { node_id: number; stack_name: string; /** Named trigger_action because TRIGGER is reserved in SQLite. */ - trigger_action: 'update' | 'deploy'; + trigger_action: 'update' | 'deploy' | 'service_update' | 'service_restore'; status: 'observing' | 'passed' | 'failed' | 'unknown'; reason: string | null; window_seconds: number; @@ -155,6 +202,30 @@ export interface HealthGateRunRow { started_at: number; ended_at: number | null; created_by: string | null; + /** Additive; legacy rows default to stack. */ + target_scope: 'stack' | 'service'; + service_name: string | null; + failure_source: 'primary' | 'collateral' | null; +} + +/** Pre-update image snapshot enabling a manual per-service restore after a service-scoped update. */ +export interface ServiceUpdateRecoveryRow { + id: string; + node_id: number; + stack_name: string; + service_name: string; + /** JSON array of pre-update replica image snapshots (imageId + repoDigest when known). */ + replicas_json: string; + majority_image_id: string; + /** Audit/UI only: the tag the majority image is retagged onto during restore, never a policy scan target. */ + declared_image_ref: string; + weak_floating_tag: number; + health_gate_id: string | null; + status: 'active' | 'restoring' | 'consumed' | 'expired' | 'invalidated'; + expires_at: number; + claim_expires_at: number | null; + created_at: number; + created_by: string | null; } /** One finding within a stored preflight run. Never carries an environment value. */ @@ -1005,6 +1076,7 @@ export class DatabaseService { has_update INTEGER DEFAULT 0, check_status TEXT NOT NULL DEFAULT 'ok', last_error TEXT, + services_json TEXT, checked_at INTEGER NOT NULL, PRIMARY KEY (node_id, stack_name) ); @@ -1525,18 +1597,44 @@ export class DatabaseService { id TEXT PRIMARY KEY, node_id INTEGER NOT NULL, stack_name TEXT NOT NULL, - trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy')), + trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy','service_update','service_restore')), status TEXT NOT NULL CHECK (status IN ('observing','passed','failed','unknown')), reason TEXT, window_seconds INTEGER NOT NULL, containers_json TEXT NOT NULL DEFAULT '[]', started_at INTEGER NOT NULL, ended_at INTEGER, - created_by TEXT + created_by TEXT, + target_scope TEXT NOT NULL DEFAULT 'stack' CHECK (target_scope IN ('stack','service')), + service_name TEXT, + failure_source TEXT CHECK (failure_source IS NULL OR failure_source IN ('primary','collateral')) ); CREATE INDEX IF NOT EXISTS idx_health_gate_runs_node_stack ON health_gate_runs(node_id, stack_name, started_at); + CREATE TABLE IF NOT EXISTS service_update_recovery ( + id TEXT PRIMARY KEY, + node_id INTEGER NOT NULL, + stack_name TEXT NOT NULL, + service_name TEXT NOT NULL, + replicas_json TEXT NOT NULL, + majority_image_id TEXT NOT NULL, + declared_image_ref TEXT NOT NULL, + weak_floating_tag INTEGER NOT NULL DEFAULT 0, + health_gate_id TEXT, + status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','restoring','consumed','expired','invalidated')), + expires_at INTEGER NOT NULL, + claim_expires_at INTEGER, + created_at INTEGER NOT NULL, + created_by TEXT + ); + CREATE INDEX IF NOT EXISTS idx_service_update_recovery_node_stack_service + ON service_update_recovery(node_id, stack_name, service_name, status); + CREATE INDEX IF NOT EXISTS idx_service_update_recovery_status_expires + ON service_update_recovery(status, expires_at); + CREATE INDEX IF NOT EXISTS idx_service_update_recovery_status_claim_expires + ON service_update_recovery(status, claim_expires_at); + CREATE TABLE IF NOT EXISTS secrets ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL UNIQUE, @@ -1664,6 +1762,11 @@ export class DatabaseService { // failed check is no longer indistinguishable from "up to date". maybeAddCol('stack_update_status', 'check_status', "TEXT NOT NULL DEFAULT 'ok'"); maybeAddCol('stack_update_status', 'last_error', 'TEXT'); + // Per-service image-update snapshot. Must run AFTER the composite-PK + // recreate above so it is never silently dropped on old installs. + maybeAddCol('stack_update_status', 'services_json', 'TEXT'); + + this.migrateHealthGateTargetSchema(); // Drop legacy SSH/TLS columns from pre-0.7 databases (no longer read or written) const legacyCols = ['host', 'port', 'ssh_port', 'ssh_user', 'ssh_password', 'ssh_key', 'tls_ca', 'tls_cert', 'tls_key']; @@ -1791,6 +1894,66 @@ export class DatabaseService { } } + /** + * Rebuild health_gate_runs when the installed CHECK still only allows + * update|deploy or target/failure columns are missing. Idempotent and + * restart-safe: drops a stale temporary table, then rebuilds in one + * better-sqlite3 transaction so an interrupted startup cannot leave + * CREATE TABLE health_gate_runs_new blocking the next boot. + */ + private migrateHealthGateTargetSchema(): void { + const tableSql = (this.db.prepare( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'health_gate_runs'" + ).get() as { sql: string } | undefined)?.sql ?? ''; + const cols = this.db.pragma('table_info(health_gate_runs)') as Array<{ name: string }>; + const colNames = new Set(cols.map(c => c.name)); + const hasTarget = colNames.has('target_scope'); + const hasFailure = colNames.has('failure_source'); + const hasWideTrigger = tableSql.includes('service_update') && tableSql.includes('service_restore'); + if (hasTarget && hasFailure && hasWideTrigger) return; + + // A previous crash between CREATE and RENAME leaves this temp table behind. + this.db.exec('DROP TABLE IF EXISTS health_gate_runs_new'); + + const targetExpr = hasTarget ? 'target_scope' : "'stack'"; + const serviceExpr = colNames.has('service_name') ? 'service_name' : 'NULL'; + const failureExpr = hasFailure ? 'failure_source' : 'NULL'; + + this.db.transaction(() => { + this.db.exec(` + CREATE TABLE health_gate_runs_new ( + id TEXT PRIMARY KEY, + node_id INTEGER NOT NULL, + stack_name TEXT NOT NULL, + trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy','service_update','service_restore')), + status TEXT NOT NULL CHECK (status IN ('observing','passed','failed','unknown')), + reason TEXT, + window_seconds INTEGER NOT NULL, + containers_json TEXT NOT NULL DEFAULT '[]', + started_at INTEGER NOT NULL, + ended_at INTEGER, + created_by TEXT, + target_scope TEXT NOT NULL DEFAULT 'stack' CHECK (target_scope IN ('stack','service')), + service_name TEXT, + failure_source TEXT CHECK (failure_source IS NULL OR failure_source IN ('primary','collateral')) + ); + INSERT INTO health_gate_runs_new ( + id, node_id, stack_name, trigger_action, status, reason, window_seconds, + containers_json, started_at, ended_at, created_by, target_scope, service_name, failure_source + ) + SELECT + id, node_id, stack_name, trigger_action, status, reason, window_seconds, + containers_json, started_at, ended_at, created_by, + ${targetExpr}, ${serviceExpr}, ${failureExpr} + FROM health_gate_runs; + DROP TABLE health_gate_runs; + ALTER TABLE health_gate_runs_new RENAME TO health_gate_runs; + CREATE INDEX IF NOT EXISTS idx_health_gate_runs_node_stack + ON health_gate_runs(node_id, stack_name, started_at); + `); + })(); + } + private migrateEncryptNodeTokens(): void { const crypto = CryptoService.getInstance(); const rows = this.db.prepare("SELECT id, api_token FROM nodes WHERE api_token != '' AND api_token IS NOT NULL").all() as Array<{ id: number; api_token: string }>; @@ -3128,11 +3291,13 @@ export class DatabaseService { public insertHealthGateRun(run: HealthGateRunRow): void { this.db.prepare( `INSERT INTO health_gate_runs - (id, node_id, stack_name, trigger_action, status, reason, window_seconds, containers_json, started_at, ended_at, created_by) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + (id, node_id, stack_name, trigger_action, status, reason, window_seconds, containers_json, + started_at, ended_at, created_by, target_scope, service_name, failure_source) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( run.id, run.node_id, run.stack_name, run.trigger_action, run.status, run.reason, run.window_seconds, run.containers_json, run.started_at, run.ended_at, run.created_by, + run.target_scope, run.service_name, run.failure_source, ); // Bounded history: keep only the 10 most recent runs per stack. this.db.prepare( @@ -3152,10 +3317,11 @@ export class DatabaseService { reason: string | null, endedAt: number, containersJson: string, + failureSource: 'primary' | 'collateral' | null = null, ): void { this.db.prepare( - 'UPDATE health_gate_runs SET status = ?, reason = ?, ended_at = ?, containers_json = ? WHERE id = ?' - ).run(status, reason, endedAt, containersJson, id); + 'UPDATE health_gate_runs SET status = ?, reason = ?, ended_at = ?, containers_json = ?, failure_source = ? WHERE id = ?' + ).run(status, reason, endedAt, containersJson, failureSource, id); } public getHealthGateRun(nodeId: number, stackName: string, id: string): HealthGateRunRow | undefined { @@ -3178,6 +3344,128 @@ export class DatabaseService { return result.changes; } + // --- Service Update Recovery --- + + public insertServiceUpdateRecovery(row: ServiceUpdateRecoveryRow): void { + this.db.prepare( + `INSERT INTO service_update_recovery + (id, node_id, stack_name, service_name, replicas_json, majority_image_id, + declared_image_ref, weak_floating_tag, health_gate_id, status, + expires_at, claim_expires_at, created_at, created_by) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + row.id, row.node_id, row.stack_name, row.service_name, row.replicas_json, row.majority_image_id, + row.declared_image_ref, row.weak_floating_tag, row.health_gate_id, row.status, + row.expires_at, row.claim_expires_at, row.created_at, row.created_by, + ); + } + + public getServiceUpdateRecovery(id: string): ServiceUpdateRecoveryRow | undefined { + return this.db.prepare( + 'SELECT * FROM service_update_recovery WHERE id = ?' + ).get(id) as ServiceUpdateRecoveryRow | undefined; + } + + /** Active, unexpired rows for one service, most recent first. */ + public listActiveServiceUpdateRecoveries(nodeId: number, stackName: string, serviceName: string): ServiceUpdateRecoveryRow[] { + return this.db.prepare( + `SELECT * FROM service_update_recovery + WHERE node_id = ? AND stack_name = ? AND service_name = ? AND status = 'active' + ORDER BY created_at DESC` + ).all(nodeId, stackName, serviceName) as ServiceUpdateRecoveryRow[]; + } + + /** Attach the update flow's own health gate run id while the row is still active. */ + public linkServiceUpdateRecoveryHealthGate(id: string, healthGateId: string): void { + this.db.prepare( + `UPDATE service_update_recovery SET health_gate_id = ? WHERE id = ? AND status = 'active'` + ).run(healthGateId, id); + } + + /** + * Atomic claim CAS for restore: active -> restoring, only when the row has + * not already expired. Returns the updated row, or undefined when another + * claim, a sweep, or a terminal status won the race. + */ + public claimServiceUpdateRecovery(id: string, claimExpiresAt: number, now: number): ServiceUpdateRecoveryRow | undefined { + const result = this.db.prepare( + `UPDATE service_update_recovery + SET status = 'restoring', claim_expires_at = ? + WHERE id = ? AND status = 'active' AND expires_at > ?` + ).run(claimExpiresAt, id, now); + if (result.changes === 0) return undefined; + return this.getServiceUpdateRecovery(id); + } + + /** Renewal CAS: only while still restoring. Cannot revive a consumed/expired/invalidated/active row. */ + public renewServiceUpdateRecoveryClaim(id: string, claimExpiresAt: number): boolean { + const result = this.db.prepare( + `UPDATE service_update_recovery SET claim_expires_at = ? WHERE id = ? AND status = 'restoring'` + ).run(claimExpiresAt, id); + return result.changes > 0; + } + + /** Restore succeeded: restoring -> consumed, optionally linking the restore's own health gate run. */ + public markServiceUpdateRecoveryConsumed(id: string, healthGateId: string | null = null): boolean { + const result = this.db.prepare( + `UPDATE service_update_recovery + SET status = 'consumed', claim_expires_at = NULL, health_gate_id = COALESCE(?, health_gate_id) + WHERE id = ? AND status = 'restoring'` + ).run(healthGateId, id); + return result.changes > 0; + } + + /** Mid-flight restore failure with the image still local: restoring -> active, claim cleared. */ + public reactivateServiceUpdateRecovery(id: string): boolean { + const result = this.db.prepare( + `UPDATE service_update_recovery SET status = 'active', claim_expires_at = NULL WHERE id = ? AND status = 'restoring'` + ).run(id); + return result.changes > 0; + } + + /** Mid-flight restore failure with the image gone: restoring -> invalidated. */ + public invalidateServiceUpdateRecovery(id: string): boolean { + const result = this.db.prepare( + `UPDATE service_update_recovery SET status = 'invalidated', claim_expires_at = NULL WHERE id = ? AND status = 'restoring'` + ).run(id); + return result.changes > 0; + } + + /** Startup/interval sweep: active rows whose TTL has lapsed. */ + public sweepExpiredActiveServiceUpdateRecoveries(now: number): number { + const result = this.db.prepare( + `UPDATE service_update_recovery SET status = 'expired' WHERE status = 'active' AND expires_at <= ?` + ).run(now); + return result.changes; + } + + /** + * Startup/interval sweep: restoring rows abandoned by a dead claim (process + * died mid-restore). A restoring row with a live claim is never expired here, + * even if its original expires_at has long passed. + */ + public sweepAbandonedRestoringServiceUpdateRecoveries(now: number): number { + const result = this.db.prepare( + `UPDATE service_update_recovery + SET status = 'expired' + WHERE status = 'restoring' AND claim_expires_at IS NOT NULL AND claim_expires_at <= ?` + ).run(now); + return result.changes; + } + + /** Image IDs currently protected from prune: active rows and restoring rows with a live claim. */ + public listHeldServiceUpdateRecoveryImageIds(nodeId: number, now: number): string[] { + const rows = this.db.prepare( + `SELECT DISTINCT majority_image_id FROM service_update_recovery + WHERE node_id = ? AND (status = 'active' OR (status = 'restoring' AND claim_expires_at > ?))` + ).all(nodeId, now) as Array<{ majority_image_id: string }>; + return rows.map(r => r.majority_image_id); + } + + public deleteServiceUpdateRecoveries(nodeId: number, stackName: string): void { + this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName); + } + // --- Notification History --- private mapNotificationRow(row: any): NotificationHistory { @@ -3569,6 +3857,7 @@ export class DatabaseService { this.db.prepare('DELETE FROM preflight_acknowledgements WHERE node_id = ?').run(id); this.db.prepare('DELETE FROM stack_exposure WHERE node_id = ?').run(id); this.db.prepare('DELETE FROM health_gate_runs WHERE node_id = ?').run(id); + this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ?').run(id); this.db.prepare('UPDATE blueprints SET pinned_node_id = NULL WHERE pinned_node_id = ?').run(id); this.deleteRoleAssignmentsByResource('node', String(id)); this.db.prepare('DELETE FROM fleet_sync_status WHERE node_id = ?').run(id); @@ -3644,6 +3933,12 @@ export class DatabaseService { // --- Stack Update Status --- + /** + * `services` is omitted by legacy (non-service-aware) callers; omitting it + * leaves the stack's existing services_json untouched (COALESCE against the + * current row) rather than erasing a per-service breakdown that a model-aware + * caller persisted on a previous check. + */ public upsertStackUpdateStatus( nodeId: number, stackName: string, @@ -3651,16 +3946,20 @@ export class DatabaseService { checkedAt: number, checkStatus: StackCheckStatus = 'ok', lastError: string | null = null, + services?: StackServiceStatus[], + generation = 0, ): void { + const servicesJson = services !== undefined ? stringifyServicesJson(services, generation) : null; this.db.prepare( - `INSERT INTO stack_update_status (node_id, stack_name, has_update, check_status, last_error, checked_at) - VALUES (?, ?, ?, ?, ?, ?) + `INSERT INTO stack_update_status (node_id, stack_name, has_update, check_status, last_error, checked_at, services_json) + VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(node_id, stack_name) DO UPDATE SET has_update = excluded.has_update, check_status = excluded.check_status, last_error = excluded.last_error, - checked_at = excluded.checked_at` - ).run(nodeId, stackName, hasUpdate ? 1 : 0, checkStatus, lastError, checkedAt); + checked_at = excluded.checked_at, + services_json = COALESCE(excluded.services_json, stack_update_status.services_json)` + ).run(nodeId, stackName, hasUpdate ? 1 : 0, checkStatus, lastError, checkedAt, servicesJson); } /** @@ -3671,15 +3970,24 @@ export class DatabaseService { * first-ever failed check inserts a row with has_update = 0 so the stack * still appears with its failure reason. */ - public recordStackCheckFailure(nodeId: number, stackName: string, lastError: string, checkedAt: number): void { + public recordStackCheckFailure( + nodeId: number, + stackName: string, + lastError: string, + checkedAt: number, + services?: StackServiceStatus[], + generation = 0, + ): void { + const servicesJson = services !== undefined ? stringifyServicesJson(services, generation) : null; this.db.prepare( - `INSERT INTO stack_update_status (node_id, stack_name, has_update, check_status, last_error, checked_at) - VALUES (?, ?, 0, 'failed', ?, ?) + `INSERT INTO stack_update_status (node_id, stack_name, has_update, check_status, last_error, checked_at, services_json) + VALUES (?, ?, 0, 'failed', ?, ?, ?) ON CONFLICT(node_id, stack_name) DO UPDATE SET check_status = 'failed', last_error = excluded.last_error, - checked_at = excluded.checked_at` - ).run(nodeId, stackName, lastError, checkedAt); + checked_at = excluded.checked_at, + services_json = COALESCE(excluded.services_json, stack_update_status.services_json)` + ).run(nodeId, stackName, lastError, checkedAt, servicesJson); } public getStackUpdateStatus(nodeId?: number): Record { @@ -3700,20 +4008,35 @@ export class DatabaseService { */ public getStackUpdateDetail(nodeId: number): Record { const rows = this.db.prepare( - 'SELECT stack_name, has_update, check_status, last_error, checked_at FROM stack_update_status WHERE node_id = ?' - ).all(nodeId) as Array<{ stack_name: string; has_update: number; check_status: string | null; last_error: string | null; checked_at: number }>; + 'SELECT stack_name, has_update, check_status, last_error, checked_at, services_json FROM stack_update_status WHERE node_id = ?' + ).all(nodeId) as Array<{ stack_name: string; has_update: number; check_status: string | null; last_error: string | null; checked_at: number; services_json: string | null }>; const result: Record = {}; for (const row of rows) { + const services = parseServicesJson(row.services_json); result[row.stack_name] = { hasUpdate: row.has_update === 1, checkStatus: (row.check_status === 'failed' || row.check_status === 'partial') ? row.check_status : 'ok', lastError: row.last_error, checkedAt: row.checked_at, + ...(services.length > 0 ? { services } : {}), }; } return result; } + /** + * Prior per-service breakdown for a single stack, used by ImageUpdateService + * to look up each service's last-known hasUpdate before reducing a fresh + * check (so a preserved value survives a partial/failed re-check). Corrupt + * or missing JSON yields an empty list. + */ + public getStackServicesJson(nodeId: number, stackName: string): StackServiceStatus[] { + const row = this.db.prepare( + 'SELECT services_json FROM stack_update_status WHERE node_id = ? AND stack_name = ?' + ).get(nodeId, stackName) as { services_json: string | null } | undefined; + return parseServicesJson(row?.services_json); + } + public clearStackUpdateStatus(nodeId: number, stackName: string): void { this.db.prepare('DELETE FROM stack_update_status WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName); } diff --git a/backend/src/services/DockerController.ts b/backend/src/services/DockerController.ts index 6b4e537f..cf1effc3 100644 --- a/backend/src/services/DockerController.ts +++ b/backend/src/services/DockerController.ts @@ -403,13 +403,56 @@ class DockerController { }; } + /** True when an image is unused and has no meaningful tag (dangling / untagged). */ + private static isDanglingImage(img: { RepoTags?: string[] | null; Containers?: number }): boolean { + if ((img.Containers ?? 0) !== 0) return false; + const tags = img.RepoTags ?? []; + if (tags.length === 0) return true; + return tags.every((t) => t === ':'); + } + + /** + * Enumerate candidate images and delete individually. Used when a recovery + * hold predicate is supplied so held images are never removed by a bulk + * Docker prune API. Reclaimed bytes come from a LayersSize before/after delta. + */ + private async pruneImagesEnumerated( + eligible: (img: { Id: string; RepoTags?: string[] | null; Containers?: number }) => boolean, + isImageHeld?: (imageId: string) => boolean, + ): Promise<{ success: boolean; reclaimedBytes: number }> { + const beforeDf = await this.safeDfSnapshot(); + const rawImages = await this.docker.listImages({ all: false }); + const candidates = (rawImages as Array<{ Id: string; RepoTags?: string[] | null; Containers?: number }>) + .filter((img) => eligible(img) && !isImageHeld?.(img.Id)); + + for (const img of candidates) { + if (isImageHeld?.(img.Id)) continue; + try { + await this.docker.getImage(img.Id).remove({ force: true }); + } catch (e) { + console.error(`[pruneImagesEnumerated] Failed to remove image ${sanitizeForLog(img.Id)}:`, e); + } + } + + const afterDf = await this.safeDfSnapshot(); + let reclaimedBytes = 0; + if (typeof beforeDf?.LayersSize === 'number' && typeof afterDf?.LayersSize === 'number') { + reclaimedBytes = Math.max(0, beforeDf.LayersSize - afterDf.LayersSize); + } + return { success: true, reclaimedBytes }; + } + // Sencho's own image, networks, and named volumes are always in use by the // running container, so Docker's server-side prune APIs (pruneContainers, // pruneImages, pruneNetworks, pruneVolumes) skip them by definition. No // extra self-guard is needed at this layer; the `managed` scope path goes // through `pruneManagedOnly`, which adds an explicit self filter for // defense-in-depth. - public async pruneSystem(target: 'containers' | 'images' | 'networks' | 'volumes', labelFilter?: string) { + public async pruneSystem( + target: 'containers' | 'images' | 'networks' | 'volumes', + labelFilter?: string, + isImageHeld?: (imageId: string) => boolean, + ) { let spaceReclaimed = 0; if (target === 'containers') { const filters: Record = {}; @@ -417,6 +460,9 @@ class DockerController { const r = await this.docker.pruneContainers({ filters }); spaceReclaimed = r.SpaceReclaimed || 0; } else if (target === 'images') { + if (isImageHeld) { + return this.pruneImagesEnumerated((img) => (img.Containers ?? 0) === 0, isImageHeld); + } // Remove all unused images, not just dangling ones const filters: Record> = { dangling: { 'false': true } }; if (labelFilter) filters.label = [labelFilter]; @@ -444,7 +490,10 @@ class DockerController { // which uses { dangling: { 'false': true } } to remove every unused image. // Used by the prune-on-update flow to reclaim the layers a pull/recreate // orphans, without touching tagged images for stopped stacks. - public async pruneDanglingImages(): Promise<{ success: boolean; reclaimedBytes: number }> { + public async pruneDanglingImages(isImageHeld?: (imageId: string) => boolean): Promise<{ success: boolean; reclaimedBytes: number }> { + if (isImageHeld) { + return this.pruneImagesEnumerated((img) => DockerController.isDanglingImage(img), isImageHeld); + } const filters: Record> = { dangling: { 'true': true } }; const r = await this.docker.pruneImages({ filters }); return { success: true, reclaimedBytes: r.SpaceReclaimed || 0 }; @@ -635,7 +684,8 @@ class DockerController { public async pruneManagedOnly( target: 'images' | 'volumes' | 'networks', - knownStackNames: string[] + knownStackNames: string[], + isImageHeld?: (imageId: string) => boolean, ): Promise<{ success: boolean; reclaimedBytes: number }> { const knownSet = new Set(knownStackNames); const projectToStack = await DockerController.resolveProjectNameMap(knownStackNames); @@ -707,6 +757,7 @@ class DockerController { // once, but the per-image formula subtracts it from every referrer). const beforeDf = await this.safeDfSnapshot(); await Promise.all(prunable.map(async (img) => { + if (isImageHeld?.(img.Id)) return; try { await this.docker.getImage(img.Id).remove({ force: true }); } catch (e) { @@ -832,6 +883,7 @@ class DockerController { scope: PruneScope, knownStackNames: string[], nodeId: number = this.nodeId, + isImageHeld?: (imageId: string) => boolean, ): Promise { const ordered = normalizePruneTargets(targets); const knownSet = new Set(knownStackNames); @@ -980,6 +1032,7 @@ class DockerController { const freeingImages = ordered.includes('containers'); for (const img of rawImages) { if (selfIdentity.isOwnImage(img.Id)) continue; + if (isImageHeld?.(img.Id)) continue; const containers = img.Containers ?? 0; const refs = imageToContainerIds.get(img.Id) ?? []; const becomesFree = freeingImages @@ -1038,8 +1091,9 @@ class DockerController { public async assertPlanFresh( plan: PrunePlan, knownStackNames: string[], + isImageHeld?: (imageId: string) => boolean, ): Promise { - const rebuilt = await this.buildPrunePlan(plan.targets, plan.scope, knownStackNames, plan.nodeId); + const rebuilt = await this.buildPrunePlan(plan.targets, plan.scope, knownStackNames, plan.nodeId, isImageHeld); if (rebuilt.fingerprint !== plan.fingerprint) return null; return rebuilt; } @@ -1051,8 +1105,9 @@ class DockerController { public async executePrunePlan( plan: PrunePlan, knownStackNames: string[], + isImageHeld?: (imageId: string) => boolean, ): Promise<{ outcomes: PruneItemOutcome[]; reclaimedBytes: number; success: boolean }> { - const fresh = await this.assertPlanFresh(plan, knownStackNames); + const fresh = await this.assertPlanFresh(plan, knownStackNames, isImageHeld); if (!fresh) throw new PrunePlanStaleError(); const knownSet = new Set(knownStackNames); @@ -1123,6 +1178,15 @@ class DockerController { } if (target === 'images') { + if (isImageHeld?.(item.id)) { + outcomes.push({ + id: item.id, + target: 'images', + status: 'skipped', + reason: 'Image held for pending service-update recovery', + }); + continue; + } if (imageIdBlocked(item.id)) { const stillReferenced = await this.imageStillReferenced(item.id); if (stillReferenced) { diff --git a/backend/src/services/DriftLedgerService.ts b/backend/src/services/DriftLedgerService.ts index 8f116b86..73f489a8 100644 --- a/backend/src/services/DriftLedgerService.ts +++ b/backend/src/services/DriftLedgerService.ts @@ -179,6 +179,65 @@ export class DriftLedgerService { return { detected: toInsert.length, resolved: toResolve.length }; } + /** + * Service-scoped counterpart to `reconcile`: only inserts/resolves findings + * for `serviceName`, leaving every other service's (and the empty/`null` + * service, i.e. network-level) open findings untouched. Used by the service + * update orchestrator so a service-scoped update never resolves drift it + * did not touch. Never pass a report already filtered to one service into + * the stack-wide `reconcile`; that would falsely resolve every other + * service's open findings. + */ + reconcileService(nodeId: number, stackName: string, serviceName: string, report: StackDriftReport): DriftReconcileResult { + if (report.status === 'unreachable' || report.parseError) { + return { detected: 0, resolved: 0 }; + } + const db = DatabaseService.getInstance(); + const now = Date.now(); + const openForService = db.getOpenDriftFindings(nodeId, stackName).filter(r => r.service === serviceName); + const openByKey = new Map(openForService.map(r => [findingKey(r.service, r.finding_type), r])); + const currentForService = report.findings.filter(f => f.service === serviceName); + const currentByKey = new Map(currentForService.map(f => [findingKey(f.service, f.kind), f])); + + const toInsert: StackDriftFinding[] = []; + for (const [key, f] of currentByKey) { + if (!openByKey.has(key)) toInsert.push(f); + } + const toResolve: StackDriftFindingRow[] = []; + for (const [key, row] of openByKey) { + if (!currentByKey.has(key)) toResolve.push(row); + } + db.getDb().transaction(() => { + db.setStackDossierDriftCheck(nodeId, stackName, now); + for (const f of toInsert) { + db.insertDriftFinding({ + node_id: nodeId, + stack_name: stackName, + service: f.service, + finding_type: f.kind, + severity: 'warning', + message: f.detail, + expected_json: f.expected !== undefined ? JSON.stringify(f.expected) : null, + actual_json: f.actual !== undefined ? JSON.stringify(f.actual) : null, + detected_at: now, + }); + } + for (const row of toResolve) { + db.resolveDriftFinding(row.id, now); + } + })(); + + if (toInsert.length > 0) { + this.recordActivity(nodeId, stackName, 'drift_detected', 'warning', + `Drift detected on ${stackName} (${serviceName}): ${toInsert.length} new finding${toInsert.length === 1 ? '' : 's'}`, now); + } + if (toResolve.length > 0) { + this.recordActivity(nodeId, stackName, 'drift_resolved', 'info', + `Drift resolved on ${stackName} (${serviceName}): ${toResolve.length} finding${toResolve.length === 1 ? '' : 's'} cleared`, now); + } + return { detected: toInsert.length, resolved: toResolve.length }; + } + /** * Build the spatial report for one stack and reconcile it into the ledger. * Used by the deploy and update success hooks (and the rollback route, which @@ -196,6 +255,22 @@ export class DriftLedgerService { } } + /** + * Build the spatial report for one stack and reconcile only `serviceName`'s + * findings into the ledger. Used by the service update orchestrator's + * success path. Best-effort: a build or reconcile failure is logged and + * swallowed so it never fails the update that triggered it. + */ + async reconcileServiceForStack(nodeId: number, stackName: string, serviceName: string): Promise { + try { + const report = await buildStackDriftReport(nodeId, stackName); + return this.reconcileService(nodeId, stackName, serviceName, report); + } catch (error) { + console.error('[DriftLedger] reconcileServiceForStack failed for %s/%s:', sanitizeForLog(stackName), sanitizeForLog(serviceName), sanitizeForLog(getErrorMessage(error, 'unknown'))); + return { detected: 0, resolved: 0 }; + } + } + /** * Write a drift transition to the stack activity timeline. History-only (no * external channel dispatch): a drift signal belongs in the activity feed, not diff --git a/backend/src/services/GitSourceService.ts b/backend/src/services/GitSourceService.ts index fa3b9899..fdaaa56c 100644 --- a/backend/src/services/GitSourceService.ts +++ b/backend/src/services/GitSourceService.ts @@ -1337,7 +1337,7 @@ export class GitSourceService { console.warn(`[GitSource] ${busy}`); return { applied: true, deployed: false, deployError: busy }; } - HealthGateService.getInstance().begin(nodeId, stackName, 'deploy', 'system:git-source'); + HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:git-source'); console.log(`[GitSource] Applied and deployed ${stackName} at ${commitSha.slice(0, 7)}`); return { applied: true, deployed: true }; } catch (e) { diff --git a/backend/src/services/HealthGateService.ts b/backend/src/services/HealthGateService.ts index c085bcfb..0e51f1ce 100644 --- a/backend/src/services/HealthGateService.ts +++ b/backend/src/services/HealthGateService.ts @@ -1,12 +1,22 @@ import { randomUUID } from 'crypto'; import DockerController from './DockerController'; import { DatabaseService, type HealthGateRunRow } from './DatabaseService'; +import { AutoHealService } from './AutoHealService'; import { sanitizeForLog } from '../utils/safeLog'; import { getErrorMessage } from '../utils/errors'; import { withTimeout } from '../utils/withTimeout'; import type { HealthGateContainer, HealthGateReport } from './updateGuard/types'; +import { getComposeCommandTimeoutMs } from './ComposeService'; const POLL_INTERVAL_MS = 5_000; +// A prepared-but-never-begun token (prepare called, mutation then failed before +// beginPrepared) is dropped after this long. Swept lazily on each prepare/ +// attach/beginPrepared call rather than on a timer, so the service leaves no +// standing interval (the observation poll timers are the only timers it arms). +// Must outlive the longest Compose mutation (compose timeout + 5m buffer). +function prepareTtlMs(): number { + return Math.max(getComposeCommandTimeoutMs(), 30 * 60_000) + 5 * 60_000; +} // Per-observe ceiling so a hung Docker socket cannot leave a poll pending // forever. Above POLL_INTERVAL_MS so a slow-but-live probe is not cut short; a // timeout counts as a poll error and three in a row resolve the gate unknown. @@ -20,6 +30,8 @@ const MAX_WINDOW_SECONDS = 600; // updates). Gates past the cap finalize immediately as unknown. const MAX_CONCURRENT_GATES = 25; +type GateRole = 'primary' | 'collateral'; + interface ObservedContainer { id: string; name: string; @@ -31,6 +43,10 @@ interface ObservedContainer { restarts: number; state: string; health: string | null; + /** `com.docker.compose.service` label, when present (service gates only). */ + service: string | null; + /** Image id the container is running (service gates check convergence on it). */ + imageId: string; } interface ActiveGate { @@ -53,6 +69,90 @@ interface ActiveGate { * or stopped can never overwrite the terminal verdict with a stale one. */ finalized: boolean; + /** 'stack' for the legacy post-mutation gate, 'service' for prepared gates. */ + targetScope: 'stack' | 'service'; + /** Named trigger persisted on the row. */ + trigger: 'update' | 'deploy' | 'service_update' | 'service_restore'; + /** Service gates only. */ + serviceName: string | null; + /** Single image id every primary replica must converge on (service gates). */ + expectedImageId: string | null; + /** Expected primary replica count from the effective model (service gates; may be 0). */ + expectedReplicas: number; + /** Sibling names eligible for regression evaluation (service gates). */ + collateralEligibleNames: Set; + /** + * Pre-mutation baselines for regression-eligible collateral siblings. + * Used to seed `expected` at arm time so a sibling that vanishes during the + * mutation gap is still tracked (and can fail) on the first poll. + */ + collateralBaselineByName: Map; + /** Role of each expected container (service gates), for failure attribution. */ + roleByName: Map; +} + +/** A pre-mutation baseline container captured at prepare time. */ +interface PreparedBaseline { + id: string; + name: string; + service: string | null; + state: string; + health: string | null; + restartCount: number; + startedAt: string | null; + imageId: string; +} + +function isRegressionEligibleSibling(baseline: PreparedBaseline): boolean { + return baseline.state === 'running' && (baseline.health === null || baseline.health === 'healthy'); +} + +function observedFromPreparedBaseline(baseline: PreparedBaseline): ObservedContainer { + return { + id: baseline.id, + name: baseline.name, + startedAt: baseline.startedAt, + restartCount: baseline.restartCount, + restarts: 0, + state: baseline.state, + health: baseline.health, + service: baseline.service, + imageId: baseline.imageId, + }; +} + +/** An in-memory prepare snapshot awaiting attachExpectedImage + beginPrepared. */ +interface PreparedGate { + token: string; + nodeId: number; + stackName: string; + serviceName: string; + trigger: 'service_update' | 'service_restore'; + windowSeconds: number; + expiresAt: number; + expectedReplicas: number; + expectedImageId: string | null; + collateralEligibleNames: Set; + primaryBaseline: PreparedBaseline[]; + collateralBaseline: PreparedBaseline[]; +} + +export interface PrepareInput { + nodeId: number; + stackName: string; + target: { scope: 'service'; serviceName: string }; + trigger: 'service_update' | 'service_restore'; + /** + * Expected primary replica count from the effective model (may be 0). Passed + * by the orchestrator, which already rendered the model, so the gate stays a + * pure observer and does not run a second `docker compose config`. + */ + expectedReplicas: number; +} + +export interface BeginPreparedResult { + runId: string | null; + observing: boolean; } /** @@ -71,6 +171,8 @@ interface ActiveGate { export class HealthGateService { private static instance: HealthGateService; private readonly active = new Map(); + /** Prepare tokens awaiting beginPrepared (service update/restore only). */ + private readonly prepared = new Map(); private started = false; public static getInstance(): HealthGateService { @@ -101,6 +203,51 @@ export class HealthGateService { for (const gate of [...this.active.values()]) { this.finalize(gate, 'unknown', 'shutdown during observation', []); } + this.prepared.clear(); + } + + private gateKey(nodeId: number, stackName: string, scope: 'stack' | 'service', serviceName: string | null): string { + return `${nodeId}:${stackName}:${scope}:${serviceName ?? '_'}`; + } + + private baselineMapFromPrepared(baselines: PreparedBaseline[]): Map { + const map = new Map(); + for (const baseline of baselines) { + map.set(baseline.name, observedFromPreparedBaseline(baseline)); + } + return map; + } + + /** + * Finalize in-flight gates that a newer operation must replace: + * - A stack-scoped begin supersedes every gate on the stack. + * - A service-scoped begin supersedes only the same service's gate and any + * stack-scoped gate; other services keep observing independently. + */ + private supersedeGatesForStack( + nodeId: number, + stackName: string, + options?: { serviceName?: string | null; stackOnly?: boolean }, + ): void { + const serviceName = options?.serviceName; + const stackOnly = options?.stackOnly === true; + for (const gate of [...this.active.values()]) { + if (gate.nodeId !== nodeId || gate.stackName !== stackName) continue; + if (serviceName !== undefined) { + // Service begin: keep other services' gates. + if (gate.targetScope === 'service' && gate.serviceName !== serviceName) continue; + } else if (stackOnly) { + if (gate.targetScope !== 'stack') continue; + } + this.finalize(gate, 'unknown', 'superseded by a newer operation', []); + } + } + + /** Drop prepare tokens whose TTL elapsed without a beginPrepared (lazy, no standing timer). */ + private sweepExpiredPrepared(now: number): void { + for (const [token, prep] of this.prepared) { + if (now >= prep.expiresAt) this.prepared.delete(token); + } } /** @@ -114,7 +261,7 @@ export class HealthGateService { * every gated update path gets the timeline marker even when the gate * itself is disabled. */ - public begin( + public beginStack( nodeId: number, stackName: string, trigger: 'update' | 'deploy', @@ -132,12 +279,10 @@ export class HealthGateService { } if (!settings.enabled) return null; - // A newer operation supersedes an in-flight gate for the same stack. - const key = `${nodeId}:${stackName}`; - const existing = this.active.get(key); - if (existing) { - this.finalize(existing, 'unknown', 'superseded by a newer update', []); - } + // A newer stack operation supersedes every in-flight gate on this stack + // (including service gates). + const key = this.gateKey(nodeId, stackName, 'stack', null); + this.supersedeGatesForStack(nodeId, stackName); const runId = randomUUID(); const startedAt = Date.now(); @@ -153,6 +298,9 @@ export class HealthGateService { started_at: startedAt, ended_at: null, created_by: actor, + target_scope: 'stack', + service_name: null, + failure_source: null, }; if (this.active.size >= MAX_CONCURRENT_GATES) { @@ -173,6 +321,14 @@ export class HealthGateService { missingLastPoll: new Set(), restartingLastPoll: new Set(), finalized: false, + targetScope: 'stack', + trigger, + serviceName: null, + expectedImageId: null, + expectedReplicas: 0, + collateralEligibleNames: new Set(), + collateralBaselineByName: new Map(), + roleByName: new Map(), }; this.active.set(key, gate); this.scheduleNextPoll(gate); @@ -180,13 +336,169 @@ export class HealthGateService { } catch (error) { // The gate is an observer; its failure must never fail the operation. console.error( - '[HealthGate] begin (%s) failed for %s on node %d:', + '[HealthGate] beginStack (%s) failed for %s on node %d:', trigger, sanitizeForLog(stackName), nodeId, error, ); return null; } } + /** @deprecated Prefer beginStack; retained as a one-PR alias for callers under migration. */ + public begin( + nodeId: number, + stackName: string, + trigger: 'update' | 'deploy', + actor: string | null, + ): string | null { + return this.beginStack(nodeId, stackName, trigger, actor); + } + + /** + * Pre-mutation snapshot for a service-scoped update/restore. Captures the + * selected service's primary replicas and the sibling collateral set (marking + * only currently-running, healthy or healthcheck-less siblings as + * regression-eligible), and returns an opaque token the orchestrator carries + * through attachExpectedImage and beginPrepared. Never throws: a Docker read + * failure yields snapshotOk=false so the orchestrator can skip observation + * rather than claim a gate with an empty baseline. + */ + public async prepare(input: PrepareInput): Promise<{ prepareToken: string; expiresAt: number; snapshotOk: boolean }> { + const now = Date.now(); + this.sweepExpiredPrepared(now); + const { nodeId, stackName, trigger } = input; + const serviceName = input.target.serviceName; + const settings = this.readSettings(); + + let baselines: PreparedBaseline[] = []; + let snapshotOk = true; + try { + baselines = await this.snapshotBaselines(nodeId, stackName); + } catch (error) { + snapshotOk = false; + console.warn('[HealthGate] prepare snapshot failed for %s/%s:', + sanitizeForLog(stackName), sanitizeForLog(serviceName), getErrorMessage(error, 'unknown')); + } + + const primaryBaseline = baselines.filter(b => b.service === serviceName); + const collateralBaseline = baselines.filter(b => b.service !== serviceName); + const collateralEligibleNames = new Set( + collateralBaseline.filter(isRegressionEligibleSibling).map(b => b.name), + ); + + const token = randomUUID(); + const expiresAt = now + prepareTtlMs(); + this.prepared.set(token, { + token, + nodeId, + stackName, + serviceName, + trigger, + windowSeconds: settings.windowSeconds, + expiresAt, + expectedReplicas: input.expectedReplicas, + expectedImageId: null, + collateralEligibleNames, + primaryBaseline, + collateralBaseline, + }); + return { prepareToken: token, expiresAt, snapshotOk }; + } + + /** Record the single image id every primary replica must converge on. No-op for an unknown/expired token. */ + public attachExpectedImage(prepareToken: string, expectedImageId: string): void { + this.sweepExpiredPrepared(Date.now()); + const prep = this.prepared.get(prepareToken); + if (!prep) { + console.warn('[HealthGate] attachExpectedImage: unknown or expired prepare token'); + return; + } + prep.expectedImageId = expectedImageId; + } + + /** + * Begin observing a prepared service gate after its mutation. Mirrors + * beginStack's nullability: returns a null runId (and observing:false) when + * the service is not started, gating is disabled, or the insert fails; a + * non-null runId with observing:false past the concurrency cap; and a non-null + * runId with observing:true once the poll is armed. The prepare token is + * consumed either way. Never throws. + */ + public beginPrepared(input: { prepareToken: string; actor: string | null }): BeginPreparedResult { + const now = Date.now(); + this.sweepExpiredPrepared(now); + const prep = this.prepared.get(input.prepareToken); + // Always consume the token so a caller cannot begin twice off one prepare. + if (prep) this.prepared.delete(input.prepareToken); + + if (!this.started || !prep) return { runId: null, observing: false }; + + try { + const db = DatabaseService.getInstance(); + const settings = this.readSettings(); + if (!settings.enabled) return { runId: null, observing: false }; + + const key = this.gateKey(prep.nodeId, prep.stackName, 'service', prep.serviceName); + // Same-service + stack gates only; sibling service observations continue. + this.supersedeGatesForStack(prep.nodeId, prep.stackName, { serviceName: prep.serviceName }); + + const runId = randomUUID(); + const startedAt = Date.now(); + const row: HealthGateRunRow = { + id: runId, + node_id: prep.nodeId, + stack_name: prep.stackName, + trigger_action: prep.trigger, + status: 'observing', + reason: null, + window_seconds: prep.windowSeconds, + containers_json: '[]', + started_at: startedAt, + ended_at: null, + created_by: input.actor, + target_scope: 'service', + service_name: prep.serviceName, + failure_source: null, + }; + + if (this.active.size >= MAX_CONCURRENT_GATES) { + db.insertHealthGateRun({ ...row, status: 'unknown', reason: 'too many concurrent observations', ended_at: startedAt }); + return { runId, observing: false }; + } + + db.insertHealthGateRun(row); + const gate: ActiveGate = { + runId, + nodeId: prep.nodeId, + stackName: prep.stackName, + windowSeconds: prep.windowSeconds, + startedAt, + timer: null, + expected: null, + consecutivePollErrors: 0, + missingLastPoll: new Set(), + restartingLastPoll: new Set(), + finalized: false, + targetScope: 'service', + trigger: prep.trigger, + serviceName: prep.serviceName, + expectedImageId: prep.expectedImageId, + expectedReplicas: prep.expectedReplicas, + collateralEligibleNames: prep.collateralEligibleNames, + collateralBaselineByName: this.baselineMapFromPrepared( + prep.collateralBaseline.filter(b => prep.collateralEligibleNames.has(b.name)), + ), + roleByName: new Map(), + }; + this.active.set(key, gate); + this.scheduleNextPoll(gate); + return { runId, observing: true }; + } catch (error) { + console.error('[HealthGate] beginPrepared failed for %s/%s:', + sanitizeForLog(prep.stackName), sanitizeForLog(prep.serviceName), error); + return { runId: null, observing: false }; + } + } + /** A specific run by id, the latest run, or the never-run sentinel. */ public getReport(nodeId: number, stackName: string, gateId?: string): HealthGateReport { const db = DatabaseService.getInstance(); @@ -197,6 +509,7 @@ export class HealthGateService { return { stack: stackName, id: null, status: 'never-run', trigger: null, reason: null, windowSeconds: null, startedAt: null, endedAt: null, containers: [], + targetScope: 'stack', serviceName: null, failureSource: null, }; } let containers: HealthGateContainer[] = []; @@ -217,6 +530,9 @@ export class HealthGateService { startedAt: row.started_at, endedAt: row.ended_at, containers, + targetScope: row.target_scope ?? 'stack', + serviceName: row.service_name ?? null, + failureSource: row.failure_source ?? null, }; } @@ -227,11 +543,15 @@ export class HealthGateService { * or corrupt the restart/missing accounting that assumes one poll at a time. */ private async poll(gate: ActiveGate): Promise { - const key = `${gate.nodeId}:${gate.stackName}`; + const key = this.gateKey(gate.nodeId, gate.stackName, gate.targetScope, gate.serviceName); // A late timer fire after supersede, stop, or finalize must do nothing. if (gate.finalized || this.active.get(key) !== gate) return; try { - await this.runPollCycle(gate, key); + if (gate.targetScope === 'service') { + await this.runServicePollCycle(gate, key); + } else { + await this.runPollCycle(gate, key); + } } finally { if (!gate.finalized && this.active.get(key) === gate) { this.scheduleNextPoll(gate); @@ -352,16 +672,201 @@ export class HealthGateService { this.finalize(gate, 'passed', null, summary); } + /** + * Poll cycle for a prepared service gate. Observes the selected service's + * primary replicas (image convergence, replica count, health/restart) and the + * regression-eligible collateral siblings, attributing a failure to + * 'primary' or 'collateral'. Unrelated containers that appear after prepare do + * not expand the observed set. + */ + private async runServicePollCycle(gate: ActiveGate, key: string): Promise { + let observed: ObservedContainer[]; + try { + observed = await withTimeout( + this.observeContainers(gate), OBSERVE_TIMEOUT_MS, 'health gate observe', + ); + } catch (error) { + gate.consecutivePollErrors += 1; + console.warn( + '[HealthGate] service poll error %d for %s/%s:', + gate.consecutivePollErrors, sanitizeForLog(gate.stackName), + sanitizeForLog(gate.serviceName ?? ''), getErrorMessage(error, 'unknown'), + ); + if (gate.consecutivePollErrors >= 3) { + this.finalize(gate, 'unknown', 'Docker became unreachable during observation', []); + } + return; + } + if (gate.finalized || this.active.get(key) !== gate) return; + gate.consecutivePollErrors = 0; + + const elapsedMs = Date.now() - gate.startedAt; + const serviceName = gate.serviceName ?? ''; + + // Arm the expected set once from post-mutation primary replicas plus the + // pre-mutation regression-eligible collateral baselines. Seeding collateral + // from the prepare baseline (not only from the first post-mutation poll) + // means a healthy sibling that vanished during the mutation gap is still + // tracked and can fail the gate. Scale-0 arms immediately with an empty + // primary set; scale>0 waits (up to the empty grace) for replicas. + if (gate.expected === null) { + const primary = observed.filter(c => c.service === serviceName); + if (gate.expectedReplicas > 0 && primary.length === 0) { + if (elapsedMs >= EMPTY_GRACE_MS) { + this.finalize(gate, 'failed', `service ${serviceName} has no running replicas to observe`, [], 'primary'); + } + return; + } + const expected = new Map(); + for (const c of primary) { + gate.roleByName.set(c.name, 'primary'); + expected.set(c.name, c); + } + const observedByName = new Map(observed.map(c => [c.name, c])); + for (const [name, baseline] of gate.collateralBaselineByName) { + if (gate.roleByName.has(name)) continue; + gate.roleByName.set(name, 'collateral'); + // Prefer the live observation when the sibling is still present; otherwise + // keep the prepare baseline so a missing sibling enters the miss path. + expected.set(name, observedByName.get(name) ?? baseline); + } + // Also pick up eligible siblings that appeared under a new name after mutation + // (unusual, but keeps prior behavior for containers still in the eligible set). + for (const c of observed) { + if (gate.collateralEligibleNames.has(c.name) && !gate.roleByName.has(c.name)) { + gate.roleByName.set(c.name, 'collateral'); + expected.set(c.name, c); + } + } + gate.expected = expected; + return; + } + + const byName = new Map(observed.map(c => [c.name, c])); + + for (const [name, baseline] of gate.expected) { + const current = byName.get(name); + if (!current) continue; + const restarted = + current.id !== baseline.id || + current.restartCount > baseline.restartCount || + (current.startedAt !== null && baseline.startedAt !== null && current.startedAt !== baseline.startedAt); + current.restarts = baseline.restarts + (restarted ? 1 : 0); + } + const summary = this.summarizeService(gate, byName); + + // Primary image convergence: a primary replica on any image id other than + // the single attached expected id fails the gate. + if (gate.expectedImageId) { + const wrongImage = observed.find( + c => c.service === serviceName && c.imageId && c.imageId !== gate.expectedImageId, + ); + if (wrongImage) { + this.finalize(gate, 'failed', `service ${serviceName} replica ${wrongImage.name} is running an unexpected image`, summary, 'primary'); + return; + } + } + + for (const [name, baseline] of gate.expected) { + const role = gate.roleByName.get(name) ?? 'collateral'; + const noun = role === 'primary' ? 'replica' : 'sibling'; + const current = byName.get(name); + if (!current) { + if (gate.missingLastPoll.has(name)) { + this.finalize(gate, 'failed', `${noun} ${name} disappeared during observation`, summary, role); + return; + } + gate.missingLastPoll.add(name); + continue; + } + gate.missingLastPoll.delete(name); + + if (current.state === 'exited' && baseline.restarts === current.restarts) { + this.finalize(gate, 'failed', `${noun} ${name} exited during observation`, summary, role); + return; + } + if (current.health === 'unhealthy') { + this.finalize(gate, 'failed', `${noun} ${name} reported unhealthy`, summary, role); + return; + } + if (current.restarts >= 2) { + this.finalize(gate, 'failed', `${noun} ${name} is restart looping`, summary, role); + return; + } + if (current.state === 'restarting') { + if (gate.restartingLastPoll.has(name)) { + this.finalize(gate, 'failed', `${noun} ${name} is stuck restarting`, summary, role); + return; + } + gate.restartingLastPoll.add(name); + } else { + gate.restartingLastPoll.delete(name); + } + gate.expected.set(name, current); + } + + if (elapsedMs < gate.windowSeconds * 1000) return; + + const stillStarting = observed.filter( + c => c.health === 'starting' && (c.service === serviceName || gate.collateralEligibleNames.has(c.name)), + ); + if (stillStarting.length > 0) { + this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary); + return; + } + const runningPrimary = observed.filter(c => c.service === serviceName && c.state === 'running'); + if (runningPrimary.length !== gate.expectedReplicas) { + this.finalize( + gate, 'failed', + `service ${serviceName} has ${runningPrimary.length} running replica(s), expected ${gate.expectedReplicas}`, + summary, 'primary', + ); + return; + } + const collateralNotRunning = [...gate.expected.keys()] + .filter(name => gate.roleByName.get(name) === 'collateral') + .filter(name => byName.get(name)?.state !== 'running'); + if (collateralNotRunning.length > 0) { + this.finalize(gate, 'failed', `sibling(s) not running at the end of the window: ${collateralNotRunning.join(', ')}`, summary, 'collateral'); + return; + } + this.finalize(gate, 'passed', null, summary); + } + + private summarizeService( + gate: ActiveGate, + current: Map, + ): HealthGateContainer[] { + if (!gate.expected) return []; + return [...gate.expected.values()].map(baseline => { + const now = current.get(baseline.name); + return { + name: baseline.name, + state: now?.state ?? 'missing', + health: now?.health ?? null, + restarts: now?.restarts ?? baseline.restarts, + service: (now ?? baseline).service ?? gate.serviceName, + role: gate.roleByName.get(baseline.name) ?? null, + }; + }); + } + private async observeContainers(gate: ActiveGate): Promise { - const docker = DockerController.getInstance(gate.nodeId).getDocker(); + return this.listStackContainers(gate.nodeId, gate.stackName); + } + + /** List and inspect a stack's containers into the gate's observation shape. */ + private async listStackContainers(nodeId: number, stackName: string): Promise { + const docker = DockerController.getInstance(nodeId).getDocker(); const listed = await docker.listContainers({ all: true, - filters: { label: [`com.docker.compose.project=${gate.stackName}`] }, + filters: { label: [`com.docker.compose.project=${stackName}`] }, }); const observed = await Promise.all( listed.map(async (info): Promise => { try { const inspect = await docker.getContainer(info.Id).inspect(); + const labels = (info.Labels ?? inspect.Config?.Labels ?? {}) as Record; return { id: info.Id, name: info.Names?.[0]?.replace(/^\//, '') ?? info.Id.slice(0, 12), @@ -370,6 +875,8 @@ export class HealthGateService { restarts: 0, state: inspect.State?.Status ?? info.State ?? 'unknown', health: inspect.State?.Health?.Status ?? null, + service: labels['com.docker.compose.service'] ?? null, + imageId: inspect.Image ?? '', }; } catch (e: unknown) { // Removed between list and inspect; the missing-container logic will @@ -382,6 +889,23 @@ export class HealthGateService { return observed.filter((c): c is ObservedContainer => c !== null); } + /** Snapshot the current per-container facts for prepare's primary/collateral partition. */ + private async snapshotBaselines(nodeId: number, stackName: string): Promise { + const observed = await withTimeout( + this.listStackContainers(nodeId, stackName), OBSERVE_TIMEOUT_MS, 'health gate prepare snapshot', + ); + return observed.map(c => ({ + id: c.id, + name: c.name, + service: c.service, + state: c.state, + health: c.health, + restartCount: c.restartCount, + startedAt: c.startedAt, + imageId: c.imageId, + })); + } + private summarize( expected: Map, current: Map, @@ -402,16 +926,29 @@ export class HealthGateService { status: 'passed' | 'failed' | 'unknown', reason: string | null, containers: HealthGateContainer[], + failureSource: 'primary' | 'collateral' | null = null, ): void { if (gate.finalized) return; gate.finalized = true; if (gate.timer) clearTimeout(gate.timer); - const key = `${gate.nodeId}:${gate.stackName}`; + const key = this.gateKey(gate.nodeId, gate.stackName, gate.targetScope, gate.serviceName); if (this.active.get(key) === gate) this.active.delete(key); + // A service gate owns its service's Auto-Heal suppression while observing; + // release it here so a failed/superseded/stopped gate never leaves auto-heal + // muted for that service. + if (gate.targetScope === 'service' && gate.serviceName) { + try { + AutoHealService.getInstance().clearSuppress(gate.nodeId, gate.stackName, gate.serviceName); + } catch (error) { + console.warn('[HealthGate] Failed to clear auto-heal suppression for %s/%s:', + sanitizeForLog(gate.stackName), sanitizeForLog(gate.serviceName), getErrorMessage(error, 'unknown')); + } + } + try { DatabaseService.getInstance().finalizeHealthGateRun( - gate.runId, status, reason, Date.now(), JSON.stringify(containers), + gate.runId, status, reason, Date.now(), JSON.stringify(containers), failureSource, ); } catch (error) { // The verdict is lost from the DB (the startup sweep will later rewrite diff --git a/backend/src/services/ImageUpdateService.ts b/backend/src/services/ImageUpdateService.ts index f0e46b92..8c958746 100644 --- a/backend/src/services/ImageUpdateService.ts +++ b/backend/src/services/ImageUpdateService.ts @@ -2,7 +2,7 @@ import path from 'path'; import YAML from 'yaml'; import { CronExpressionParser } from 'cron-parser'; import DockerController from './DockerController'; -import { DatabaseService } from './DatabaseService'; +import { DatabaseService, type StackCheckStatus, type StackServiceStatus } from './DatabaseService'; import { FileSystemService } from './FileSystemService'; import { RegistryService } from './RegistryService'; import { NodeRegistry } from './NodeRegistry'; @@ -12,6 +12,7 @@ import { parseImageRef, selectLocalRepoDigest, compareLocalToRemoteTag } from '. import { isDebugEnabled } from '../utils/debug'; import { getErrorMessage } from '../utils/errors'; import { sanitizeForLog } from '../utils/safeLog'; +import { buildEffectiveServiceModel } from './effectiveServiceModel'; const BACKFILL_KEY = 'image_update_notifications_backfilled'; @@ -243,11 +244,124 @@ export async function loadStackBuildServices(nodeId: number, stackName: string): return extractBuildServicesFromCompose(composeContent); } +// ─── Per-service reduction (model-based status) ───────────────────────────── + +export interface ServiceReduction { + status: StackServiceStatus; + confirmedUpdateThisRun: boolean; +} + +export function reduceServiceStatus( + service: string, + declaredImage: string | null, + runtimeImages: string[], + imageUpdateMap: Map, + prior: StackServiceStatus | undefined, +): ServiceReduction { + const dedupedRuntime = [...new Set(runtimeImages)].sort(); + const refs = new Set(); + if (declaredImage) refs.add(declaredImage); + for (const ref of dedupedRuntime) refs.add(ref); + + if (refs.size === 0) { + return { + status: { service, image: declaredImage, hasUpdate: false, checkStatus: 'not_checkable', lastError: null }, + confirmedUpdateThisRun: false, + }; + } + + const checkableResults: ImageCheckResult[] = []; + for (const ref of refs) { + const result = imageUpdateMap.get(ref); + if (!result || result.notCheckable) continue; + checkableResults.push(result); + } + + if (checkableResults.length === 0) { + return { + status: { service, image: declaredImage, hasUpdate: false, checkStatus: 'not_checkable', lastError: null }, + confirmedUpdateThisRun: false, + }; + } + + const errored = checkableResults.filter((r) => r.error !== undefined); + const confirmedUpdateThisRun = checkableResults.some( + (r) => r.error === undefined && r.hasUpdate === true, + ); + + if (errored.length === checkableResults.length) { + const priorHasUpdate = prior?.hasUpdate ?? false; + return { + status: { + service, + image: declaredImage, + ...(dedupedRuntime.length > 0 ? { runtimeImages: dedupedRuntime } : {}), + hasUpdate: priorHasUpdate, + checkStatus: 'failed', + lastError: errored[0].error ?? 'Update check failed', + }, + confirmedUpdateThisRun: false, + }; + } + + const checkStatus: StackServiceStatus['checkStatus'] = errored.length > 0 ? 'partial' : 'ok'; + const lastError = errored.length > 0 ? (errored[0].error ?? null) : null; + const hasUpdate = checkStatus === 'partial' + ? (confirmedUpdateThisRun || (prior?.hasUpdate ?? false)) + : confirmedUpdateThisRun; + + return { + status: { + service, + image: declaredImage, + ...(dedupedRuntime.length > 0 ? { runtimeImages: dedupedRuntime } : {}), + hasUpdate, + checkStatus, + lastError, + }, + confirmedUpdateThisRun, + }; +} + +export function aggregateServiceCheckStatus(services: StackServiceStatus[]): StackCheckStatus { + if (services.length === 0) return 'ok'; + const checkable = services.filter((s) => s.checkStatus !== 'not_checkable'); + if (checkable.length === 0) return 'ok'; + const failedCount = checkable.filter((s) => s.checkStatus === 'failed').length; + if (failedCount === checkable.length) return 'failed'; + if (checkable.some((s) => s.checkStatus === 'partial') + || checkable.some((s) => s.checkStatus === 'failed')) { + return 'partial'; + } + return 'ok'; +} + +export function buildAvailabilityNotifyMessage(stackName: string, services: StackServiceStatus[]): string { + const named = services.filter((s) => s.hasUpdate).map((s) => s.service).sort(); + if (named.length === 0 || services.length <= 1) { + return `Stack "${stackName}" has image updates available.`; + } + if (named.length === 1) { + return `Stack "${stackName}" has image updates available for service: ${named[0]}.`; + } + return `Stack "${stackName}" has image updates available for services: ${named.join(', ')}.`; +} + +function stackStatusLastError(services: StackServiceStatus[]): string | null { + for (const svc of services) { + if (svc.checkStatus !== 'not_checkable' && svc.lastError) return svc.lastError; + } + return null; +} + // ─── Service ────────────────────────────────────────────────────────────────── export class ImageUpdateService { private static instance: ImageUpdateService; + /** Per-stack write serialization and generation for stale-writer discard. */ + private stackWriteState = new Map; generation: number }>(); + private static readonly MIN_INTERVAL_MINUTES = 15; private static readonly MAX_INTERVAL_MINUTES = 1440; // 24 hours private static readonly DEFAULT_INTERVAL_MINUTES = 120; // 2 hours @@ -594,9 +708,10 @@ export class ImageUpdateService { } // Phase 3: Container augmentation (captures actual deployed image tags) + let allContainers: Awaited> = []; try { - const containers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers'); - for (const c of containers) { + allContainers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers'); + for (const c of allContainers) { // Prefer the pinned project label (== stackName for Sencho-deployed // stacks, including multi-file / context-dir ones where // --project-directory would otherwise change the working-dir @@ -635,6 +750,14 @@ export class ImageUpdateService { } // Phase 4: Deduplicate and check all unique images + // Reserve write generations before the slow registry checks so a concurrent + // recheckStack that finishes first keeps its write (stale full-scan commits + // are dropped in withStackWriteLock). + const writeGenerations = new Map(); + for (const stackName of stackImages.keys()) { + writeGenerations.set(stackName, this.reserveStackWriteGeneration(nodeId, stackName)); + } + const allImages = new Set(); for (const imgs of stackImages.values()) for (const img of imgs) allImages.add(img); @@ -662,54 +785,36 @@ export class ImageUpdateService { // Write status for ALL stacks (including those with no pullable images) const now = Date.now(); let updatesFound = 0; - const newlyUpdated: string[] = []; - for (const [stackName, images] of stackImages) { - // Tally only checkable images: a not-checkable image (locally built, - // or a bare digest ref) is neither a pass nor a failure. - const checkable = Array.from(images) - .map(img => imageUpdateMap.get(img)) - .filter((r): r is ImageCheckResult => !!r && !r.notCheckable); - const errored = checkable.filter(r => r.error !== undefined); - const confirmedHasUpdate = checkable.some(r => r.error === undefined && r.hasUpdate === true); - - // Every checkable image failed: status is undeterminable. Preserve the - // last-known has_update so a transient registry outage neither erases a - // real update nor flaps the notification state. - if (checkable.length > 0 && errored.length === checkable.length) { - db.recordStackCheckFailure(nodeId, stackName, errored[0].error ?? 'Update check failed', now); - continue; - } - - const checkStatus = errored.length > 0 ? 'partial' : 'ok'; - const lastError = errored.length > 0 ? (errored[0].error ?? null) : null; - // Only a fully-ok check is authoritative enough to lower has_update to - // false. On a partial check some image could not be reached, so a - // previously confirmed update is preserved rather than erased (which - // would also re-fire the notification when that image recovers). - const hasUpdate = checkStatus === 'partial' - ? (confirmedHasUpdate || previousState[stackName] === true) - : confirmedHasUpdate; - - if (hasUpdate) { + const newlyUpdated: Array<{ stackName: string; message: string }> = []; + for (const [stackName] of stackImages) { + const outcome = await this.writeStackUpdateStatus( + nodeId, + stackName, + stackImages.get(stackName) ?? new Set(), + imageUpdateMap, + previousState, + now, + allContainers, + db, + writeGenerations.get(stackName) ?? this.reserveStackWriteGeneration(nodeId, stackName), + ); + if (outcome.committed && outcome.hasUpdate) { updatesFound++; - // Notify only on state transition: was false/absent, now true if (!isBackfilled || !previousState[stackName]) { - newlyUpdated.push(stackName); + newlyUpdated.push({ stackName, message: outcome.notifyMessage }); } } - db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now, checkStatus, lastError); } // Dispatch notifications for stacks that newly have updates if (newlyUpdated.length > 0) { const notifier = NotificationService.getInstance(); - for (const stackName of newlyUpdated) { + for (const { stackName, message } of newlyUpdated) { try { await notifier.dispatchAlert( 'info', 'image_update_available', - // Node-neutral body: the hub bell badge attributes remotes. - `Stack "${stackName}" has image updates available.`, + message, { stackName, actor: 'system:image-update' }, ); } catch (e) { @@ -751,6 +856,212 @@ export class ImageUpdateService { } } + /** + * Re-check a single stack after a service-scoped update or restore. On a + * render failure the prior row is left untouched and a warning is returned. + */ + public async recheckStack(nodeId: number, stackName: string): Promise<{ warning: string | null }> { + const generation = this.reserveStackWriteGeneration(nodeId, stackName); + const db = DatabaseService.getInstance(); + const docker = DockerController.getInstance(nodeId); + const model = await buildEffectiveServiceModel(nodeId, stackName); + if (!model.renderable) { + return { warning: model.error }; + } + + let containers: Array<{ Image?: string; Labels?: Record }> = []; + try { + containers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers'); + } catch (e) { + console.warn( + '[ImageUpdateService] recheckStack container read failed for %s: %s', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(e, 'unknown')), + ); + } + + const refs = new Set(); + const runtimeByService = this.runtimeImagesByService(stackName, containers); + for (const spec of model.services) { + if (spec.declaredImage) refs.add(spec.declaredImage); + for (const ref of runtimeByService.get(spec.name) ?? []) refs.add(ref); + } + + const imageUpdateMap = new Map(); + for (const imageRef of refs) { + try { + imageUpdateMap.set(imageRef, await this.checkImage(docker, imageRef)); + } catch (e) { + imageUpdateMap.set(imageRef, { hasUpdate: false, error: getErrorMessage(e, 'Update check failed') }); + } + await sleep(ImageUpdateService.INTER_IMAGE_DELAY_MS); + } + + const priorByService = new Map(db.getStackServicesJson(nodeId, stackName).map((s) => [s.service, s])); + const reductions = model.services.map((spec) => reduceServiceStatus( + spec.name, + spec.declaredImage, + runtimeByService.get(spec.name) ?? [], + imageUpdateMap, + priorByService.get(spec.name), + )); + const services = reductions.map((r) => r.status); + const checkStatus = aggregateServiceCheckStatus(services); + const hasUpdate = services.some((s) => s.hasUpdate); + const lastError = stackStatusLastError(services); + const now = Date.now(); + + await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => { + if (checkStatus === 'failed') { + db.recordStackCheckFailure(nodeId, stackName, lastError ?? 'Update check failed', now, services, gen); + } else { + db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now, checkStatus, lastError, services, gen); + } + }); + + return { warning: null }; + } + + private stackWriteKey(nodeId: number, stackName: string): string { + return `${nodeId}:${stackName}`; + } + + /** Bump the per-stack write generation before async registry work so a later + * slower scan cannot commit after a newer recheck reserved a higher gen. */ + private reserveStackWriteGeneration(nodeId: number, stackName: string): number { + const key = this.stackWriteKey(nodeId, stackName); + let state = this.stackWriteState.get(key); + if (!state) { + state = { chain: Promise.resolve(), generation: 0 }; + this.stackWriteState.set(key, state); + } + state.generation += 1; + return state.generation; + } + + private async withStackWriteLock( + nodeId: number, + stackName: string, + generation: number, + write: (generation: number) => void | Promise, + ): Promise { + const key = this.stackWriteKey(nodeId, stackName); + let state = this.stackWriteState.get(key); + if (!state) { + state = { chain: Promise.resolve(), generation }; + this.stackWriteState.set(key, state); + } + let committed = false; + state.chain = state.chain.then(async () => { + const current = this.stackWriteState.get(key); + // A newer reservation supersedes this writer; drop the stale commit. + if (!current || generation < current.generation) return; + await write(generation); + committed = true; + }); + await state.chain; + return committed; + } + + private runtimeImagesByService( + stackName: string, + containers: Array<{ Image?: string; Labels?: Record }>, + ): Map { + const out = new Map(); + for (const c of containers) { + if (c.Labels?.['com.docker.compose.project'] !== stackName) continue; + const service = c.Labels?.['com.docker.compose.service']; + if (!service) continue; + const imageRef = c.Image ?? ''; + if (!imageRef || imageRef.startsWith('sha256:')) continue; + const list = out.get(service) ?? []; + list.push(imageRef); + out.set(service, list); + } + return out; + } + + private async writeStackUpdateStatus( + nodeId: number, + stackName: string, + images: Set, + imageUpdateMap: Map, + previousState: Record, + checkedAt: number, + containers: Array<{ Image?: string; Labels?: Record }>, + db: DatabaseService, + generation: number, + ): Promise<{ hasUpdate: boolean; notifyMessage: string; committed: boolean }> { + const model = await buildEffectiveServiceModel(nodeId, stackName); + if (model.renderable) { + const priorByService = new Map( + DatabaseService.getInstance().getStackServicesJson(nodeId, stackName).map((s) => [s.service, s]), + ); + const runtimeByService = this.runtimeImagesByService(stackName, containers); + const reductions = model.services.map((spec) => reduceServiceStatus( + spec.name, + spec.declaredImage, + runtimeByService.get(spec.name) ?? [], + imageUpdateMap, + priorByService.get(spec.name), + )); + const services = reductions.map((r) => r.status); + const checkStatus = aggregateServiceCheckStatus(services); + const hasUpdate = services.some((s) => s.hasUpdate); + const lastError = stackStatusLastError(services); + const notifyMessage = buildAvailabilityNotifyMessage(stackName, services); + + const committed = await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => { + if (checkStatus === 'failed') { + db.recordStackCheckFailure( + nodeId, stackName, lastError ?? 'Update check failed', checkedAt, services, gen, + ); + } else { + db.upsertStackUpdateStatus( + nodeId, stackName, hasUpdate, checkedAt, checkStatus, lastError, services, gen, + ); + } + }); + + return { hasUpdate, notifyMessage, committed }; + } + + const checkable = Array.from(images) + .map((img) => imageUpdateMap.get(img)) + .filter((r): r is ImageCheckResult => !!r && !r.notCheckable); + const errored = checkable.filter((r) => r.error !== undefined); + const confirmedHasUpdate = checkable.some((r) => r.error === undefined && r.hasUpdate === true); + + if (checkable.length > 0 && errored.length === checkable.length) { + const committed = await this.withStackWriteLock(nodeId, stackName, generation, async () => { + db.recordStackCheckFailure( + nodeId, stackName, errored[0].error ?? 'Update check failed', checkedAt, + ); + }); + return { + hasUpdate: previousState[stackName] === true, + notifyMessage: buildAvailabilityNotifyMessage(stackName, []), + committed, + }; + } + + const checkStatus: StackCheckStatus = errored.length > 0 ? 'partial' : 'ok'; + const lastError = errored.length > 0 ? (errored[0].error ?? null) : null; + const hasUpdate = checkStatus === 'partial' + ? (confirmedHasUpdate || previousState[stackName] === true) + : confirmedHasUpdate; + + const committed = await this.withStackWriteLock(nodeId, stackName, generation, async () => { + db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, checkedAt, checkStatus, lastError); + }); + + return { + hasUpdate, + notifyMessage: buildAvailabilityNotifyMessage(stackName, []), + committed, + }; + } + public async checkImage(docker: DockerController, imageRef: string): Promise { const parsed = parseImageRef(imageRef); // A bare digest ref (sha256:...) has no tag to track upstream; not applicable. diff --git a/backend/src/services/SchedulerService.ts b/backend/src/services/SchedulerService.ts index ae7b5657..d1f40710 100644 --- a/backend/src/services/SchedulerService.ts +++ b/backend/src/services/SchedulerService.ts @@ -5,9 +5,11 @@ import { LicenseService } from './LicenseService'; import { PROXY_TIER_HEADER, deployProvenanceHeaders } from './license-headers'; import DockerController from './DockerController'; import { ComposeService } from './ComposeService'; +import { StackUpdateOrchestrator } from './StackUpdateOrchestrator'; import { StackOpLockService, stackOpSkipMessage as skipMessage } from './StackOpLockService'; import { FileSystemService } from './FileSystemService'; import { HealthGateService } from './HealthGateService'; +import { ServiceUpdateRecoveryService } from './ServiceUpdateRecoveryService'; import { ImageUpdateService } from './ImageUpdateService'; import type { ImageCheckResult } from './ImageUpdateService'; import { isDebugEnabled } from '../utils/debug'; @@ -706,12 +708,13 @@ export class SchedulerService { ? (JSON.parse(task.prune_targets) as string[]).filter((t): t is PruneTarget => allTargets.includes(t as PruneTarget)) : [...allTargets]; const labelFilter = task.prune_label_filter || undefined; + const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(nodeId); const results: string[] = []; const failures: string[] = []; for (const target of targets) { try { - const result = await docker.pruneSystem(target, labelFilter); + const result = await docker.pruneSystem(target, labelFilter, isImageHeld); results.push(`${target}: ${result.reclaimedBytes ?? 0} bytes reclaimed`); } catch (error: unknown) { const msg = error instanceof Error ? error.message : String(error); @@ -766,12 +769,11 @@ export class SchedulerService { const db = DatabaseService.getInstance(); const docker = DockerController.getInstance(task.node_id); const imageUpdateService = ImageUpdateService.getInstance(); - const compose = ComposeService.getInstance(task.node_id); const results: string[] = []; for (const stackName of stackNames) { try { - const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, compose, db, isFleet || isWildcard); + const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, db, isFleet || isWildcard); results.push(output); } catch (e) { const msg = getErrorMessage(e, String(e)); @@ -1028,7 +1030,6 @@ export class SchedulerService { nodeId: number, docker: DockerController, imageUpdateService: ImageUpdateService, - compose: ComposeService, db: DatabaseService, isWildcard = false ): Promise { @@ -1096,11 +1097,14 @@ export class SchedulerService { const atomic = true; const lock = await StackOpLockService.getInstance().runExclusive( nodeId, stackName, 'update', 'system', - () => compose.updateStack(stackName, undefined, atomic), + () => StackUpdateOrchestrator.getInstance().execute( + { nodeId, stackName, target: { scope: 'stack' }, trigger: 'scheduled', actor: 'system:scheduler' }, + { atomic, terminalWs: null }, + ), ); if (!lock.ran) return skipMessage(stackName, lock.existing.action); db.clearStackUpdateStatus(nodeId, stackName); - HealthGateService.getInstance().begin(nodeId, stackName, 'update', 'system:scheduler'); + HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler'); this.safeDispatch( 'info', diff --git a/backend/src/services/ServiceUpdateRecoveryService.ts b/backend/src/services/ServiceUpdateRecoveryService.ts new file mode 100644 index 00000000..17779eff --- /dev/null +++ b/backend/src/services/ServiceUpdateRecoveryService.ts @@ -0,0 +1,280 @@ +import { randomUUID } from 'crypto'; +import { DatabaseService, type ServiceUpdateRecoveryRow } from './DatabaseService'; +import { getComposeCommandTimeoutMs } from './ComposeService'; +import { getErrorMessage } from '../utils/errors'; + +const SWEEP_INTERVAL_MS = 5 * 60_000; +const INITIAL_SWEEP_DELAY_MS = 30_000; +const CLAIM_RENEWAL_INTERVAL_MS = 5 * 60_000; +const MIN_CLAIM_WINDOW_MS = 30 * 60_000; +const CLAIM_RENEWAL_BUFFER_MS = 5 * 60_000; +const RECOVERY_TTL_BUFFER_MS = 30 * 60_000; +const MIN_RECOVERY_WINDOW_SECONDS = 90; +const DIGEST_PIN_PATTERN = /@sha256:[a-f0-9]{64}$/i; + +/** A single pre-update replica snapshot for one service. */ +export interface ServiceReplicaSnapshot { + imageId: string; + /** Repo digest captured from the running container, when known. */ + repoDigest: string | null; +} + +export interface CreateServiceUpdateRecoveryInput { + nodeId: number; + stackName: string; + serviceName: string; + replicas: ServiceReplicaSnapshot[]; + /** EffectiveServiceSpec.declaredImage; null for a pure-build service with no image field. */ + declaredImageRef: string | null; + createdBy: string | null; +} + +export type ServiceUpdateRecoveryUnavailableReason = + | 'no_replicas' + | 'majority_tie' + | 'build_only' + | 'digest_pinned_declared_ref' + | 'missing_local_id'; + +export type CreateServiceUpdateRecoveryResult = + | { eligible: true; row: ServiceUpdateRecoveryRow } + | { eligible: false; reason: ServiceUpdateRecoveryUnavailableReason }; + +/** + * Pre-update image snapshots that let an operator manually restore a single + * service after a service-scoped update, without touching its siblings. + * + * Purely a data/lifecycle layer: it decides whether a snapshot is eligible to + * persist, holds the claimed row's image from prune while a restore is in + * flight, and sweeps stale rows. It never runs Compose or touches Docker; the + * update/restore orchestration (StackUpdateOrchestrator) calls into this + * service at the right points and is responsible for policy, health gating, + * and Auto-Heal suppression around those calls. + */ +export class ServiceUpdateRecoveryService { + private static instance: ServiceUpdateRecoveryService; + private started = false; + private intervalId: NodeJS.Timeout | null = null; + private initialTimer: NodeJS.Timeout | null = null; + private readonly renewalTimers = new Map(); + + private constructor() {} + + public static getInstance(): ServiceUpdateRecoveryService { + if (!ServiceUpdateRecoveryService.instance) { + ServiceUpdateRecoveryService.instance = new ServiceUpdateRecoveryService(); + } + return ServiceUpdateRecoveryService.instance; + } + + public start(): void { + this.started = true; + if (this.initialTimer || this.intervalId) return; + this.initialTimer = setTimeout(() => { + this.sweep(); + this.intervalId = setInterval(() => this.sweep(), SWEEP_INTERVAL_MS); + }, INITIAL_SWEEP_DELAY_MS); + } + + /** Clear the sweep timer and every in-flight claim renewal loop. No callbacks fire after this returns. */ + public stop(): void { + this.started = false; + if (this.initialTimer) { + clearTimeout(this.initialTimer); + this.initialTimer = null; + } + if (this.intervalId) { + clearInterval(this.intervalId); + this.intervalId = null; + } + for (const timer of this.renewalTimers.values()) clearInterval(timer); + this.renewalTimers.clear(); + } + + /** Expire active rows past their TTL and restoring rows abandoned by a dead claim. Never throws. */ + public sweep(): void { + try { + const now = Date.now(); + const db = DatabaseService.getInstance(); + const expiredActive = db.sweepExpiredActiveServiceUpdateRecoveries(now); + const expiredAbandoned = db.sweepAbandonedRestoringServiceUpdateRecoveries(now); + if (expiredActive > 0 || expiredAbandoned > 0) { + console.log( + `[ServiceUpdateRecovery] Swept ${expiredActive} expired active and ${expiredAbandoned} abandoned restoring record(s)`, + ); + } + } catch (error) { + console.error('[ServiceUpdateRecovery] Sweep failed:', getErrorMessage(error, 'unknown')); + } + } + + /** + * Decide whether a pre-update snapshot is eligible to persist and, if so, + * insert it. Unavailable per §8: no replicas, a majority tie, a pure-build + * service (no declared image ref), a digest-pinned declared ref, or a + * replica missing its local image id. + */ + public createIfEligible(input: CreateServiceUpdateRecoveryInput): CreateServiceUpdateRecoveryResult { + const validReplicas = input.replicas.filter(r => r.imageId.trim().length > 0); + if (validReplicas.length === 0) { + return { eligible: false, reason: input.replicas.length === 0 ? 'no_replicas' : 'missing_local_id' }; + } + + const majority = this.computeMajorityImage(validReplicas); + if (!majority) return { eligible: false, reason: 'majority_tie' }; + + if (!input.declaredImageRef) return { eligible: false, reason: 'build_only' }; + if (DIGEST_PIN_PATTERN.test(input.declaredImageRef)) return { eligible: false, reason: 'digest_pinned_declared_ref' }; + + const now = Date.now(); + const row: ServiceUpdateRecoveryRow = { + id: randomUUID(), + node_id: input.nodeId, + stack_name: input.stackName, + service_name: input.serviceName, + replicas_json: JSON.stringify(input.replicas), + majority_image_id: majority.imageId, + declared_image_ref: input.declaredImageRef, + weak_floating_tag: majority.weakFloatingTag ? 1 : 0, + health_gate_id: null, + status: 'active', + expires_at: now + this.activeRecoveryTtlMs() + RECOVERY_TTL_BUFFER_MS, + claim_expires_at: null, + created_at: now, + created_by: input.createdBy, + }; + DatabaseService.getInstance().insertServiceUpdateRecovery(row); + return { eligible: true, row }; + } + + /** Active, unexpired recovery rows for one service (drives `recoveryAvailable` in the update/restore responses). */ + public listActive(nodeId: number, stackName: string, serviceName: string): ServiceUpdateRecoveryRow[] { + return DatabaseService.getInstance().listActiveServiceUpdateRecoveries(nodeId, stackName, serviceName); + } + + public get(id: string): ServiceUpdateRecoveryRow | undefined { + return DatabaseService.getInstance().getServiceUpdateRecovery(id); + } + + /** Attach the update flow's own health gate run id while the row is still active. */ + public linkHealthGate(id: string, healthGateId: string): void { + DatabaseService.getInstance().linkServiceUpdateRecoveryHealthGate(id, healthGateId); + } + + /** Atomic claim: active -> restoring. Returns the claimed row, or undefined if another actor won the race. */ + public claim(id: string): ServiceUpdateRecoveryRow | undefined { + const now = Date.now(); + return DatabaseService.getInstance().claimServiceUpdateRecovery(id, this.nextClaimExpiry(now), now); + } + + /** + * Start the mandatory 5-minute renewal loop for a claimed row (RECOVERY-CLAIM-2). + * Idempotent per id. Each tick CAS-extends claim_expires_at only while the row + * is still restoring; a renewal that finds the row no longer restoring (superseded, + * consumed, or force-expired) stops itself rather than reviving a terminal row. + */ + public startClaimRenewal(id: string): void { + if (!this.started || this.renewalTimers.has(id)) return; + const timer = setInterval(() => { + try { + const renewed = DatabaseService.getInstance().renewServiceUpdateRecoveryClaim(id, this.nextClaimExpiry(Date.now())); + if (!renewed) this.stopClaimRenewal(id); + } catch (error) { + console.error('[ServiceUpdateRecovery] Claim renewal failed for %s:', id, getErrorMessage(error, 'unknown')); + } + }, CLAIM_RENEWAL_INTERVAL_MS); + this.renewalTimers.set(id, timer); + } + + /** Stop the renewal loop for a row. Callers must call this in the restore operation's `finally` regardless of outcome. */ + public stopClaimRenewal(id: string): void { + const timer = this.renewalTimers.get(id); + if (timer) { + clearInterval(timer); + this.renewalTimers.delete(id); + } + } + + /** Restore succeeded: restoring -> consumed, optionally linking the restore's own health gate run. */ + public markConsumed(id: string, healthGateId: string | null = null): boolean { + return DatabaseService.getInstance().markServiceUpdateRecoveryConsumed(id, healthGateId); + } + + /** Mid-flight restore failure with the image still local: restoring -> active, claim cleared. */ + public reactivate(id: string): boolean { + return DatabaseService.getInstance().reactivateServiceUpdateRecovery(id); + } + + /** Mid-flight restore failure with the image gone: restoring -> invalidated. */ + public invalidate(id: string): boolean { + return DatabaseService.getInstance().invalidateServiceUpdateRecovery(id); + } + + /** + * Image IDs currently held for this node (active rows and restoring rows + * with a live claim). Returns null when the held set cannot be read so + * callers can fail closed (skip prune) instead of treating the miss as empty. + */ + public getHeldImageIds(nodeId: number): Set | null { + try { + return new Set(DatabaseService.getInstance().listHeldServiceUpdateRecoveryImageIds(nodeId, Date.now())); + } catch (error) { + console.warn( + '[ServiceUpdateRecovery] Failed to compute held image ids for node %d:', nodeId, getErrorMessage(error, 'unknown'), + ); + return null; + } + } + + /** + * A predicate a pruner can call immediately before deleting each candidate + * image. Re-reads the held set on every call (rather than snapshotting it + * once) so a snapshot that becomes eligible between plan and delete is + * still honored. When the held set cannot be read, returns true for every + * id so prune skips deletes (fail closed). + */ + public buildHeldImagePredicate(nodeId: number): (imageId: string) => boolean { + return (imageId: string) => { + const held = this.getHeldImageIds(nodeId); + if (held === null) return true; + return held.has(imageId); + }; + } + + private nextClaimExpiry(now: number): number { + return now + Math.max(getComposeCommandTimeoutMs(), MIN_CLAIM_WINDOW_MS) + CLAIM_RENEWAL_BUFFER_MS; + } + + /** + * Active recovery must outlive both the Compose mutation and the post-update + * health window. A raised SENCHO_COMPOSE_COMMAND_TIMEOUT_MS must not leave the + * snapshot eligible for sweep while Compose is still running. + */ + private activeRecoveryTtlMs(): number { + return Math.max(getComposeCommandTimeoutMs(), this.readRecoveryWindowSeconds() * 1000); + } + + private readRecoveryWindowSeconds(): number { + try { + const raw = parseInt(DatabaseService.getInstance().getGlobalSettings()['health_gate_window_seconds'] ?? '', 10); + return Number.isFinite(raw) ? Math.max(raw, MIN_RECOVERY_WINDOW_SECONDS) : MIN_RECOVERY_WINDOW_SECONDS; + } catch (error) { + console.warn('[ServiceUpdateRecovery] Settings read failed; using default window:', getErrorMessage(error, 'unknown')); + return MIN_RECOVERY_WINDOW_SECONDS; + } + } + + /** Majority image id among replicas, or null on a tie (no single winner). */ + private computeMajorityImage(replicas: ServiceReplicaSnapshot[]): { imageId: string; weakFloatingTag: boolean } | null { + const counts = new Map(); + for (const replica of replicas) { + counts.set(replica.imageId, (counts.get(replica.imageId) ?? 0) + 1); + } + const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1]); + if (ranked.length === 0) return null; + if (ranked.length > 1 && ranked[0][1] === ranked[1][1]) return null; + const winner = ranked[0][0]; + const weakFloatingTag = !replicas.some(r => r.imageId === winner && r.repoDigest); + return { imageId: winner, weakFloatingTag }; + } +} diff --git a/backend/src/services/StackOpMetricsService.ts b/backend/src/services/StackOpMetricsService.ts index aff6c6bd..902955df 100644 --- a/backend/src/services/StackOpMetricsService.ts +++ b/backend/src/services/StackOpMetricsService.ts @@ -11,6 +11,11 @@ export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update'; +export interface StackOpRecordMeta { + targetScope?: 'stack' | 'service'; + serviceName?: string | null; +} + interface StackOpStats { count: number; successCount: number; @@ -22,6 +27,10 @@ interface StackOpStats { * computed from this window on demand. */ recentSamples: number[]; + /** Most recent target scope recorded for this bucket (additive; stack default). */ + lastTargetScope: 'stack' | 'service'; + /** Most recent service name when lastTargetScope is service; otherwise null. */ + lastServiceName: string | null; } export interface StackOpSnapshotEntry { @@ -33,6 +42,8 @@ export interface StackOpSnapshotEntry { avgMs: number; p50Ms: number; p95Ms: number; + targetScope: 'stack' | 'service'; + serviceName: string | null; } const MAX_SAMPLES = 1000; @@ -66,12 +77,26 @@ export class StackOpMetricsService { * Record one completed op. Call from the route layer after the lifecycle * call resolves or rejects; `ok=false` for the rejection path. */ - public record(nodeId: number, action: StackOpAction, durationMs: number, ok: boolean): void { + public record( + nodeId: number, + action: StackOpAction, + durationMs: number, + ok: boolean, + meta?: StackOpRecordMeta, + ): void { if (!Number.isFinite(durationMs) || durationMs < 0) return; const k = this.key(nodeId, action); let s = this.stats.get(k); if (!s) { - s = { count: 0, successCount: 0, errorCount: 0, totalMs: 0, recentSamples: [] }; + s = { + count: 0, + successCount: 0, + errorCount: 0, + totalMs: 0, + recentSamples: [], + lastTargetScope: 'stack', + lastServiceName: null, + }; this.stats.set(k, s); } s.count += 1; @@ -84,6 +109,12 @@ export class StackOpMetricsService { // and this path is once-per-stack-op (low cadence). s.recentSamples.shift(); } + if (meta?.targetScope) s.lastTargetScope = meta.targetScope; + if (meta?.targetScope === 'service') { + s.lastServiceName = meta.serviceName ?? null; + } else if (meta?.targetScope === 'stack') { + s.lastServiceName = null; + } } public snapshot(): StackOpSnapshotEntry[] { @@ -102,6 +133,8 @@ export class StackOpMetricsService { avgMs: s.count === 0 ? 0 : Math.round(s.totalMs / s.count), p50Ms: percentile(sorted, 0.5), p95Ms: percentile(sorted, 0.95), + targetScope: s.lastTargetScope, + serviceName: s.lastServiceName, }); } // Stable ordering: nodeId asc, then action asc. diff --git a/backend/src/services/StackUpdateOrchestrator.ts b/backend/src/services/StackUpdateOrchestrator.ts new file mode 100644 index 00000000..fb1379fd --- /dev/null +++ b/backend/src/services/StackUpdateOrchestrator.ts @@ -0,0 +1,684 @@ +import WebSocket from 'ws'; +import DockerController from './DockerController'; +import { ComposeService } from './ComposeService'; +import { HealthGateService } from './HealthGateService'; +import { AutoHealService } from './AutoHealService'; +import { ImageUpdateService } from './ImageUpdateService'; +import { + ServiceUpdateRecoveryService, + type ServiceReplicaSnapshot, +} from './ServiceUpdateRecoveryService'; +import type { ServiceUpdateRecoveryRow } from './DatabaseService'; +import { buildEffectiveServiceModel } from './effectiveServiceModel'; +import { DriftLedgerService } from './DriftLedgerService'; +import { MeshService } from './MeshService'; +import { DatabaseService } from './DatabaseService'; +import { enforcePolicyForImageRefs, type PolicyEnforcementOptions } from './PolicyEnforcement'; +import { describePolicyBlock } from '../helpers/policyGate'; +import { getErrorMessage } from '../utils/errors'; +import { sanitizeForLog } from '../utils/safeLog'; +import type { EffectiveServiceSpec } from './effectiveServiceModel'; + +export type UpdateTarget = + | { scope: 'stack' } + | { scope: 'service'; serviceName: string }; + +export type UpdateTrigger = + | 'manual' | 'scheduled' | 'automatic' | 'blueprint' | 'webhook' | 'bulk'; + +export interface UpdateOperationContext { + nodeId: number; + stackName: string; + target: UpdateTarget; + trigger: UpdateTrigger; + actor: string | null; +} + +export interface StackComposeOptions { + atomic: boolean; + terminalWs?: WebSocket | null; +} + +export interface ServiceUpdateOptions { + policyOptions: PolicyEnforcementOptions; + terminalWs?: WebSocket | null; + /** Restore only: binds the operation to a recovery snapshot. */ + recoveryId?: string; +} + +export type OrchestratorResult = + | { kind: 'stack_compose_done' } + | { + kind: 'service_done'; + serviceName: string; + healthGateId: string | null; + observing: boolean; + recoveryId: string | null; + recoveryAvailable: boolean; + previousImageId?: string | null; + newImageId?: string | null; + recheckWarning?: string; + } + | { + kind: 'service_failed'; + code: string; + error: string; + serviceName?: string; + mutationStage?: string; + recoveryId?: string | null; + }; + +function serviceFailed( + code: string, + error: string, + extra?: { serviceName?: string; mutationStage?: string; recoveryId?: string | null }, +): Extract { + return { kind: 'service_failed', code, error, ...extra }; +} + +/** Shorten a local image id for activity copy (sha256:abcdef… → abcdef…). */ +export function shortImageId(imageId: string | null | undefined): string | null { + if (!imageId) return null; + const bare = imageId.startsWith('sha256:') ? imageId.slice(7) : imageId; + return bare.slice(0, 12); +} + +/** + * Post-mutation replica convergence rules shared by update and restore. + * Requires the exact expected count of inspected running image ids and a + * single shared image before success (independent of optional health gating). + */ +export function evaluateServiceReplicaConvergence( + serviceName: string, + expectedReplicas: number, + ids: string[], + inspectErrors: number, +): + | { kind: 'converged'; imageId: string | null } + | { kind: 'divergent'; error: string } + | { kind: 'inspect_failed'; error: string } { + if (expectedReplicas === 0) { + return { kind: 'converged', imageId: null }; + } + if (inspectErrors > 0 && ids.length === 0) { + return { + kind: 'inspect_failed', + error: `Could not inspect replicas for service "${serviceName}" after the update.`, + }; + } + const distinct = new Set(ids); + if (distinct.size === 0) { + return { kind: 'divergent', error: `Service "${serviceName}" has no running replicas after the update.` }; + } + if (ids.length !== expectedReplicas) { + return { + kind: 'divergent', + error: `Service "${serviceName}" has ${ids.length} running replica(s); expected ${expectedReplicas}.`, + }; + } + if (distinct.size > 1) { + return { kind: 'divergent', error: `Service "${serviceName}" replicas did not converge on a single image.` }; + } + return { kind: 'converged', imageId: [...distinct][0] }; +} + +/** Parse a tag image ref into a Docker tag() repo + tag pair. */ +function splitImageRef(ref: string): { repo: string; tag: string } { + const lastSlash = ref.lastIndexOf('/'); + const lastColon = ref.lastIndexOf(':'); + if (lastColon > lastSlash) { + return { repo: ref.slice(0, lastColon), tag: ref.slice(lastColon + 1) }; + } + return { repo: ref, tag: 'latest' }; +} + +/** + * Single entry point for stack-scope Compose updates and the manual + * service-scoped update/restore flows. + * + * Ownership boundaries (§3): the five stack update callers keep every + * side effect (policy, lock, clear status, invalidate, broadcast, notify, + * post-scan, and the post-success `beginStack`); this orchestrator's stack + * branch only runs `ComposeService.updateStack`. The service branch (nested + * manual routes only) owns policy, the recovery snapshot, the pre-mutation + * health gate (prepare/attach/beginPrepared), Auto-Heal suppression, and the + * per-service recheck; the route holds the stack lock and maps the result to + * HTTP. Expected failures never throw past `execute`; they return a + * `service_failed` result. + */ +export class StackUpdateOrchestrator { + private static instance: StackUpdateOrchestrator; + + public static getInstance(): StackUpdateOrchestrator { + if (!StackUpdateOrchestrator.instance) { + StackUpdateOrchestrator.instance = new StackUpdateOrchestrator(); + } + return StackUpdateOrchestrator.instance; + } + + public async execute( + ctx: UpdateOperationContext, + options: StackComposeOptions | ServiceUpdateOptions, + ): Promise { + if (ctx.target.scope === 'stack') { + return this.executeStack(ctx, options as StackComposeOptions); + } + // Service scope: automation never selects this branch. + if (ctx.trigger !== 'manual') { + throw new Error('Service-scoped updates require a manual trigger'); + } + const serviceOptions = options as ServiceUpdateOptions; + if (serviceOptions.recoveryId) { + return this.executeRestore(ctx, ctx.target.serviceName, serviceOptions); + } + return this.executeServiceUpdate(ctx, ctx.target.serviceName, serviceOptions); + } + + /** + * Stack branch: run the full-stack Compose update only. Side effects and the + * post-success `beginStack(..., 'update')` stay with the caller. + */ + private async executeStack( + ctx: UpdateOperationContext, + options: StackComposeOptions, + ): Promise { + await ComposeService.getInstance(ctx.nodeId).updateStack( + ctx.stackName, options.terminalWs ?? undefined, options.atomic, + ); + return { kind: 'stack_compose_done' }; + } + + private async executeServiceUpdate( + ctx: UpdateOperationContext, + serviceName: string, + options: ServiceUpdateOptions, + ): Promise { + const { nodeId, stackName } = ctx; + + const loaded = await this.loadServiceSpec(nodeId, stackName, serviceName); + if (!loaded.ok) return loaded.result; + const { spec, services } = loaded; + + if (services.length <= 1) { + return serviceFailed( + 'service_update_single_service', + 'Service-scoped updates require a stack with more than one service.', + ); + } + if (!spec.hasBuild && !spec.declaredImage) { + return serviceFailed( + 'service_not_updatable', + `Service "${serviceName}" has neither an image nor a build, so it cannot be updated.`, + ); + } + + AutoHealService.getInstance().suppress(nodeId, stackName, serviceName); + let observing = false; + let recoveryId: string | null = null; + let recoveryAvailable = false; + try { + const policyFailure = await this.checkPolicyOrFail( + stackName, nodeId, serviceName, spec.declaredImage ? [spec.declaredImage] : [], options.policyOptions, 'update', + ); + if (policyFailure) return policyFailure; + + const prepared = await HealthGateService.getInstance().prepare({ + nodeId, stackName, + target: { scope: 'service', serviceName }, + trigger: 'service_update', + expectedReplicas: spec.expectedReplicas, + }); + const prepareToken = prepared.prepareToken; + const prepareSnapshotOk = prepared.snapshotOk; + + const replicas = await this.snapshotServiceReplicas(nodeId, stackName, serviceName); + const recovery = ServiceUpdateRecoveryService.getInstance().createIfEligible({ + nodeId, stackName, serviceName, + replicas, + declaredImageRef: spec.declaredImage, + createdBy: ctx.actor, + }); + const previousImageId = recovery.eligible + ? recovery.row.majority_image_id + : (replicas[0]?.imageId ?? null); + if (recovery.eligible) { + recoveryId = recovery.row.id; + recoveryAvailable = true; + } + + try { + await ComposeService.getInstance(nodeId).updateService( + stackName, serviceName, spec.hasBuild, options.terminalWs ?? undefined, + ); + } catch (composeError) { + return serviceFailed( + 'service_update_compose_failed', + getErrorMessage(composeError, 'Service update failed'), + { serviceName, mutationStage: 'compose', recoveryId }, + ); + } + + const observed = await this.observePreparedService( + prepareToken, nodeId, stackName, serviceName, spec.expectedReplicas, ctx.actor, + 'service_update_replica_divergence', recoveryId, prepareSnapshotOk, + ); + if (!observed.ok) return observed.result; + observing = observed.observing; + const healthGateId = observed.healthGateId; + + if (recoveryId && healthGateId) { + try { + ServiceUpdateRecoveryService.getInstance().linkHealthGate(recoveryId, healthGateId); + } catch (linkError) { + console.warn( + '[StackUpdateOrchestrator] Failed to link recovery %s to gate %s: %s', + sanitizeForLog(recoveryId), sanitizeForLog(healthGateId), + sanitizeForLog(getErrorMessage(linkError, 'unknown')), + ); + } + } + const recheck = await ImageUpdateService.getInstance().recheckStack(nodeId, stackName); + await DriftLedgerService.getInstance().reconcileServiceForStack(nodeId, stackName, serviceName); + await this.refreshMeshIfEnabled(nodeId, stackName); + + const warnings = [observed.gateWarning, recheck.warning].filter((w): w is string => !!w); + return { + kind: 'service_done', + serviceName, + healthGateId, + observing, + recoveryId, + recoveryAvailable, + previousImageId, + newImageId: observed.newImageId ?? null, + recheckWarning: warnings.length > 0 ? warnings.join(' ') : undefined, + }; + } finally { + if (!observing) { + AutoHealService.getInstance().clearSuppress(nodeId, stackName, serviceName); + } + } + } + + private async executeRestore( + ctx: UpdateOperationContext, + serviceName: string, + options: ServiceUpdateOptions, + ): Promise { + const { nodeId, stackName } = ctx; + const recoveryId = options.recoveryId as string; + + const recovery = ServiceUpdateRecoveryService.getInstance().get(recoveryId); + if ( + !recovery || + recovery.node_id !== nodeId || + recovery.stack_name !== stackName || + recovery.service_name !== serviceName + ) { + return serviceFailed('recovery_not_found', 'No matching recovery snapshot for this service.'); + } + if (recovery.status !== 'active') { + return serviceFailed( + 'recovery_not_restorable', + `This recovery snapshot is ${recovery.status} and can no longer be restored.`, + { recoveryId }, + ); + } + + AutoHealService.getInstance().suppress(nodeId, stackName, serviceName); + let observing = false; + let claimed = false; + let consumed = false; + try { + const loaded = await this.loadServiceSpec(nodeId, stackName, serviceName, recoveryId); + if (!loaded.ok) return loaded.result; + const { spec, services } = loaded; + if (services.length <= 1) { + return serviceFailed( + 'service_update_single_service', + 'Service-scoped restore requires a stack with more than one service.', + { recoveryId }, + ); + } + + const scanTarget = await this.resolveRestoreScanTarget(nodeId, recovery); + const policyFailure = await this.checkPolicyOrFail( + stackName, nodeId, serviceName, [scanTarget], options.policyOptions, 'rollback', recoveryId, + ); + if (policyFailure) return policyFailure; + + const prepared = await HealthGateService.getInstance().prepare({ + nodeId, stackName, + target: { scope: 'service', serviceName }, + trigger: 'service_restore', + expectedReplicas: spec.expectedReplicas, + }); + const prepareToken = prepared.prepareToken; + const prepareSnapshotOk = prepared.snapshotOk; + + const claimedRow = ServiceUpdateRecoveryService.getInstance().claim(recoveryId); + if (!claimedRow) { + return serviceFailed( + 'recovery_claim_failed', + 'This recovery snapshot is being restored by another operation.', + { recoveryId }, + ); + } + claimed = true; + ServiceUpdateRecoveryService.getInstance().startClaimRenewal(recoveryId); + + const previousImageId = (await this.snapshotServiceReplicas(nodeId, stackName, serviceName))[0]?.imageId ?? null; + + try { + const { repo, tag } = splitImageRef(recovery.declared_image_ref); + await DockerController.getInstance(nodeId).getDocker() + .getImage(recovery.majority_image_id).tag({ repo, tag }); + } catch (retagError) { + return serviceFailed( + 'restore_retag_failed', + getErrorMessage(retagError, 'Could not retag the recovery image'), + { serviceName, mutationStage: 'retag', recoveryId }, + ); + } + try { + await ComposeService.getInstance(nodeId).recreateServiceFromLocal( + stackName, serviceName, options.terminalWs ?? undefined, + ); + } catch (composeError) { + return serviceFailed( + 'restore_compose_failed', + getErrorMessage(composeError, 'Service restore failed'), + { serviceName, mutationStage: 'compose', recoveryId }, + ); + } + + const observed = await this.observePreparedService( + prepareToken, nodeId, stackName, serviceName, spec.expectedReplicas, ctx.actor, + 'restore_replica_divergence', recoveryId, prepareSnapshotOk, + ); + if (!observed.ok) return observed.result; + observing = observed.observing; + const healthGateId = observed.healthGateId; + + try { + ServiceUpdateRecoveryService.getInstance().markConsumed(recoveryId, healthGateId); + consumed = true; + } catch (consumeError) { + console.warn( + '[StackUpdateOrchestrator] Failed to mark recovery %s consumed: %s', + sanitizeForLog(recoveryId), sanitizeForLog(getErrorMessage(consumeError, 'unknown')), + ); + // Compose already succeeded; treat as consumed for lease/cleanup purposes so + // we do not reactivate a snapshot that may no longer match the running image. + consumed = true; + } + + const recheck = await ImageUpdateService.getInstance().recheckStack(nodeId, stackName); + await this.refreshMeshIfEnabled(nodeId, stackName); + + const warnings = [observed.gateWarning, recheck.warning].filter((w): w is string => !!w); + return { + kind: 'service_done', + serviceName, + healthGateId, + observing, + recoveryId, + recoveryAvailable: false, + previousImageId, + newImageId: observed.newImageId ?? null, + recheckWarning: warnings.length > 0 ? warnings.join(' ') : undefined, + }; + } finally { + ServiceUpdateRecoveryService.getInstance().stopClaimRenewal(recoveryId); + if (claimed && !consumed) { + const stillLocal = await this.imageIsLocal(nodeId, recovery.majority_image_id); + if (stillLocal) { + ServiceUpdateRecoveryService.getInstance().reactivate(recoveryId); + } else { + ServiceUpdateRecoveryService.getInstance().invalidate(recoveryId); + } + } + if (!observing) { + AutoHealService.getInstance().clearSuppress(nodeId, stackName, serviceName); + } + } + } + + /** Best-effort mesh alias refresh when this stack is opted into Mesh. */ + private async refreshMeshIfEnabled(nodeId: number, stackName: string): Promise { + try { + if (!DatabaseService.getInstance().isMeshStackEnabled(nodeId, stackName)) return; + await MeshService.getInstance().refreshAliasCache(); + } catch (error) { + console.warn( + '[StackUpdateOrchestrator] Mesh alias refresh failed for %s on node %d: %s', + sanitizeForLog(stackName), nodeId, sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + } + } + + private async loadServiceSpec( + nodeId: number, + stackName: string, + serviceName: string, + recoveryId?: string | null, + ): Promise< + | { ok: true; spec: EffectiveServiceSpec; services: EffectiveServiceSpec[] } + | { ok: false; result: Extract } + > { + const model = await buildEffectiveServiceModel(nodeId, stackName); + if (!model.renderable) { + return { ok: false, result: serviceFailed(model.code, model.error, { recoveryId }) }; + } + const spec = model.services.find(s => s.name === serviceName); + if (!spec) { + return { + ok: false, + result: serviceFailed( + 'service_not_found', + `Service "${serviceName}" is not declared in this stack.`, + { recoveryId }, + ), + }; + } + return { ok: true, spec, services: model.services }; + } + + private async checkPolicyOrFail( + stackName: string, + nodeId: number, + serviceName: string, + refs: string[], + policyOptions: PolicyEnforcementOptions, + action: 'update' | 'rollback', + recoveryId?: string | null, + ): Promise | null> { + if (refs.length === 0) return null; + const gate = await enforcePolicyForImageRefs(stackName, nodeId, refs, policyOptions); + if (gate.ok || gate.bypassed) return null; + return serviceFailed( + 'policy_blocked', + `Service "${serviceName}": ${describePolicyBlock(gate.policy, gate.violations, action)}`, + { serviceName, mutationStage: 'policy', recoveryId }, + ); + } + + private async observePreparedService( + prepareToken: string, + nodeId: number, + stackName: string, + serviceName: string, + expectedReplicas: number, + actor: string | null, + divergenceCode: string, + recoveryId: string | null, + prepareSnapshotOk: boolean, + ): Promise< + | { ok: true; healthGateId: string | null; observing: boolean; newImageId: string | null; gateWarning?: string } + | { ok: false; result: Extract } + > { + const convergence = await this.resolvePrimaryImage(nodeId, stackName, serviceName, expectedReplicas); + if (convergence.kind === 'inspect_failed') { + return { + ok: false, + result: serviceFailed('replica_inspect_failed', convergence.error, { + serviceName, mutationStage: 'inspect', recoveryId, + }), + }; + } + if (convergence.kind === 'divergent') { + return { + ok: false, + result: serviceFailed(divergenceCode, convergence.error, { + serviceName, mutationStage: 'inspect', recoveryId, + }), + }; + } + if (!prepareSnapshotOk) { + return { + ok: true, + healthGateId: null, + observing: false, + newImageId: convergence.imageId, + gateWarning: 'Health gate skipped: pre-update container snapshot was unavailable.', + }; + } + const begin = this.beginPrepared(prepareToken, convergence.imageId, actor); + return { + ok: true, + healthGateId: begin.runId, + observing: begin.observing, + newImageId: convergence.imageId, + }; + } + + /** Attach the single converged image id (scale>0) and begin the prepared gate. */ + private beginPrepared( + prepareToken: string, + imageId: string | null, + actor: string | null, + ): { runId: string | null; observing: boolean } { + if (imageId) { + HealthGateService.getInstance().attachExpectedImage(prepareToken, imageId); + } + return HealthGateService.getInstance().beginPrepared({ prepareToken, actor }); + } + + /** + * Post-mutation replica convergence check. Returns the single shared image id + * when every expected primary replica agrees, null for a scale-0 service, or a + * typed divergence when replicas are missing or run 2+ distinct images. + */ + private async resolvePrimaryImage( + nodeId: number, + stackName: string, + serviceName: string, + expectedReplicas: number, + ): Promise< + | { kind: 'converged'; imageId: string | null } + | { kind: 'divergent'; error: string } + | { kind: 'inspect_failed'; error: string } + > { + if (expectedReplicas === 0) { + return { kind: 'converged', imageId: null }; + } + const inspected = await this.inspectServiceContainers(nodeId, stackName, serviceName); + return evaluateServiceReplicaConvergence( + serviceName, expectedReplicas, inspected.ids, inspected.inspectErrors, + ); + } + + private async inspectPrimaryImageIds(nodeId: number, stackName: string, serviceName: string): Promise { + return (await this.inspectServiceContainers(nodeId, stackName, serviceName)).ids; + } + + /** Snapshot the current per-replica image ids + repo digests for the recovery row. */ + private async snapshotServiceReplicas( + nodeId: number, + stackName: string, + serviceName: string, + ): Promise { + const docker = DockerController.getInstance(nodeId).getDocker(); + const snapshots: ServiceReplicaSnapshot[] = []; + for (const imageId of (await this.inspectServiceContainers(nodeId, stackName, serviceName)).ids) { + let repoDigest: string | null = null; + try { + const image = await docker.getImage(imageId).inspect(); + const digests = (image.RepoDigests ?? []) as string[]; + repoDigest = digests.length > 0 ? digests[0] : null; + } catch { + // The digest is optional context; a missing image inspect leaves it null. + } + snapshots.push({ imageId, repoDigest }); + } + return snapshots; + } + + private async inspectServiceContainers( + nodeId: number, + stackName: string, + serviceName: string, + ): Promise<{ ids: string[]; inspectErrors: number }> { + const docker = DockerController.getInstance(nodeId).getDocker(); + const listed = await docker.listContainers({ + all: true, + filters: { label: [`com.docker.compose.project=${stackName}`, `com.docker.compose.service=${serviceName}`] }, + }); + const ids: string[] = []; + let inspectErrors = 0; + for (const info of listed) { + try { + const inspect = await docker.getContainer(info.Id).inspect(); + // Convergence requires running replicas; exited/created leftovers must + // not count toward the expected replica total or the shared image id. + if (inspect.State?.Status !== 'running') continue; + if (inspect.Image) ids.push(inspect.Image); + } catch (error) { + if ((error as { statusCode?: number })?.statusCode === 404) continue; + inspectErrors += 1; + console.warn('[Orchestrator] Replica inspect failed for %s/%s:', + sanitizeForLog(stackName), sanitizeForLog(serviceName), getErrorMessage(error, 'unknown')); + } + } + return { ids, inspectErrors }; + } + + /** + * Immutable restore scan target: the captured repo digest when it resolves + * locally to the majority image id, otherwise the majority image id directly. + * Never the moving declared tag (RESTORE-POLICY-TARGET-2). + */ + private async resolveRestoreScanTarget(nodeId: number, recovery: ServiceUpdateRecoveryRow): Promise { + let replicas: ServiceReplicaSnapshot[] = []; + try { + const parsed: unknown = JSON.parse(recovery.replicas_json); + if (Array.isArray(parsed)) replicas = parsed as ServiceReplicaSnapshot[]; + } catch { + // Corrupt snapshot: fall back to the majority image id below. + } + const withDigest = replicas.find(r => r.imageId === recovery.majority_image_id && r.repoDigest); + if (withDigest?.repoDigest) { + const resolvedId = await this.resolveImageId(nodeId, withDigest.repoDigest); + if (resolvedId === recovery.majority_image_id) return withDigest.repoDigest; + } + return recovery.majority_image_id; + } + + private async resolveImageId(nodeId: number, ref: string): Promise { + try { + const inspect = await DockerController.getInstance(nodeId).getDocker().getImage(ref).inspect(); + return inspect.Id ?? null; + } catch { + return null; + } + } + + private async imageIsLocal(nodeId: number, imageId: string): Promise { + try { + await DockerController.getInstance(nodeId).getDocker().getImage(imageId).inspect(); + return true; + } catch { + return false; + } + } +} diff --git a/backend/src/services/UpdateGuardService.ts b/backend/src/services/UpdateGuardService.ts index 8c01c41f..a36c5d12 100644 --- a/backend/src/services/UpdateGuardService.ts +++ b/backend/src/services/UpdateGuardService.ts @@ -3,7 +3,9 @@ import DockerController from './DockerController'; import { DatabaseService } from './DatabaseService'; import { FileSystemService } from './FileSystemService'; import { ComposeDoctorService } from './ComposeDoctorService'; -import { UpdatePreviewService, isMovingTag } from './UpdatePreviewService'; +import { UpdatePreviewService, isMovingTag, filterPreviewForService } from './UpdatePreviewService'; +import { buildEffectiveServiceModel, type EffectiveServiceModelResult } from './effectiveServiceModel'; +import { filterContainersByComposeService } from '../helpers/composeServiceMatch'; import { withTimeout } from '../utils/withTimeout'; import { getErrorMessage } from '../utils/errors'; import { sanitizeForLog } from '../utils/safeLog'; @@ -17,9 +19,12 @@ import { diskSignal, driftSignal, healthchecksSignal, + isTroubledContainer, preflightSignal, + serviceSignal, updatePreviewSignal, type Errored, + type ServiceMembership, } from './updateGuard/readiness'; import type { ContainerProbe, RollbackReadinessReport, UpdateReadinessReport } from './updateGuard/types'; @@ -30,6 +35,19 @@ import type { ContainerProbe, RollbackReadinessReport, UpdateReadinessReport } f // of failing the report. const INPUT_TIMEOUT_MS = 3_000; +/** + * Thrown by `computeUpdateReadiness` when a `serviceName` is given for a + * stack that has one (or zero) declared services: service-scoped readiness + * requires a real choice among siblings. The route maps this to a 400, + * distinct from the 500 a genuine computation failure gets. + */ +export class SingleServiceUpdateReadinessError extends Error { + constructor() { + super('Service-scoped readiness requires a stack with more than one service.'); + this.name = 'SingleServiceUpdateReadinessError'; + } +} + /** * Computes update readiness and rollback readiness for a stack, on demand, * from existing per-feature stores (preflight runs, drift findings, the atomic @@ -49,16 +67,18 @@ export class UpdateGuardService { /** * Probe the stack's containers via the compose project label, normalized for * the pure scoring functions. Throws on Docker errors; callers map that to - * the 'error' sentinel. + * the 'error' sentinel. When `serviceName` is given, narrows to that + * service's containers before inspecting (labeled containers, per §6). */ - async probeContainers(nodeId: number, stackName: string): Promise { + async probeContainers(nodeId: number, stackName: string, serviceName?: string): Promise { const docker = DockerController.getInstance(nodeId).getDocker(); const listed = await docker.listContainers({ all: true, filters: { label: [`com.docker.compose.project=${stackName}`] }, }); + const scoped = serviceName ? filterContainersByComposeService(listed, serviceName) : listed; const probes = await Promise.all( - listed.map(async (info): Promise => { + scoped.map(async (info): Promise => { const name = info.Names?.[0]?.replace(/^\//, '') ?? info.Id.slice(0, 12); let inspect: Awaited['inspect']>>; try { @@ -86,17 +106,40 @@ export class UpdateGuardService { return probes.filter((p): p is ContainerProbe => p !== null); } - async computeUpdateReadiness(nodeId: number, stackName: string): Promise { + /** + * When `serviceName` is set, scopes the verdict-affecting inputs (labeled + * containers, healthchecks, the update preview, and drift) to that service + * only; the stack guardrails (preflight, disk, backup, Docker reachability) + * stay unchanged (§6). A render failure or missing service fails closed via + * the added `service` signal; a single-service stack throws instead of + * scoping (the caller/route maps that to a 400). + */ + async computeUpdateReadiness(nodeId: number, stackName: string, serviceName?: string): Promise { const db = DatabaseService.getInstance(); const now = Date.now(); - const [preflight, drift, containers, preview, backup, disk] = await Promise.all([ + let model: EffectiveServiceModelResult | null = null; + if (serviceName) { + model = await buildEffectiveServiceModel(nodeId, stackName); + if (model.renderable && model.services.length <= 1) { + throw new SingleServiceUpdateReadinessError(); + } + } + + const [preflight, drift, containers, siblings, preview, backup, disk] = await Promise.all([ this.collect('preflight', stackName, async () => ComposeDoctorService.getInstance().getLatest(nodeId, stackName)), - this.collect('drift', stackName, async () => db.getOpenDriftFindings(nodeId, stackName).length), + this.collect('drift', stackName, async () => + db.getOpenDriftFindings(nodeId, stackName).filter(f => !serviceName || f.service === serviceName).length), this.collect('containers', stackName, () => - withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness container probe')), - this.collect('update preview', stackName, () => - withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness update preview')), + withTimeout(this.probeContainers(nodeId, stackName, serviceName), INPUT_TIMEOUT_MS, 'readiness container probe')), + serviceName + ? this.collect('sibling containers', stackName, () => + withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness sibling probe')) + : Promise.resolve([]), + this.collect('update preview', stackName, async () => { + const full = await withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness update preview'); + return serviceName ? filterPreviewForService(full, serviceName) : full; + }), this.collect('backup info', stackName, () => FileSystemService.getInstance(nodeId).getBackupInfo(stackName)), this.collect('disk', stackName, () => this.readDiskUsage()), ]); @@ -114,8 +157,56 @@ export class UpdateGuardService { backupSlotSignal(backup, now), diskSignal(typeof disk === 'number' ? { usePercent: disk, limitPercent } : 'error'), ]; + if (serviceName) { + signals.push(serviceSignal(this.resolveServiceMembership(model, serviceName))); + } - return { stack: stackName, computedAt: now, verdict: aggregateVerdict(signals), signals }; + const advisories = serviceName + ? [ + this.siblingHealthAdvisory(containers, siblings), + this.dependencyAdvisory(model, serviceName), + ].filter((note): note is string => note !== null) + : []; + + return { + stack: stackName, + computedAt: now, + verdict: aggregateVerdict(signals), + signals, + serviceName: serviceName ?? null, + advisories, + }; + } + + /** Model-membership facts for the selected service; 'error' when the model failed to render. */ + private resolveServiceMembership(model: EffectiveServiceModelResult | null, serviceName: string): ServiceMembership | Errored { + if (!model || !model.renderable) return 'error'; + const spec = model.services.find(s => s.name === serviceName); + if (!spec) return { found: false }; + return { found: true, hasBuild: spec.hasBuild, declaredImage: spec.declaredImage, expectedReplicas: spec.expectedReplicas }; + } + + /** Advisory only (§6): a sibling already unhealthy before the update never blocks the selected service's verdict. */ + private siblingHealthAdvisory(selected: ContainerProbe[] | Errored, all: ContainerProbe[] | Errored): string | null { + if (selected === 'error' || all === 'error') return null; + const selectedNames = new Set(selected.map(c => c.name)); + const troubled = all.filter(c => !selectedNames.has(c.name) && isTroubledContainer(c)); + if (troubled.length === 0) return null; + const names = troubled.map(c => c.name).join(', '); + return `Other containers in this stack are already unhealthy: ${names}. This does not block the selected service.`; + } + + /** Advisory only (§6): dependsOn relationships are informational; this update never recreates siblings. */ + private dependencyAdvisory(model: EffectiveServiceModelResult | null, serviceName: string): string | null { + if (!model || !model.renderable) return null; + const spec = model.services.find(s => s.name === serviceName); + const dependsOn = spec?.dependsOn ?? []; + const dependents = model.services.filter(s => s.dependsOn.includes(serviceName)).map(s => s.name); + const parts: string[] = []; + if (dependsOn.length > 0) parts.push(`depends on ${dependsOn.join(', ')}`); + if (dependents.length > 0) parts.push(`is a dependency of ${dependents.join(', ')}`); + if (parts.length === 0) return null; + return `This service ${parts.join(' and ')}. Those services are not restarted by this update.`; } async computeRollbackReadiness(nodeId: number, stackName: string): Promise { diff --git a/backend/src/services/UpdatePreviewService.ts b/backend/src/services/UpdatePreviewService.ts index c2854baf..42cdc0ae 100644 --- a/backend/src/services/UpdatePreviewService.ts +++ b/backend/src/services/UpdatePreviewService.ts @@ -287,6 +287,13 @@ export function buildSummary( }; } +/** Filter a full-stack preview down to one service's images and recompute the summary from that subset. */ +export function filterPreviewForService(preview: UpdatePreview, serviceName: string): UpdatePreview { + const images = preview.images.filter(i => i.service === serviceName); + const buildServices = preview.build_services.filter(s => s === serviceName); + return buildSummary(preview.stack_name, images, buildServices); +} + export class UpdatePreviewService { private static instance: UpdatePreviewService; diff --git a/backend/src/services/WebhookService.ts b/backend/src/services/WebhookService.ts index a61884e8..9417420e 100644 --- a/backend/src/services/WebhookService.ts +++ b/backend/src/services/WebhookService.ts @@ -1,5 +1,6 @@ import crypto from 'crypto'; import { ComposeService } from './ComposeService'; +import { StackUpdateOrchestrator } from './StackUpdateOrchestrator'; import { StackOpLockService, stackOpSkipMessage, type StackOpAction } from './StackOpLockService'; import { DatabaseService, type Webhook } from './DatabaseService'; import { FileSystemService } from './FileSystemService'; @@ -162,7 +163,7 @@ export class WebhookService { atomic, { source: 'webhook', actor: 'system:webhook' }, ); - HealthGateService.getInstance().begin(nodeId, stackName, 'deploy', 'system:webhook'); + HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:webhook'); break; case 'restart': await compose.runCommand(stackName, 'restart'); @@ -179,8 +180,11 @@ export class WebhookService { nodeId, buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }), ); - await compose.updateStack(stackName, undefined, atomic); - HealthGateService.getInstance().begin(nodeId, stackName, 'update', 'system:webhook'); + await StackUpdateOrchestrator.getInstance().execute( + { nodeId, stackName, target: { scope: 'stack' }, trigger: 'webhook', actor: 'system:webhook' }, + { atomic: atomic ?? false, terminalWs: null }, + ); + HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook'); break; } }, diff --git a/backend/src/services/effectiveServiceModel.ts b/backend/src/services/effectiveServiceModel.ts new file mode 100644 index 00000000..ed74330f --- /dev/null +++ b/backend/src/services/effectiveServiceModel.ts @@ -0,0 +1,144 @@ +/** + * Effective Service Model: per-service facts that service-scoped update and + * restore key off of (declared image, build presence, expected replica + * count, dependencies, healthcheck presence), derived from the + * FULLY-MERGED effective model (`docker compose config --format json`) + * instead of a single compose file, so a multi-file Git source's overrides + * are always reflected. + * + * Fail closed: unlike the read-only facts readers in this module family + * (effectiveAnatomy, storage inventory, etc.), this model backs mutation and + * readiness decisions, so a render failure must never fall back to a + * root-file parse. A caller that mutates or gates on `renderable: false` + * would risk pulling the wrong image or under/over-recreating replicas + * against a stale guess. + * + * Reads only the structural fields a mutation needs; it never reads + * `environment`, `command`, `entrypoint`, `labels`, `secrets`, or `configs`. + */ +import { ComposeService } from './ComposeService'; +import { parseMissingRequiredVars } from '../helpers/envVarParse'; +import { getErrorMessage } from '../utils/errors'; +import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; + +const MAX_RENDER_ERROR = 600; + +export interface EffectiveServiceSpec { + name: string; + declaredImage: string | null; + hasBuild: boolean; + /** May be 0 (explicit `scale: 0` or `deploy.replicas: 0`); defaults to 1 when neither is set, matching Compose's own default. */ + expectedReplicas: number; + dependsOn: string[]; + hasHealthcheck: boolean; +} + +export type EffectiveServiceModelResult = + | { renderable: true; services: EffectiveServiceSpec[] } + | { renderable: false; code: 'effective_model_render_failed'; error: string }; + +function asString(v: unknown): string | undefined { + if (typeof v === 'string') return v; + if (typeof v === 'number') return String(v); + return undefined; +} + +function asNumber(v: unknown): number | undefined { + if (typeof v === 'number' && Number.isFinite(v)) return v; + return undefined; +} + +/** + * `depends_on` always renders as a map keyed by target service name (Compose + * normalizes the short list form into the same map shape), so a plain key + * extraction covers both authoring styles; the list branch is a defensive + * fallback for a hand-built fixture. + */ +function dependsOnNames(value: unknown): string[] { + if (Array.isArray(value)) { + return value.map(asString).filter((s): s is string => s !== undefined); + } + if (value && typeof value === 'object') return Object.keys(value as Record); + return []; +} + +/** + * Expected replica count mirrors Compose's own `ServiceConfig.GetScale()`: + * the top-level `scale` field wins when set (Compose keeps it in sync with + * `deploy.replicas`), else `deploy.replicas`, else the Compose default of 1. + */ +function expectedReplicasOf(svc: Record, deploy: Record | undefined): number { + return asNumber(svc.scale) ?? asNumber(deploy?.replicas) ?? 1; +} + +function parseServiceSpec(name: string, raw: unknown): EffectiveServiceSpec { + const svc = (raw ?? {}) as Record; + const deploy = (svc.deploy && typeof svc.deploy === 'object') ? svc.deploy as Record : undefined; + const healthcheck = svc.healthcheck; + const hasHealthcheck = !!healthcheck + && typeof healthcheck === 'object' + && (healthcheck as Record).disable !== true; + return { + name, + declaredImage: asString(svc.image) ?? null, + hasBuild: !!svc.build && typeof svc.build === 'object', + expectedReplicas: expectedReplicasOf(svc, deploy), + dependsOn: dependsOnNames(svc.depends_on), + hasHealthcheck, + }; +} + +/** Parse the raw `docker compose config --format json` output into the per-service specs. Tolerant of missing/garbage fields: an empty or malformed `services` map yields an empty list rather than throwing. */ +function parseEffectiveServices(parsed: unknown): EffectiveServiceSpec[] { + const root = (parsed && typeof parsed === 'object') ? parsed as Record : {}; + const rawServices = (root.services && typeof root.services === 'object' && !Array.isArray(root.services)) + ? root.services as Record + : {}; + return Object.entries(rawServices).map(([name, svc]) => parseServiceSpec(name, svc)); +} + +/** + * Render the fully-merged effective Compose model for a stack and extract the + * per-service specs service-scoped update/restore key off of. Mirrors the + * render-error handling of the other effective-facts readers (redacted, + * never raw stderr or an exception), but never falls back to a root-file + * parse: a render failure leaves `renderable: false` with no services, so a + * caller that skips the `renderable` check cannot silently mutate against a + * stale or partial model. + */ +export async function buildEffectiveServiceModel(nodeId: number, stackName: string): Promise { + try { + const result = await ComposeService.getInstance(nodeId).renderConfig(stackName); + if (result.rendered !== null) { + try { + return { renderable: true, services: parseEffectiveServices(JSON.parse(result.rendered)) }; + } catch (parseErr) { + // JSON.parse errors carry no file content, so the message is safe to log. + console.warn('[EffectiveServiceModel] Effective model parse failed for %s:', + sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(parseErr, 'unknown'))); + return { + renderable: false, + code: 'effective_model_render_failed', + error: 'Sencho could not parse the rendered Compose model.', + }; + } + } + + // Raw stderr can echo file content/secrets and is never surfaced; only the + // names of any missing required variables, otherwise a generic nudge. + const missing = parseMissingRequiredVars(result.stderr); + const error = missing.length + ? `Required variable${missing.length > 1 ? 's' : ''} ${missing.join(', ')} ${missing.length > 1 ? 'have' : 'has'} no value, so the effective model cannot be rendered.` + : 'Sencho could not render the effective Compose model. Check the compose and env files for a YAML syntax error, an unresolved include or merge, or a required variable with no value.'; + return { renderable: false, code: 'effective_model_render_failed', error }; + } catch (err) { + // Spawn failure (docker unavailable), or an unexpected throw before/inside the + // render. Leave a sanitized breadcrumb so a non-spawn bug is not invisible, then + // redact the surfaced message defensively. + console.warn('[EffectiveServiceModel] Render failed for %s:', + sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown'))); + const error = redactSensitiveText(getErrorMessage(err, 'docker compose could not be started.')).slice(0, MAX_RENDER_ERROR).trim() + || 'Sencho could not run docker compose on this node.'; + return { renderable: false, code: 'effective_model_render_failed', error }; + } +} diff --git a/backend/src/services/updateGuard/readiness.ts b/backend/src/services/updateGuard/readiness.ts index 0274e869..854f30d5 100644 --- a/backend/src/services/updateGuard/readiness.ts +++ b/backend/src/services/updateGuard/readiness.ts @@ -67,6 +67,11 @@ export function driftSignal(input: number | Errored): ReadinessSignal { return { ...base, status: 'ok', affectsVerdict: true, detail: 'No open drift findings.' }; } +/** A container already unhealthy, restarting, or crash-exited before any update runs. */ +export function isTroubledContainer(c: ContainerProbe): boolean { + return c.health === 'unhealthy' || c.state === 'restarting' || (c.state === 'exited' && (c.exitCode ?? 0) !== 0); +} + export function containersSignal(input: ContainerProbe[] | Errored): ReadinessSignal { const base = { id: 'containers' as const, title: 'Current containers' }; if (input === 'error') { @@ -75,9 +80,7 @@ export function containersSignal(input: ContainerProbe[] | Errored): ReadinessSi if (input.length === 0) { return { ...base, status: 'warning', affectsVerdict: true, detail: 'The stack is not running; updating will start it.' }; } - const troubled = input.filter( - c => c.health === 'unhealthy' || c.state === 'restarting' || (c.state === 'exited' && (c.exitCode ?? 0) !== 0), - ); + const troubled = input.filter(isTroubledContainer); if (troubled.length > 0) { const names = troubled.map(c => c.name).join(', '); return { @@ -162,6 +165,50 @@ export function buildServicesSignal(buildServices: string[] | Errored): Readines }; } +/** Model-membership facts for the selected service, gathered from EffectiveServiceModel. */ +export type ServiceMembership = + | { found: false } + | { found: true; hasBuild: boolean; declaredImage: string | null; expectedReplicas: number }; + +/** Matches `ServiceUpdateRecoveryService`'s digest-pin check (a recovery snapshot is never eligible for a digest-pinned declared ref). */ +const DIGEST_PIN_RE = /@sha256:[a-f0-9]{64}$/i; + +/** + * Selected-service signal for a service-scoped readiness check: model + * membership and whether the update would leave a rollback recovery snapshot + * behind. Render failure or a missing service fails closed (blocked), never + * unknown, so a service-scoped check cannot silently proceed against a stale + * or absent model. + */ +export function serviceSignal(input: ServiceMembership | Errored): ReadinessSignal { + const base = { id: 'service' as const, title: 'Selected service' }; + if (input === 'error') { + return { ...base, status: 'blocked', affectsVerdict: true, detail: 'The compose model could not be rendered, so the selected service cannot be validated. Resolve the compose error before updating.' }; + } + if (!input.found) { + return { ...base, status: 'blocked', affectsVerdict: true, detail: 'The selected service is not declared in this stack\u2019s compose model.' }; + } + if (!input.hasBuild && !input.declaredImage) { + return { ...base, status: 'blocked', affectsVerdict: true, detail: 'The selected service has neither an image nor a build, so it cannot be updated.' }; + } + const digestPinned = input.declaredImage !== null && DIGEST_PIN_RE.test(input.declaredImage); + if (!input.declaredImage || digestPinned) { + const reason = !input.declaredImage ? 'it builds from source with no declared image' : 'its declared image is pinned to a digest'; + return { + ...base, + status: 'warning', + affectsVerdict: true, + detail: `Declared with ${input.expectedReplicas} expected replica${input.expectedReplicas === 1 ? '' : 's'}. No rollback recovery snapshot will be available for this update because ${reason}.`, + }; + } + return { + ...base, + status: 'ok', + affectsVerdict: true, + detail: `Declared with ${input.expectedReplicas} expected replica${input.expectedReplicas === 1 ? '' : 's'}; a rollback recovery snapshot will be captured before the update.`, + }; +} + export function backupSlotSignal( input: { exists: boolean; timestamp: number | null } | Errored, now: number, diff --git a/backend/src/services/updateGuard/types.ts b/backend/src/services/updateGuard/types.ts index 1bfd7adb..7b756f82 100644 --- a/backend/src/services/updateGuard/types.ts +++ b/backend/src/services/updateGuard/types.ts @@ -6,7 +6,7 @@ export type SignalStatus = 'ok' | 'warning' | 'attention' | 'blocked' | 'unknown /** One input to the readiness verdict (preflight, drift, containers, ...). */ export interface ReadinessSignal { - id: 'preflight' | 'drift' | 'containers' | 'healthchecks' | 'update_preview' | 'build_services' | 'backup_slot' | 'disk'; + id: 'preflight' | 'drift' | 'containers' | 'healthchecks' | 'update_preview' | 'build_services' | 'backup_slot' | 'disk' | 'service'; status: SignalStatus; /** Short headline ("Compose Doctor", "Running containers"). */ title: string; @@ -25,6 +25,13 @@ export interface UpdateReadinessReport { computedAt: number; verdict: ReadinessVerdict; signals: ReadinessSignal[]; + /** The service this report is scoped to, or null for a full-stack check. */ + serviceName: string | null; + /** + * Non-blocking notes that never affect `verdict` (sibling health, + * dependsOn relationships). Always empty for a full-stack check. + */ + advisories: string[]; } /** State of one rollback readiness item. */ @@ -72,6 +79,8 @@ export interface HealthGateContainer { state: string; health: string | null; restarts: number; + service?: string | null; + role?: 'primary' | 'collateral' | null; } /** Payload of GET /:stackName/health-gate ('never-run' when no run exists). */ @@ -79,12 +88,15 @@ export interface HealthGateReport { stack: string; id: string | null; status: HealthGateStatus | 'never-run'; - trigger: 'update' | 'deploy' | null; + trigger: 'update' | 'deploy' | 'service_update' | 'service_restore' | null; reason: string | null; windowSeconds: number | null; startedAt: number | null; endedAt: number | null; containers: HealthGateContainer[]; + targetScope: 'stack' | 'service'; + serviceName: string | null; + failureSource: 'primary' | 'collateral' | null; } /** Categories a failed stack deploy or update can be classified into. */ diff --git a/docs/features/alerts-notifications.mdx b/docs/features/alerts-notifications.mdx index 782763ca..906f43fb 100644 --- a/docs/features/alerts-notifications.mdx +++ b/docs/features/alerts-notifications.mdx @@ -338,7 +338,7 @@ The entire host threshold evaluation can be silenced per node from **Settings · ### Image update availability -`info`/`image_update_available`: `Stack "" has image updates available.` By default, checks every two hours in interval mode, with a two-minute startup delay. The cadence can be changed or replaced with a cron schedule. Manual refresh carries a two-minute cooldown. Notifies on **state transition only**; the first run also emits catch-up notifications for stacks already known to have updates. +`info`/`image_update_available`: `Stack "" has image updates available.` On multi-service stacks the message can name the services that have updates. By default, checks every two hours in interval mode, with a two-minute startup delay. The cadence can be changed or replaced with a cron schedule. Manual refresh carries a two-minute cooldown. Notifies on **state transition only**; the first run also emits catch-up notifications for stacks already known to have updates. ### Auto-update execution diff --git a/docs/features/auto-update-policies.mdx b/docs/features/auto-update-policies.mdx index 40857c9e..9e503e55 100644 --- a/docs/features/auto-update-policies.mdx +++ b/docs/features/auto-update-policies.mdx @@ -71,6 +71,8 @@ Once the cause is resolved, the next check (on the interval, or via **Recheck**) 2. Skim the card grid. The badge tells you the risk at a glance: `Safe · patch` is green, `Review · minor` is amber, `Blocked · major` is red, and a digest-only rebuild on a non-semver tag shows the gray `Digest rebuild` badge. 3. For a safe update, click **Apply now** on the card to pull and recreate the stack immediately. 4. For a major bump, review the changelog preview and the upstream release notes. **Apply now** is disabled on the readiness board for blocked cards; to apply a major bump after review, use the stack's lifecycle **Update** action (right-click the stack in the sidebar, or open the kebab menu and choose **Update**, or click **Deploy** in the stack editor). + +On multi-service stacks, the Updates view can also apply a single service when that service has a confirmed image update. Scheduled auto-update, webhook pull, and bulk update always refresh the full stack. 5. Use **Recheck** in the hero to force an immediate registry poll across every reachable node. A 2-minute per-node cooldown applies, and the toast tells you how many nodes were triggered, rate-limited, or failed. ## Risk badges diff --git a/docs/features/dashboard.mdx b/docs/features/dashboard.mdx index db8c418c..e6da7619 100644 --- a/docs/features/dashboard.mdx +++ b/docs/features/dashboard.mdx @@ -62,7 +62,7 @@ A mono table of every stack discovered in the active node's `COMPOSE_DIR`, sorte | Column | Description | |--------|-------------| | **Status dot** | Green when the stack is running and its 10-minute peak CPU is under 80%, amber when peak CPU is at or above 80%, rose when any container has exited or peak CPU is at or above 90% | -| **STACK** | Stack name with an orange "Update available" badge when a newer image has been detected. The badge appears regardless of the sidebar indicator setting. | +| **STACK** | Stack name with an orange update badge when a newer image has been detected. When per-service status is known, the badge names the outdated service (or a count for several). The badge appears regardless of the sidebar indicator setting. | | **HOST** | Active node this stack belongs to | | **UP** | How long the oldest running container has been up, in compact units (`s` / `m` / `h` / `d`); a stopped or never-started stack reads `--` | | **CPU** | Latest aggregate CPU across the stack's containers | diff --git a/docs/features/health-gated-updates.mdx b/docs/features/health-gated-updates.mdx index 707c8e4e..fe051904 100644 --- a/docs/features/health-gated-updates.mdx +++ b/docs/features/health-gated-updates.mdx @@ -34,6 +34,10 @@ Admins also see whether a [fleet snapshot](/features/fleet-backups) covers this Every verdict keeps the **Update now** button enabled. The dialog informs the decision; it does not make it for you. +On stacks with more than one Compose service, you can also open readiness for a single service from that service's header. The dialog still checks stack-wide guardrails (Compose Doctor, disk, backup, Docker), but container health, pending change, and recovery signals focus on the selected service. Sibling and dependency issues stay advisory. The toolbar **Update** action and the Updates view's stack **Apply** always update the full stack. + +When the selected service is build-backed and has no registry update, the proceed button reads **Rebuild** instead of **Update**. A per-service update badge appears only when a registry update is confirmed for that service. + Update readiness dialog showing the verdict chip, signal rows for Compose Doctor, Drift, Current containers, Healthcheck coverage, Pending update, Rollback backup and Node disk, the fleet snapshot row with its checkbox, and the Cancel and Update now buttons @@ -50,6 +54,8 @@ During the window, the gate checks every container that came up with the stack: A clear failure ends the observation immediately. Passing requires the full window, and a healthcheck still in its start period when the window ends records the verdict as unknown rather than claiming success. +After a **service-scoped** update or restore, the gate still watches the whole project, but it attributes failures: a problem on the updated service is a primary failure; a previously healthy sibling that regresses during the window is a collateral failure. Siblings that were already unhealthy before the update do not fail the gate on their own. + The deploy progress modal treats the gate verdict as the real result: while the gate observes, the status reads **Verifying health** rather than claiming success, and the auto-close waits. A passed gate shows the success state; a failed or unknown gate becomes the headline result with its reason. Closing the modal never stops the observation; the gate runs server-side and its result appears on the stack timeline either way, as **health gate passed** or **health gate failed** events alongside the **update started** marker. The live verifying and recovery view is part of the deploy progress panel. If you have turned that panel off, the in-browser gate view does not appear, but the gate still runs on the node and records its verdict on the stack timeline. diff --git a/docs/features/node-compatibility.mdx b/docs/features/node-compatibility.mdx index 8248c636..f4e5a11f 100644 --- a/docs/features/node-compatibility.mdx +++ b/docs/features/node-compatibility.mdx @@ -79,6 +79,7 @@ Every Sencho release ships with a static list of capabilities. The current list | `compose-doctor` | Compose Doctor preflight | | `compose-networking` | Stack Networking tab and node Networking overview | | `guided-external-network-preflight` | Guided missing-external-network check before deploy | +| `service-scoped-update` | Per-service update, rebuild, and restore on multi-service stacks | `vulnerability-scanning` is advertised only when the Trivy binary is present on the node. If a node does not have Trivy installed, it omits this capability and the scanning UI is replaced by a lock card. diff --git a/docs/features/sidebar.mdx b/docs/features/sidebar.mdx index cfe99d50..31000f5e 100644 --- a/docs/features/sidebar.mdx +++ b/docs/features/sidebar.mdx @@ -68,7 +68,7 @@ Each row gives you everything you need to read the stack at a glance, in a fixed - **Stack name** in mono type, truncated with an ellipsis when the row gets tight. - **Label dots** to the right of the name. Up to three colored dots represent the stack's labels. If a stack carries more than three labels, a **+N** counter appears for the extras. - **Trailing indicator** in its own fixed-width slot at the far right of the name area, showing the highest-priority signal present: - - Pulsing orange dot: an image update is available. Hover for the "Update available" tooltip. + - Pulsing orange dot: an image update is available. Hover for the update tooltip; when per-service status is known, the tooltip names the outdated service(s). - Muted circle-alert icon: the last registry check failed (registry unreachable, missing credentials, or rate-limited). Hover to read the failure reason. - Git branch icon: a Git source update is pending and no image update is available. - **Hover kebab** on the right edge. On desktop, hover the row to reveal a vertical three-dot menu that opens the same actions as right-clicking. On touch viewports the kebab is always visible. diff --git a/docs/features/stack-management.mdx b/docs/features/stack-management.mdx index c9ce8df4..b2082379 100644 --- a/docs/features/stack-management.mdx +++ b/docs/features/stack-management.mdx @@ -220,7 +220,15 @@ Each row includes: - **Meta line.** Uptime (`up 12 hours`) and the primary port mapping (`8989 → 8989/tcp`). - **Open link.** When the container publishes a port, the mapping itself is a link (`8989 → 8989/tcp ↗`) that opens the service in a new tab, with a **Copy URL** button beside it. The address uses the active node's host and switches to `https` for port 443. Recognised multi-port apps open their web path automatically (for example, Plex opens `/web`). - **Live stat tiles.** Three tiles show CPU, memory, and network I/O with a rolling sparkline. The sparkline uses the cyan data color and refreshes roughly every 1.5 seconds. -- **Action icons.** The image source links button (see above), plus shortcuts to **View logs**, open a bash shell, and the per-service kebab. +- **Action icons.** The image source links button (see above), plus shortcuts to **View logs**, open a bash shell, and (on single-service stacks) the per-container Start / Stop / Restart kebab. + +### Multi-service stacks + +When a stack declares more than one Compose service, each service gets its own header above its container cards. That header owns **Update service** or **Rebuild service** (when eligible), the update badge when a registry update is confirmed for that service, and **Start / Stop / Restart** for every replica of the service. Child container cards keep logs, shell, ports, and metrics only. + +Build-backed services without a registry update show **Rebuild** wording and no update badge. The editor toolbar **Update** action still updates the entire stack. + +Single-service stacks keep the existing flat container layout; service headers do not appear. ## Logs viewer diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 7144aaf6..d28d8986 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1772,10 +1772,21 @@ paths: now, from the stored preflight result, open drift findings, live container health, the pending image change, the rollback backup slot, and node disk headroom. Advisory only: no verdict blocks an update. - Requires `stack:read` permission. + Requires `stack:read` permission. Pass `service` to scope the verdict + to one Compose service on a multi-service stack. parameters: - $ref: "#/components/parameters/stackName" - $ref: "#/components/parameters/nodeId" + - name: service + in: query + required: false + schema: + type: string + description: | + When set, readiness is scoped to this Compose service (multi-service + stacks only). Stack-wide guardrails still apply; sibling and + dependency signals become advisory. Single-service stacks reject + this parameter with 400. responses: "200": description: Readiness report. @@ -1810,6 +1821,12 @@ paths: type: string affectsVerdict: type: boolean + "400": + description: Invalid service scope. + content: + application/json: + schema: + $ref: "#/components/schemas/Error" "401": $ref: "#/components/responses/Unauthorized" "403": @@ -1817,6 +1834,238 @@ paths: "500": $ref: "#/components/responses/InternalError" + /api/stacks/{stackName}/effective-services: + get: + operationId: getStackEffectiveServices + tags: [Stacks] + summary: List effective Compose services + description: | + Returns the fail-closed effective service model for the stack (declared + image, build presence, expected replicas, depends_on, healthcheck). + Requires `stack:read` permission. + parameters: + - $ref: "#/components/parameters/stackName" + - $ref: "#/components/parameters/nodeId" + responses: + "200": + description: Effective model result. + content: + application/json: + schema: + type: object + required: [renderable] + properties: + renderable: + type: boolean + services: + type: array + items: + type: object + required: [name, declaredImage, hasBuild, expectedReplicas, dependsOn, hasHealthcheck] + properties: + name: + type: string + declaredImage: + type: string + nullable: true + hasBuild: + type: boolean + expectedReplicas: + type: integer + dependsOn: + type: array + items: + type: string + hasHealthcheck: + type: boolean + code: + type: string + error: + type: string + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "500": + $ref: "#/components/responses/InternalError" + + /api/stacks/{stackName}/services/{serviceName}/update: + post: + operationId: updateStackService + tags: [Stacks] + summary: Update or rebuild one Compose service + description: | + Manually pulls or builds one declared service, then recreates only that + service with `--no-deps --force-recreate`. Multi-service stacks only. + Requires `stack:deploy`. Automation paths continue to update the full stack. + parameters: + - $ref: "#/components/parameters/stackName" + - name: serviceName + in: path + required: true + schema: + type: string + - $ref: "#/components/parameters/nodeId" + responses: + "200": + description: Service update completed. + content: + application/json: + schema: + type: object + required: [serviceName, healthGateId, observing, recoveryAvailable] + properties: + serviceName: + type: string + healthGateId: + type: string + nullable: true + observing: + type: boolean + recoveryId: + type: string + nullable: true + recoveryAvailable: + type: boolean + recheckWarning: + type: string + "400": + description: Service update rejected (single-service stack, missing service, or unavailable capability). + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "403": + $ref: "#/components/responses/Forbidden" + "500": + description: Service update failed. + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + + /api/stacks/{stackName}/services/{serviceName}/recovery: + get: + operationId: getStackServiceRecovery + tags: [Stacks] + summary: Latest active service recovery snapshot + description: | + Returns the newest active, unexpired recovery row for one service, or + `recovery: null` when none is available. Used so Restore stays + discoverable when Deploy Progress is disabled or dismissed. + Requires `stack:deploy`. + parameters: + - $ref: "#/components/parameters/stackName" + - name: serviceName + in: path + required: true + schema: + type: string + - $ref: "#/components/parameters/nodeId" + responses: + "200": + description: Recovery lookup completed. + content: + application/json: + schema: + type: object + required: [recovery] + properties: + recovery: + oneOf: + - type: "null" + - type: object + required: [id, status, expiresAt, createdAt] + properties: + id: + type: string + status: + type: string + healthGateId: + type: string + nullable: true + expiresAt: + type: integer + createdAt: + type: integer + majorityImageId: + type: string + declaredImageRef: + type: string + "400": + description: Capability unavailable. + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + + /api/stacks/{stackName}/services/{serviceName}/restore: + post: + operationId: restoreStackService + tags: [Stacks] + summary: Restore one Compose service from a recovery snapshot + description: | + Retags the captured majority image onto the declared tag and recreates + only that service. Body must include `recoveryId` from a prior service + update. Requires `stack:deploy`. + parameters: + - $ref: "#/components/parameters/stackName" + - name: serviceName + in: path + required: true + schema: + type: string + - $ref: "#/components/parameters/nodeId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [recoveryId] + properties: + recoveryId: + type: string + responses: + "200": + description: Service restore completed. + content: + application/json: + schema: + type: object + required: [serviceName, healthGateId, observing, recoveryAvailable] + properties: + serviceName: + type: string + healthGateId: + type: string + nullable: true + observing: + type: boolean + recoveryId: + type: string + nullable: true + recoveryAvailable: + type: boolean + "400": + description: Restore rejected (missing recoveryId or invalid binding). + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "403": + $ref: "#/components/responses/Forbidden" + "500": + description: Service restore failed. + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /api/stacks/{stackName}/rollback-readiness: get: operationId: getStackRollbackReadiness @@ -1908,7 +2157,7 @@ paths: trigger: type: string nullable: true - enum: [update, deploy, null] + enum: [update, deploy, service_update, service_restore, null] reason: type: string nullable: true @@ -1937,6 +2186,25 @@ paths: nullable: true restarts: type: integer + service: + type: string + nullable: true + role: + type: string + nullable: true + enum: [primary, collateral, null] + targetScope: + type: string + enum: [stack, service] + description: Defaults to stack for legacy runs. + serviceName: + type: string + nullable: true + failureSource: + type: string + nullable: true + enum: [primary, collateral, null] + description: Set on failed service-scoped runs; null for stack and successful runs. "401": $ref: "#/components/responses/Unauthorized" "403": diff --git a/frontend/src/components/AutoUpdateReadinessView.tsx b/frontend/src/components/AutoUpdateReadinessView.tsx index 0e642dc4..8b2cc687 100644 --- a/frontend/src/components/AutoUpdateReadinessView.tsx +++ b/frontend/src/components/AutoUpdateReadinessView.tsx @@ -12,6 +12,9 @@ import { useIsMobile } from '@/hooks/use-is-mobile'; import { Masthead, Kicker } from '@/components/mobile/mobile-ui'; import { ImageSourceMenu } from './ImageSourceMenu'; import type { ScheduledTask } from '@/types/scheduling'; +import { SERVICE_SCOPED_UPDATE_CAPABILITY } from '@/lib/capabilities'; +import { requestServiceUpdate } from '@/lib/serviceUpdate'; +import { useDeployFeedback } from '@/context/DeployFeedbackContext'; type SemverBump = 'none' | 'patch' | 'minor' | 'major' | 'unknown'; @@ -38,11 +41,22 @@ interface UpdatePreview { update_kind: UpdateKind; blocked: boolean; blocked_reason: string | null; + has_build_services?: boolean; + rebuild_available?: boolean; }; + build_services?: string[]; rollback_target: string | null; changelog: string | null; } +function declaredServiceCount(preview: UpdatePreview | null | undefined): number { + if (!preview) return 0; + const names = new Set(); + for (const img of preview.images) names.add(img.service); + for (const name of preview.build_services ?? []) names.add(name); + return names.size; +} + export interface StackCard { stack: string; nodeId: number; @@ -55,6 +69,9 @@ export interface StackCard { // and the hero "ready to apply automatically" count. Manual Apply now is // schedule-independent and does not read this field. autoUpdateEnabled: boolean; + // Name of the service currently applying a per-service update on this card, + // or null when none is in flight. Distinct from `applying` (full-stack). + applyingService: string | null; } interface NodeGroup { @@ -186,17 +203,25 @@ function VersionDiff({ current, next }: { current: string | null; next: string | function StackReadinessCard({ card, + canServiceUpdate = false, onApply, + onApplyService, }: { card: StackCard; + canServiceUpdate?: boolean; onApply: (stack: string, nodeId: number) => void; + onApplyService?: (stack: string, nodeId: number, serviceName: string) => void; }) { - const { stack, nodeId, preview, previewLoaded, scheduledTask, applying, autoUpdateEnabled } = card; + const { stack, nodeId, preview, previewLoaded, scheduledTask, applying, applyingService, autoUpdateEnabled } = card; const loading = !previewLoaded; const failed = previewLoaded && preview === null; const blocked = preview?.summary.blocked ?? false; const bump = preview?.summary.semver_bump ?? 'none'; - const updatingImageCount = preview?.images.filter(i => i.has_update).length ?? 0; + const updatingImages = preview?.images.filter(i => i.has_update) ?? []; + const updatingImageCount = updatingImages.length; + // Multi-service only: count declared Compose services (image-backed and + // build-only), not preview.images.length (shared tags collapse that list). + const showServiceApply = canServiceUpdate && declaredServiceCount(preview) > 1 && updatingImageCount > 0; const nextRun = scheduledTask?.next_run_at ?? null; return ( @@ -267,6 +292,25 @@ function StackReadinessCard({ )} + {showServiceApply && ( +
+ {updatingImages.map(img => ( +
+ {img.service} + +
+ ))} +
+ )} +
{nextRun ? ( @@ -285,7 +329,7 @@ function StackReadinessCard({
{group.cards.map(card => ( - + ))}
@@ -412,11 +466,23 @@ function NodeGroupSection({ /** One-up readiness card for the phone screen. Reuses RiskBadge + VersionDiff * and the same apply/disabled logic as the desktop card. Exported for tests. */ -export function MobileReadinessCard({ card, onApply }: { card: StackCard; onApply: (stack: string, nodeId: number) => void }) { - const { stack, nodeId, preview, previewLoaded, scheduledTask, applying, autoUpdateEnabled } = card; +export function MobileReadinessCard({ + card, + canServiceUpdate = false, + onApply, + onApplyService, +}: { + card: StackCard; + canServiceUpdate?: boolean; + onApply: (stack: string, nodeId: number) => void; + onApplyService?: (stack: string, nodeId: number, serviceName: string) => void; +}) { + const { stack, nodeId, preview, previewLoaded, scheduledTask, applying, applyingService, autoUpdateEnabled } = card; const failed = previewLoaded && preview === null; const blocked = preview?.summary.blocked ?? false; const bump = preview?.summary.semver_bump ?? 'none'; + const updatingImages = preview?.images.filter(i => i.has_update) ?? []; + const showServiceApply = canServiceUpdate && declaredServiceCount(preview) > 1 && updatingImages.length > 0; const nextRun = scheduledTask?.next_run_at ?? null; const changelog = preview?.changelog ?? 'No changelog available from the registry yet.'; const dot = changelog.indexOf('.'); @@ -458,6 +524,24 @@ export function MobileReadinessCard({ card, onApply }: { card: StackCard; onAppl
{lead && {lead}}{rest}
+ {showServiceApply && ( +
+ {updatingImages.map(img => ( +
+ {img.service} + +
+ ))} +
+ )}
{nextRun ? <>{formatClock(nextRun)} · {formatRelative(nextRun)} : (blocked ? 'Held for review' : 'No schedule')} @@ -466,7 +550,7 @@ export function MobileReadinessCard({ card, onApply }: { card: StackCard; onAppl size="sm" variant={blocked ? 'outline' : 'default'} onClick={() => onApply(stack, nodeId)} - disabled={blocked || applying} + disabled={blocked || applying || applyingService !== null} className="gap-1.5" >
) : ( - groups.map(group => ) + groups.map(group => ( + + )) )}
@@ -903,7 +1079,13 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps) ) : (
{groups.map(group => ( - + ))}
)} diff --git a/frontend/src/components/DeployFeedbackModal.tsx b/frontend/src/components/DeployFeedbackModal.tsx index 47e03b86..e5c69ce4 100644 --- a/frontend/src/components/DeployFeedbackModal.tsx +++ b/frontend/src/components/DeployFeedbackModal.tsx @@ -17,6 +17,8 @@ import { StructuredLogRow } from '@/components/log-rendering/StructuredLogRow'; import TerminalComponent from '@/components/Terminal'; import { useDeployFeedback, VERB_LABELS, type HealthGateUiState } from '@/context/DeployFeedbackContext'; import { useDeployFeedbackStyle } from '@/hooks/use-deploy-feedback-style'; +import { requestServiceRestore } from '@/lib/serviceUpdate'; +import { toast } from '@/components/ui/toast-store'; const AUTO_CLOSE_SECONDS = 4; @@ -364,15 +366,62 @@ export function DeployFeedbackModal({ isMinimized, onMinimize }: DeployFeedbackM // Re-renders every second via the elapsed-time interval, so the observing // elapsed count stays live without its own timer. function HealthGateBanner({ gate }: { gate: HealthGateUiState }) { + const { runWithLog } = useDeployFeedback(); + const [restoring, setRestoring] = useState(false); const elapsed = gate.startedAt ? Math.max(0, Math.floor((Date.now() - gate.startedAt) / 1000)) : 0; const windowLabel = gate.windowSeconds ? ` of ${gate.windowSeconds}s` : ''; + // Service-scoped gates name the service; a full-stack gate keeps the + // existing "containers"/"the stack" phrasing unchanged. + const scopeSubject = gate.serviceName ? `service "${gate.serviceName}"` : 'containers'; + const collateralNote = gate.failureSource === 'collateral' ? ' A dependent service triggered the failure.' : ''; + const canRestoreService = + gate.targetScope === 'service' + && !!gate.serviceName + && !!gate.recoveryId + && gate.status === 'failed'; + + const onRestoreService = useCallback(async () => { + if (!gate.serviceName || !gate.recoveryId || restoring) return; + setRestoring(true); + try { + await runWithLog( + { stackName: gate.stackName, action: 'update', nodeId: gate.nodeId, serviceName: gate.serviceName }, + async (started, ds) => { + await started; + const result = await requestServiceRestore({ + nodeId: gate.nodeId, + stackName: gate.stackName, + serviceName: gate.serviceName as string, + recoveryId: gate.recoveryId as string, + deploySessionId: ds, + }); + if (!result.ok) { + toast.error(result.error); + return { ok: false as const, errorMessage: result.error }; + } + if (result.healthGateId && result.observing) { + toast.info(`Service "${gate.serviceName}" restored. Verifying health...`); + } else { + toast.success(`Service "${gate.serviceName}" restored successfully!`); + } + return { + ok: true as const, + healthGateId: result.observing ? result.healthGateId : null, + recoveryId: result.recoveryId, + }; + }, + ); + } finally { + setRestoring(false); + } + }, [gate, restoring, runWithLog]); if (gate.status === 'observing') { return (

- Health gate: observing containers ({elapsed}s{windowLabel}). Closing this panel does not stop the observation. + Health gate: observing {scopeSubject} ({elapsed}s{windowLabel}). Closing this panel does not stop the observation.

); @@ -382,7 +431,7 @@ function HealthGateBanner({ gate }: { gate: HealthGateUiState }) {

- Health gate passed: containers stayed healthy through the observation window. + Health gate passed: {scopeSubject} stayed healthy through the observation window.

); @@ -391,9 +440,26 @@ function HealthGateBanner({ gate }: { gate: HealthGateUiState }) { return (
-

- Health gate failed{gate.reason ? `: ${gate.reason}` : ''}. Rollback options are available on the stack. -

+
+

+ Health gate failed{gate.reason ? `: ${gate.reason}` : ''}.{collateralNote} + {canRestoreService + ? ` Restore "${gate.serviceName}" from the recovery snapshot, or inspect the stack.` + : ' Rollback options are available on the stack.'} +

+ {canRestoreService && ( + + )} +
); } diff --git a/frontend/src/components/EditorLayout.tsx b/frontend/src/components/EditorLayout.tsx index dcefb763..81deb221 100644 --- a/frontend/src/components/EditorLayout.tsx +++ b/frontend/src/components/EditorLayout.tsx @@ -48,6 +48,7 @@ import { useTopNavMode } from '@/hooks/use-top-nav-mode'; import { useTopNavQuickLinks } from '@/hooks/use-top-nav-quick-links'; import { getAppNavItem } from '@/lib/navigation/appNavRegistry'; import { useStackMuteActions } from '@/hooks/useMuteRuleActions'; +import { useServiceUpdateStatus } from '@/hooks/useServiceUpdateStatus'; import { toast } from '@/components/ui/toast-store'; import { useIsMobile } from '@/hooks/use-is-mobile'; import { MobileTabBar } from './MobileTabBar'; @@ -112,6 +113,8 @@ export default function EditorLayout() { backupInfo, isEditing, editingCompose, setEditingCompose, + effectiveServices, + serviceUpdateInProgress, } = editorState; const stackListState = useStackListState(); @@ -258,6 +261,8 @@ export default function EditorLayout() { activeNode?.id ?? null, ); + const serviceUpdateStatuses = useServiceUpdateStatus(stackUpdates, selectedFile); + const stackActions = useStackActions({ editorState, stackListState, @@ -271,6 +276,7 @@ export default function EditorLayout() { diffPreviewEnabled, hasUpdateGuard: hasCapability('update-guard'), hasGuidedExternalNetworkPreflight: hasCapability('guided-external-network-preflight'), + hasServiceScopedUpdate: hasCapability('service-scoped-update'), canEditStack: (stackNameOrFilename) => { const stackName = stackNameOrFilename.replace(/\.(ya?ml)$/, ''); return can('stack:edit', 'stack', stackName); @@ -639,6 +645,13 @@ export default function EditorLayout() { openLogViewer={stackActions.openLogViewer} openBashModal={stackActions.openBashModal} serviceAction={stackActions.serviceAction} + effectiveServices={effectiveServices} + serviceUpdateStatuses={serviceUpdateStatuses} + serviceUpdateInProgress={serviceUpdateInProgress} + onRequestServiceUpdate={(serviceName, mode) => { + if (!selectedFile) return; + void stackActions.requestServiceUpdate(selectedFile, serviceName, mode); + }} setActiveTab={setActiveTab} setLogsMode={setLogsMode} setEditingCompose={setEditingCompose} diff --git a/frontend/src/components/EditorLayout/EditorView.tsx b/frontend/src/components/EditorLayout/EditorView.tsx index b1b62ab9..57c2d93b 100644 --- a/frontend/src/components/EditorLayout/EditorView.tsx +++ b/frontend/src/components/EditorLayout/EditorView.tsx @@ -47,6 +47,8 @@ import type { NotificationItem } from '../dashboard/types'; import type { Node } from '@/context/NodeContext'; import type { useAuth } from '@/context/AuthContext'; import type { useStackMuteActions } from '@/hooks/useMuteRuleActions'; +import type { EffectiveServiceSpec } from '@/types/effectiveServices'; +import type { StackServiceUpdateStatus } from '@/types/imageUpdates'; export interface ContainerInfo { Id: string; @@ -176,6 +178,14 @@ export interface EditorViewProps { action: 'start' | 'stop' | 'restart', serviceName: string, ) => Promise; + // Declared-service facts for the multi-service header split (§12). Empty + // on single-service stacks and older remotes (capability-gated fetch), so + // ContainersHealth falls back to the flat single-service layout. Optional + // so callers/tests that never deal in services can omit them. + effectiveServices?: EffectiveServiceSpec[]; + serviceUpdateStatuses?: StackServiceUpdateStatus[]; + serviceUpdateInProgress?: { service: string; mode: 'update' | 'rebuild' } | null; + onRequestServiceUpdate?: (serviceName: string, mode: 'update' | 'rebuild') => void; // UI state setters setActiveTab: (tab: 'compose' | 'env' | 'files') => void; @@ -263,6 +273,10 @@ export function EditorView(props: EditorViewProps) { openLogViewer, openBashModal, serviceAction, + effectiveServices = [], + serviceUpdateStatuses = [], + serviceUpdateInProgress = null, + onRequestServiceUpdate, setActiveTab, setLogsMode, setEditingCompose, @@ -368,6 +382,11 @@ export function EditorView(props: EditorViewProps) { }); }; + // Declared-service headers (§12) need the same expandable, scroll-wrapped + // layout as a multi-container stack even when only one container of a + // multi-service stack is currently running. + const isMultiContainerLayout = safeContainers.length > 1 || effectiveServices.length > 1; + // Below md, render the segmented full-screen mobile detail instead of the // desktop two-pane grid. All hooks above run unconditionally before this // branch so hook order stays stable across breakpoints. @@ -386,7 +405,7 @@ export function EditorView(props: EditorViewProps) { {/* Command Center Card (identity + health strip). Hidden when the logs are expanded so the logs pane fills the column. */} {!logsExpanded && ( - 1 && !containersExpanded ? 'flex flex-col min-h-0 max-h-[42%]' : safeContainers.length > 1 && containersExpanded ? 'flex flex-col flex-1 min-h-0' : 'shrink-0'}`}> +
@@ -438,7 +457,7 @@ export function EditorView(props: EditorViewProps) { panelStartedAt={panelStartedAt} variant="band" /> - {safeContainers.length > 1 ? ( + {isMultiContainerLayout ? ( @@ -477,7 +504,7 @@ export function EditorView(props: EditorViewProps) { {/* Logs Section (fills remaining left-column height). On multi- container stacks a min-h guarantees logs are never hidden. Hidden when containers are expanded to fill the column. */} - {!containersExpanded && (safeContainers.length > 1 ? ( + {!containersExpanded && (isMultiContainerLayout ? (
diff --git a/frontend/src/components/EditorLayout/ShellOverlays.tsx b/frontend/src/components/EditorLayout/ShellOverlays.tsx index 8e683077..b7dfb8de 100644 --- a/frontend/src/components/EditorLayout/ShellOverlays.tsx +++ b/frontend/src/components/EditorLayout/ShellOverlays.tsx @@ -124,6 +124,8 @@ export function ShellOverlays({ open={updateReadiness !== null} stackName={updateReadiness?.stackName ?? ''} nodeId={updateReadiness?.nodeId ?? null} + serviceName={updateReadiness?.serviceName} + mode={updateReadiness?.mode} onCancel={() => setUpdateReadiness(null)} onProceed={() => updateReadiness?.proceed()} /> diff --git a/frontend/src/components/EditorLayout/__tests__/ContainersHealth.test.tsx b/frontend/src/components/EditorLayout/__tests__/ContainersHealth.test.tsx index 7b95779e..2cf4dfe8 100644 --- a/frontend/src/components/EditorLayout/__tests__/ContainersHealth.test.tsx +++ b/frontend/src/components/EditorLayout/__tests__/ContainersHealth.test.tsx @@ -1,5 +1,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { render, screen, fireEvent, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; vi.mock('@/lib/clipboard', () => ({ copyToClipboard: vi.fn().mockResolvedValue(undefined) })); vi.mock('../../Terminal', () => ({ default: () => null })); @@ -10,6 +11,8 @@ import { ContainersHealth } from '../editor-view-blocks'; import { copyToClipboard } from '@/lib/clipboard'; import type { ContainerInfo } from '../EditorView'; import type { Node } from '@/context/NodeContext'; +import type { EffectiveServiceSpec } from '@/types/effectiveServices'; +import type { StackServiceUpdateStatus } from '@/types/imageUpdates'; const LOCAL_NODE = { id: 1, type: 'local' } as Node; @@ -221,3 +224,229 @@ describe('density toggle and summary strip', () => { expect(screen.queryByRole('button', { name: 'Compact view' })).toBeNull(); }); }); + +describe('declared-service headers (multi-service only)', () => { + function makeContainer(overrides: Partial = {}): ContainerInfo { + return { + Id: overrides.Id || 'abc', + Names: overrides.Names || ['/app'], + State: overrides.State || 'running', + Status: overrides.Status || 'Up 1 hour', + Image: overrides.Image || 'nginx', + ...overrides, + } as unknown as ContainerInfo; + } + + function spec(overrides: Partial = {}): EffectiveServiceSpec { + return { + name: 'web', + declaredImage: 'nginx:latest', + hasBuild: false, + expectedReplicas: 1, + dependsOn: [], + hasHealthcheck: false, + ...overrides, + }; + } + + function status(overrides: Partial = {}): StackServiceUpdateStatus { + return { + service: 'web', + image: 'nginx:latest', + hasUpdate: false, + checkStatus: 'ok', + lastError: null, + ...overrides, + }; + } + + it('renders no declared-service header for a single effective service (unchanged single-service UX)', () => { + render( + , + ); + // No grouped "X/Y running" service header; the flat per-container card + // layout (with its own pre-existing "Service actions" menu) is unchanged. + expect(screen.queryByText(/running$/)).toBeNull(); + expect(screen.getByLabelText('Service actions')).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'View logs' })).toBeInTheDocument(); + }); + + it('renders one header per declared service and groups containers under it', () => { + render( + , + ); + expect(screen.getByText('web')).toBeInTheDocument(); + expect(screen.getByText('db')).toBeInTheDocument(); + expect(screen.getAllByLabelText('Service actions')).toHaveLength(2); + // Per-container service menu is hidden inside a multi-service header group. + expect(screen.getAllByLabelText('Open bash shell')).toHaveLength(2); + }); + + it('shows the Update badge only for the service with a confirmed update', () => { + render( + , + ); + expect(screen.getByText('Update', { selector: 'span' })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: /^Update$/ })).toBeInTheDocument(); + }); + + it('uses Rebuild wording with no badge for a build-backed service without a detected update', () => { + const onRequestServiceUpdate = vi.fn(); + render( + , + ); + expect(screen.queryByText('Update', { selector: 'span' })).toBeNull(); + const rebuildBtn = screen.getByRole('button', { name: /^Rebuild$/ }); + expect(rebuildBtn).toBeInTheDocument(); + fireEvent.click(rebuildBtn); + expect(onRequestServiceUpdate).toHaveBeenCalledWith('web', 'rebuild'); + }); + + it('moves Start/Stop/Restart to the declared-service header menu', async () => { + const user = userEvent.setup(); + const serviceAction = vi.fn(); + render( + , + ); + await user.click(screen.getAllByLabelText('Service actions')[0]); + await user.click(await screen.findByRole('menuitem', { name: 'Restart service' })); + expect(serviceAction).toHaveBeenCalledWith('restart', 'web'); + }); + + it('still surfaces summary strip and density toggle on multi-service stacks', () => { + render( + , + ); + expect(screen.getByText(/3 containers/i)).toBeInTheDocument(); + expect(screen.getByText(/2 up/i)).toBeInTheDocument(); + expect(screen.getByText(/1 paused/i)).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Compact view' })).toHaveAttribute('aria-pressed', 'true'); + expect(screen.getByRole('button', { name: 'Detailed view' })).toBeInTheDocument(); + }); + + it('toggles detailed sparklines on the multi-service path', () => { + render( + , + ); + expect(screen.queryByText('cpu')).toBeNull(); + fireEvent.click(screen.getByRole('button', { name: 'Detailed view' })); + expect(screen.getAllByText('cpu')).toHaveLength(2); + }); + + it('surfaces expand control on multi-service stacks when wired', () => { + const onToggle = vi.fn(); + render( + , + ); + fireEvent.click(screen.getByRole('button', { name: 'Expand containers' })); + expect(onToggle).toHaveBeenCalledTimes(1); + }); +}); diff --git a/frontend/src/components/EditorLayout/editor-view-blocks.tsx b/frontend/src/components/EditorLayout/editor-view-blocks.tsx index e68b2d19..e8952d98 100644 --- a/frontend/src/components/EditorLayout/editor-view-blocks.tsx +++ b/frontend/src/components/EditorLayout/editor-view-blocks.tsx @@ -46,6 +46,8 @@ import StructuredLogViewer from '../StructuredLogViewer'; import type { Node } from '@/context/NodeContext'; import type { useAuth } from '@/context/AuthContext'; import type { ContainerInfo, ContainerStatsEntry, StackAction } from './EditorView'; +import type { EffectiveServiceSpec } from '@/types/effectiveServices'; +import type { StackServiceUpdateStatus } from '@/types/imageUpdates'; const extractUptime = (status: string | undefined): string | null => { if (!status) return null; @@ -304,6 +306,15 @@ export interface ContainersHealthProps { openLogViewer: (containerId: string, containerName: string) => void; openBashModal: (containerId: string, containerName: string) => void; serviceAction: (action: 'start' | 'stop' | 'restart', serviceName: string) => Promise; + // Declared Compose services from the effective model. Multi-service + // headers (owning Update/Rebuild + badge + Start/Stop/Restart) render only + // when this has more than one entry; empty/single leaves the flat + // container-card layout below untouched. Optional so callers that never + // deal in services (and existing tests) can omit them. + effectiveServices?: EffectiveServiceSpec[]; + serviceUpdateStatuses?: StackServiceUpdateStatus[]; + serviceUpdateInProgress?: { service: string; mode: 'update' | 'rebuild' } | null; + onRequestServiceUpdate?: (serviceName: string, mode: 'update' | 'rebuild') => void; containersExpanded?: boolean; onToggleContainersExpand?: () => void; } @@ -319,16 +330,23 @@ export function ContainersHealth({ openLogViewer, openBashModal, serviceAction, + effectiveServices = [], + serviceUpdateStatuses = [], + serviceUpdateInProgress = null, + onRequestServiceUpdate, containersExpanded, onToggleContainersExpand, }: ContainersHealthProps) { + // Multi-service only (§12): a single-service stack keeps the existing flat + // layout untouched, including its per-container Start/Stop/Restart kebab. + const isMultiService = effectiveServices.length > 1; const [copiedUrlId, setCopiedUrlId] = useState(null); const copiedUrlTimerRef = useRef(null); // Compact mode hides sparkline grids across all containers for a denser // list. Detailed mode (default) shows CPU / Mem / Net per container. const [density, setDensity] = useState<'compact' | 'detailed'>( - safeContainers.length > 1 ? 'compact' : 'detailed', -); + safeContainers.length > 1 ? 'compact' : 'detailed', + ); useEffect(() => () => { if (copiedUrlTimerRef.current !== null) window.clearTimeout(copiedUrlTimerRef.current); }, []); @@ -343,102 +361,85 @@ export function ContainersHealth({ }, 1500); }).catch(() => { /* clipboard unavailable */ }); }, []); - return ( -
- {containerStatsError && safeContainers.length > 0 && ( -
+ + // Summary strip + density/expand toggles: multi-container stacks only, + // whether the body is flat or grouped by declared service. + const total = safeContainers.length; + const running = safeContainers.filter(c => c.State === 'running').length; + const unhealthy = safeContainers.filter(c => c.healthStatus === 'unhealthy').length; + const paused = safeContainers.filter(c => c.State === 'paused').length; + const densityToolbar = total > 1 ? ( +
+
+ {total} container{total !== 1 ? 's' : ''} + {running} up + {paused > 0 && {paused} paused} + {unhealthy > 0 && {unhealthy} unhealthy} +
+
+
- - Stats unavailable - + - {containerStatsError} + Compact view + + + + + + Detailed view + + + {onToggleContainersExpand && ( + + + + + + {containersExpanded ? 'Collapse containers' : 'Expand containers'} + + + )}
- )} - {safeContainers.length === 0 ? ( -
No containers running for this stack.
- ) : ( - <> - {/* Summary strip + density toggle appear only for multi-container - stacks; single-container stacks keep the original layout. */} - {safeContainers.length > 1 && (() => { - const total = safeContainers.length; - const running = safeContainers.filter(c => c.State === 'running').length; - const unhealthy = safeContainers.filter(c => c.healthStatus === 'unhealthy').length; - const paused = safeContainers.filter(c => c.State === 'paused').length; - return ( -
-
- {total} container{total !== 1 ? 's' : ''} - {running} up - {paused > 0 && {paused} paused} - {unhealthy > 0 && {unhealthy} unhealthy} -
-
-
- - - - - - Compact view - - - - - - - - Detailed view - - - {onToggleContainersExpand && ( - - - - - - {containersExpanded ? 'Collapse containers' : 'Expand containers'} - - - )} -
-
-
- ); - })()} -
- {safeContainers.map(container => { +
+
+ ) : null; + + // One container card. `hideServiceMenu` drops the per-container + // Start/Stop/Restart kebab on multi-service stacks, where the declared- + // service header above owns that action instead (§12 point 4: child cards + // keep only logs, shell, ports, metrics). + const renderContainerCard = (container: ContainerInfo, hideServiceMenu: boolean) => { let mainPort: number | undefined; let mainPortPrivate: number | undefined; let mainPortProto: string | undefined; @@ -574,7 +575,7 @@ export function ContainersHealth({ )} - {container.Service && ( + {!hideServiceMenu && container.Service && ( + + {replicaCopy} + + + )} + + + + + + {isServiceActive ? ( + <> + serviceAction('restart', spec.name)}> + Restart service + + serviceAction('stop', spec.name)}> + Stop service + + + ) : ( + serviceAction('start', spec.name)}> + Start service + + )} + + +
+
+ {group.length > 0 ? ( +
+ {group.map(container => renderContainerCard(container, true))} +
+ ) : ( +
+ No containers running for this service. +
+ )} +
+ ); })}
- + ) : safeContainers.length === 0 ? ( +
No containers running for this stack.
+ ) : ( +
+ {safeContainers.map(container => renderContainerCard(container, false))} +
)}
); diff --git a/frontend/src/components/EditorLayout/failed-gate-recovery.test.ts b/frontend/src/components/EditorLayout/failed-gate-recovery.test.ts index 42519901..2df0034b 100644 --- a/frontend/src/components/EditorLayout/failed-gate-recovery.test.ts +++ b/frontend/src/components/EditorLayout/failed-gate-recovery.test.ts @@ -2,8 +2,8 @@ import { describe, it, expect } from 'vitest'; import { classifyFailedGate } from './failed-gate-recovery'; import type { HealthGateUiState } from '@/context/DeployFeedbackContext'; -type Gate = Pick; -const gate = (over: Partial = {}): Gate => ({ status: 'failed', nodeId: null, stackName: 'web', ...over }); +type Gate = Pick; +const gate = (over: Partial = {}): Gate => ({ status: 'failed', nodeId: null, stackName: 'web', targetScope: 'stack', ...over }); describe('classifyFailedGate', () => { it('skips when there is no gate', () => { @@ -43,4 +43,8 @@ describe('classifyFailedGate', () => { it('reports no-file when the node and file list match but no stack file matches the name yet', () => { expect(classifyFailedGate(gate({ nodeId: 3, stackName: 'web' }), 3, 3, ['other.yml'])).toEqual({ kind: 'no-file' }); }); + + it('skips a failed service-scoped gate: the stack rollback recovery does not apply to a single service', () => { + expect(classifyFailedGate(gate({ targetScope: 'service' }), null, null, ['web.yml'])).toEqual({ kind: 'skip' }); + }); }); diff --git a/frontend/src/components/EditorLayout/failed-gate-recovery.ts b/frontend/src/components/EditorLayout/failed-gate-recovery.ts index 29b6ace6..3c832905 100644 --- a/frontend/src/components/EditorLayout/failed-gate-recovery.ts +++ b/frontend/src/components/EditorLayout/failed-gate-recovery.ts @@ -17,6 +17,12 @@ import type { HealthGateUiState } from '@/context/DeployFeedbackContext'; * file matches its name yet (the list may be mid-refresh). The caller leaves it * unhandled so the effect retries once the files land. * - `record`: record a recovery entry against `stackFile`. + * + * A service-scoped gate (`targetScope === 'service'`) always classifies as + * `skip`: the stack-level rollback recovery this feeds (RecoveryChip/Panel, + * keyed off the stack's own `rollback_target`) does not know how to restore a + * single service, so routing a service gate's failure into it would offer a + * rollback action that does not correspond to what actually happened. */ export type FailedGateOutcome = | { kind: 'skip' } @@ -24,12 +30,13 @@ export type FailedGateOutcome = | { kind: 'record'; stackFile: string }; export function classifyFailedGate( - healthGate: Pick | null, + healthGate: Pick | null, activeNodeId: number | null, filesNodeId: number | null, files: string[], ): FailedGateOutcome { if (!healthGate || healthGate.status !== 'failed') return { kind: 'skip' }; + if (healthGate.targetScope === 'service') return { kind: 'skip' }; // Record only while on the gate's node AND with that node's file list loaded. if (healthGate.nodeId !== activeNodeId || healthGate.nodeId !== filesNodeId) return { kind: 'skip' }; const stackFile = files.find(f => f.replace(/\.(yml|yaml)$/, '') === healthGate.stackName); diff --git a/frontend/src/components/EditorLayout/hooks/useEditorViewState.ts b/frontend/src/components/EditorLayout/hooks/useEditorViewState.ts index 835362ee..9fe30ca2 100644 --- a/frontend/src/components/EditorLayout/hooks/useEditorViewState.ts +++ b/frontend/src/components/EditorLayout/hooks/useEditorViewState.ts @@ -1,5 +1,6 @@ import { useEffect, useState } from 'react'; import type { ContainerInfo } from '../EditorView'; +import type { EffectiveServiceSpec } from '@/types/effectiveServices'; export const LOGS_MODE_STORAGE_KEY = 'sencho.stackView.logsMode'; @@ -30,6 +31,16 @@ export function useEditorViewState() { const [envFiles, setEnvFiles] = useState([]); const [selectedEnvFile, setSelectedEnvFile] = useState(''); const [containers, setContainers] = useState([]); + // Declared-service facts for the loaded stack, from the effective Compose + // model. Empty for a single-service stack, an older node without the + // service-scoped-update capability, or a render failure; all three cases + // fail closed to the legacy per-container layout (no declared-service + // headers), so this array doubles as the multi-service gate. + const [effectiveServices, setEffectiveServices] = useState([]); + // The declared service currently running a manual update/rebuild, so the + // owning header can show a busy state. Only one at a time, mirroring the + // single `loadingAction` for stack-level operations. + const [serviceUpdateInProgress, setServiceUpdateInProgress] = useState<{ service: string; mode: 'update' | 'rebuild' } | null>(null); const [activeTab, setActiveTab] = useState('compose'); const [logsMode, setLogsMode] = useState(readLogsMode); @@ -56,6 +67,8 @@ export function useEditorViewState() { envFiles, setEnvFiles, selectedEnvFile, setSelectedEnvFile, containers, setContainers, + effectiveServices, setEffectiveServices, + serviceUpdateInProgress, setServiceUpdateInProgress, activeTab, setActiveTab, logsMode, setLogsMode, gitSourceOpen, setGitSourceOpen, diff --git a/frontend/src/components/EditorLayout/hooks/useOverlayState.ts b/frontend/src/components/EditorLayout/hooks/useOverlayState.ts index 2e5532df..8ddfabd5 100644 --- a/frontend/src/components/EditorLayout/hooks/useOverlayState.ts +++ b/frontend/src/components/EditorLayout/hooks/useOverlayState.ts @@ -122,13 +122,18 @@ export function useOverlayState() { const [policyBypassing, setPolicyBypassing] = useState(false); // Pre-update readiness dialog. `proceed` runs the actual update when the - // user confirms; opened by useStackActions.requestStackUpdate. `nodeId` is - // captured at open time so both the readiness fetch and the update run against - // the same node even if the active node changes while the dialog is open. + // user confirms; opened by useStackActions.requestStackUpdate (full stack) + // or requestServiceUpdate (a single declared service). `nodeId` is captured + // at open time so both the readiness fetch and the update run against the + // same node even if the active node changes while the dialog is open. + // `serviceName`/`mode` are set only for a service-scoped update; absent + // means the full-stack readiness check, unchanged from before. const [updateReadiness, setUpdateReadiness] = useState<{ stackName: string; stackFile: string; nodeId: number | null; + serviceName?: string; + mode?: 'update' | 'rebuild'; proceed: () => void; } | null>(null); diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts index 2933533f..011ace82 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts @@ -50,6 +50,10 @@ function makeEditorState(over: Partial = {}): EditorState { setGitSourcePendingMap: vi.fn(), setComposeEtag: vi.fn(), setEnvEtag: vi.fn(), + effectiveServices: [], + setEffectiveServices: vi.fn(), + serviceUpdateInProgress: null, + setServiceUpdateInProgress: vi.fn(), }; return { ...base, ...over } as unknown as EditorState; } diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.ts index 1d1dfaf9..74d90e83 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.ts @@ -11,6 +11,8 @@ import { } from '@/lib/hydrationTiming'; import { toast } from '@/components/ui/toast-store'; import { buildServiceUrl, openServiceUrl } from '@/lib/serviceUrl'; +import { requestServiceUpdate as postServiceUpdate, requestServiceRestore as postServiceRestore } from '@/lib/serviceUpdate'; +import type { EffectiveServiceModelResult } from '@/types/effectiveServices'; import type { useEditorViewState } from './useEditorViewState'; import type { useStackListState } from './useStackListState'; import type { useViewNavigationState } from './useViewNavigationState'; @@ -153,6 +155,11 @@ interface UseStackActionsOptions { // Active node advertises guided external-network preflight. Absent capability // keeps legacy deploy (no GET). Advertised-but-broken fails closed. hasGuidedExternalNetworkPreflight?: boolean; + // Active node advertises service-scoped updates. Gates both the + // effective-services fetch (skipped entirely on an older node, so + // effectiveServices stays empty and no declared-service headers render) + // and the manual per-service update/rebuild action. + hasServiceScopedUpdate?: boolean; // Target-aware stack:edit check. Pass the loaded stack identity (folder name // or compose path); callers strip extensions when comparing to RBAC stack // names. Evaluated against the load target so post-load auto-edit is not @@ -379,6 +386,7 @@ export function useStackActions(options: UseStackActionsOptions) { diffPreviewEnabled, hasUpdateGuard = false, hasGuidedExternalNetworkPreflight = false, + hasServiceScopedUpdate = false, canEditStack, canOfferVolumeRemoval = false, } = options; @@ -493,6 +501,8 @@ export function useStackActions(options: UseStackActionsOptions) { editorState.setSelectedEnvFile(''); editorState.setEnvExists(false); editorState.setContainers([]); + editorState.setEffectiveServices([]); + editorState.setServiceUpdateInProgress(null); editorState.setIsEditing(false); }; @@ -692,6 +702,32 @@ export function useStackActions(options: UseStackActionsOptions) { } }; + // Declared-service facts for the multi-service headers. Skipped entirely + // without the capability so an older remote node never sees the extra + // request; a render failure or non-ok response also fails closed to an + // empty list, which keeps the legacy single-service layout. + const loadEffectiveServicesState = async (filename: string, signal?: AbortSignal) => { + if (!hasServiceScopedUpdate) { + editorState.setEffectiveServices([]); + return; + } + const stackName = filename.replace(/\.(yml|yaml)$/, ''); + try { + const res = await apiFetch(`/stacks/${stackName}/effective-services`, { signal }); + if (signal?.aborted) return; + if (!res.ok) { + editorState.setEffectiveServices([]); + return; + } + const data = await res.json() as EffectiveServiceModelResult; + if (signal?.aborted) return; + editorState.setEffectiveServices(data.renderable ? data.services : []); + } catch (err) { + if (isAbortError(err)) return; + editorState.setEffectiveServices([]); + } + }; + const applyEditorRouteState = (tab: EditorTab) => { editorState.setActiveTab(tab); editorState.setEditingCompose(true); @@ -764,6 +800,7 @@ export function useStackActions(options: UseStackActionsOptions) { }; } await loadBackupState(filename, signal); + await loadEffectiveServicesState(filename, signal); // Post-load auto-edit evaluates permission for the loaded target, not // the previously selected stack (selectedFile was just updated above). if (options?.startInComposeEdit && canEditStack(filename)) { @@ -789,6 +826,7 @@ export function useStackActions(options: UseStackActionsOptions) { editorState.setOriginalEnvContent(''); editorState.setEnvEtag(null); editorState.setContainers([]); + editorState.setEffectiveServices([]); return { ok: false }; } finally { if (!signal.aborted) { @@ -1586,6 +1624,115 @@ export function useStackActions(options: UseStackActionsOptions) { await run(); }; + // Single entry point for a manual service-scoped update/rebuild (declared- + // service header, Updates view per-service Apply). Uses the same deploy- + // feedback session as full-stack Update so progress streams and the health + // gate is polled; siblings are not intentionally recreated. + const requestServiceUpdate = async ( + stackFile: string, + serviceName: string, + mode: 'update' | 'rebuild' = 'update', + ): Promise => { + if (stackListState.isStackBusy(stackFile) || editorState.serviceUpdateInProgress) return; + const stackName = stackFile.replace(/\.(yml|yaml)$/, ''); + const opNodeId = activeNode?.id ?? null; + const run = async () => { + editorState.setServiceUpdateInProgress({ service: serviceName, mode }); + try { + await runWithLog({ stackName, action: 'update', nodeId: opNodeId, serviceName }, async (started, ds) => { + await started; + const result = await postServiceUpdate({ + nodeId: opNodeId, + stackName, + serviceName, + mode, + deploySessionId: ds, + }); + if (!result.ok) { + toast.error(result.error); + return { ok: false as const, errorMessage: result.error }; + } + const verb = mode === 'rebuild' ? 'rebuilt' : 'updated'; + if (result.healthGateId && result.observing) { + toast.info(`Service "${serviceName}" ${verb}. Verifying health...`); + } else { + toast.success(`Service "${serviceName}" ${verb} successfully!`); + } + if (result.recheckWarning) toast.info(result.recheckWarning); + stackListState.fetchImageUpdates(); + await refreshSelectedContainers(stackName, stackFile); + stackListState.recordActionSuccess(stackFile); + return { + ok: true as const, + healthGateId: result.observing ? result.healthGateId : null, + recoveryId: result.recoveryId, + }; + }); + } finally { + editorState.setServiceUpdateInProgress(null); + stackListState.refreshStacks(true); + } + }; + if (hasUpdateGuard) { + overlayState.setUpdateReadiness({ + stackName, + stackFile, + nodeId: opNodeId, + serviceName, + mode, + proceed: () => { + overlayState.setUpdateReadiness(null); + void run(); + }, + }); + return; + } + await run(); + }; + + const requestServiceRestore = async ( + stackFile: string, + serviceName: string, + recoveryId: string, + ): Promise => { + if (stackListState.isStackBusy(stackFile) || editorState.serviceUpdateInProgress) return; + const stackName = stackFile.replace(/\.(yml|yaml)$/, ''); + const opNodeId = activeNode?.id ?? null; + editorState.setServiceUpdateInProgress({ service: serviceName, mode: 'update' }); + try { + await runWithLog({ stackName, action: 'update', nodeId: opNodeId, serviceName }, async (started, ds) => { + await started; + const result = await postServiceRestore({ + nodeId: opNodeId, + stackName, + serviceName, + recoveryId, + deploySessionId: ds, + }); + if (!result.ok) { + toast.error(result.error); + return { ok: false as const, errorMessage: result.error }; + } + if (result.healthGateId && result.observing) { + toast.info(`Service "${serviceName}" restored. Verifying health...`); + } else { + toast.success(`Service "${serviceName}" restored successfully!`); + } + stackListState.fetchImageUpdates(); + await refreshSelectedContainers(stackName, stackFile); + stackListState.recordActionSuccess(stackFile); + return { + ok: true as const, + healthGateId: result.observing ? result.healthGateId : null, + recoveryId: result.recoveryId, + }; + }); + } finally { + editorState.setServiceUpdateInProgress(null); + stackListState.refreshStacks(true); + } + }; + const updateStack = async (e?: React.MouseEvent) => { e?.preventDefault(); e?.stopPropagation(); @@ -1969,6 +2116,8 @@ export function useStackActions(options: UseStackActionsOptions) { serviceAction, updateStack, requestStackUpdate, + requestServiceUpdate, + requestServiceRestore, deleteStack, attemptLeaveEditor, attemptPopstateNavigation, diff --git a/frontend/src/components/__tests__/AutoUpdateReadinessView.test.tsx b/frontend/src/components/__tests__/AutoUpdateReadinessView.test.tsx index a993ab6a..07616334 100644 --- a/frontend/src/components/__tests__/AutoUpdateReadinessView.test.tsx +++ b/frontend/src/components/__tests__/AutoUpdateReadinessView.test.tsx @@ -8,15 +8,34 @@ import { describe, it, expect, vi, afterEach } from 'vitest'; import { render, screen, act, waitFor, fireEvent } from '@testing-library/react'; vi.mock('@/lib/api', () => ({ apiFetch: vi.fn(), fetchForNode: vi.fn() })); +vi.mock('@/lib/serviceUpdate', () => ({ + requestServiceUpdate: vi.fn(), +})); vi.mock('@/components/ui/toast-store', () => ({ toast: { error: vi.fn(), success: vi.fn(), warning: vi.fn(), info: vi.fn(), loading: vi.fn(), dismiss: vi.fn() }, })); vi.mock('@/hooks/use-is-mobile', () => ({ useIsMobile: () => false })); +vi.mock('@/context/DeployFeedbackContext', () => ({ + useDeployFeedback: () => ({ + runWithLog: async (_params: unknown, fn: (started: Promise, ds: string) => Promise) => + fn(Promise.resolve(), 'test-session'), + }), +})); +// nodeMeta/refreshNodeMeta must be stable across renders (matching the real +// NodeContext), or a fresh Map/fn on every useNodes() call churns the +// loadReadiness useCallback identity and re-triggers its effect forever. +const mockNodeMeta = new Map(); +const mockRefreshNodeMeta = vi.fn(); vi.mock('@/context/NodeContext', () => ({ - useNodes: () => ({ nodes: [{ id: 1, name: 'Local', type: 'local', status: 'online' }] }), + useNodes: () => ({ + nodes: [{ id: 1, name: 'Local', type: 'local', status: 'online' }], + nodeMeta: mockNodeMeta, + refreshNodeMeta: mockRefreshNodeMeta, + }), })); import { apiFetch, fetchForNode } from '@/lib/api'; +import { requestServiceUpdate } from '@/lib/serviceUpdate'; import AutoUpdateReadinessView, { MobileReadinessCard, CadenceStrip, type StackCard } from '../AutoUpdateReadinessView'; function card(over: Partial = {}): StackCard { @@ -25,6 +44,7 @@ function card(over: Partial = {}): StackCard { nodeId: 1, previewLoaded: true, applying: false, + applyingService: null, autoUpdateEnabled: true, scheduledTask: null, preview: { @@ -83,6 +103,48 @@ it('enables Apply when no schedule covers the stack', () => { expect(apply()).toBeEnabled(); }); +it('offers per-service Apply when build-only companions make the stack multi-service', () => { + const onApplyService = vi.fn(); + render( + , + ); + const serviceApply = screen.getByRole('button', { name: /^Apply$/i }); + expect(serviceApply).toBeEnabled(); + fireEvent.click(serviceApply); + expect(onApplyService).toHaveBeenCalledWith('nextcloud', 1, 'app'); +}); + /** * The desktop StackReadinessCard is not exported, so its Apply-now gating is * covered through a full-view render (useIsMobile is mocked false). A safe @@ -97,6 +159,8 @@ describe('AutoUpdateReadinessView desktop Apply now', () => { afterEach(() => { mockedFetch.mockReset(); mockedFetchForNode.mockReset(); + mockNodeMeta.clear(); + vi.mocked(requestServiceUpdate).mockReset(); }); it('enables Apply for a safe update with no covering schedule', async () => { @@ -122,6 +186,97 @@ describe('AutoUpdateReadinessView desktop Apply now', () => { // apply automatically": that still requires a covering schedule. expect(screen.getByText(/0 of 1 ready to apply automatically/)).toBeInTheDocument(); }); + + it('applies a single service and refreshes the authoritative update preview', async () => { + mockNodeMeta.set(1, { + version: '1.0.0', + capabilities: ['service-scoped-update'], + fetchedAt: Date.now(), + }); + const multiPreview = { + stack_name: 'nextcloud', + images: [ + { + service: 'app', + image: 'nextcloud:27', + current_tag: '27.1.4', + next_tag: '27.1.5', + has_update: true, + semver_bump: 'patch' as const, + }, + { + service: 'redis', + image: 'redis:7', + current_tag: '7.2', + next_tag: '7.2', + has_update: false, + semver_bump: 'none' as const, + }, + ], + summary: { + has_update: true, + primary_image: 'nextcloud', + current_tag: '27.1.4', + next_tag: '27.1.5', + semver_bump: 'patch' as const, + update_kind: 'tag' as const, + blocked: false, + blocked_reason: null, + }, + rollback_target: null, + changelog: 'Fixes.', + }; + const refreshedPreview = { + ...multiPreview, + images: multiPreview.images.map((img) => ( + img.service === 'app' ? { ...img, has_update: false, current_tag: '27.1.5', next_tag: '27.1.5' } : img + )), + summary: { ...multiPreview.summary, has_update: false, current_tag: '27.1.5' }, + }; + mockedFetch.mockImplementation((url: string) => { + if (url === '/image-updates/fleet') { + return Promise.resolve({ ok: true, json: async () => ({ '1': { nextcloud: true } }) }); + } + if (url.startsWith('/scheduled-tasks')) { + return Promise.resolve({ ok: true, json: async () => [] }); + } + return Promise.resolve({ ok: true, json: async () => ({}) }); + }); + mockedFetchForNode.mockImplementation((url: string) => { + if (String(url).includes('/update-preview')) { + const call = mockedFetchForNode.mock.calls.filter((c) => String(c[0]).includes('/update-preview')).length; + return Promise.resolve({ + ok: true, + json: async () => (call <= 1 ? multiPreview : refreshedPreview), + }); + } + return Promise.resolve({ ok: true, json: async () => ({}) }); + }); + vi.mocked(requestServiceUpdate).mockResolvedValue({ + ok: true, + mode: 'update', + serviceName: 'app', + healthGateId: null, + observing: false, + recoveryId: null, + recoveryAvailable: false, + }); + + render(); + const serviceApply = await screen.findByRole('button', { name: /^Apply$/i }); + await act(async () => { fireEvent.click(serviceApply); }); + + await waitFor(() => { + expect(requestServiceUpdate).toHaveBeenCalledWith(expect.objectContaining({ + stackName: 'nextcloud', + serviceName: 'app', + mode: 'update', + })); + }); + await waitFor(() => { + expect(mockedFetchForNode.mock.calls.filter((c) => String(c[0]).includes('/update-preview')).length).toBeGreaterThanOrEqual(2); + }); + }); }); /** diff --git a/frontend/src/components/__tests__/DeployFeedbackModal.test.tsx b/frontend/src/components/__tests__/DeployFeedbackModal.test.tsx index f0b19a47..01d55d57 100644 --- a/frontend/src/components/__tests__/DeployFeedbackModal.test.tsx +++ b/frontend/src/components/__tests__/DeployFeedbackModal.test.tsx @@ -5,7 +5,15 @@ import { DeployFeedbackProvider, useDeployFeedback } from '@/context/DeployFeedb import { DeployFeedbackModal } from '../DeployFeedbackModal'; vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() })); +vi.mock('@/lib/serviceUpdate', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + requestServiceRestore: vi.fn(), + }; +}); import { apiFetch } from '@/lib/api'; +import { requestServiceRestore } from '@/lib/serviceUpdate'; // Lets a test simulate a mid-stream drop (onReady then onError) so the panel // reaches 'streaming' with progressUnavailable set. @@ -15,12 +23,21 @@ const ctl = vi.hoisted(() => ({ drop: false, lastNodeId: undefined as number | n // the stream connected on mount so the panel reaches the 'streaming' state, and // records the captured nodeId it was mounted with. vi.mock('@/components/Terminal', () => { - const MockTerminal = ({ onReady, onError, nodeId }: { onReady?: () => void; onError?: () => void; nodeId?: number | null }) => { + const MockTerminal = ({ + onReady, onError, nodeId, deploySessionId, + }: { + onReady?: () => void; + onError?: () => void; + nodeId?: number | null; + deploySessionId?: string | null; + }) => { ctl.lastNodeId = nodeId; + // Re-fire on each deploy session so a Restore (second runWithLog) can + // release its progress-stream gate the same way the first update does. React.useEffect(() => { onReady?.(); if (ctl.drop) onError?.(); - }, [onReady, onError]); + }, [onReady, onError, deploySessionId]); return null; }; return { default: MockTerminal }; @@ -28,19 +45,28 @@ vi.mock('@/components/Terminal', () => { // Resolver for the in-flight operation, assigned inside the run callback (async, // after render) so the test can leave it pending or settle it on demand. -let resolveRun: ((r: { ok: boolean; errorMessage?: string; healthGateId?: string | null }) => void) | null = null; +let resolveRun: ((r: { + ok: boolean; + errorMessage?: string; + healthGateId?: string | null; + recoveryId?: string | null; +}) => void) | null = null; // The runWithLog promise itself, so a test can await full result propagation. let runOuter: Promise | null = null; // Node the driver captures for the operation; default local, overridden per test. let driverNodeId: number | null = null; +let driverServiceName: string | undefined; function Driver() { const { runWithLog } = useDeployFeedback(); React.useEffect(() => { - runOuter = runWithLog({ stackName: 'web', action: 'update', nodeId: driverNodeId }, async (started) => { - await started; - return new Promise<{ ok: boolean; errorMessage?: string; healthGateId?: string | null }>((res) => { resolveRun = res; }); - }); + runOuter = runWithLog( + { stackName: 'web', action: 'update', nodeId: driverNodeId, serviceName: driverServiceName }, + async (started) => { + await started; + return new Promise((res) => { resolveRun = res; }); + }, + ); }, [runWithLog]); return null; } @@ -60,7 +86,14 @@ async function renderStreaming() { type GateStatus = 'observing' | 'passed' | 'failed' | 'unknown'; -function routeGateApi(responses: Array<{ id: string; status: GateStatus; reason?: string | null }>) { +function routeGateApi(responses: Array<{ + id: string; + status: GateStatus; + reason?: string | null; + serviceName?: string | null; + targetScope?: 'stack' | 'service'; + failureSource?: 'primary' | 'collateral' | null; +}>) { let call = 0; vi.mocked(apiFetch).mockImplementation((url: string) => { if (!String(url).includes('/health-gate')) { @@ -71,6 +104,7 @@ function routeGateApi(responses: Array<{ id: string; status: GateStatus; reason? return Promise.resolve(new Response(JSON.stringify({ stack: 'web', id: r.id, status: r.status, trigger: 'update', reason: r.reason ?? null, windowSeconds: 90, startedAt: Date.now(), endedAt: null, containers: [], + targetScope: r.targetScope ?? 'stack', serviceName: r.serviceName ?? null, failureSource: r.failureSource ?? null, }), { status: 200 })); }); } @@ -83,8 +117,10 @@ describe('DeployFeedbackModal health gate', () => { ctl.drop = false; ctl.lastNodeId = undefined; driverNodeId = null; + driverServiceName = undefined; vi.mocked(apiFetch).mockReset(); vi.mocked(apiFetch).mockResolvedValue(new Response('{}', { status: 200 })); + vi.mocked(requestServiceRestore).mockReset(); }); afterEach(() => { vi.useRealTimers(); @@ -152,6 +188,55 @@ describe('DeployFeedbackModal health gate', () => { expect(screen.getByTestId('deploy-feedback-modal')).toBeInTheDocument(); }); + it('names the service in the banner for a service-scoped gate and notes a collateral failure', async () => { + routeGateApi([{ + id: 'gate-1', status: 'failed', reason: 'service web has no running replicas to observe', + serviceName: 'web', targetScope: 'service', failureSource: 'collateral', + }]); + await succeedWithGate('gate-1'); + await act(async () => { await vi.advanceTimersByTimeAsync(100); }); + expect(screen.getByTestId('health-gate-banner')).toHaveAttribute('data-status', 'failed'); + expect(screen.getByText(/A dependent service triggered the failure/)).toBeInTheDocument(); + }); + + it('offers Restore for a failed service gate and calls requestServiceRestore with the recovery id', async () => { + driverServiceName = 'api'; + vi.mocked(requestServiceRestore).mockResolvedValue({ + ok: true, + mode: 'update', + serviceName: 'api', + healthGateId: null, + observing: false, + recoveryId: 'rec-1', + recoveryAvailable: false, + }); + routeGateApi([{ + id: 'gate-1', status: 'failed', reason: 'service api reported unhealthy', + serviceName: 'api', targetScope: 'service', failureSource: 'primary', + }]); + await renderStreaming(); + await act(async () => { await vi.advanceTimersByTimeAsync(60); }); + await act(async () => { + resolveRun?.({ ok: true, healthGateId: 'gate-1', recoveryId: 'rec-1' }); + await runOuter; + }); + await act(async () => { await vi.advanceTimersByTimeAsync(100); }); + const restoreBtn = screen.getByTestId('service-restore-from-gate'); + expect(restoreBtn).toBeInTheDocument(); + await act(async () => { + restoreBtn.click(); + }); + // Restore starts a fresh runWithLog; Terminal remounts for the new + // deploySessionId and releases the gate after the 50ms handshake. + await act(async () => { await vi.advanceTimersByTimeAsync(60); }); + await act(async () => { await Promise.resolve(); await Promise.resolve(); }); + expect(requestServiceRestore).toHaveBeenCalledWith(expect.objectContaining({ + stackName: 'web', + serviceName: 'api', + recoveryId: 'rec-1', + })); + }); + it('gives up with an unknown verdict after repeated poll failures', async () => { vi.mocked(apiFetch).mockImplementation((url: string) => { if (String(url).includes('/health-gate')) { diff --git a/frontend/src/components/dashboard/StackHealthTable.tsx b/frontend/src/components/dashboard/StackHealthTable.tsx index 89b7419f..f5273af7 100644 --- a/frontend/src/components/dashboard/StackHealthTable.tsx +++ b/frontend/src/components/dashboard/StackHealthTable.tsx @@ -7,6 +7,7 @@ import type { StackStatusEntry, MetricPoint, StackCpuSeries } from './types'; import type { StackUpdateInfo } from '@/types/imageUpdates'; import { aggregateCurrentUsage } from './aggregateCurrentUsage'; import { classifyRow, type RowState } from './classifyRow'; +import { updateAvailableBadge, updateAvailableLabel } from '@/lib/updateAvailableLabel'; interface StackHealthTableProps { stackStatuses: Record; @@ -125,6 +126,9 @@ export function StackHealthTable({ source: entry.source ?? 'local', mainPort: entry.mainPort ?? null, hasUpdate: stackUpdates[file]?.hasUpdate ?? false, + outdatedServices: (stackUpdates[file]?.services ?? []) + .filter((s) => s.hasUpdate) + .map((s) => s.service), }; }); }, [stackStatuses, stackAggregates, stackCpuSeries, stackUpdates]); @@ -245,8 +249,11 @@ export function StackHealthTable({ {row.name} {row.hasUpdate && ( - - Update available + + {updateAvailableBadge(row.outdatedServices)} )} diff --git a/frontend/src/components/sidebar/StackList.tsx b/frontend/src/components/sidebar/StackList.tsx index 892b3f87..086c8e82 100644 --- a/frontend/src/components/sidebar/StackList.tsx +++ b/frontend/src/components/sidebar/StackList.tsx @@ -253,6 +253,9 @@ export function StackList(props: StackListProps & StackListBulkProps) { isActive={selectedFile === file} labels={stackLabelMap[file] ?? []} hasUpdate={stackUpdates[file]?.hasUpdate ?? false} + outdatedServices={(stackUpdates[file]?.services ?? []) + .filter((s) => s.hasUpdate) + .map((s) => s.service)} checkStatus={stackUpdates[file]?.checkStatus} lastError={stackUpdates[file]?.lastError ?? undefined} hasGitPending={!!gitSourcePendingMap[file]} diff --git a/frontend/src/components/sidebar/StackRow.tsx b/frontend/src/components/sidebar/StackRow.tsx index 1387579c..b082717b 100644 --- a/frontend/src/components/sidebar/StackRow.tsx +++ b/frontend/src/components/sidebar/StackRow.tsx @@ -8,6 +8,7 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/comp import { sidebarRowActive, sidebarRowBase, sidebarRowCheckboxSlot } from './sidebar-styles'; import { statusText, statusColor } from './stack-status-utils'; import type { StackRowStatus } from './stack-status-utils'; +import { updateAvailableLabel } from '@/lib/updateAvailableLabel'; interface StackRowProps { file: string; @@ -20,6 +21,8 @@ interface StackRowProps { isActive: boolean; labels: Label[]; hasUpdate: boolean; + /** Outdated service names for the update tooltip; empty keeps the generic label. */ + outdatedServices?: string[]; // Last image-update check outcome. 'failed' surfaces a muted "couldn't check" // indicator so an undeterminable check is not mistaken for "up to date". checkStatus?: CheckStatus; @@ -48,7 +51,7 @@ function RowTooltip({ trigger, label }: { trigger: ReactNode; label: string }) { export function StackRow(props: StackRowProps) { const { file, displayName, status, running, total, isBusy, isActive, - hasUpdate, checkStatus, lastError, hasGitPending, onSelect, kebabSlot, + hasUpdate, outdatedServices, checkStatus, lastError, hasGitPending, onSelect, kebabSlot, bulkMode = false, isSelected = false, onToggleSelect, } = props; @@ -113,7 +116,7 @@ export function StackRow(props: StackRowProps) { )} - label="Update available" + label={updateAvailableLabel(outdatedServices)} /> ) : checkStatus === 'failed' ? ( { expect(container.querySelector('.bg-update')).not.toBeNull(); }); + it('names outdated services in the update tooltip', async () => { + render(); + fireEvent.pointerMove(screen.getByTestId('stack-trailing-update')); + expect((await screen.findAllByText('Update available: api, worker')).length).toBeGreaterThan(0); + }); + it('shows no trailing indicator for a clean ok check with no update', () => { const { container } = render(); expect(container.querySelector('.lucide-alert-circle')).toBeNull(); diff --git a/frontend/src/components/stack/UpdateReadinessDialog.tsx b/frontend/src/components/stack/UpdateReadinessDialog.tsx index 13222795..ebc37774 100644 --- a/frontend/src/components/stack/UpdateReadinessDialog.tsx +++ b/frontend/src/components/stack/UpdateReadinessDialog.tsx @@ -97,6 +97,10 @@ interface UpdateReadinessDialogProps { * while the dialog is open cannot mismatch the readiness from the update. */ nodeId: number | null; + /** Set only for a service-scoped update; absent means the full stack. */ + serviceName?: string; + /** Update vs rebuild copy for a service-scoped update. Ignored for the full stack. */ + mode?: 'update' | 'rebuild'; onCancel: () => void; /** Caller closes the dialog and starts the update. */ onProceed: () => void; @@ -108,7 +112,7 @@ interface UpdateReadinessDialogProps { * the single hard block. A slow or failed readiness fetch degrades to an * unknown verdict so this dialog can never strand the update path. */ -export function UpdateReadinessDialog({ open, stackName, nodeId, onCancel, onProceed }: UpdateReadinessDialogProps) { +export function UpdateReadinessDialog({ open, stackName, nodeId, serviceName, mode = 'update', onCancel, onProceed }: UpdateReadinessDialogProps) { const { isAdmin } = useAuth(); const [report, setReport] = useState(null); @@ -136,7 +140,10 @@ export function UpdateReadinessDialog({ open, stackName, nodeId, onCancel, onPro const load = async () => { try { - const res = await apiFetch(`/stacks/${stackName}/update-readiness`, { nodeId, signal: controller.signal }); + const path = serviceName + ? `/stacks/${stackName}/update-readiness?service=${encodeURIComponent(serviceName)}` + : `/stacks/${stackName}/update-readiness`; + const res = await apiFetch(path, { nodeId, signal: controller.signal }); if (!res.ok) { const unreachable = res.status === 502 || res.status === 503 || res.status === 504; setReport(UNKNOWN_FALLBACK(unreachable @@ -184,7 +191,7 @@ export function UpdateReadinessDialog({ open, stackName, nodeId, onCancel, onPro clearTimeout(timer); controller.abort(); }; - }, [open, stackName, nodeId, isAdmin]); + }, [open, stackName, nodeId, isAdmin, serviceName]); const proceed = async () => { if (snapshotFirst) { @@ -221,9 +228,11 @@ export function UpdateReadinessDialog({ open, stackName, nodeId, onCancel, onPro return ( { if (!next && !working) onCancel(); }} size="lg"> {!report || !verdict || !VerdictIcon ? ( @@ -297,7 +306,7 @@ export function UpdateReadinessDialog({ open, stackName, nodeId, onCancel, onPro void proceed(); }} > - {working ? 'Creating snapshot…' : 'Update now'} + {working ? 'Creating snapshot…' : (mode === 'rebuild' ? 'Rebuild now' : 'Update now')} } /> diff --git a/frontend/src/context/DeployFeedbackContext.tsx b/frontend/src/context/DeployFeedbackContext.tsx index 59764762..847dd473 100644 --- a/frontend/src/context/DeployFeedbackContext.tsx +++ b/frontend/src/context/DeployFeedbackContext.tsx @@ -3,6 +3,8 @@ import { apiFetch } from '../lib/api'; import { type ParsedLogRow, parseLogChunk } from '../components/log-rendering/composeLogParser'; import { useDeployFeedbackEnabled } from '../hooks/use-deploy-feedback-enabled'; import { readDeployFeedbackStyle } from '../hooks/use-deploy-feedback-style'; +import { toast } from '../components/ui/toast-store'; +import { fetchActiveServiceRecovery, requestServiceRestore } from '../lib/serviceUpdate'; export type ActionVerb = 'deploy' | 'update' | 'down' | 'restart' | 'stop' | 'install' | 'scan'; @@ -59,6 +61,8 @@ interface RunResult { * node never returns one, so no gate UI appears. */ healthGateId?: string | null; + /** Service-scoped recovery snapshot id, when one was captured. */ + recoveryId?: string | null; } /** Post-update health gate state for the current deploy session. */ @@ -74,6 +78,14 @@ export interface HealthGateUiState { reason: string | null; windowSeconds: number | null; startedAt: number | null; + /** 'service' for a service-scoped update/restore gate; 'stack' otherwise. Absent on older gates. */ + targetScope?: 'stack' | 'service'; + /** Set only for a service-scoped gate; null/absent for a full-stack gate. */ + serviceName?: string | null; + /** Which side of a service-scoped gate failed; null/absent for full-stack gates and non-failures. */ + failureSource?: 'primary' | 'collateral' | null; + /** Recovery snapshot to offer restore after a failed service gate. */ + recoveryId?: string | null; } const GATE_POLL_INTERVAL_MS = 4_000; @@ -84,6 +96,8 @@ export interface RunWithLogParams { action: ActionVerb; /** Node the operation runs on (null = local), for node-scoped surfaces. */ nodeId: number | null; + /** When set, this is a service-scoped update/restore; gate polling uses targetScope=service. */ + serviceName?: string; } interface DeployFeedbackContextValue { @@ -164,9 +178,22 @@ export function DeployFeedbackProvider({ children }: { children: React.ReactNode const [logRows, setLogRows] = useState([]); const [lastOutputAt, setLastOutputAt] = useState(0); - // Poll timer for the current session's health gate; cleared on panel close - // and whenever a new session starts. + // Poll timer for the current session's health gate; cleared when a watch + // ends or a newer session starts. Closing the panel no longer stops a + // service-scoped watch: recovery must stay discoverable after dismiss. const gatePollRef = useRef | null>(null); + const healthGateRef = useRef(null); + const panelOpenRef = useRef(false); + /** True when gate polling runs without the Deploy Progress panel (disabled setting or after dismiss). */ + const silentGateRef = useRef(false); + const offerRestoreToastRef = useRef<(gate: HealthGateUiState) => void>(() => {}); + + useEffect(() => { + healthGateRef.current = healthGate; + }, [healthGate]); + useEffect(() => { + panelOpenRef.current = panelState.isOpen; + }, [panelState.isOpen]); const stopGatePolling = useCallback(() => { if (gatePollRef.current !== null) { @@ -245,16 +272,31 @@ export function DeployFeedbackProvider({ children }: { children: React.ReactNode }, []); const onPanelClose = useCallback(() => { - sessionIdRef.current += 1; + const gate = healthGateRef.current; + const keepServiceWatch = !!gate + && gate.targetScope === 'service' + && (gate.status === 'observing' || gate.status === 'failed'); + settleStartRef.current = null; streamReadyRef.current = false; - // The gate keeps observing server-side; only this session's poll stops. - stopGatePolling(); - setHealthGate(null); setPanelState(DEFAULT_PANEL_STATE); setMinimized(false); setBannerActive(false); setLogRows([]); + + if (keepServiceWatch) { + // Keep polling (or a failed gate with recovery) so Restore stays reachable. + silentGateRef.current = true; + if (gate.status === 'failed') { + offerRestoreToastRef.current(gate); + } + return; + } + + sessionIdRef.current += 1; + silentGateRef.current = false; + stopGatePolling(); + setHealthGate(null); }, [stopGatePolling]); // Poll the by-id gate endpoint until a terminal status. The id-scoped read @@ -262,9 +304,24 @@ export function DeployFeedbackProvider({ children }: { children: React.ReactNode // instead of this session showing a newer run's result. Mirrors the backend // gate's own degradation: repeated failures (or an absurdly long poll) // resolve to an honest client-side unknown rather than observing forever. - const startGatePolling = useCallback((stackName: string, nodeId: number | null, gateId: string, trigger: 'update' | 'deploy', mySession: number) => { + const startGatePolling = useCallback(( + stackName: string, + nodeId: number | null, + gateId: string, + trigger: 'update' | 'deploy', + mySession: number, + options?: { serviceName?: string; recoveryId?: string | null; silent?: boolean }, + ) => { stopGatePolling(); - setHealthGate({ stackName, nodeId, gateId, trigger, status: 'observing', reason: null, windowSeconds: null, startedAt: null }); + silentGateRef.current = options?.silent === true; + const targetScope = options?.serviceName ? 'service' : 'stack'; + setHealthGate({ + stackName, nodeId, gateId, trigger, status: 'observing', reason: null, windowSeconds: null, startedAt: null, + targetScope, + serviceName: options?.serviceName ?? null, + failureSource: null, + recoveryId: options?.recoveryId ?? null, + }); let strikes = 0; // Single-flight: skip a tick while one request is outstanding so two // overlapping responses cannot land out of order. @@ -299,6 +356,9 @@ export function DeployFeedbackProvider({ children }: { children: React.ReactNode reason: string | null; windowSeconds: number | null; startedAt: number | null; + targetScope?: 'stack' | 'service'; + serviceName?: string | null; + failureSource?: 'primary' | 'collateral' | null; } : null; if (sessionIdRef.current !== mySession || settled) return; @@ -311,10 +371,33 @@ export function DeployFeedbackProvider({ children }: { children: React.ReactNode return; } strikes = 0; - setHealthGate({ stackName, nodeId, gateId, trigger, status: report.status, reason: report.reason, windowSeconds: report.windowSeconds, startedAt: report.startedAt }); - if (report.status !== 'observing') { + const status: HealthGateUiState['status'] = + report.status === 'observing' || report.status === 'passed' + || report.status === 'failed' || report.status === 'unknown' + ? report.status + : 'unknown'; + const nextGate: HealthGateUiState = { + stackName, nodeId, gateId, trigger, + status, + reason: report.reason, + windowSeconds: report.windowSeconds, startedAt: report.startedAt, + targetScope: report.targetScope ?? targetScope, + serviceName: report.serviceName ?? options?.serviceName ?? null, + failureSource: report.failureSource ?? null, + recoveryId: healthGateRef.current?.recoveryId ?? options?.recoveryId ?? null, + }; + setHealthGate(nextGate); + if (status !== 'observing') { settled = true; stopGatePolling(); + if ( + status === 'failed' + && nextGate.targetScope === 'service' + && nextGate.serviceName + && (silentGateRef.current || !panelOpenRef.current) + ) { + offerRestoreToastRef.current(nextGate); + } } } catch (e) { strikes += 1; @@ -328,6 +411,76 @@ export function DeployFeedbackProvider({ children }: { children: React.ReactNode gatePollRef.current = setInterval(() => { void tick(); }, GATE_POLL_INTERVAL_MS); }, [stopGatePolling]); + // Keep the toast helper current without re-creating startGatePolling on every render. + useEffect(() => { + offerRestoreToastRef.current = (gate: HealthGateUiState) => { + const serviceName = gate.serviceName; + if (!serviceName) return; + toast.error( + `Health gate failed for service "${serviceName}"${gate.reason ? `: ${gate.reason}` : ''}.`, + { + duration: 120_000, + action: { + label: 'Restore', + onClick: () => { + void (async () => { + let recoveryId = gate.recoveryId ?? null; + if (!recoveryId) { + const lookup = await fetchActiveServiceRecovery({ + nodeId: gate.nodeId, + stackName: gate.stackName, + serviceName, + }); + if (!lookup.ok) { + toast.error(lookup.error); + return; + } + recoveryId = lookup.recovery?.id ?? null; + } + if (!recoveryId) { + toast.error(`No recovery snapshot is available for "${serviceName}".`); + return; + } + const loadingId = toast.loading(`Restoring "${serviceName}"...`); + try { + const result = await requestServiceRestore({ + nodeId: gate.nodeId, + stackName: gate.stackName, + serviceName, + recoveryId, + }); + toast.dismiss(loadingId); + if (!result.ok) { + toast.error(result.error); + return; + } + if (result.healthGateId && result.observing) { + toast.info(`Service "${serviceName}" restored. Verifying health...`); + sessionIdRef.current += 1; + startGatePolling( + gate.stackName, + gate.nodeId, + result.healthGateId, + 'update', + sessionIdRef.current, + { serviceName, recoveryId: result.recoveryId, silent: true }, + ); + } else { + toast.success(`Service "${serviceName}" restored successfully`); + setHealthGate(null); + } + } catch (error) { + toast.dismiss(loadingId); + toast.error(error instanceof Error ? error.message : `Failed to restore "${serviceName}"`); + } + })(); + }, + }, + }, + ); + }; + }, [startGatePolling]); + const runWithLog = useCallback( async ( params: RunWithLogParams, @@ -344,7 +497,26 @@ export function DeployFeedbackProvider({ children }: { children: React.ReactNode const deploySessionId = Array.from(idBytes, (b) => b.toString(16).padStart(2, '0')).join(''); if (!isEnabled) { - return run(Promise.resolve(), deploySessionId); + const result = await run(Promise.resolve(), deploySessionId); + // Service-scoped updates still need gate polling and Restore discovery + // when Deploy Progress is off; open no panel, watch silently. + if ( + result.ok + && result.healthGateId + && params.serviceName + && (params.action === 'update' || params.action === 'deploy') + ) { + sessionIdRef.current += 1; + startGatePolling( + params.stackName, + params.nodeId, + result.healthGateId, + params.action === 'deploy' ? 'deploy' : 'update', + sessionIdRef.current, + { serviceName: params.serviceName, recoveryId: result.recoveryId, silent: true }, + ); + } + return result; } // Read the persisted style synchronously so this deploy uses the style in @@ -419,7 +591,10 @@ export function DeployFeedbackProvider({ children }: { children: React.ReactNode errorMessage: result.ok ? undefined : result.errorMessage, })); if (result.ok && result.healthGateId && (params.action === 'update' || params.action === 'deploy')) { - startGatePolling(params.stackName, params.nodeId, result.healthGateId, params.action, mySession); + startGatePolling(params.stackName, params.nodeId, result.healthGateId, params.action, mySession, { + serviceName: params.serviceName, + recoveryId: result.recoveryId, + }); } } diff --git a/frontend/src/context/__tests__/DeployFeedbackContext.test.tsx b/frontend/src/context/__tests__/DeployFeedbackContext.test.tsx index aa15d32a..a53f0cd1 100644 --- a/frontend/src/context/__tests__/DeployFeedbackContext.test.tsx +++ b/frontend/src/context/__tests__/DeployFeedbackContext.test.tsx @@ -21,9 +21,11 @@ describe('DeployFeedbackContext', () => { beforeEach(() => { localStorage.setItem(DEPLOY_FEEDBACK_KEY, 'true'); vi.mocked(apiFetch).mockReset(); + vi.useRealTimers(); }); afterEach(() => { localStorage.clear(); + vi.useRealTimers(); }); it('releases the deploy when the progress stream fails before connecting', async () => { @@ -149,6 +151,90 @@ describe('DeployFeedbackContext', () => { expect(result.current.panelState.isOpen).toBe(false); }); + it('silently polls a service health gate when Deploy Progress is disabled', async () => { + vi.useFakeTimers(); + try { + localStorage.setItem(DEPLOY_FEEDBACK_KEY, 'false'); + vi.mocked(apiFetch).mockImplementation(async (url: string) => { + if (String(url).includes('/health-gate')) { + return new Response(JSON.stringify({ + id: 'gate-svc', status: 'observing', reason: null, windowSeconds: 90, startedAt: Date.now(), + targetScope: 'service', serviceName: 'api', failureSource: null, + }), { status: 200 }); + } + return new Response('{}', { status: 200 }); + }); + const { result } = renderHook(() => useDeployFeedback(), { wrapper }); + + await act(async () => { + await result.current.runWithLog( + { stackName: 'web', action: 'update', nodeId: null, serviceName: 'api' }, + async (started) => { + await started; + return { ok: true, healthGateId: 'gate-svc', recoveryId: 'rec-1' }; + }, + ); + }); + + expect(result.current.panelState.isOpen).toBe(false); + expect(result.current.healthGate).toMatchObject({ + gateId: 'gate-svc', serviceName: 'api', recoveryId: 'rec-1', status: 'observing', + }); + await act(async () => { await vi.advanceTimersByTimeAsync(4_000); }); + expect(apiFetch).toHaveBeenCalledWith( + expect.stringContaining('/stacks/web/health-gate?gateId=gate-svc'), + expect.anything(), + ); + } finally { + vi.useRealTimers(); + } + }); + + it('keeps service gate recovery after the panel is closed', async () => { + vi.useFakeTimers(); + try { + vi.mocked(apiFetch).mockImplementation(async (url: string) => { + if (String(url).includes('/health-gate')) { + return new Response(JSON.stringify({ + id: 'gate-svc', status: 'observing', reason: null, windowSeconds: 90, startedAt: Date.now(), + targetScope: 'service', serviceName: 'api', failureSource: null, + }), { status: 200 }); + } + return new Response('{}', { status: 200 }); + }); + const { result } = renderHook(() => useDeployFeedback(), { wrapper }); + + let outer: Promise | undefined; + await act(async () => { + outer = result.current.runWithLog( + { stackName: 'web', action: 'update', nodeId: null, serviceName: 'api' }, + async (started) => { + await started; + return { ok: true, healthGateId: 'gate-svc', recoveryId: 'rec-keep' }; + }, + ); + await Promise.resolve(); + }); + // onTerminalReady schedules the deploy gate release after 50ms. + await act(async () => { + result.current.onTerminalReady(); + await vi.advanceTimersByTimeAsync(60); + await outer; + await Promise.resolve(); + await Promise.resolve(); + }); + expect(result.current.healthGate?.recoveryId).toBe('rec-keep'); + + act(() => { result.current.onPanelClose(); }); + expect(result.current.panelState.isOpen).toBe(false); + expect(result.current.healthGate).toMatchObject({ + gateId: 'gate-svc', recoveryId: 'rec-keep', status: 'observing', + }); + } finally { + vi.useRealTimers(); + } + }); + it('caps log rows and marks the truncation point', () => { const { result } = renderHook(() => useDeployFeedback(), { wrapper }); diff --git a/frontend/src/hooks/useServiceUpdateStatus.ts b/frontend/src/hooks/useServiceUpdateStatus.ts new file mode 100644 index 00000000..457635d2 --- /dev/null +++ b/frontend/src/hooks/useServiceUpdateStatus.ts @@ -0,0 +1,26 @@ +import { useMemo } from 'react'; +import type { StackUpdateInfo, StackServiceUpdateStatus } from '@/types/imageUpdates'; + +/** + * Per-service update status for one stack, selected from the + * `useImageUpdates` map (`detail.services`). Returns an empty array when the + * stack has no persisted per-service breakdown yet (older check, or a stack + * that has never been checked). + */ +export function useServiceUpdateStatus( + stackUpdates: Record, + stackFile: string | null, +): StackServiceUpdateStatus[] { + return useMemo(() => { + if (!stackFile) return []; + return stackUpdates[stackFile]?.services ?? []; + }, [stackUpdates, stackFile]); +} + +/** Look up one service's status by name from a per-stack breakdown. */ +export function findServiceUpdateStatus( + services: StackServiceUpdateStatus[], + serviceName: string, +): StackServiceUpdateStatus | undefined { + return services.find((s) => s.service === serviceName); +} diff --git a/frontend/src/lib/__tests__/updateAvailableLabel.test.ts b/frontend/src/lib/__tests__/updateAvailableLabel.test.ts new file mode 100644 index 00000000..0b3405dd --- /dev/null +++ b/frontend/src/lib/__tests__/updateAvailableLabel.test.ts @@ -0,0 +1,25 @@ +import { describe, it, expect } from 'vitest'; +import { updateAvailableBadge, updateAvailableLabel } from '@/lib/updateAvailableLabel'; + +describe('updateAvailableLabel', () => { + it('uses the generic label when no services are named', () => { + expect(updateAvailableLabel()).toBe('Update available'); + expect(updateAvailableLabel([])).toBe('Update available'); + }); + + it('names one or a few outdated services', () => { + expect(updateAvailableLabel(['api'])).toBe('Update available: api'); + expect(updateAvailableLabel(['api', 'db', 'worker'])).toBe('Update available: api, db, worker'); + }); + + it('summarizes larger service sets by count', () => { + expect(updateAvailableLabel(['a', 'b', 'c', 'd'])).toBe('Update available: 4 services'); + }); +}); + +describe('updateAvailableBadge', () => { + it('stays compact for the Stack Health column', () => { + expect(updateAvailableBadge(['api'])).toBe('Update: api'); + expect(updateAvailableBadge(['api', 'db'])).toBe('2 updates'); + }); +}); diff --git a/frontend/src/lib/capabilities.ts b/frontend/src/lib/capabilities.ts index 33c7c5ee..ea62818b 100644 --- a/frontend/src/lib/capabilities.ts +++ b/frontend/src/lib/capabilities.ts @@ -34,9 +34,11 @@ export const CAPABILITIES = [ 'cross-node-rbac', 'stack-down-remove-volumes', 'guided-external-network-preflight', + 'service-scoped-update', ] as const; export type Capability = (typeof CAPABILITIES)[number]; export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes' as const satisfies Capability; export const GUIDED_EXTERNAL_NETWORK_PREFLIGHT_CAPABILITY = 'guided-external-network-preflight' as const satisfies Capability; +export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability; diff --git a/frontend/src/lib/serviceUpdate.ts b/frontend/src/lib/serviceUpdate.ts new file mode 100644 index 00000000..e6430ee9 --- /dev/null +++ b/frontend/src/lib/serviceUpdate.ts @@ -0,0 +1,223 @@ +import { apiFetch, withDeploySession } from './api'; + +export interface RequestServiceUpdateParams { + nodeId: number | null; + stackName: string; + serviceName: string; + /** Caller's intent only (Update vs Rebuild copy); the backend route and + * orchestrator path are the same either way. Defaults to 'update'. */ + mode?: 'update' | 'rebuild'; + /** Deploy-feedback session id so Compose output streams to the panel. */ + deploySessionId?: string; +} + +export interface ServiceUpdateSuccess { + ok: true; + mode: 'update' | 'rebuild'; + serviceName: string; + healthGateId: string | null; + observing: boolean; + recoveryId: string | null; + recoveryAvailable: boolean; + recheckWarning?: string; +} + +export interface ServiceUpdateFailure { + ok: false; + mode: 'update' | 'rebuild'; + error: string; + code?: string; + serviceName?: string; + mutationStage?: string; + recoveryId?: string; + status?: number; +} + +export type ServiceUpdateResult = ServiceUpdateSuccess | ServiceUpdateFailure; + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null; + +/** + * Single entry point for a manual service-scoped update/rebuild, mirroring + * `POST /stacks/:stackName/services/:serviceName/update`'s response shape + * (see `sendServiceResult` in backend/src/routes/stacks.ts). + */ +export async function requestServiceUpdate(params: RequestServiceUpdateParams): Promise { + const { nodeId, stackName, serviceName, mode = 'update', deploySessionId } = params; + try { + const res = await apiFetch( + `/stacks/${encodeURIComponent(stackName)}/services/${encodeURIComponent(serviceName)}/update`, + withDeploySession(deploySessionId ?? '', { method: 'POST', nodeId }), + ); + const body: unknown = await res.json().catch(() => null); + if (!res.ok) { + const error = isRecord(body) && typeof body.error === 'string' + ? body.error + : `Failed to update service "${serviceName}"`; + return { + ok: false, + mode, + error, + code: isRecord(body) && typeof body.code === 'string' ? body.code : undefined, + serviceName: isRecord(body) && typeof body.serviceName === 'string' ? body.serviceName : undefined, + mutationStage: isRecord(body) && typeof body.mutationStage === 'string' ? body.mutationStage : undefined, + recoveryId: isRecord(body) && typeof body.recoveryId === 'string' ? body.recoveryId : undefined, + status: res.status, + }; + } + if (!isRecord(body) || typeof body.serviceName !== 'string') { + return { ok: false, mode, error: 'Unexpected response from the service update', status: res.status }; + } + return { + ok: true, + mode, + serviceName: body.serviceName, + healthGateId: typeof body.healthGateId === 'string' ? body.healthGateId : null, + observing: body.observing === true, + recoveryId: typeof body.recoveryId === 'string' ? body.recoveryId : null, + recoveryAvailable: body.recoveryAvailable === true, + recheckWarning: typeof body.recheckWarning === 'string' ? body.recheckWarning : undefined, + }; + } catch (error) { + return { + ok: false, + mode, + error: error instanceof Error ? error.message : `Failed to update service "${serviceName}"`, + }; + } +} + +export interface RequestServiceRestoreParams { + nodeId: number | null; + stackName: string; + serviceName: string; + recoveryId: string; + deploySessionId?: string; +} + +/** + * Restore one Compose service from a recovery snapshot captured during a + * prior service-scoped update (`POST .../services/:serviceName/restore`). + */ +export async function requestServiceRestore(params: RequestServiceRestoreParams): Promise { + const { nodeId, stackName, serviceName, recoveryId, deploySessionId } = params; + try { + const res = await apiFetch( + `/stacks/${encodeURIComponent(stackName)}/services/${encodeURIComponent(serviceName)}/restore`, + withDeploySession(deploySessionId ?? '', { + method: 'POST', + nodeId, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ recoveryId }), + }), + ); + const body: unknown = await res.json().catch(() => null); + if (!res.ok) { + const error = isRecord(body) && typeof body.error === 'string' + ? body.error + : `Failed to restore service "${serviceName}"`; + return { + ok: false, + mode: 'update', + error, + code: isRecord(body) && typeof body.code === 'string' ? body.code : undefined, + serviceName: isRecord(body) && typeof body.serviceName === 'string' ? body.serviceName : undefined, + mutationStage: isRecord(body) && typeof body.mutationStage === 'string' ? body.mutationStage : undefined, + recoveryId: isRecord(body) && typeof body.recoveryId === 'string' ? body.recoveryId : undefined, + status: res.status, + }; + } + if (!isRecord(body) || typeof body.serviceName !== 'string') { + return { ok: false, mode: 'update', error: 'Unexpected response from the service restore', status: res.status }; + } + return { + ok: true, + mode: 'update', + serviceName: body.serviceName, + healthGateId: typeof body.healthGateId === 'string' ? body.healthGateId : null, + observing: body.observing === true, + recoveryId: typeof body.recoveryId === 'string' ? body.recoveryId : null, + recoveryAvailable: body.recoveryAvailable === true, + recheckWarning: typeof body.recheckWarning === 'string' ? body.recheckWarning : undefined, + }; + } catch (error) { + return { + ok: false, + mode: 'update', + error: error instanceof Error ? error.message : `Failed to restore service "${serviceName}"`, + }; + } +} + +export interface ActiveServiceRecovery { + id: string; + status: string; + healthGateId: string | null; + expiresAt: number; + createdAt: number; + majorityImageId: string; + declaredImageRef: string; +} + +/** + * Fetch the newest active recovery snapshot for a service, if any. Used when + * Deploy Progress is disabled or dismissed so Restore remains discoverable. + * Distinguishes "none" from lookup failure so the UI does not claim the + * snapshot is missing when the request actually failed. + */ +export type FetchActiveServiceRecoveryResult = + | { ok: true; recovery: ActiveServiceRecovery | null } + | { ok: false; error: string }; + +export async function fetchActiveServiceRecovery(params: { + nodeId: number | null; + stackName: string; + serviceName: string; +}): Promise { + const { nodeId, stackName, serviceName } = params; + try { + const res = await apiFetch( + `/stacks/${encodeURIComponent(stackName)}/services/${encodeURIComponent(serviceName)}/recovery`, + { method: 'GET', nodeId }, + ); + const body: unknown = await res.json().catch(() => null); + if (!res.ok) { + const error = isRecord(body) && typeof body.error === 'string' + ? body.error + : `Failed to look up recovery for "${serviceName}"`; + console.warn('[serviceUpdate] recovery lookup failed:', res.status, error); + return { ok: false, error }; + } + if (!isRecord(body)) return { ok: true, recovery: null }; + if (body.recovery === null || body.recovery === undefined) { + return { ok: true, recovery: null }; + } + if (!isRecord(body.recovery)) { + console.warn('[serviceUpdate] recovery lookup returned an unexpected payload'); + return { ok: false, error: `Unexpected recovery response for "${serviceName}"` }; + } + const row = body.recovery; + const id = row.id; + if (typeof id !== 'string') { + console.warn('[serviceUpdate] recovery lookup returned an unexpected payload'); + return { ok: false, error: `Unexpected recovery response for "${serviceName}"` }; + } + return { + ok: true, + recovery: { + id, + status: typeof row.status === 'string' ? row.status : 'active', + healthGateId: typeof row.healthGateId === 'string' ? row.healthGateId : null, + expiresAt: typeof row.expiresAt === 'number' ? row.expiresAt : 0, + createdAt: typeof row.createdAt === 'number' ? row.createdAt : 0, + majorityImageId: typeof row.majorityImageId === 'string' ? row.majorityImageId : '', + declaredImageRef: typeof row.declaredImageRef === 'string' ? row.declaredImageRef : '', + }, + }; + } catch (error) { + const message = error instanceof Error ? error.message : `Failed to look up recovery for "${serviceName}"`; + console.warn('[serviceUpdate] recovery lookup failed:', message); + return { ok: false, error: message }; + } +} diff --git a/frontend/src/lib/updateAvailableLabel.ts b/frontend/src/lib/updateAvailableLabel.ts new file mode 100644 index 00000000..3177ec26 --- /dev/null +++ b/frontend/src/lib/updateAvailableLabel.ts @@ -0,0 +1,16 @@ +/** Tooltip / badge copy naming outdated services when the breakdown is known. */ +export function updateAvailableLabel(outdatedServices?: string[]): string { + const names = (outdatedServices ?? []).filter(Boolean); + if (names.length === 0) return 'Update available'; + if (names.length === 1) return `Update available: ${names[0]}`; + if (names.length <= 3) return `Update available: ${names.join(', ')}`; + return `Update available: ${names.length} services`; +} + +/** Compact Stack Health badge; full detail stays in the title attribute. */ +export function updateAvailableBadge(outdatedServices?: string[]): string { + const names = (outdatedServices ?? []).filter(Boolean); + if (names.length === 0) return 'Update available'; + if (names.length === 1) return `Update: ${names[0]}`; + return `${names.length} updates`; +} diff --git a/frontend/src/types/effectiveServices.ts b/frontend/src/types/effectiveServices.ts new file mode 100644 index 00000000..6c6a849c --- /dev/null +++ b/frontend/src/types/effectiveServices.ts @@ -0,0 +1,20 @@ +/** + * Per-service facts from the fully-merged effective Compose model, as + * returned by `GET /api/stacks/:stackName/effective-services`. Mirrors + * `backend/src/services/effectiveServiceModel.ts`; service-scoped update + * gating (multi-service headers, eligible-for-update, rebuild vs update + * wording, expected replica count) reads these fields. + */ +export interface EffectiveServiceSpec { + name: string; + declaredImage: string | null; + hasBuild: boolean; + /** May be 0 (explicit `scale: 0` or `deploy.replicas: 0`); defaults to 1 when neither is set. */ + expectedReplicas: number; + dependsOn: string[]; + hasHealthcheck: boolean; +} + +export type EffectiveServiceModelResult = + | { renderable: true; services: EffectiveServiceSpec[] } + | { renderable: false; code: 'effective_model_render_failed'; error: string }; diff --git a/frontend/src/types/imageUpdates.ts b/frontend/src/types/imageUpdates.ts index 0097768a..24255937 100644 --- a/frontend/src/types/imageUpdates.ts +++ b/frontend/src/types/imageUpdates.ts @@ -34,6 +34,23 @@ export interface ImageUpdateStatus { */ export type CheckStatus = 'ok' | 'partial' | 'failed'; +/** + * Per-service check outcome, as returned in `StackUpdateInfo.services`. + * Mirrors the backend's `StackServiceStatus`. Distinct from `CheckStatus`: + * a service with no checkable image (build-only, no declared image) is + * `not_checkable`, which never counts as a check failure at the stack level. + */ +export type ServiceCheckStatus = 'ok' | 'partial' | 'failed' | 'not_checkable'; + +export interface StackServiceUpdateStatus { + service: string; + image: string | null; + runtimeImages?: string[]; + hasUpdate: boolean; + checkStatus: ServiceCheckStatus; + lastError: string | null; +} + /** * Rich per-stack update status from `GET /api/image-updates/detail`. `lastError` * carries the failure reason when `checkStatus` is 'failed' or 'partial'. @@ -43,4 +60,6 @@ export interface StackUpdateInfo { checkStatus: CheckStatus; lastError: string | null; checkedAt: number; + /** Per-service breakdown; absent when the stack has no persisted per-service data yet. */ + services?: StackServiceUpdateStatus[]; }