From 719180f156c49bea48ad2b319d81cb9823cfb9fa Mon Sep 17 00:00:00 2001 From: Anso Date: Sun, 26 Jul 2026 03:09:21 -0400 Subject: [PATCH] fix(fleet): verify update status before removing readiness cards (#1697) * fix(fleet): verify update status before removing readiness cards Full-stack Apply now rechecks persisted status after the health gate starts, reloads the live preview before dropping a card, and invalidates the hub fleet aggregation so cleared updates cannot resurrect from a stale cache. Closes #1686 * fix(fleet): align persisted update status with preview semver detection Share digest-plus-tag detection so post-Apply sidebar status matches Fleet and Anatomy. * fix(fleet): keep tag-only updates advisory for Compose automation Expose digestUpdate vs tagUpdate from checkImage so scheduled and API auto-update only apply same-tag digest drift Compose can pull. * docs: clarify scheduled auto-update applies digest drift only Document that higher pinned tags stay advisory until Compose is changed, matching schedule and Run Now behavior. * docs: require Compose pin edits for higher-tag advisories Stop recommending Apply now or Update as remedies that cannot rewrite a pinned image tag. * docs: clarify Apply now pulls pinned tags only Align the detection-cadence bullet with digest-rebuild vs higher-tag guidance. * fix(fleet): keep tag advisories after apply and scheduled updates Tag-only previews were treated as cleared on Fleet reload, and scheduled/ Run Now paths wiped status without rechecking. Align post-update verification with the manual Apply path (health gate first, recheck, no blind clear) and block digest apply when sibling image checks failed. * fix(fleet): clear eslint unused-arg and containers assignment --- .../__tests__/auto-update-digest-gate.test.ts | 38 ++ .../image-update-detect-cross-surface.test.ts | 97 +++++ .../__tests__/image-update-service.test.ts | 127 +++++- .../__tests__/image-updates-routes.test.ts | 112 +++++- .../src/__tests__/scheduler-service.test.ts | 103 ++++- .../stack-update-post-recheck.test.ts | 193 +++++++++ backend/src/helpers/autoUpdateDigestGate.ts | 13 + backend/src/routes/imageUpdates.ts | 38 +- backend/src/routes/stacks.ts | 38 +- backend/src/services/ImageUpdateService.ts | 62 ++- backend/src/services/SchedulerService.ts | 42 +- docs/features/auto-update-policies.mdx | 53 ++- docs/openapi.yaml | 6 + .../components/AutoUpdateReadinessView.tsx | 96 ++++- .../AutoUpdateReadinessView.test.tsx | 367 ++++++++++++++++++ .../src/lib/updatePreviewActionability.ts | 9 +- 16 files changed, 1320 insertions(+), 74 deletions(-) create mode 100644 backend/src/__tests__/auto-update-digest-gate.test.ts create mode 100644 backend/src/__tests__/image-update-detect-cross-surface.test.ts create mode 100644 backend/src/__tests__/stack-update-post-recheck.test.ts diff --git a/backend/src/__tests__/auto-update-digest-gate.test.ts b/backend/src/__tests__/auto-update-digest-gate.test.ts new file mode 100644 index 00000000..45a6f7dd --- /dev/null +++ b/backend/src/__tests__/auto-update-digest-gate.test.ts @@ -0,0 +1,38 @@ +import { describe, it, expect } from 'vitest'; +import { + createAutoUpdateDigestGateState, + messageWhenDigestApplyBlockedByCheckErrors, + recordAutoUpdateImageCheck, +} from '../helpers/autoUpdateDigestGate'; + +describe('autoUpdateDigestGate', () => { + it('blocks digest apply when sibling check errors exist', () => { + const state = createAutoUpdateDigestGateState(); + recordAutoUpdateImageCheck(state, 'nginx:latest', { + hasUpdate: true, + digestUpdate: true, + tagUpdate: false, + }); + recordAutoUpdateImageCheck(state, 'redis:latest', { + hasUpdate: false, + digestUpdate: false, + tagUpdate: false, + checkStatus: 'failed', + error: 'registry timeout', + }); + + const msg = messageWhenDigestApplyBlockedByCheckErrors('web', state); + expect(msg).toContain('image check(s) failed'); + expect(msg).toContain('registry timeout'); + }); + + it('does not block when digest update exists without check errors', () => { + const state = createAutoUpdateDigestGateState(); + recordAutoUpdateImageCheck(state, 'nginx:latest', { + hasUpdate: true, + digestUpdate: true, + tagUpdate: false, + }); + expect(messageWhenDigestApplyBlockedByCheckErrors('web', state)).toBeNull(); + }); +}); diff --git a/backend/src/__tests__/image-update-detect-cross-surface.test.ts b/backend/src/__tests__/image-update-detect-cross-surface.test.ts new file mode 100644 index 00000000..f59fd014 --- /dev/null +++ b/backend/src/__tests__/image-update-detect-cross-surface.test.ts @@ -0,0 +1,97 @@ +import { describe, it, expect, vi } from 'vitest'; +import { detectImageUpdate } from '../services/imageUpdateDetect'; +import { computeImagePreview } from '../services/UpdatePreviewService'; +import type { DigestComparisonResult } from '../services/registry-api'; + +const PLATFORM = { os: 'linux', architecture: 'amd64' }; +const LOCAL_DIGEST = `sha256:${'a'.repeat(64)}`; +const CREDENTIALS = { username: 'u', password: 'p' }; +const IMAGE = 'nginx:1.2.3'; + +/** + * COR-1 regression: persisted sidebar status (via detectImageUpdate / + * checkImage) and Fleet/Anatomy preview (computeImagePreview) must agree when + * the declared-tag digest matches but a higher semantic tag exists. + */ +describe('cross-surface update detection (digest match + higher semver)', () => { + async function runDetectionAndPreview( + comparison: DigestComparisonResult, + tags: string[], + localDigests: string[] = [LOCAL_DIGEST], + ) { + const compareDigest = vi.fn().mockResolvedValue(comparison); + const listRegistryTagsResult = vi.fn().mockResolvedValue({ ok: true, tags }); + + const detection = await detectImageUpdate({ + localDigests, + platform: PLATFORM, + registry: 'registry-1.docker.io', + repo: 'library/nginx', + tag: '1.2.3', + credentials: CREDENTIALS, + deps: { compareDigest, listRegistryTagsResult }, + }); + + const preview = await computeImagePreview('app', IMAGE, { + getCredentials: vi.fn().mockResolvedValue(CREDENTIALS), + getLocalDigest: vi.fn().mockResolvedValue({ digests: localDigests, platform: PLATFORM, emptyReason: null }), + compareDigest, + listRegistryTagsResult, + }); + + return { detection, preview, compareDigest }; + } + + it('shared detector and preview both report hasUpdate for app:1.2.3 when 1.2.4 exists', async () => { + const { detection, preview } = await runDetectionAndPreview( + { kind: 'match' }, + ['1.2.3', '1.2.4'], + ); + + expect(detection.hasUpdate).toBe(true); + expect(detection.nextTag).toBe('1.2.4'); + expect(detection.digestUpdate).toBe(false); + expect(preview.has_update).toBe(true); + expect(preview.next_tag).toBe('1.2.4'); + expect(preview.has_update).toBe(detection.hasUpdate); + }); + + it('shared detector and preview both clear when digest matches and no higher tag exists', async () => { + const { detection, preview } = await runDetectionAndPreview({ kind: 'match' }, ['1.2.3']); + + expect(detection.hasUpdate).toBe(false); + expect(preview.has_update).toBe(false); + expect(preview.has_update).toBe(detection.hasUpdate); + }); + + it('shared detector and preview both report hasUpdate when digest errors but 1.2.4 exists', async () => { + const { detection, preview } = await runDetectionAndPreview( + { kind: 'error', reason: 'Registry unreachable' }, + ['1.2.3', '1.2.4'], + ); + + expect(detection.hasUpdate).toBe(true); + expect(detection.nextTag).toBe('1.2.4'); + expect(detection.digestUpdate).toBe(false); + expect(detection.digestError).toBe('Registry unreachable'); + expect(preview.has_update).toBe(true); + expect(preview.next_tag).toBe('1.2.4'); + expect(preview.has_update).toBe(detection.hasUpdate); + }); + + it('shared detector and preview both report hasUpdate with no local digests when 1.2.4 exists', async () => { + const { detection, preview, compareDigest } = await runDetectionAndPreview( + { kind: 'update' }, + ['1.2.3', '1.2.4'], + [], + ); + + expect(compareDigest).not.toHaveBeenCalled(); + expect(detection.hasUpdate).toBe(true); + expect(detection.digestUpdate).toBe(false); + expect(detection.nextTag).toBe('1.2.4'); + expect(preview.has_update).toBe(true); + expect(preview.next_tag).toBe('1.2.4'); + expect(preview.has_update).toBe(detection.hasUpdate); + }); +}); diff --git a/backend/src/__tests__/image-update-service.test.ts b/backend/src/__tests__/image-update-service.test.ts index 302dcf40..347218ab 100644 --- a/backend/src/__tests__/image-update-service.test.ts +++ b/backend/src/__tests__/image-update-service.test.ts @@ -239,6 +239,7 @@ describe('ImageUpdateService - checkImage surfaces the comparison resolver outco beforeEach(() => { vi.clearAllMocks(); + mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: [] }); (ImageUpdateService as any).instance = undefined; service = ImageUpdateService.getInstance(); }); @@ -255,6 +256,16 @@ describe('ImageUpdateService - checkImage surfaces the comparison resolver outco }), } as any); + const dockerWithNginxSemver = (repoDigests: string[] = [`registry-1.docker.io/library/nginx@${LOCAL_DIGEST}`]) => ({ + getDocker: () => ({ + getImage: () => ({ inspect: vi.fn().mockResolvedValue({ + RepoDigests: repoDigests, + Os: 'linux', + Architecture: 'amd64', + }) }), + }), + } as any); + it('surfaces the specific failure reason (not a generic "unreachable") as the check error', async () => { mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'error', reason: 'Authentication failed for ghcr.io/linuxserver/radarr:latest' }); const result = await service.checkImage(dockerWithLocalDigest(LOCAL_DIGEST), 'ghcr.io/linuxserver/radarr:latest'); @@ -286,6 +297,20 @@ describe('ImageUpdateService - checkImage surfaces the comparison resolver outco ); }); + it('reports an update when the declared tag digest matches but a higher semver tag exists', async () => { + mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'match' }); + mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: ['1.2.3', '1.2.4'] }); + const result = await service.checkImage(dockerWithNginxSemver(), 'nginx:1.2.3'); + expect(result).toMatchObject({ hasUpdate: true, digestUpdate: false, tagUpdate: true, checkStatus: 'ok' }); + }); + + it('reports an update when digest comparison errors but a higher semver tag exists', async () => { + mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'error', reason: 'Registry unreachable' }); + mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: ['1.2.3', '1.2.4'] }); + const result = await service.checkImage(dockerWithNginxSemver(), 'nginx:1.2.3'); + expect(result).toMatchObject({ hasUpdate: true, digestUpdate: false, tagUpdate: true, checkStatus: 'ok' }); + }); + it('forwards every matching RepoDigest (stale index ahead of current) to the comparison resolver', async () => { const STALE = `sha256:${'f'.repeat(64)}`; const CURRENT = `sha256:${'e'.repeat(64)}`; @@ -339,6 +364,7 @@ services: beforeEach(() => { vi.clearAllMocks(); + mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: [] }); (ImageUpdateService as any).instance = undefined; mockGetSystemState.mockReturnValue('1'); mockGetStacks.mockResolvedValue(['stackA']); @@ -1667,7 +1693,7 @@ services: }); describe('recheckStack', () => { - it('persists a fresh per-service reduction and returns no warning on success', async () => { + it('returns still_present when a checkable service still has an update', async () => { mockBuildEffectiveServiceModel.mockResolvedValueOnce({ renderable: true, services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')], @@ -1680,7 +1706,10 @@ services: const result = await service.recheckStack(1, 'stackA'); - expect(result).toEqual({ warning: null }); + expect(result).toEqual({ + outcome: 'still_present', + warning: 'The update command completed, but Sencho still detects an available image update.', + }); expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith( 1, 'stackA', true, expect.any(Number), 'ok', null, [ @@ -1691,13 +1720,103 @@ services: ); }); - it('returns a warning and leaves the prior row untouched when the model cannot render', async () => { + it('returns cleared when every checkable service is up to date', async () => { + mockBuildEffectiveServiceModel.mockResolvedValueOnce({ + renderable: true, + services: [specFor('web', 'web:latest')], + }); + mockGetAllContainers.mockResolvedValue([ + { Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } }, + ]); + const service = ImageUpdateService.getInstance(); + (service as any).checkImage = vi.fn().mockResolvedValue({ hasUpdate: false }); + + const result = await service.recheckStack(1, 'stackA'); + + expect(result).toEqual({ outcome: 'cleared', warning: null }); + expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith( + 1, 'stackA', false, expect.any(Number), 'ok', null, + expect.any(Array), + expect.any(Number), + ); + }); + + it('returns verification_failed and leaves the prior row untouched when the model cannot render', async () => { mockBuildEffectiveServiceModel.mockResolvedValueOnce({ renderable: false, code: 'effective_model_render_failed', error: 'no model in test' }); const service = ImageUpdateService.getInstance(); const result = await service.recheckStack(1, 'stackA'); - expect(result).toEqual({ warning: 'no model in test' }); + expect(result).toEqual({ outcome: 'verification_failed', warning: 'no model in test' }); + expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled(); + expect(mockRecordStackCheckFailure).not.toHaveBeenCalled(); + }); + + it('returns verification_incomplete and preserves prior hasUpdate on a fully failed check', async () => { + mockBuildEffectiveServiceModel.mockResolvedValueOnce({ + renderable: true, + services: [specFor('web', 'web:latest')], + }); + mockGetAllContainers.mockResolvedValue([ + { Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } }, + ]); + mockGetStackServicesJson.mockReturnValueOnce([ + { service: 'web', image: 'web:latest', hasUpdate: true, checkStatus: 'ok', lastError: null }, + ]); + const service = ImageUpdateService.getInstance(); + (service as any).checkImage = vi.fn().mockResolvedValue({ + hasUpdate: false, + error: 'registry timeout', + }); + + const result = await service.recheckStack(1, 'stackA'); + + expect(result).toEqual({ + outcome: 'verification_incomplete', + warning: 'The update command completed, but Sencho could not fully verify whether an image update remains.', + }); + expect(mockRecordStackCheckFailure).toHaveBeenCalled(); + expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled(); + }); + + it('returns verification_incomplete when the write lock discards a stale commit', async () => { + mockBuildEffectiveServiceModel.mockResolvedValueOnce({ + renderable: true, + services: [specFor('web', 'web:latest')], + }); + mockGetAllContainers.mockResolvedValue([ + { Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } }, + ]); + const service = ImageUpdateService.getInstance(); + (service as any).checkImage = vi.fn().mockResolvedValue({ hasUpdate: false }); + (service as any).withStackWriteLock = vi.fn().mockResolvedValue(false); + + const result = await service.recheckStack(1, 'stackA'); + + expect(result).toEqual({ + outcome: 'verification_incomplete', + warning: 'The update command completed, but Sencho could not fully verify whether an image update remains.', + }); + expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled(); + expect(mockRecordStackCheckFailure).not.toHaveBeenCalled(); + }); + + it('returns verification_incomplete when container listing fails', async () => { + mockBuildEffectiveServiceModel.mockResolvedValueOnce({ + renderable: true, + services: [specFor('web', 'web:latest')], + }); + mockGetAllContainers.mockRejectedValueOnce(new Error('docker socket down')); + const service = ImageUpdateService.getInstance(); + (service as any).checkImage = vi.fn().mockResolvedValue({ hasUpdate: false }); + + const result = await service.recheckStack(1, 'stackA'); + + expect(result).toEqual({ + outcome: 'verification_incomplete', + warning: 'The update command completed, but Sencho could not fully verify whether an image update remains.', + }); + expect((service as any).checkImage).not.toHaveBeenCalled(); expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled(); }); }); diff --git a/backend/src/__tests__/image-updates-routes.test.ts b/backend/src/__tests__/image-updates-routes.test.ts index 98004bca..1d93f5d9 100644 --- a/backend/src/__tests__/image-updates-routes.test.ts +++ b/backend/src/__tests__/image-updates-routes.test.ts @@ -428,13 +428,22 @@ describe('POST /api/auto-update/execute', () => { const { ComposeService } = await import('../services/ComposeService'); const { HealthGateService } = await import('../services/HealthGateService'); const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + const callOrder: string[] = []; const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack') .mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never); const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') .mockResolvedValue({ hasUpdate: true, digestUpdate: true } as never); const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); - const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-au'); + const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack') + .mockImplementation(async () => { + callOrder.push('recheckStack'); + return { outcome: 'cleared', warning: null } as never; + }); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockImplementation(() => { + callOrder.push('beginStack'); + return 'gate-au'; + }); try { const res = await request(app) .post('/api/auto-update/execute') @@ -442,12 +451,113 @@ describe('POST /api/auto-update/execute', () => { .send({ target: 'auto-upd-gate' }); expect(res.status).toBe(200); expect(updateSpy).toHaveBeenCalledWith('auto-upd-gate', undefined, true); + expect(recheckSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-gate'); expect(beginSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-gate', 'update', `auto-update:${TEST_USERNAME}`); + expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack')); } finally { containersSpy.mockRestore(); checkSpy.mockRestore(); updateSpy.mockRestore(); + recheckSpy.mockRestore(); beginSpy.mockRestore(); } }); + + it('skips Compose apply for tag-only availability without clearing status', async () => { + const DockerController = (await import('../services/DockerController')).default; + const { ImageUpdateService } = await import('../services/ImageUpdateService'); + const { ComposeService } = await import('../services/ComposeService'); + const { DatabaseService } = await import('../services/DatabaseService'); + const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + + const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack') + .mockResolvedValue([{ Id: 'c1', Image: 'nginx:1.2.3' }] as never); + const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') + .mockResolvedValue({ hasUpdate: true, digestUpdate: false, tagUpdate: true } as never); + const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); + const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack'); + const clearSpy = vi.spyOn(DatabaseService.getInstance(), 'clearStackUpdateStatus'); + try { + const res = await request(app) + .post('/api/auto-update/execute') + .set('Cookie', adminCookie) + .send({ target: 'auto-upd-tag-only' }); + expect(res.status).toBe(200); + expect(res.body.result).toContain('Compose pin unchanged'); + expect(updateSpy).not.toHaveBeenCalled(); + expect(recheckSpy).not.toHaveBeenCalled(); + expect(clearSpy).not.toHaveBeenCalledWith(nodeId, 'auto-upd-tag-only'); + } finally { + containersSpy.mockRestore(); + checkSpy.mockRestore(); + updateSpy.mockRestore(); + recheckSpy.mockRestore(); + clearSpy.mockRestore(); + } + }); + + it('skips digest apply when a sibling image check failed', async () => { + const DockerController = (await import('../services/DockerController')).default; + const { ImageUpdateService } = await import('../services/ImageUpdateService'); + const { ComposeService } = await import('../services/ComposeService'); + + const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack') + .mockResolvedValue([ + { Id: 'c1', Image: 'nginx:latest' }, + { Id: 'c2', Image: 'redis:latest' }, + ] as never); + const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') + .mockResolvedValueOnce({ hasUpdate: true, digestUpdate: true, tagUpdate: false } as never) + .mockResolvedValueOnce({ hasUpdate: false, error: 'registry timeout', checkStatus: 'failed' } as never); + const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); + const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack'); + try { + const res = await request(app) + .post('/api/auto-update/execute') + .set('Cookie', adminCookie) + .send({ target: 'auto-upd-check-err' }); + expect(res.status).toBe(200); + expect(res.body.result).toContain('image check(s) failed'); + expect(updateSpy).not.toHaveBeenCalled(); + expect(recheckSpy).not.toHaveBeenCalled(); + } finally { + containersSpy.mockRestore(); + checkSpy.mockRestore(); + updateSpy.mockRestore(); + recheckSpy.mockRestore(); + } + }); + + it('still applies when checkImage reports same-tag digestUpdate', async () => { + const DockerController = (await import('../services/DockerController')).default; + const { ImageUpdateService } = await import('../services/ImageUpdateService'); + const { ComposeService } = await import('../services/ComposeService'); + const { DatabaseService } = await import('../services/DatabaseService'); + const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + + const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack') + .mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never); + const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') + .mockResolvedValue({ hasUpdate: true, digestUpdate: true, tagUpdate: false } as never); + const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); + const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack') + .mockResolvedValue({ outcome: 'still_present', warning: null } as never); + const clearSpy = vi.spyOn(DatabaseService.getInstance(), 'clearStackUpdateStatus'); + try { + const res = await request(app) + .post('/api/auto-update/execute') + .set('Cookie', adminCookie) + .send({ target: 'auto-upd-digest' }); + expect(res.status).toBe(200); + expect(updateSpy).toHaveBeenCalledWith('auto-upd-digest', undefined, true); + expect(recheckSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-digest'); + expect(clearSpy).not.toHaveBeenCalledWith(nodeId, 'auto-upd-digest'); + } finally { + containersSpy.mockRestore(); + checkSpy.mockRestore(); + updateSpy.mockRestore(); + recheckSpy.mockRestore(); + clearSpy.mockRestore(); + } + }); }); diff --git a/backend/src/__tests__/scheduler-service.test.ts b/backend/src/__tests__/scheduler-service.test.ts index 2ae5968c..dba10002 100644 --- a/backend/src/__tests__/scheduler-service.test.ts +++ b/backend/src/__tests__/scheduler-service.test.ts @@ -18,8 +18,8 @@ const { mockStartContainer, mockStopContainer, mockPruneSystem, mockUpdateStack, mockGetStacks, mockGetStackContent, mockGetEnvContent, - mockCheckImage, - mockDispatchAlert, + mockCheckImage, mockRecheckStack, + mockDispatchAlert, mockBroadcastEvent, mockGetProxyTarget, mockIsTrivyAvailable, mockScanAllNodeImages, @@ -59,7 +59,9 @@ const { mockGetStackContent: vi.fn().mockResolvedValue(''), mockGetEnvContent: vi.fn().mockResolvedValue(''), mockCheckImage: vi.fn().mockResolvedValue({ hasUpdate: false }), + mockRecheckStack: vi.fn().mockResolvedValue({ outcome: 'cleared', warning: null }), mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), + mockBroadcastEvent: vi.fn(), mockGetProxyTarget: vi.fn().mockReturnValue(null), mockIsTrivyAvailable: vi.fn().mockReturnValue(true), mockScanAllNodeImages: vi.fn().mockResolvedValue({ @@ -167,14 +169,18 @@ vi.mock('../services/ImageUpdateService', () => ({ ImageUpdateService: { getInstance: () => ({ checkImage: mockCheckImage, + recheckStack: mockRecheckStack, }), }, + UPDATE_VERIFICATION_INCOMPLETE_WARNING: + 'The update command completed, but Sencho could not fully verify whether an image update remains.', })); vi.mock('../services/NotificationService', () => ({ NotificationService: { getInstance: () => ({ dispatchAlert: mockDispatchAlert, + broadcastEvent: mockBroadcastEvent, }), }, })); @@ -795,7 +801,47 @@ describe('SchedulerService - executeUpdate', () => { await svc.triggerTask(80); expect(mockUpdateStack).toHaveBeenCalledWith('web-app', undefined, true); - expect(mockClearStackUpdateStatus).toHaveBeenCalledWith(1, 'web-app'); + expect(mockRecheckStack).toHaveBeenCalledWith(1, 'web-app'); + expect(mockClearStackUpdateStatus).not.toHaveBeenCalled(); + expect(mockBroadcastEvent).toHaveBeenCalledWith(expect.objectContaining({ + type: 'state-invalidate', + scope: 'image-updates', + nodeId: 1, + stackName: 'web-app', + })); + }); + + it('blocks scheduled Compose apply when a sibling image check failed', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 186, + name: 'update-check-errors', + action: 'update', + cron_expression: '0 4 * * *', + enabled: true, + target_id: 'web-app', + node_id: 1, + created_by: 'admin', + last_status: null, + }); + mockGetContainersByStack.mockResolvedValue([ + { Id: 'c1', Image: 'nginx:latest' }, + { Id: 'c2', Image: 'redis:latest' }, + ]); + mockCheckImage + .mockResolvedValueOnce({ hasUpdate: true, digestUpdate: true, tagUpdate: false }) + .mockResolvedValueOnce({ hasUpdate: false, error: 'registry timeout', checkStatus: 'failed' }); + + await SchedulerService.getInstance().triggerTask(186); + + expect(mockUpdateStack).not.toHaveBeenCalled(); + expect(mockRecheckStack).not.toHaveBeenCalled(); + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + 1, + expect.objectContaining({ + status: 'success', + output: expect.stringContaining('image check(s) failed'), + }), + ); }); it('runs a scheduled update on the community tier (no paid gate)', async () => { @@ -828,7 +874,15 @@ describe('SchedulerService - executeUpdate', () => { it('begins a health gate after a scheduled update succeeds', async () => { const { HealthGateService } = await import('../services/HealthGateService'); - const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-1'); + const callOrder: string[] = []; + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockImplementation(() => { + callOrder.push('beginStack'); + return 'gate-1'; + }); + mockRecheckStack.mockImplementation(async () => { + callOrder.push('recheckStack'); + return { outcome: 'cleared', warning: null }; + }); try { mockGetScheduledTask.mockReturnValue({ id: 83, @@ -847,6 +901,8 @@ describe('SchedulerService - executeUpdate', () => { await SchedulerService.getInstance().triggerTask(83); expect(beginSpy).toHaveBeenCalledWith(1, 'web-app', 'update', 'system:scheduler'); + expect(mockRecheckStack).toHaveBeenCalledWith(1, 'web-app'); + expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack')); } finally { beginSpy.mockRestore(); } @@ -875,6 +931,45 @@ describe('SchedulerService - executeUpdate', () => { expect(mockUpdateStack).not.toHaveBeenCalled(); }); + + it('skips Compose apply for tag-only availability (pinned semver cannot be rewritten)', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 185, + name: 'update-tag-only', + action: 'update', + cron_expression: '0 4 * * *', + enabled: true, + target_id: 'web-app', + node_id: 1, + created_by: 'admin', + last_status: null, + }); + mockGetContainersByStack.mockResolvedValue([ + { Id: 'c1', Image: 'nginx:1.2.3' }, + ]); + // Higher tag is visible (hasUpdate) but not actionable via Compose pull. + mockCheckImage.mockResolvedValue({ hasUpdate: true, digestUpdate: false, tagUpdate: true }); + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(84); + + expect(mockUpdateStack).not.toHaveBeenCalled(); + expect(mockClearStackUpdateStatus).not.toHaveBeenCalled(); + expect(mockDispatchAlert).not.toHaveBeenCalledWith( + 'info', + 'image_update_applied', + expect.any(String), + expect.anything(), + ); + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + 1, + expect.objectContaining({ + status: 'success', + output: expect.stringContaining('Compose pin unchanged'), + }), + ); + }); + it('handles wildcard target (*) by updating all stacks', async () => { mockGetScheduledTask.mockReturnValue({ id: 82, diff --git a/backend/src/__tests__/stack-update-post-recheck.test.ts b/backend/src/__tests__/stack-update-post-recheck.test.ts new file mode 100644 index 00000000..4baf2367 --- /dev/null +++ b/backend/src/__tests__/stack-update-post-recheck.test.ts @@ -0,0 +1,193 @@ +/** + * Manual POST /api/stacks/:name/update must start the health gate before + * registry recheck, never blind-clear update status, and keep Compose success + * as HTTP 200 even when recheck throws. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import request from 'supertest'; +import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; + +const { + mockExecute, + mockRecheckStack, + mockBeginStack, + mockClearStackUpdateStatus, + mockBroadcastEvent, + mockDispatchAlert, +} = vi.hoisted(() => ({ + mockExecute: vi.fn(), + mockRecheckStack: vi.fn(), + mockBeginStack: vi.fn(), + mockClearStackUpdateStatus: vi.fn(), + mockBroadcastEvent: vi.fn(), + mockDispatchAlert: vi.fn(), +})); + +vi.mock('../services/StackUpdateOrchestrator', () => ({ + StackUpdateOrchestrator: { + getInstance: () => ({ execute: mockExecute }), + }, + shortImageId: (id: string) => id.slice(0, 12), +})); + +vi.mock('../services/ImageUpdateService', async () => { + const actual = await vi.importActual( + '../services/ImageUpdateService', + ); + return { + ...actual, + ImageUpdateService: { + getInstance: () => ({ recheckStack: mockRecheckStack }), + }, + }; +}); + +vi.mock('../services/HealthGateService', () => ({ + HealthGateService: { + getInstance: () => ({ + beginStack: mockBeginStack, + }), + }, +})); + +vi.mock('../services/FileSystemService', () => ({ + FileSystemService: { + getInstance: () => ({ + getBaseDir: () => '/tmp/compose', + hasComposeFile: vi.fn().mockResolvedValue(true), + }), + }, +})); + +vi.mock('../helpers/policyGate', async () => { + const actual = await vi.importActual( + '../helpers/policyGate', + ); + return { + ...actual, + runPolicyGate: vi.fn().mockResolvedValue(true), + triggerPostDeployScan: vi.fn().mockResolvedValue(undefined), + }; +}); + +let tmpDir: string; +let app: import('express').Express; +let authCookie: string; +let clearSpy: ReturnType | undefined; +let broadcastSpy: ReturnType | undefined; +const callOrder: string[] = []; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + authCookie = await loginAsTestAdmin(app); + + const { DatabaseService } = await import('../services/DatabaseService'); + const { NotificationService } = await import('../services/NotificationService'); + clearSpy = vi.spyOn(DatabaseService.getInstance(), 'clearStackUpdateStatus').mockImplementation((...args) => { + callOrder.push('clearStackUpdateStatus'); + return mockClearStackUpdateStatus(...args); + }); + broadcastSpy = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent').mockImplementation((...args) => { + callOrder.push('broadcastEvent'); + return mockBroadcastEvent(...args); + }); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockImplementation((...args) => { + callOrder.push(`dispatchAlert:${String(args[2] ?? args[0])}`); + return mockDispatchAlert(...args) ?? Promise.resolve({ persisted: true }); + }); +}); + +afterAll(() => { + clearSpy?.mockRestore(); + broadcastSpy?.mockRestore(); + vi.restoreAllMocks(); + cleanupTestDb(tmpDir); +}); + +beforeEach(() => { + callOrder.length = 0; + mockExecute.mockReset(); + mockRecheckStack.mockReset(); + mockBeginStack.mockReset(); + mockClearStackUpdateStatus.mockReset(); + mockBroadcastEvent.mockReset(); + mockDispatchAlert.mockReset().mockResolvedValue({ persisted: true }); + + mockExecute.mockImplementation(async () => { + callOrder.push('execute'); + return { kind: 'stack_compose_done', recoveryId: null }; + }); + mockBeginStack.mockImplementation(() => { + callOrder.push('beginStack'); + return 'gate-1'; + }); + mockRecheckStack.mockImplementation(async () => { + callOrder.push('recheckStack'); + return { outcome: 'cleared', warning: null }; + }); +}); + +describe('POST /api/stacks/:name/update post-compose verification', () => { + it('starts the health gate before recheck, skips clear, and broadcasts after recheck', async () => { + const res = await request(app) + .post('/api/stacks/web/update') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ status: 'Update completed', healthGateId: 'gate-1' }); + expect(res.body.recheckWarning).toBeUndefined(); + expect(callOrder.indexOf('execute')).toBeLessThan(callOrder.indexOf('beginStack')); + expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack')); + expect(callOrder.indexOf('recheckStack')).toBeLessThan(callOrder.indexOf('broadcastEvent')); + expect(callOrder).not.toContain('clearStackUpdateStatus'); + expect(mockClearStackUpdateStatus).not.toHaveBeenCalled(); + }); + + it('returns recheckWarning when the update condition remains', async () => { + mockRecheckStack.mockImplementation(async () => { + callOrder.push('recheckStack'); + return { + outcome: 'still_present', + warning: 'The update command completed, but Sencho still detects an available image update.', + }; + }); + + const res = await request(app) + .post('/api/stacks/web/update') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + + expect(res.status).toBe(200); + expect(res.body.recheckWarning).toBe( + 'The update command completed, but Sencho still detects an available image update.', + ); + }); + + it('keeps HTTP 200 and success notification when recheck throws after Compose', async () => { + mockRecheckStack.mockImplementation(async () => { + callOrder.push('recheckStack'); + throw new Error('registry blew up'); + }); + + const res = await request(app) + .post('/api/stacks/web/update') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + + expect(res.status).toBe(200); + expect(res.body.healthGateId).toBe('gate-1'); + expect(res.body.recheckWarning).toMatch(/could not fully verify/i); + expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack')); + expect(callOrder).toContain('broadcastEvent'); + // Success path still notifies; failure notification must not fire. + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'info', + 'image_update_applied', + expect.any(String), + expect.objectContaining({ stackName: 'web' }), + ); + expect(mockDispatchAlert.mock.calls.some((c) => c[0] === 'error' && c[1] === 'deploy_failure')).toBe(false); + }); +}); diff --git a/backend/src/helpers/autoUpdateDigestGate.ts b/backend/src/helpers/autoUpdateDigestGate.ts index 7dfd8ae2..5aa0a2b7 100644 --- a/backend/src/helpers/autoUpdateDigestGate.ts +++ b/backend/src/helpers/autoUpdateDigestGate.ts @@ -40,6 +40,19 @@ export function recordAutoUpdateImageCheck( } } +/** + * Operator message when a sibling image check failed and a full-stack Compose + * update must not run (it would pull/recreate the unverified image as + * collateral). Null when digest apply may proceed. + */ +export function messageWhenDigestApplyBlockedByCheckErrors( + stackName: string, + state: Pick, +): string | null { + if (!state.hasDigestUpdate || state.checkErrors.length === 0) return null; + return `Stack "${stackName}": WARNING - digest update available but ${state.checkErrors.length} image check(s) failed; skipped auto-update (${state.checkErrors.join('; ')}).`; +} + /** Operator message when no digest-actionable update was found. */ export function messageWhenNoDigestUpdate( stackName: string, diff --git a/backend/src/routes/imageUpdates.ts b/backend/src/routes/imageUpdates.ts index 6b2d9801..c1e7db74 100644 --- a/backend/src/routes/imageUpdates.ts +++ b/backend/src/routes/imageUpdates.ts @@ -5,13 +5,13 @@ import DockerController from '../services/DockerController'; import { DatabaseService } from '../services/DatabaseService'; import { NodeRegistry } from '../services/NodeRegistry'; import { CacheService } from '../services/CacheService'; -import { FLEET_UPDATE_CACHE_KEY } from '../helpers/fleetUpdateCache'; import { createAutoUpdateDigestGateState, + messageWhenDigestApplyBlockedByCheckErrors, messageWhenNoDigestUpdate, recordAutoUpdateImageCheck, } from '../helpers/autoUpdateDigestGate'; -import { ImageUpdateService } from '../services/ImageUpdateService'; +import { ImageUpdateService, UPDATE_VERIFICATION_INCOMPLETE_WARNING } from '../services/ImageUpdateService'; import { FileSystemService } from '../services/FileSystemService'; import { StackUpdateOrchestrator } from '../services/StackUpdateOrchestrator'; import { StackOpLockService, stackOpSkipMessage } from '../services/StackOpLockService'; @@ -21,6 +21,8 @@ import { HealthGateService } from '../services/HealthGateService'; import { authMiddleware } from '../middleware/auth'; import { requireAdmin } from '../middleware/tierGates'; import { buildPolicyGateOptions } from '../helpers/policyGate'; +import { FLEET_UPDATE_CACHE_KEY, invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache'; +import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; import { summarizeBlockReasons } from '../utils/policy-risk'; import { isValidStackName } from '../utils/validation'; import { sanitizeForLog } from '../utils/safeLog'; @@ -311,7 +313,7 @@ imageUpdatesRouter.post('/fleet/refresh', authMiddleware, async (_req: Request, } } - CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY); + invalidateFleetUpdateCache(); res.json({ triggered, rateLimited, failed }); }); @@ -349,7 +351,6 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp const docker = DockerController.getInstance(req.nodeId); const imageUpdateService = ImageUpdateService.getInstance(); - const db = DatabaseService.getInstance(); const atomic = true; const results: string[] = []; @@ -388,6 +389,11 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp results.push(messageWhenNoDigestUpdate(stackName, gate, imageRefs.length)); continue; } + const checkErrorBlock = messageWhenDigestApplyBlockedByCheckErrors(stackName, gate); + if (checkErrorBlock) { + results.push(checkErrorBlock); + continue; + } const { updatedImages } = gate; @@ -421,7 +427,9 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp results.push(stackOpSkipMessage(stackName, lock.existing.action)); continue; } - db.clearStackUpdateStatus(req.nodeId, stackName); + + // Health observation starts immediately after Compose; registry recheck is + // isolated so a verification failure cannot turn Compose success into a failure. const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`); const orchResult = lock.result; const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; @@ -430,6 +438,21 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId); } + // Recheck persists digest-cleared / tag-advisory state. Do not blind-clear. + let recheckWarning: string | undefined; + try { + const recheck = await imageUpdateService.recheckStack(req.nodeId, stackName); + if (recheck.warning) recheckWarning = recheck.warning; + } catch (recheckErr) { + console.warn( + '[AutoUpdate] Post-update recheck failed for %s: %s', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(recheckErr, 'unknown')), + ); + recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING; + } + + invalidateNodeCaches(req.nodeId); NotificationService.getInstance().broadcastEvent({ type: 'state-invalidate', scope: 'image-updates', @@ -446,7 +469,8 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp { stackName, actor: 'system:image-update' }, ); - results.push(`Stack "${stackName}": updated (${updatedImages.join(', ')}).`); + const base = `Stack "${stackName}": updated (${updatedImages.join(', ')}).`; + results.push(recheckWarning ? `${base} ${recheckWarning}` : base); } catch (e) { const msg = getErrorMessage(e, String(e)); results.push(`Stack "${stackName}" failed: ${msg}`); @@ -454,7 +478,7 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp } } - CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY); + invalidateFleetUpdateCache(); res.json({ result: results.join('\n') }); } catch (error) { const msg = getErrorMessage(error, 'Auto-update execution failed'); diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index e7b814e9..8d1469ee 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -18,8 +18,6 @@ import DockerController, { type BulkStackInfo } from '../services/DockerControll import { DatabaseService, type StackDossierFields } from '../services/DatabaseService'; import { CacheService, type CacheFetchOutcome } from '../services/CacheService'; import { UpdatePreviewService, isAuthoritativeNegativePreview } from '../services/UpdatePreviewService'; -import { ImageUpdateService } from '../services/ImageUpdateService'; -import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache'; import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService'; import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement'; import { buildStackDriftReport, type DriftFindingKind, type StackDriftReport } from '../services/DriftDetectionService'; @@ -58,6 +56,11 @@ import { buildPolicyGateOptions, runPolicyGate, triggerPostDeployScan, describeP import { parseComposePreview, type ComposePreview } from '../helpers/composePreview'; import { filterContainersByComposeService } from '../helpers/composeServiceMatch'; import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; +import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache'; +import { + ImageUpdateService, + UPDATE_VERIFICATION_INCOMPLETE_WARNING, +} from '../services/ImageUpdateService'; import { parseComposeSelection, defaultEnvPath } from '../helpers/gitSourceSelection'; import { resolveStackEnvSources, discoverStackLocalEnvFiles } from '../helpers/envFileResolution'; import { STACK_STATUSES_CACHE_TTL_MS } from '../helpers/constants'; @@ -2313,7 +2316,26 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { { nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'manual', actor: req.user?.username ?? null }, { atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) }, ); - DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName); + // Health observation starts immediately after Compose; registry recheck is + // isolated so a verification failure cannot turn Compose success into 500. + ok = true; + const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null); + const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; + linkStackUpdateRecoveryGate(recoveryId, healthGateId); + + let recheckWarning: string | undefined; + try { + const recheck = await ImageUpdateService.getInstance().recheckStack(req.nodeId, stackName); + if (recheck.warning) recheckWarning = recheck.warning; + } catch (recheckErr) { + console.warn( + '[Stacks] Post-update recheck failed for %s: %s', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(recheckErr, 'unknown')), + ); + recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING; + } + invalidateFleetUpdateCache(); invalidateNodeCaches(req.nodeId); NotificationService.getInstance().broadcastEvent({ @@ -2326,11 +2348,11 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { }); dlog(`[Stacks] Update completed: ${sanitizeForLog(stackName)}`); if (debug) console.debug(`[Stacks:debug] Update finished in ${Date.now() - t0}ms`); - ok = true; - const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null); - const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; - linkStackUpdateRecoveryGate(recoveryId, healthGateId); - res.json({ status: 'Update completed', healthGateId }); + res.json({ + status: 'Update completed', + healthGateId, + ...(recheckWarning ? { recheckWarning } : {}), + }); notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, req.user?.username ?? 'system'); if (!skipScan) { triggerPostDeployScan(stackName, req.nodeId).catch(err => diff --git a/backend/src/services/ImageUpdateService.ts b/backend/src/services/ImageUpdateService.ts index 91c8a2ea..630f4ec5 100644 --- a/backend/src/services/ImageUpdateService.ts +++ b/backend/src/services/ImageUpdateService.ts @@ -17,6 +17,25 @@ import { buildEffectiveServiceModel } from './effectiveServiceModel'; const BACKFILL_KEY = 'image_update_notifications_backfilled'; +/** Post-update scanner reconciliation outcome for a single stack. */ +export type StackRecheckOutcome = + | 'cleared' + | 'still_present' + | 'verification_incomplete' + | 'verification_failed'; + +export interface StackRecheckResult { + outcome: StackRecheckOutcome; + /** Present when the update condition remains or could not be verified. */ + warning: string | null; +} + +export const UPDATE_STILL_PRESENT_WARNING = + 'The update command completed, but Sencho still detects an available image update.'; + +export const UPDATE_VERIFICATION_INCOMPLETE_WARNING = + 'The update command completed, but Sencho could not fully verify whether an image update remains.'; + export interface ImageCheckResult { hasUpdate: boolean; /** Same-tag registry digest drift; Compose pull can apply without pin change. */ @@ -884,19 +903,23 @@ export class ImageUpdateService { } /** - * Re-check a single stack after a service-scoped update or restore. On a - * render failure the prior row is left untouched and a warning is returned. + * Re-check a single stack after a service-scoped update or restore, or + * after a manual full-stack update. On a render failure the prior row is + * left untouched and a verification_failed result is returned. */ - public async recheckStack(nodeId: number, stackName: string): Promise<{ warning: string | null }> { + public async recheckStack(nodeId: number, stackName: string): Promise { const generation = this.reserveStackWriteGeneration(nodeId, stackName); const db = DatabaseService.getInstance(); const docker = DockerController.getInstance(nodeId); const model = await buildEffectiveServiceModel(nodeId, stackName); if (!model.renderable) { - return { warning: model.error }; + return { + outcome: 'verification_failed', + warning: model.error || UPDATE_VERIFICATION_INCOMPLETE_WARNING, + }; } - let containers: Array<{ Image?: string; Labels?: Record }> = []; + let containers: Array<{ Image?: string; Labels?: Record }>; try { containers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers'); } catch (e) { @@ -905,6 +928,12 @@ export class ImageUpdateService { sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(e, 'unknown')), ); + // Do not clear or upsert from declared-image-only checks: runtime + // digests were never observed, so "cleared" would be a false negative. + return { + outcome: 'verification_incomplete', + warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING, + }; } const refs = new Set(); @@ -942,15 +971,34 @@ export class ImageUpdateService { const lastError = stackStatusLastError(services); const now = Date.now(); - await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => { + const committed = await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => { if (checkStatus === 'failed') { db.recordStackCheckFailure(nodeId, stackName, lastError ?? 'Update check failed', now, services, gen); } else { db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now, checkStatus, lastError, services, gen); } }); + // A newer scanner reservation dropped this write; do not report cleared. + if (!committed) { + return { + outcome: 'verification_incomplete', + warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING, + }; + } - return { warning: null }; + if (checkStatus === 'partial' || checkStatus === 'failed') { + return { + outcome: 'verification_incomplete', + warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING, + }; + } + if (hasUpdate) { + return { + outcome: 'still_present', + warning: UPDATE_STILL_PRESENT_WARNING, + }; + } + return { outcome: 'cleared', warning: null }; } private stackWriteKey(nodeId: number, stackName: string): string { diff --git a/backend/src/services/SchedulerService.ts b/backend/src/services/SchedulerService.ts index 25feff95..c68f1f7b 100644 --- a/backend/src/services/SchedulerService.ts +++ b/backend/src/services/SchedulerService.ts @@ -10,12 +10,15 @@ import { StackOpLockService, stackOpSkipMessage as skipMessage } from './StackOp import { FileSystemService } from './FileSystemService'; import { HealthGateService } from './HealthGateService'; import { ServiceUpdateRecoveryService } from './ServiceUpdateRecoveryService'; -import { ImageUpdateService } from './ImageUpdateService'; import { createAutoUpdateDigestGateState, + messageWhenDigestApplyBlockedByCheckErrors, messageWhenNoDigestUpdate, recordAutoUpdateImageCheck, } from '../helpers/autoUpdateDigestGate'; +import { ImageUpdateService, UPDATE_VERIFICATION_INCOMPLETE_WARNING } from './ImageUpdateService'; +import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache'; +import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; import { isDebugEnabled } from '../utils/debug'; import { getErrorMessage } from '../utils/errors'; import { formatNoTargetError } from '../utils/remoteTarget'; @@ -770,14 +773,13 @@ export class SchedulerService { console.log(`[SchedulerService] executeUpdate: ${stackNames.length} stack(s) to check, fleet=${isFleet}, wildcard=${isWildcard}`); } - const db = DatabaseService.getInstance(); const docker = DockerController.getInstance(task.node_id); const imageUpdateService = ImageUpdateService.getInstance(); const results: string[] = []; for (const stackName of stackNames) { try { - const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, db, isFleet || isWildcard); + const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, isFleet || isWildcard); results.push(output); } catch (e) { const msg = getErrorMessage(e, String(e)); @@ -1034,7 +1036,6 @@ export class SchedulerService { nodeId: number, docker: DockerController, imageUpdateService: ImageUpdateService, - db: DatabaseService, isWildcard = false ): Promise { const containers = await docker.getContainersByStack(stackName); @@ -1076,6 +1077,8 @@ export class SchedulerService { if (!gate.hasDigestUpdate) { return messageWhenNoDigestUpdate(stackName, gate, imageRefs.length); } + const checkErrorBlock = messageWhenDigestApplyBlockedByCheckErrors(stackName, gate); + if (checkErrorBlock) return checkErrorBlock; const { updatedImages } = gate; @@ -1096,7 +1099,9 @@ export class SchedulerService { ), ); if (!lock.ran) return skipMessage(stackName, lock.existing.action); - db.clearStackUpdateStatus(nodeId, stackName); + + // Health observation starts immediately after Compose; registry recheck is + // isolated so a verification failure cannot turn Compose success into a failure. const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler'); const orchResult = lock.result; const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; @@ -1105,6 +1110,30 @@ export class SchedulerService { StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId); } + // Recheck persists digest-cleared / tag-advisory state. Do not blind-clear. + let recheckWarning: string | undefined; + try { + const recheck = await imageUpdateService.recheckStack(nodeId, stackName); + if (recheck.warning) recheckWarning = recheck.warning; + } catch (recheckErr) { + console.warn( + `[SchedulerService] Post-update recheck failed for ${sanitizeForLog(stackName)}:`, + sanitizeForLog(getErrorMessage(recheckErr, 'unknown')), + ); + recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING; + } + + invalidateFleetUpdateCache(); + invalidateNodeCaches(nodeId); + NotificationService.getInstance().broadcastEvent({ + type: 'state-invalidate', + scope: 'image-updates', + nodeId, + stackName, + action: 'stack-updated', + ts: Date.now(), + }); + this.safeDispatch( 'info', 'image_update_applied', @@ -1112,7 +1141,8 @@ export class SchedulerService { stackName ); - return `Stack "${stackName}": updated (${updatedImages.join(', ')}).`; + const base = `Stack "${stackName}": updated (${updatedImages.join(', ')}).`; + return recheckWarning ? `${base} ${recheckWarning}` : base; } private async executeScan(task: ScheduledTask): Promise<{ output: string; failed: number }> { diff --git a/docs/features/auto-update-policies.mdx b/docs/features/auto-update-policies.mdx index 48d35fed..2834cd42 100644 --- a/docs/features/auto-update-policies.mdx +++ b/docs/features/auto-update-policies.mdx @@ -39,8 +39,8 @@ Sencho polls your registries on a configurable schedule to detect available imag Detection is separate from applying updates: - **Registry detection** is this interval. It only looks for newer images and raises an "update available" notification the first time a stack goes from up to date to having an update. -- **Scheduled auto-update tasks** apply updates on their own cron schedule, independent of the detection interval. See [Scheduling auto-updates](#scheduling-auto-updates). -- **Apply now** updates a single stack on demand, regardless of either schedule. +- **Scheduled auto-update tasks** pull and recreate on their own cron schedule when the Compose-pinned tag has same-tag digest drift. A higher semver tag that is not yet written into Compose stays advisory on the readiness board and sidebar. See [Scheduling auto-updates](#scheduling-auto-updates). +- **Apply now** pulls and recreates the tags currently written in Compose (same-tag digest rebuilds). It does not rewrite a higher semver pin. The interval is node-scoped: each node runs its own scanner on its own cadence, and across a fleet Sencho staggers the runs slightly so the nodes do not all poll at the same instant. Setting the interval requires an admin account. @@ -68,38 +68,39 @@ Once the cause is resolved, the next check (on the interval, or via **Recheck**) ## Workflow 1. Open **Update** from the top nav strip. -2. Skim the card grid. The badge tells you the risk at a glance: `Safe · patch` is green, `Review · minor` is amber, `Blocked · major` is red, a digest-only rebuild on a non-semver tag shows the gray `Digest rebuild` badge, `Check uncertain` appears when the last preview check was incomplete or failed, and `Newer tag · edit Compose` marks a higher tag that requires editing the Compose pin (Compose pull cannot rewrite pins). -3. For a safe, Compose-actionable update (digest drift or local rebuild), click **Apply now** on the card to pull and recreate the stack immediately. **Apply now** stays disabled for tag-only advisories and for uncertain checks. -4. For a major bump, review the changelog preview and the upstream release notes. **Apply now** is disabled on the readiness board for blocked cards; to apply a major bump after review, use the stack's lifecycle **Update** action (right-click the stack in the sidebar, or open the kebab menu and choose **Update**, or click **Deploy** in the stack editor). +2. Skim the card grid. The badge tells you the risk at a glance: `Safe · patch` is green, `Review · minor` is amber, `Blocked · major` is red, a digest-only rebuild on a non-semver tag shows the gray `Digest rebuild` badge, `Check uncertain` appears when the last preview check was incomplete or failed, and `Newer tag · edit Compose` marks a higher tag that requires editing the Compose pin. +3. For a **Digest rebuild** (same Compose tag, new registry content), click **Apply now** on the card to pull and recreate the stack immediately. **Apply now** stays disabled for tag-only advisories and for uncertain checks. +4. For a higher pinned semver tag (a version diff such as `1.2.3` → `1.2.4`, including major bumps), edit the Compose `image:` reference to the next tag, then deploy the stack from the editor. **Apply now** and the lifecycle **Update** action pull the currently pinned tag only; they do not rewrite the pin. +5. For a major bump, review the changelog preview and upstream release notes before changing the pin and deploying. **Apply now** is disabled on blocked cards. -On multi-service stacks, the Updates view can also apply a single service when that service has a confirmed image update. Scheduled auto-update, webhook pull, and bulk update always refresh the full stack. -5. Use **Recheck** in the hero to force an immediate registry poll across every reachable node. A 2-minute per-node cooldown applies, and the toast tells you how many nodes were triggered, rate-limited, or failed. +On multi-service stacks, the Updates view can also apply a single service when that service has a confirmed same-tag digest update. Scheduled auto-update, webhook pull, and bulk update always refresh the full stack. +6. Use **Recheck** in the hero to force an immediate registry poll across every reachable node. A 2-minute per-node cooldown applies, and the toast tells you how many nodes were triggered, rate-limited, or failed. -**Apply now** runs the same update path as every other update trigger on a stack: Sencho takes an atomic backup of the compose and env files first, pulls the updated registry images (or rebuilds from source for services that declare `build:`, then pulls the rest), and recreates the containers. See [Controlling a running stack](/features/stack-management#controlling-a-running-stack) for exactly how a build-aware update differs from a plain pull. Once the containers are up, the [post-update health gate](/features/health-gated-updates#the-post-update-health-gate) observes them for a configurable window and records a passed, failed, or unknown verdict on the stack's timeline, the same as a manual update from the editor toolbar or sidebar. +**Apply now** runs the same update path as other Compose-pull triggers on a stack: Sencho takes an atomic backup of the compose and env files first, pulls the updated registry images for the tags already written in Compose (or rebuilds from source for services that declare `build:`, then pulls the rest), and recreates the containers. See [Controlling a running stack](/features/stack-management#controlling-a-running-stack) for exactly how a build-aware update differs from a plain pull. Once the containers are up, the [post-update health gate](/features/health-gated-updates#the-post-update-health-gate) observes them for a configurable window and records a passed, failed, or unknown verdict on the stack's timeline, the same as a manual update from the editor toolbar or sidebar. After Apply finishes, Sencho rechecks the stack's update status and keeps the readiness card when an image update is still detected or verification cannot confirm clearance. ## Risk badges | Badge | Color | When it appears | |-------|-------|-----------------| -| `Safe · patch` | Green (Shield icon) | Patch-level semver bump (e.g. `1.2.3` to `1.2.4`) | -| `Review · minor` | Amber (AlertTriangle icon) | Minor semver bump (e.g. `1.2.3` to `1.3.0`) | -| `Blocked · major` | Red (ShieldAlert icon) | Major semver bump (e.g. `1.2.3` to `2.0.0`). **Apply now** is disabled; the card surfaces the reason "Major version jumps require human review before applying." | -| `Digest rebuild` | Gray | Non-semver tag (e.g. `main`, `stable`) with an updated digest (tag name unchanged; content changed; see [Tags vs digests](/features/vulnerability-scanning#tags-vs-digests)) | -| `Check uncertain` | Amber | Preview check incomplete or failed; **Apply now** stays disabled until a full successful check | -| `Newer tag · edit Compose` | Amber | A higher tag exists than the Compose pin; edit Compose (do not use **Apply now**) | +| `Safe · patch` | Green (Shield icon) | Patch-level semver bump (e.g. `1.2.3` to `1.2.4`). Resolve by editing the Compose pin, then deploying. | +| `Review · minor` | Amber (AlertTriangle icon) | Minor semver bump (e.g. `1.2.3` to `1.3.0`). Resolve by editing the Compose pin, then deploying. | +| `Blocked · major` | Red (ShieldAlert icon) | Major semver bump (e.g. `1.2.3` to `2.0.0`). **Apply now** is disabled; the card surfaces the reason "Major version jumps require human review before applying." After review, edit the Compose pin, then deploy. | +| `Digest rebuild` | Gray | Non-semver tag (e.g. `main`, `stable`) with an updated digest (tag name unchanged; content changed; see [Tags vs digests](/features/vulnerability-scanning#tags-vs-digests)). **Apply now** pulls this rebuild. | +| `Check uncertain` | Amber | Preview check incomplete or failed; **Apply now** stays disabled until a full successful check. | +| `Newer tag · edit Compose` | Amber | A higher tag exists than the Compose pin; edit Compose (do not use **Apply now**). | + +Blocked major bumps still surface in scheduled check runs so you stay informed, but **Apply now** stays disabled until you review them, change the Compose pin, and deploy. A separate inline `Rebuild available` label replaces the version diff when only the digest changed (same tag, new image). The risk badge on those cards still reflects the underlying semver classification reported by the registry. -Blocked updates still surface in scheduled check runs so you stay informed, but the apply button is disabled until you review them manually. - ## Per-stack control Auto-update is opt-in per stack. A stack participates in unattended updates only when an enabled scheduled task covers it. To leave a stack out (databases, self-built images, anything pinned to a fixed tag), simply do not create a schedule for it. - **Per-stack schedule.** Create a **Auto-update Stack** task targeting that stack alone. Only this stack is updated when the cron fires. -- **Fleet-wide schedule.** Create a **Auto-update All Stacks on Node** task targeting a node. Every stack on that node is checked and updated when new images are available. If you do not want every stack covered, create per-stack schedules instead. +- **Fleet-wide schedule.** Create a **Auto-update All Stacks on Node** task targeting a node. Every stack on that node is checked when the cron fires, and stacks with same-tag digest drift are pulled and recreated. If you do not want every stack covered, create per-stack schedules instead. - **Stack list dot.** Image-update *detection* runs on the configured interval (every 2 hours by default) regardless of whether any schedule is configured. The sidebar dot and the readiness board still show available updates so you can decide what to do with them. -- **Manual updates are always available.** The lifecycle **Update** action on a stack applies an update on demand, independent of any scheduled task. +- **Manual updates are always available.** The lifecycle **Update** action pulls and recreates the tags currently written in Compose, independent of any scheduled task. It does not rewrite a higher-tag advisory into the Compose file. ## Cleaning up after updates @@ -121,7 +122,9 @@ Auto-update is a first-class action in the Schedules view. To create a recurring The task lives alongside restart, prune, snapshot, and scan tasks in the same timeline and table. Run history, notifications, and the Run Now button behave the same as for every other scheduled action. See [Scheduled Operations](/features/scheduled-operations) for details. -A scheduled run applies an update the same way **Apply now** does: an atomic backup first, then the pull or build-aware rebuild, then the [post-update health gate](/features/health-gated-updates) observing the result. A schedule never bypasses a [deploy enforcement](/features/deploy-enforcement) policy: a run that a policy would block fails with the policy violation recorded in the run history, rather than applying partway. +A scheduled run (including **Run Now**) only auto-applies when Compose already pins the tag that has new registry content: same-tag digest drift, the gray **Digest rebuild** case. Sencho then follows the same update path as **Apply now**: an atomic backup first, then the pull or build-aware rebuild, then the [post-update health gate](/features/health-gated-updates) observing the result. + +When detection finds a higher semver tag while Compose still pins the older one (for example Compose says `nginx:1.2.3` and the registry also publishes `1.2.4`), the readiness board and sidebar keep showing the advisory update. The schedule does not rewrite the Compose image reference, so it skips Compose pull and recreate until you change the pin in Compose and deploy. A schedule never bypasses a [deploy enforcement](/features/deploy-enforcement) policy: a run that a policy would block fails with the policy violation recorded in the run history, rather than applying partway. ## Multi-node support @@ -150,7 +153,7 @@ A stack that mixes registry images and `build:` services still gets a card, scor Sencho reads changelog metadata from the registry's manifest and OCI annotations. Registries that do not publish this metadata (most private registries and many self-hosted ones) render the card without a changelog. The risk badge is still accurate because it is computed from the tag itself. - The stack has a major version bump and is blocked on the readiness board by policy: major updates never auto-apply without human review. To apply after reviewing the upstream release notes, use the stack's lifecycle **Update** action from the sidebar kebab or right-click menu, or open the stack editor and click **Deploy**. + The stack has a major version bump and is blocked on the readiness board by policy: major updates never auto-apply without human review. After reviewing the upstream release notes, edit the Compose `image:` pin to the reviewed next tag, then deploy the stack from the editor. **Apply now** and lifecycle **Update** pull the currently pinned tag only; they do not bump the pin. The registry call is either still pending or it failed. Click **Recheck** in the hero to retry. If the stack uses private-registry credentials, confirm they are still valid in **Settings > Registries**. @@ -159,7 +162,15 @@ A stack that mixes registry images and `build:` services still gets a card, scor Image update detection runs on the configured interval (every 2 hours by default) on each node, and the readiness board uses the same cached status. Trigger **Recheck** to force a fresh check across every reachable node, or shorten the interval under **Settings > Automation > Image update checks**. - No schedule covers that stack. Open **Schedules** in the top nav, create a new **Auto-update Stack** task targeting the stack (or an **Auto-update All Stacks on Node** task on its node), and pick a cron. The next firing will include the stack, or you can trigger an immediate run from the row. + First confirm a schedule covers that stack: open **Schedules**, create or enable an **Auto-update Stack** task (or an **Auto-update All Stacks on Node** task on its node), and pick a cron. The next firing includes the stack, or you can trigger an immediate run from the row. + + If a schedule already covers the stack and the run history says newer tags are available but the Compose pin is unchanged, detection found a higher tag than the image reference in Compose. Scheduled auto-update only pulls same-tag digest rebuilds; it does not bump `image:1.2.3` to `image:1.2.4` for you. Edit the Compose image pin to the next tag, then deploy. + + + The registry publishes a higher pinned semver tag than the one written in Compose. The readiness board and sidebar keep that advisory visible. Scheduled auto-update leaves Compose as the source of truth, so it skips pull and recreate until the pin is updated. Edit the Compose `image:` reference to the next tag, then deploy the stack. **Apply now** and lifecycle **Update** cannot rewrite the pin. + + + Compose still pins the older tag. Apply pulls and recreates that pinned tag only. To move to the next tag shown on the card, edit the Compose `image:` reference, then deploy. One or more nodes that are marked online in your fleet did not respond within the request timeout. Pending updates from those nodes are not shown until they come back. Check the node's status from the Fleet view and the network path between this Sencho instance and the unreachable node. diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 5259dfbd..35ded748 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1775,6 +1775,12 @@ paths: Id of the post-update health gate observation started for this update, for use with the health-gate endpoint. Null when the health gate is disabled on the node. + recheckWarning: + type: string + description: | + Present when Compose succeeded but post-update image + verification still reports an available update or could + not fully verify clearance. Optional for older nodes. "403": $ref: "#/components/responses/Forbidden" "500": diff --git a/frontend/src/components/AutoUpdateReadinessView.tsx b/frontend/src/components/AutoUpdateReadinessView.tsx index e271723e..e1ada189 100644 --- a/frontend/src/components/AutoUpdateReadinessView.tsx +++ b/frontend/src/components/AutoUpdateReadinessView.tsx @@ -100,6 +100,9 @@ export interface StackCard { // Name of the service currently applying a per-service update on this card, // or null when none is in flight. Distinct from `applying` (full-stack). applyingService: string | null; + // Post-Apply verification note when Compose succeeded but clearance could + // not be confirmed (distinct from a failed preview fetch). + verificationNote: string | null; } interface NodeGroup { @@ -275,9 +278,10 @@ function StackReadinessCard({ onApply: (stack: string, nodeId: number) => void; onApplyService?: (stack: string, nodeId: number, serviceName: string) => void; }) { - const { stack, nodeId, preview, previewLoaded, scheduledTask, applying, applyingService, autoUpdateEnabled } = card; + const { stack, nodeId, preview, previewLoaded, scheduledTask, applying, applyingService, autoUpdateEnabled, verificationNote } = card; const loading = !previewLoaded; - const failed = previewLoaded && preview === null; + const uncertain = previewLoaded && !!verificationNote; + const failed = previewLoaded && preview === null && !verificationNote; const blocked = preview?.summary.blocked ?? false; const bump = preview?.summary.semver_bump ?? 'none'; const updatingImages = preview?.images.filter(i => i.has_update) ?? []; @@ -328,6 +332,10 @@ function StackReadinessCard({ {loading ? (
Checking registry...
+ ) : uncertain ? ( +
+ {verificationNote} +
) : failed ? (
Preview failed. Registry may be unreachable. @@ -584,8 +592,9 @@ export function MobileReadinessCard({ onApply: (stack: string, nodeId: number) => void; onApplyService?: (stack: string, nodeId: number, serviceName: string) => void; }) { - const { stack, nodeId, preview, previewLoaded, scheduledTask, applying, applyingService, autoUpdateEnabled } = card; - const failed = previewLoaded && preview === null; + const { stack, nodeId, preview, previewLoaded, scheduledTask, applying, applyingService, autoUpdateEnabled, verificationNote } = card; + const uncertain = previewLoaded && !!verificationNote; + const failed = previewLoaded && preview === null && !verificationNote; const blocked = preview?.summary.blocked ?? false; const bump = preview?.summary.semver_bump ?? 'none'; const updatingImages = preview?.images.filter(i => i.has_update) ?? []; @@ -628,6 +637,8 @@ export function MobileReadinessCard({ {!previewLoaded ? (
Checking registry...
+ ) : uncertain ? ( +
{verificationNote}
) : failed ? (
Preview failed. Registry may be unreachable.
) : (() => { @@ -901,6 +912,7 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps) applying: false, applyingService: null, autoUpdateEnabled: scheduledTask !== null, + verificationNote: null, }; }); initialGroups.push({ @@ -1148,8 +1160,26 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps) ...g, cards: g.cards.map(c => predicate(c) ? { ...c, ...patch } : c), }))); + const matchCard = (c: StackCard) => c.stack === stack && c.nodeId === nodeId; + const removeCard = () => setGroups(prev => prev + .map(g => g.nodeId === nodeId + ? { ...g, cards: g.cards.filter(c => c.stack !== stack) } + : g) + .filter(g => g.cards.length > 0)); + const retainPreviewFailed = () => setCardField(matchCard, { + applying: false, + preview: null, + previewLoaded: true, + verificationNote: null, + }); + const retainUncertain = (note: string) => setCardField(matchCard, { + applying: false, + preview: null, + previewLoaded: true, + verificationNote: note, + }); - setCardField(c => c.stack === stack && c.nodeId === nodeId, { applying: true }); + setCardField(matchCard, { applying: true, verificationNote: null }); const loadingId = toast.loading(`Applying update to ${stack}...`); try { const res = await fetchForNode( @@ -1161,15 +1191,57 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps) const data = await res.json().catch(() => ({ error: 'Update failed' })); throw new Error(data.error ?? 'Update failed'); } - toast.success(`${stack} updated successfully`); - setGroups(prev => prev - .map(g => g.nodeId === nodeId - ? { ...g, cards: g.cards.filter(c => c.stack !== stack) } - : g) - .filter(g => g.cards.length > 0)); + const body = await res.json().catch(() => ({})) as { recheckWarning?: unknown }; + const recheckWarning = typeof body.recheckWarning === 'string' ? body.recheckWarning : undefined; + if (recheckWarning) toast.info(recheckWarning); + else toast.success(`${stack} updated successfully`); + + // Authoritative live preview decides card removal. When it disagrees with + // a backend recheckWarning (preview cleared, persisted check uncertain), + // keep an uncertain card so Fleet does not diverge from the sidebar. + try { + const previewRes = await fetchForNode( + `/stacks/${encodeURIComponent(stack)}/update-preview`, + nodeId, + ); + if (!previewRes.ok) { + retainPreviewFailed(); + return; + } + const next = await previewRes.json() as UpdatePreview; + if (typeof next?.summary?.has_update !== 'boolean') { + retainPreviewFailed(); + return; + } + // Drop only when the live preview proves nothing remains (tag-only + // advisories stay pending via isClearedUpdatePreview). + const cleared = isAuthoritativeNegativePreview(next) || isClearedUpdatePreview(next); + if (!cleared) { + if (next.summary.has_update && !recheckWarning) { + toast.info( + 'The update command completed, but Sencho still detects an available image update.', + ); + } + setCardField(matchCard, { + applying: false, + preview: next, + previewLoaded: true, + verificationNote: recheckWarning ?? null, + }); + return; + } + if (recheckWarning) { + retainUncertain(recheckWarning); + return; + } + removeCard(); + } catch (previewErr) { + console.error('[AutoUpdate] post-Apply preview reconciliation failed', previewErr); + retainPreviewFailed(); + } } catch (err) { toast.error((err as Error)?.message || 'Update failed'); - setCardField(c => c.stack === stack && c.nodeId === nodeId, { applying: false }); + setCardField(matchCard, { applying: false }); } finally { toast.dismiss(loadingId); } diff --git a/frontend/src/components/__tests__/AutoUpdateReadinessView.test.tsx b/frontend/src/components/__tests__/AutoUpdateReadinessView.test.tsx index b91dea56..c7e59524 100644 --- a/frontend/src/components/__tests__/AutoUpdateReadinessView.test.tsx +++ b/frontend/src/components/__tests__/AutoUpdateReadinessView.test.tsx @@ -40,6 +40,7 @@ vi.mock('@/context/NodeContext', () => ({ import { apiFetch, fetchForNode } from '@/lib/api'; import { requestServiceUpdate } from '@/lib/serviceUpdate'; +import { toast } from '@/components/ui/toast-store'; import AutoUpdateReadinessView, { MobileReadinessCard, CadenceStrip, @@ -49,6 +50,7 @@ import { isActionableUpdatePreview, isClearedUpdatePreview, isReviewRequiredUpdatePreview, + isTagOnlyAdvisory, isVerificationOnlyPreview, } from '@/lib/updatePreviewActionability'; import { isAuthoritativeNegativePreview } from '@/types/imageUpdates'; @@ -62,6 +64,7 @@ function card(over: Partial = {}): StackCard { applyingService: null, autoUpdateEnabled: true, scheduledTask: null, + verificationNote: null, preview: { stack_name: 'nextcloud', images: [{ @@ -288,6 +291,29 @@ describe('verification preview helpers', () => { expect(isClearedUpdatePreview(preview)).toBe(false); expect(isActionableUpdatePreview(preview)).toBe(false); }); + + it('does not treat a tag-only advisory as cleared (Fleet must keep the card)', () => { + const preview = { + ...previewSummary({ + has_update: true, + update_kind: 'tag', + semver_bump: 'patch', + next_tag: '1.31.3', + current_tag: '1.25.3', + }), + images: [{ + service: 'web', + image: 'nginx:1.25.3', + has_update: true, + digest_update: false, + tag_update: true, + check_status: 'ok', + }], + }; + expect(isTagOnlyAdvisory(preview)).toBe(true); + expect(isActionableUpdatePreview(preview)).toBe(false); + expect(isClearedUpdatePreview(preview)).toBe(false); + }); }); it('enables Apply for a safe, non-blocked update', () => { @@ -509,6 +535,12 @@ describe('AutoUpdateReadinessView desktop Apply now', () => { mockedFetchForNode.mockReset(); mockNodeMeta.clear(); vi.mocked(requestServiceUpdate).mockReset(); + vi.mocked(toast.info).mockReset(); + vi.mocked(toast.success).mockReset(); + vi.mocked(toast.error).mockReset(); + vi.mocked(toast.loading).mockReset(); + vi.mocked(toast.dismiss).mockReset(); + mockNodes.splice(0, mockNodes.length, { id: 1, name: 'Local', type: 'local', status: 'online' }); }); it('enables Apply for a safe update with no covering schedule', async () => { @@ -633,6 +665,274 @@ describe('AutoUpdateReadinessView desktop Apply now', () => { }); }); + const basePreview = { + stack_name: 'nextcloud', + images: [{ + service: 'app', + image: 'nextcloud:27', + current_tag: '27.1.4', + next_tag: '27.1.4', + has_update: true, + digest_update: true, + tag_update: false, + semver_bump: 'patch' as const, + check_status: 'ok' as const, + check_error: null, + digest_error: null, + }], + summary: { + has_update: true, + primary_image: 'nextcloud', + current_tag: '27.1.4', + next_tag: '27.1.4', + semver_bump: 'patch' as const, + update_kind: 'digest' as const, + blocked: false, + blocked_reason: null, + rebuild_available: false, + check_status: 'ok' as const, + verification_failed: false, + verification_error: null, + }, + rollback_target: null, + changelog: 'Fixes.', + }; + + function mockFleetLoad(fleetMap: Record>) { + mockedFetch.mockImplementation((url: string) => { + if (url === '/image-updates/fleet') { + return Promise.resolve({ ok: true, json: async () => fleetMap }); + } + if (url.startsWith('/scheduled-tasks')) { + return Promise.resolve({ ok: true, json: async () => [] }); + } + return Promise.resolve({ ok: true, json: async () => ({}) }); + }); + } + + it('removes the card only after a cleared preview with no recheckWarning', async () => { + mockFleetLoad({ '1': { nextcloud: true } }); + const cleared = { + ...basePreview, + images: basePreview.images.map((img) => ({ ...img, has_update: false, digest_update: false })), + summary: { ...basePreview.summary, has_update: false }, + }; + mockedFetchForNode.mockImplementation((url: string, _nodeId?: number, init?: { method?: string }) => { + if (String(url).includes('/update') && !String(url).includes('update-preview') && init?.method === 'POST') { + return Promise.resolve({ ok: true, json: async () => ({ status: 'Update completed' }) }); + } + if (String(url).includes('/update-preview')) { + const previewCalls = mockedFetchForNode.mock.calls.filter((c) => String(c[0]).includes('/update-preview')).length; + return Promise.resolve({ + ok: true, + json: async () => (previewCalls <= 1 ? basePreview : cleared), + }); + } + return Promise.resolve({ ok: true, json: async () => ({}) }); + }); + + render(); + const applyBtn = await screen.findByRole('button', { name: /Apply now/i }); + await act(async () => { fireEvent.click(applyBtn); }); + + await waitFor(() => { + expect(screen.queryByRole('button', { name: /Apply now/i })).not.toBeInTheDocument(); + }); + const postIdx = mockedFetchForNode.mock.calls.findIndex( + (c) => String(c[0]).includes('/stacks/nextcloud/update') && !String(c[0]).includes('update-preview'), + ); + const previewAfter = mockedFetchForNode.mock.calls.findIndex( + (c, i) => i > postIdx && String(c[0]).includes('/update-preview'), + ); + expect(postIdx).toBeGreaterThanOrEqual(0); + expect(previewAfter).toBeGreaterThan(postIdx); + expect(mockedFetchForNode.mock.calls[postIdx][1]).toBe(1); + expect(mockedFetchForNode.mock.calls[previewAfter][1]).toBe(1); + }); + + it('retains the card and warns when the preview still reports an update', async () => { + mockFleetLoad({ '1': { nextcloud: true } }); + mockedFetchForNode.mockImplementation((url: string, _nodeId?: number, init?: { method?: string }) => { + if (String(url).includes('/update') && !String(url).includes('update-preview') && init?.method === 'POST') { + return Promise.resolve({ ok: true, json: async () => ({ status: 'Update completed' }) }); + } + if (String(url).includes('/update-preview')) { + return Promise.resolve({ ok: true, json: async () => basePreview }); + } + return Promise.resolve({ ok: true, json: async () => ({}) }); + }); + + render(); + const applyBtn = await screen.findByRole('button', { name: /Apply now/i }); + await act(async () => { fireEvent.click(applyBtn); }); + + await waitFor(() => { + expect(screen.getByRole('button', { name: /Apply now/i })).toBeEnabled(); + }); + expect(toast.info).toHaveBeenCalledWith( + 'The update command completed, but Sencho still detects an available image update.', + ); + }); + + it('retains the card when post-Apply preview is partial (not an authoritative clear)', async () => { + mockFleetLoad({ '1': { nextcloud: true } }); + const partialNegative = { + ...basePreview, + images: basePreview.images.map((img) => ({ + ...img, + has_update: false, + digest_update: false, + tag_update: false, + check_status: 'partial' as const, + check_error: 'registry timeout', + })), + summary: { + ...basePreview.summary, + has_update: false, + check_status: 'partial' as const, + }, + }; + mockedFetchForNode.mockImplementation((url: string, _nodeId?: number, init?: { method?: string }) => { + if (String(url).includes('/update') && !String(url).includes('update-preview') && init?.method === 'POST') { + return Promise.resolve({ ok: true, json: async () => ({ status: 'Update completed' }) }); + } + if (String(url).includes('/update-preview')) { + const previewCalls = mockedFetchForNode.mock.calls.filter((c) => String(c[0]).includes('/update-preview')).length; + return Promise.resolve({ + ok: true, + json: async () => (previewCalls <= 1 ? basePreview : partialNegative), + }); + } + return Promise.resolve({ ok: true, json: async () => ({}) }); + }); + + render(); + const applyBtn = await screen.findByRole('button', { name: /Apply now/i }); + await act(async () => { fireEvent.click(applyBtn); }); + + await waitFor(() => { + expect(screen.getAllByText('nextcloud').length).toBeGreaterThan(0); + }); + expect(screen.queryByText(/Everything is up to date/)).toBeNull(); + expect(screen.getByText(/1 update pending/)).toBeInTheDocument(); + expect(isAuthoritativeNegativePreview(partialNegative)).toBe(false); + expect(isClearedUpdatePreview(partialNegative)).toBe(false); + }); + + it('retains an unknown card when recheckWarning disagrees with a cleared preview', async () => { + mockFleetLoad({ '1': { nextcloud: true } }); + const cleared = { + ...basePreview, + images: basePreview.images.map((img) => ({ ...img, has_update: false, digest_update: false })), + summary: { ...basePreview.summary, has_update: false }, + }; + mockedFetchForNode.mockImplementation((url: string, _nodeId?: number, init?: { method?: string }) => { + if (String(url).includes('/update') && !String(url).includes('update-preview') && init?.method === 'POST') { + return Promise.resolve({ + ok: true, + json: async () => ({ + status: 'Update completed', + recheckWarning: 'The update command completed, but Sencho still detects an available image update.', + }), + }); + } + if (String(url).includes('/update-preview')) { + const previewCalls = mockedFetchForNode.mock.calls.filter((c) => String(c[0]).includes('/update-preview')).length; + return Promise.resolve({ + ok: true, + json: async () => (previewCalls <= 1 ? basePreview : cleared), + }); + } + return Promise.resolve({ ok: true, json: async () => ({}) }); + }); + + render(); + const applyBtn = await screen.findByRole('button', { name: /Apply now/i }); + await act(async () => { fireEvent.click(applyBtn); }); + + await waitFor(() => { + expect(screen.getByText( + 'The update command completed, but Sencho still detects an available image update.', + )).toBeInTheDocument(); + }); + expect(screen.getByText('nextcloud')).toBeInTheDocument(); + expect(screen.queryByText(/Preview failed/i)).not.toBeInTheDocument(); + expect(screen.queryByRole('button', { name: /Apply now/i })).not.toBeInTheDocument(); + expect(toast.info).toHaveBeenCalledWith( + 'The update command completed, but Sencho still detects an available image update.', + ); + }); + + it('retains an unknown card when the post-Apply preview request fails', async () => { + mockFleetLoad({ '1': { nextcloud: true } }); + mockedFetchForNode.mockImplementation((url: string, _nodeId?: number, init?: { method?: string }) => { + if (String(url).includes('/update') && !String(url).includes('update-preview') && init?.method === 'POST') { + return Promise.resolve({ ok: true, json: async () => ({ status: 'Update completed' }) }); + } + if (String(url).includes('/update-preview')) { + const previewCalls = mockedFetchForNode.mock.calls.filter((c) => String(c[0]).includes('/update-preview')).length; + if (previewCalls <= 1) { + return Promise.resolve({ ok: true, json: async () => basePreview }); + } + return Promise.resolve({ ok: false, status: 500, json: async () => ({ error: 'boom' }) }); + } + return Promise.resolve({ ok: true, json: async () => ({}) }); + }); + + render(); + const applyBtn = await screen.findByRole('button', { name: /Apply now/i }); + await act(async () => { fireEvent.click(applyBtn); }); + + await waitFor(() => { + expect(screen.getByText(/Preview failed/i)).toBeInTheDocument(); + }); + expect(screen.getByText('nextcloud')).toBeInTheDocument(); + expect(screen.queryByRole('button', { name: /Apply now/i })).not.toBeInTheDocument(); + }); + + it('pins full-stack Apply POST and preview to a remote card nodeId', async () => { + mockNodes.splice(0, mockNodes.length, + { id: 1, name: 'Local', type: 'local', status: 'online' }, + { id: 2, name: 'Remote', type: 'remote', status: 'online' }, + ); + mockFleetLoad({ '2': { nextcloud: true } }); + const cleared = { + ...basePreview, + images: basePreview.images.map((img) => ({ ...img, has_update: false, digest_update: false })), + summary: { ...basePreview.summary, has_update: false }, + }; + mockedFetchForNode.mockImplementation((url: string, _nodeId?: number, init?: { method?: string }) => { + if (String(url).includes('/update') && !String(url).includes('update-preview') && init?.method === 'POST') { + return Promise.resolve({ ok: true, json: async () => ({ status: 'Update completed' }) }); + } + if (String(url).includes('/update-preview')) { + const previewCalls = mockedFetchForNode.mock.calls.filter((c) => String(c[0]).includes('/update-preview')).length; + return Promise.resolve({ + ok: true, + json: async () => (previewCalls <= 1 ? basePreview : cleared), + }); + } + return Promise.resolve({ ok: true, json: async () => ({}) }); + }); + + render(); + const applyBtn = await screen.findByRole('button', { name: /Apply now/i }); + await act(async () => { fireEvent.click(applyBtn); }); + + await waitFor(() => { + expect(screen.queryByRole('button', { name: /Apply now/i })).not.toBeInTheDocument(); + }); + const postCall = mockedFetchForNode.mock.calls.find( + (c) => String(c[0]).includes('/stacks/nextcloud/update') && !String(c[0]).includes('update-preview'), + ); + const postApplyPreview = mockedFetchForNode.mock.calls.filter( + (c) => String(c[0]).includes('/update-preview') && c[1] === 2, + ); + expect(postCall?.[1]).toBe(2); + expect(postApplyPreview.length).toBeGreaterThanOrEqual(2); + mockNodes.splice(0, mockNodes.length, { id: 1, name: 'Local', type: 'local', status: 'online' }); + }); + it('holds the desktop full-stack Apply for review, but keeps per-service Apply enabled, when a confirmed update sits alongside another image failing verification', async () => { mockNodeMeta.set(1, { version: '1.0.0', @@ -1026,6 +1326,73 @@ describe('AutoUpdateReadinessView check-failed advisory', () => { expect(screen.queryByText(/ready to apply automatically/)).toBeNull(); }); + it('keeps a tag-only advisory card on load instead of treating it as cleared', async () => { + mockedFetch.mockImplementation((url: string) => { + if (url === '/image-updates/fleet') { + return Promise.resolve({ ok: true, json: async () => ({ '1': { nginx: true } }) }); + } + if (url.startsWith('/scheduled-tasks')) { + return Promise.resolve({ ok: true, json: async () => [] }); + } + if (url === '/image-updates/detail') { + return Promise.resolve({ + ok: true, + json: async () => ({ + nginx: { hasUpdate: true, checkStatus: 'ok', lastError: null, checkedAt: 1 }, + }), + }); + } + return Promise.resolve({ ok: true, json: async () => ({}) }); + }); + mockedFetchForNode.mockImplementation((url: string) => { + if (String(url).includes('/update-preview')) { + return Promise.resolve({ + ok: true, + json: async () => ({ + stack_name: 'nginx', + images: [{ + service: 'web', + image: 'nginx:1.25.3', + current_tag: '1.25.3', + next_tag: '1.31.3', + has_update: true, + digest_update: false, + tag_update: true, + semver_bump: 'minor', + check_status: 'ok', + check_error: null, + digest_error: null, + }], + summary: { + has_update: true, + primary_image: 'nginx', + current_tag: '1.25.3', + next_tag: '1.31.3', + semver_bump: 'minor', + update_kind: 'tag', + blocked: false, + blocked_reason: null, + rebuild_available: false, + check_status: 'ok', + verification_failed: false, + verification_error: null, + }, + rollback_target: null, + changelog: null, + }), + }); + } + return Promise.resolve({ ok: true, json: async () => null }); + }); + + render(); + + expect(await screen.findByText(/1 update pending/)).toBeInTheDocument(); + expect(screen.getAllByText('nginx').length).toBeGreaterThan(0); + expect(screen.queryByText(/Everything is up to date/)).toBeNull(); + expect(screen.getByRole('button', { name: /Apply now/i })).toBeDisabled(); + }); + it('keeps a sticky card instead of silently clearing it when a remote sends a legacy preview missing verification_failed entirely', async () => { // Same sticky-fleet shape as the cleared-preview test above, but the // fresh preview response is missing verification_failed/rebuild_available diff --git a/frontend/src/lib/updatePreviewActionability.ts b/frontend/src/lib/updatePreviewActionability.ts index 97ae3750..903a2eb9 100644 --- a/frontend/src/lib/updatePreviewActionability.ts +++ b/frontend/src/lib/updatePreviewActionability.ts @@ -171,9 +171,9 @@ export function isServiceApplyActionable( } /** - * Fresh preview successfully proved there is nothing to apply, and the stack - * is not held for major-bump review. Sticky fleet booleans must not keep these - * cards pending. + * Fresh preview successfully proved there is nothing pending (no digest + * rebuild, no higher-tag advisory). Tag-only advisories set has_update and + * must not clear: Fleet keeps showing them even though Apply is disabled. */ export function isClearedUpdatePreview( preview: UpdatePreviewActionInput | null | undefined, @@ -184,5 +184,6 @@ export function isClearedUpdatePreview( if (isPreviewUncertain(preview)) return false; if (isVerificationOnlyPreview(preview)) return false; if (isReviewRequiredUpdatePreview(preview)) return false; - return !isActionableUpdatePreview(preview); + if (preview.summary.has_update || preview.summary.rebuild_available) return false; + return summaryCheckOk(preview.summary); }