fix(fleet): verify update status before removing readiness cards (#1697)

* fix(fleet): verify update status before removing readiness cards

Full-stack Apply now rechecks persisted status after the health gate starts, reloads the live preview before dropping a card, and invalidates the hub fleet aggregation so cleared updates cannot resurrect from a stale cache.

Closes #1686

* fix(fleet): align persisted update status with preview semver detection

Share digest-plus-tag detection so post-Apply sidebar status matches Fleet and Anatomy.

* fix(fleet): keep tag-only updates advisory for Compose automation

Expose digestUpdate vs tagUpdate from checkImage so scheduled and API auto-update only apply same-tag digest drift Compose can pull.

* docs: clarify scheduled auto-update applies digest drift only

Document that higher pinned tags stay advisory until Compose is changed, matching schedule and Run Now behavior.

* docs: require Compose pin edits for higher-tag advisories

Stop recommending Apply now or Update as remedies that cannot rewrite a pinned image tag.

* docs: clarify Apply now pulls pinned tags only

Align the detection-cadence bullet with digest-rebuild vs higher-tag guidance.

* fix(fleet): keep tag advisories after apply and scheduled updates

Tag-only previews were treated as cleared on Fleet reload, and scheduled/
Run Now paths wiped status without rechecking. Align post-update verification
with the manual Apply path (health gate first, recheck, no blind clear) and
block digest apply when sibling image checks failed.

* fix(fleet): clear eslint unused-arg and containers assignment
This commit is contained in:
Anso
2026-07-26 03:09:21 -04:00
committed by GitHub
parent bb7c76ba46
commit 719180f156
16 changed files with 1320 additions and 74 deletions
@@ -0,0 +1,38 @@
import { describe, it, expect } from 'vitest';
import {
createAutoUpdateDigestGateState,
messageWhenDigestApplyBlockedByCheckErrors,
recordAutoUpdateImageCheck,
} from '../helpers/autoUpdateDigestGate';
describe('autoUpdateDigestGate', () => {
it('blocks digest apply when sibling check errors exist', () => {
const state = createAutoUpdateDigestGateState();
recordAutoUpdateImageCheck(state, 'nginx:latest', {
hasUpdate: true,
digestUpdate: true,
tagUpdate: false,
});
recordAutoUpdateImageCheck(state, 'redis:latest', {
hasUpdate: false,
digestUpdate: false,
tagUpdate: false,
checkStatus: 'failed',
error: 'registry timeout',
});
const msg = messageWhenDigestApplyBlockedByCheckErrors('web', state);
expect(msg).toContain('image check(s) failed');
expect(msg).toContain('registry timeout');
});
it('does not block when digest update exists without check errors', () => {
const state = createAutoUpdateDigestGateState();
recordAutoUpdateImageCheck(state, 'nginx:latest', {
hasUpdate: true,
digestUpdate: true,
tagUpdate: false,
});
expect(messageWhenDigestApplyBlockedByCheckErrors('web', state)).toBeNull();
});
});
@@ -0,0 +1,97 @@
import { describe, it, expect, vi } from 'vitest';
import { detectImageUpdate } from '../services/imageUpdateDetect';
import { computeImagePreview } from '../services/UpdatePreviewService';
import type { DigestComparisonResult } from '../services/registry-api';
const PLATFORM = { os: 'linux', architecture: 'amd64' };
const LOCAL_DIGEST = `sha256:${'a'.repeat(64)}`;
const CREDENTIALS = { username: 'u', password: 'p' };
const IMAGE = 'nginx:1.2.3';
/**
* COR-1 regression: persisted sidebar status (via detectImageUpdate /
* checkImage) and Fleet/Anatomy preview (computeImagePreview) must agree when
* the declared-tag digest matches but a higher semantic tag exists.
*/
describe('cross-surface update detection (digest match + higher semver)', () => {
async function runDetectionAndPreview(
comparison: DigestComparisonResult,
tags: string[],
localDigests: string[] = [LOCAL_DIGEST],
) {
const compareDigest = vi.fn().mockResolvedValue(comparison);
const listRegistryTagsResult = vi.fn().mockResolvedValue({ ok: true, tags });
const detection = await detectImageUpdate({
localDigests,
platform: PLATFORM,
registry: 'registry-1.docker.io',
repo: 'library/nginx',
tag: '1.2.3',
credentials: CREDENTIALS,
deps: { compareDigest, listRegistryTagsResult },
});
const preview = await computeImagePreview('app', IMAGE, {
getCredentials: vi.fn().mockResolvedValue(CREDENTIALS),
getLocalDigest: vi.fn().mockResolvedValue({ digests: localDigests, platform: PLATFORM, emptyReason: null }),
compareDigest,
listRegistryTagsResult,
});
return { detection, preview, compareDigest };
}
it('shared detector and preview both report hasUpdate for app:1.2.3 when 1.2.4 exists', async () => {
const { detection, preview } = await runDetectionAndPreview(
{ kind: 'match' },
['1.2.3', '1.2.4'],
);
expect(detection.hasUpdate).toBe(true);
expect(detection.nextTag).toBe('1.2.4');
expect(detection.digestUpdate).toBe(false);
expect(preview.has_update).toBe(true);
expect(preview.next_tag).toBe('1.2.4');
expect(preview.has_update).toBe(detection.hasUpdate);
});
it('shared detector and preview both clear when digest matches and no higher tag exists', async () => {
const { detection, preview } = await runDetectionAndPreview({ kind: 'match' }, ['1.2.3']);
expect(detection.hasUpdate).toBe(false);
expect(preview.has_update).toBe(false);
expect(preview.has_update).toBe(detection.hasUpdate);
});
it('shared detector and preview both report hasUpdate when digest errors but 1.2.4 exists', async () => {
const { detection, preview } = await runDetectionAndPreview(
{ kind: 'error', reason: 'Registry unreachable' },
['1.2.3', '1.2.4'],
);
expect(detection.hasUpdate).toBe(true);
expect(detection.nextTag).toBe('1.2.4');
expect(detection.digestUpdate).toBe(false);
expect(detection.digestError).toBe('Registry unreachable');
expect(preview.has_update).toBe(true);
expect(preview.next_tag).toBe('1.2.4');
expect(preview.has_update).toBe(detection.hasUpdate);
});
it('shared detector and preview both report hasUpdate with no local digests when 1.2.4 exists', async () => {
const { detection, preview, compareDigest } = await runDetectionAndPreview(
{ kind: 'update' },
['1.2.3', '1.2.4'],
[],
);
expect(compareDigest).not.toHaveBeenCalled();
expect(detection.hasUpdate).toBe(true);
expect(detection.digestUpdate).toBe(false);
expect(detection.nextTag).toBe('1.2.4');
expect(preview.has_update).toBe(true);
expect(preview.next_tag).toBe('1.2.4');
expect(preview.has_update).toBe(detection.hasUpdate);
});
});
@@ -239,6 +239,7 @@ describe('ImageUpdateService - checkImage surfaces the comparison resolver outco
beforeEach(() => {
vi.clearAllMocks();
mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: [] });
(ImageUpdateService as any).instance = undefined;
service = ImageUpdateService.getInstance();
});
@@ -255,6 +256,16 @@ describe('ImageUpdateService - checkImage surfaces the comparison resolver outco
}),
} as any);
const dockerWithNginxSemver = (repoDigests: string[] = [`registry-1.docker.io/library/nginx@${LOCAL_DIGEST}`]) => ({
getDocker: () => ({
getImage: () => ({ inspect: vi.fn().mockResolvedValue({
RepoDigests: repoDigests,
Os: 'linux',
Architecture: 'amd64',
}) }),
}),
} as any);
it('surfaces the specific failure reason (not a generic "unreachable") as the check error', async () => {
mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'error', reason: 'Authentication failed for ghcr.io/linuxserver/radarr:latest' });
const result = await service.checkImage(dockerWithLocalDigest(LOCAL_DIGEST), 'ghcr.io/linuxserver/radarr:latest');
@@ -286,6 +297,20 @@ describe('ImageUpdateService - checkImage surfaces the comparison resolver outco
);
});
it('reports an update when the declared tag digest matches but a higher semver tag exists', async () => {
mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'match' });
mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: ['1.2.3', '1.2.4'] });
const result = await service.checkImage(dockerWithNginxSemver(), 'nginx:1.2.3');
expect(result).toMatchObject({ hasUpdate: true, digestUpdate: false, tagUpdate: true, checkStatus: 'ok' });
});
it('reports an update when digest comparison errors but a higher semver tag exists', async () => {
mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'error', reason: 'Registry unreachable' });
mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: ['1.2.3', '1.2.4'] });
const result = await service.checkImage(dockerWithNginxSemver(), 'nginx:1.2.3');
expect(result).toMatchObject({ hasUpdate: true, digestUpdate: false, tagUpdate: true, checkStatus: 'ok' });
});
it('forwards every matching RepoDigest (stale index ahead of current) to the comparison resolver', async () => {
const STALE = `sha256:${'f'.repeat(64)}`;
const CURRENT = `sha256:${'e'.repeat(64)}`;
@@ -339,6 +364,7 @@ services:
beforeEach(() => {
vi.clearAllMocks();
mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: [] });
(ImageUpdateService as any).instance = undefined;
mockGetSystemState.mockReturnValue('1');
mockGetStacks.mockResolvedValue(['stackA']);
@@ -1667,7 +1693,7 @@ services:
});
describe('recheckStack', () => {
it('persists a fresh per-service reduction and returns no warning on success', async () => {
it('returns still_present when a checkable service still has an update', async () => {
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
renderable: true,
services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')],
@@ -1680,7 +1706,10 @@ services:
const result = await service.recheckStack(1, 'stackA');
expect(result).toEqual({ warning: null });
expect(result).toEqual({
outcome: 'still_present',
warning: 'The update command completed, but Sencho still detects an available image update.',
});
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(
1, 'stackA', true, expect.any(Number), 'ok', null,
[
@@ -1691,13 +1720,103 @@ services:
);
});
it('returns a warning and leaves the prior row untouched when the model cannot render', async () => {
it('returns cleared when every checkable service is up to date', async () => {
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
renderable: true,
services: [specFor('web', 'web:latest')],
});
mockGetAllContainers.mockResolvedValue([
{ Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } },
]);
const service = ImageUpdateService.getInstance();
(service as any).checkImage = vi.fn().mockResolvedValue({ hasUpdate: false });
const result = await service.recheckStack(1, 'stackA');
expect(result).toEqual({ outcome: 'cleared', warning: null });
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(
1, 'stackA', false, expect.any(Number), 'ok', null,
expect.any(Array),
expect.any(Number),
);
});
it('returns verification_failed and leaves the prior row untouched when the model cannot render', async () => {
mockBuildEffectiveServiceModel.mockResolvedValueOnce({ renderable: false, code: 'effective_model_render_failed', error: 'no model in test' });
const service = ImageUpdateService.getInstance();
const result = await service.recheckStack(1, 'stackA');
expect(result).toEqual({ warning: 'no model in test' });
expect(result).toEqual({ outcome: 'verification_failed', warning: 'no model in test' });
expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled();
expect(mockRecordStackCheckFailure).not.toHaveBeenCalled();
});
it('returns verification_incomplete and preserves prior hasUpdate on a fully failed check', async () => {
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
renderable: true,
services: [specFor('web', 'web:latest')],
});
mockGetAllContainers.mockResolvedValue([
{ Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } },
]);
mockGetStackServicesJson.mockReturnValueOnce([
{ service: 'web', image: 'web:latest', hasUpdate: true, checkStatus: 'ok', lastError: null },
]);
const service = ImageUpdateService.getInstance();
(service as any).checkImage = vi.fn().mockResolvedValue({
hasUpdate: false,
error: 'registry timeout',
});
const result = await service.recheckStack(1, 'stackA');
expect(result).toEqual({
outcome: 'verification_incomplete',
warning: 'The update command completed, but Sencho could not fully verify whether an image update remains.',
});
expect(mockRecordStackCheckFailure).toHaveBeenCalled();
expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled();
});
it('returns verification_incomplete when the write lock discards a stale commit', async () => {
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
renderable: true,
services: [specFor('web', 'web:latest')],
});
mockGetAllContainers.mockResolvedValue([
{ Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } },
]);
const service = ImageUpdateService.getInstance();
(service as any).checkImage = vi.fn().mockResolvedValue({ hasUpdate: false });
(service as any).withStackWriteLock = vi.fn().mockResolvedValue(false);
const result = await service.recheckStack(1, 'stackA');
expect(result).toEqual({
outcome: 'verification_incomplete',
warning: 'The update command completed, but Sencho could not fully verify whether an image update remains.',
});
expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled();
expect(mockRecordStackCheckFailure).not.toHaveBeenCalled();
});
it('returns verification_incomplete when container listing fails', async () => {
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
renderable: true,
services: [specFor('web', 'web:latest')],
});
mockGetAllContainers.mockRejectedValueOnce(new Error('docker socket down'));
const service = ImageUpdateService.getInstance();
(service as any).checkImage = vi.fn().mockResolvedValue({ hasUpdate: false });
const result = await service.recheckStack(1, 'stackA');
expect(result).toEqual({
outcome: 'verification_incomplete',
warning: 'The update command completed, but Sencho could not fully verify whether an image update remains.',
});
expect((service as any).checkImage).not.toHaveBeenCalled();
expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled();
});
});
@@ -428,13 +428,22 @@ describe('POST /api/auto-update/execute', () => {
const { ComposeService } = await import('../services/ComposeService');
const { HealthGateService } = await import('../services/HealthGateService');
const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!;
const callOrder: string[] = [];
const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack')
.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never);
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
.mockResolvedValue({ hasUpdate: true, digestUpdate: true } as never);
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-au');
const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack')
.mockImplementation(async () => {
callOrder.push('recheckStack');
return { outcome: 'cleared', warning: null } as never;
});
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockImplementation(() => {
callOrder.push('beginStack');
return 'gate-au';
});
try {
const res = await request(app)
.post('/api/auto-update/execute')
@@ -442,12 +451,113 @@ describe('POST /api/auto-update/execute', () => {
.send({ target: 'auto-upd-gate' });
expect(res.status).toBe(200);
expect(updateSpy).toHaveBeenCalledWith('auto-upd-gate', undefined, true);
expect(recheckSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-gate');
expect(beginSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-gate', 'update', `auto-update:${TEST_USERNAME}`);
expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack'));
} finally {
containersSpy.mockRestore();
checkSpy.mockRestore();
updateSpy.mockRestore();
recheckSpy.mockRestore();
beginSpy.mockRestore();
}
});
it('skips Compose apply for tag-only availability without clearing status', async () => {
const DockerController = (await import('../services/DockerController')).default;
const { ImageUpdateService } = await import('../services/ImageUpdateService');
const { ComposeService } = await import('../services/ComposeService');
const { DatabaseService } = await import('../services/DatabaseService');
const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!;
const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack')
.mockResolvedValue([{ Id: 'c1', Image: 'nginx:1.2.3' }] as never);
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
.mockResolvedValue({ hasUpdate: true, digestUpdate: false, tagUpdate: true } as never);
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack');
const clearSpy = vi.spyOn(DatabaseService.getInstance(), 'clearStackUpdateStatus');
try {
const res = await request(app)
.post('/api/auto-update/execute')
.set('Cookie', adminCookie)
.send({ target: 'auto-upd-tag-only' });
expect(res.status).toBe(200);
expect(res.body.result).toContain('Compose pin unchanged');
expect(updateSpy).not.toHaveBeenCalled();
expect(recheckSpy).not.toHaveBeenCalled();
expect(clearSpy).not.toHaveBeenCalledWith(nodeId, 'auto-upd-tag-only');
} finally {
containersSpy.mockRestore();
checkSpy.mockRestore();
updateSpy.mockRestore();
recheckSpy.mockRestore();
clearSpy.mockRestore();
}
});
it('skips digest apply when a sibling image check failed', async () => {
const DockerController = (await import('../services/DockerController')).default;
const { ImageUpdateService } = await import('../services/ImageUpdateService');
const { ComposeService } = await import('../services/ComposeService');
const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack')
.mockResolvedValue([
{ Id: 'c1', Image: 'nginx:latest' },
{ Id: 'c2', Image: 'redis:latest' },
] as never);
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
.mockResolvedValueOnce({ hasUpdate: true, digestUpdate: true, tagUpdate: false } as never)
.mockResolvedValueOnce({ hasUpdate: false, error: 'registry timeout', checkStatus: 'failed' } as never);
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack');
try {
const res = await request(app)
.post('/api/auto-update/execute')
.set('Cookie', adminCookie)
.send({ target: 'auto-upd-check-err' });
expect(res.status).toBe(200);
expect(res.body.result).toContain('image check(s) failed');
expect(updateSpy).not.toHaveBeenCalled();
expect(recheckSpy).not.toHaveBeenCalled();
} finally {
containersSpy.mockRestore();
checkSpy.mockRestore();
updateSpy.mockRestore();
recheckSpy.mockRestore();
}
});
it('still applies when checkImage reports same-tag digestUpdate', async () => {
const DockerController = (await import('../services/DockerController')).default;
const { ImageUpdateService } = await import('../services/ImageUpdateService');
const { ComposeService } = await import('../services/ComposeService');
const { DatabaseService } = await import('../services/DatabaseService');
const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!;
const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack')
.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never);
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
.mockResolvedValue({ hasUpdate: true, digestUpdate: true, tagUpdate: false } as never);
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack')
.mockResolvedValue({ outcome: 'still_present', warning: null } as never);
const clearSpy = vi.spyOn(DatabaseService.getInstance(), 'clearStackUpdateStatus');
try {
const res = await request(app)
.post('/api/auto-update/execute')
.set('Cookie', adminCookie)
.send({ target: 'auto-upd-digest' });
expect(res.status).toBe(200);
expect(updateSpy).toHaveBeenCalledWith('auto-upd-digest', undefined, true);
expect(recheckSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-digest');
expect(clearSpy).not.toHaveBeenCalledWith(nodeId, 'auto-upd-digest');
} finally {
containersSpy.mockRestore();
checkSpy.mockRestore();
updateSpy.mockRestore();
recheckSpy.mockRestore();
clearSpy.mockRestore();
}
});
});
@@ -18,8 +18,8 @@ const {
mockStartContainer, mockStopContainer, mockPruneSystem,
mockUpdateStack,
mockGetStacks, mockGetStackContent, mockGetEnvContent,
mockCheckImage,
mockDispatchAlert,
mockCheckImage, mockRecheckStack,
mockDispatchAlert, mockBroadcastEvent,
mockGetProxyTarget,
mockIsTrivyAvailable,
mockScanAllNodeImages,
@@ -59,7 +59,9 @@ const {
mockGetStackContent: vi.fn().mockResolvedValue(''),
mockGetEnvContent: vi.fn().mockResolvedValue(''),
mockCheckImage: vi.fn().mockResolvedValue({ hasUpdate: false }),
mockRecheckStack: vi.fn().mockResolvedValue({ outcome: 'cleared', warning: null }),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
mockBroadcastEvent: vi.fn(),
mockGetProxyTarget: vi.fn().mockReturnValue(null),
mockIsTrivyAvailable: vi.fn().mockReturnValue(true),
mockScanAllNodeImages: vi.fn().mockResolvedValue({
@@ -167,14 +169,18 @@ vi.mock('../services/ImageUpdateService', () => ({
ImageUpdateService: {
getInstance: () => ({
checkImage: mockCheckImage,
recheckStack: mockRecheckStack,
}),
},
UPDATE_VERIFICATION_INCOMPLETE_WARNING:
'The update command completed, but Sencho could not fully verify whether an image update remains.',
}));
vi.mock('../services/NotificationService', () => ({
NotificationService: {
getInstance: () => ({
dispatchAlert: mockDispatchAlert,
broadcastEvent: mockBroadcastEvent,
}),
},
}));
@@ -795,7 +801,47 @@ describe('SchedulerService - executeUpdate', () => {
await svc.triggerTask(80);
expect(mockUpdateStack).toHaveBeenCalledWith('web-app', undefined, true);
expect(mockClearStackUpdateStatus).toHaveBeenCalledWith(1, 'web-app');
expect(mockRecheckStack).toHaveBeenCalledWith(1, 'web-app');
expect(mockClearStackUpdateStatus).not.toHaveBeenCalled();
expect(mockBroadcastEvent).toHaveBeenCalledWith(expect.objectContaining({
type: 'state-invalidate',
scope: 'image-updates',
nodeId: 1,
stackName: 'web-app',
}));
});
it('blocks scheduled Compose apply when a sibling image check failed', async () => {
mockGetScheduledTask.mockReturnValue({
id: 186,
name: 'update-check-errors',
action: 'update',
cron_expression: '0 4 * * *',
enabled: true,
target_id: 'web-app',
node_id: 1,
created_by: 'admin',
last_status: null,
});
mockGetContainersByStack.mockResolvedValue([
{ Id: 'c1', Image: 'nginx:latest' },
{ Id: 'c2', Image: 'redis:latest' },
]);
mockCheckImage
.mockResolvedValueOnce({ hasUpdate: true, digestUpdate: true, tagUpdate: false })
.mockResolvedValueOnce({ hasUpdate: false, error: 'registry timeout', checkStatus: 'failed' });
await SchedulerService.getInstance().triggerTask(186);
expect(mockUpdateStack).not.toHaveBeenCalled();
expect(mockRecheckStack).not.toHaveBeenCalled();
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
1,
expect.objectContaining({
status: 'success',
output: expect.stringContaining('image check(s) failed'),
}),
);
});
it('runs a scheduled update on the community tier (no paid gate)', async () => {
@@ -828,7 +874,15 @@ describe('SchedulerService - executeUpdate', () => {
it('begins a health gate after a scheduled update succeeds', async () => {
const { HealthGateService } = await import('../services/HealthGateService');
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-1');
const callOrder: string[] = [];
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockImplementation(() => {
callOrder.push('beginStack');
return 'gate-1';
});
mockRecheckStack.mockImplementation(async () => {
callOrder.push('recheckStack');
return { outcome: 'cleared', warning: null };
});
try {
mockGetScheduledTask.mockReturnValue({
id: 83,
@@ -847,6 +901,8 @@ describe('SchedulerService - executeUpdate', () => {
await SchedulerService.getInstance().triggerTask(83);
expect(beginSpy).toHaveBeenCalledWith(1, 'web-app', 'update', 'system:scheduler');
expect(mockRecheckStack).toHaveBeenCalledWith(1, 'web-app');
expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack'));
} finally {
beginSpy.mockRestore();
}
@@ -875,6 +931,45 @@ describe('SchedulerService - executeUpdate', () => {
expect(mockUpdateStack).not.toHaveBeenCalled();
});
it('skips Compose apply for tag-only availability (pinned semver cannot be rewritten)', async () => {
mockGetScheduledTask.mockReturnValue({
id: 185,
name: 'update-tag-only',
action: 'update',
cron_expression: '0 4 * * *',
enabled: true,
target_id: 'web-app',
node_id: 1,
created_by: 'admin',
last_status: null,
});
mockGetContainersByStack.mockResolvedValue([
{ Id: 'c1', Image: 'nginx:1.2.3' },
]);
// Higher tag is visible (hasUpdate) but not actionable via Compose pull.
mockCheckImage.mockResolvedValue({ hasUpdate: true, digestUpdate: false, tagUpdate: true });
const svc = SchedulerService.getInstance();
await svc.triggerTask(84);
expect(mockUpdateStack).not.toHaveBeenCalled();
expect(mockClearStackUpdateStatus).not.toHaveBeenCalled();
expect(mockDispatchAlert).not.toHaveBeenCalledWith(
'info',
'image_update_applied',
expect.any(String),
expect.anything(),
);
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
1,
expect.objectContaining({
status: 'success',
output: expect.stringContaining('Compose pin unchanged'),
}),
);
});
it('handles wildcard target (*) by updating all stacks', async () => {
mockGetScheduledTask.mockReturnValue({
id: 82,
@@ -0,0 +1,193 @@
/**
* Manual POST /api/stacks/:name/update must start the health gate before
* registry recheck, never blind-clear update status, and keep Compose success
* as HTTP 200 even when recheck throws.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
import request from 'supertest';
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
const {
mockExecute,
mockRecheckStack,
mockBeginStack,
mockClearStackUpdateStatus,
mockBroadcastEvent,
mockDispatchAlert,
} = vi.hoisted(() => ({
mockExecute: vi.fn(),
mockRecheckStack: vi.fn(),
mockBeginStack: vi.fn(),
mockClearStackUpdateStatus: vi.fn(),
mockBroadcastEvent: vi.fn(),
mockDispatchAlert: vi.fn(),
}));
vi.mock('../services/StackUpdateOrchestrator', () => ({
StackUpdateOrchestrator: {
getInstance: () => ({ execute: mockExecute }),
},
shortImageId: (id: string) => id.slice(0, 12),
}));
vi.mock('../services/ImageUpdateService', async () => {
const actual = await vi.importActual<typeof import('../services/ImageUpdateService')>(
'../services/ImageUpdateService',
);
return {
...actual,
ImageUpdateService: {
getInstance: () => ({ recheckStack: mockRecheckStack }),
},
};
});
vi.mock('../services/HealthGateService', () => ({
HealthGateService: {
getInstance: () => ({
beginStack: mockBeginStack,
}),
},
}));
vi.mock('../services/FileSystemService', () => ({
FileSystemService: {
getInstance: () => ({
getBaseDir: () => '/tmp/compose',
hasComposeFile: vi.fn().mockResolvedValue(true),
}),
},
}));
vi.mock('../helpers/policyGate', async () => {
const actual = await vi.importActual<typeof import('../helpers/policyGate')>(
'../helpers/policyGate',
);
return {
...actual,
runPolicyGate: vi.fn().mockResolvedValue(true),
triggerPostDeployScan: vi.fn().mockResolvedValue(undefined),
};
});
let tmpDir: string;
let app: import('express').Express;
let authCookie: string;
let clearSpy: ReturnType<typeof vi.spyOn> | undefined;
let broadcastSpy: ReturnType<typeof vi.spyOn> | undefined;
const callOrder: string[] = [];
beforeAll(async () => {
tmpDir = await setupTestDb();
({ app } = await import('../index'));
authCookie = await loginAsTestAdmin(app);
const { DatabaseService } = await import('../services/DatabaseService');
const { NotificationService } = await import('../services/NotificationService');
clearSpy = vi.spyOn(DatabaseService.getInstance(), 'clearStackUpdateStatus').mockImplementation((...args) => {
callOrder.push('clearStackUpdateStatus');
return mockClearStackUpdateStatus(...args);
});
broadcastSpy = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent').mockImplementation((...args) => {
callOrder.push('broadcastEvent');
return mockBroadcastEvent(...args);
});
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockImplementation((...args) => {
callOrder.push(`dispatchAlert:${String(args[2] ?? args[0])}`);
return mockDispatchAlert(...args) ?? Promise.resolve({ persisted: true });
});
});
afterAll(() => {
clearSpy?.mockRestore();
broadcastSpy?.mockRestore();
vi.restoreAllMocks();
cleanupTestDb(tmpDir);
});
beforeEach(() => {
callOrder.length = 0;
mockExecute.mockReset();
mockRecheckStack.mockReset();
mockBeginStack.mockReset();
mockClearStackUpdateStatus.mockReset();
mockBroadcastEvent.mockReset();
mockDispatchAlert.mockReset().mockResolvedValue({ persisted: true });
mockExecute.mockImplementation(async () => {
callOrder.push('execute');
return { kind: 'stack_compose_done', recoveryId: null };
});
mockBeginStack.mockImplementation(() => {
callOrder.push('beginStack');
return 'gate-1';
});
mockRecheckStack.mockImplementation(async () => {
callOrder.push('recheckStack');
return { outcome: 'cleared', warning: null };
});
});
describe('POST /api/stacks/:name/update post-compose verification', () => {
it('starts the health gate before recheck, skips clear, and broadcasts after recheck', async () => {
const res = await request(app)
.post('/api/stacks/web/update')
.set('Cookie', authCookie)
.send({ skip_scan: true });
expect(res.status).toBe(200);
expect(res.body).toMatchObject({ status: 'Update completed', healthGateId: 'gate-1' });
expect(res.body.recheckWarning).toBeUndefined();
expect(callOrder.indexOf('execute')).toBeLessThan(callOrder.indexOf('beginStack'));
expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack'));
expect(callOrder.indexOf('recheckStack')).toBeLessThan(callOrder.indexOf('broadcastEvent'));
expect(callOrder).not.toContain('clearStackUpdateStatus');
expect(mockClearStackUpdateStatus).not.toHaveBeenCalled();
});
it('returns recheckWarning when the update condition remains', async () => {
mockRecheckStack.mockImplementation(async () => {
callOrder.push('recheckStack');
return {
outcome: 'still_present',
warning: 'The update command completed, but Sencho still detects an available image update.',
};
});
const res = await request(app)
.post('/api/stacks/web/update')
.set('Cookie', authCookie)
.send({ skip_scan: true });
expect(res.status).toBe(200);
expect(res.body.recheckWarning).toBe(
'The update command completed, but Sencho still detects an available image update.',
);
});
it('keeps HTTP 200 and success notification when recheck throws after Compose', async () => {
mockRecheckStack.mockImplementation(async () => {
callOrder.push('recheckStack');
throw new Error('registry blew up');
});
const res = await request(app)
.post('/api/stacks/web/update')
.set('Cookie', authCookie)
.send({ skip_scan: true });
expect(res.status).toBe(200);
expect(res.body.healthGateId).toBe('gate-1');
expect(res.body.recheckWarning).toMatch(/could not fully verify/i);
expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack'));
expect(callOrder).toContain('broadcastEvent');
// Success path still notifies; failure notification must not fire.
expect(mockDispatchAlert).toHaveBeenCalledWith(
'info',
'image_update_applied',
expect.any(String),
expect.objectContaining({ stackName: 'web' }),
);
expect(mockDispatchAlert.mock.calls.some((c) => c[0] === 'error' && c[1] === 'deploy_failure')).toBe(false);
});
});
@@ -40,6 +40,19 @@ export function recordAutoUpdateImageCheck(
}
}
/**
* Operator message when a sibling image check failed and a full-stack Compose
* update must not run (it would pull/recreate the unverified image as
* collateral). Null when digest apply may proceed.
*/
export function messageWhenDigestApplyBlockedByCheckErrors(
stackName: string,
state: Pick<AutoUpdateDigestGateState, 'hasDigestUpdate' | 'checkErrors'>,
): string | null {
if (!state.hasDigestUpdate || state.checkErrors.length === 0) return null;
return `Stack "${stackName}": WARNING - digest update available but ${state.checkErrors.length} image check(s) failed; skipped auto-update (${state.checkErrors.join('; ')}).`;
}
/** Operator message when no digest-actionable update was found. */
export function messageWhenNoDigestUpdate(
stackName: string,
+31 -7
View File
@@ -5,13 +5,13 @@ import DockerController from '../services/DockerController';
import { DatabaseService } from '../services/DatabaseService';
import { NodeRegistry } from '../services/NodeRegistry';
import { CacheService } from '../services/CacheService';
import { FLEET_UPDATE_CACHE_KEY } from '../helpers/fleetUpdateCache';
import {
createAutoUpdateDigestGateState,
messageWhenDigestApplyBlockedByCheckErrors,
messageWhenNoDigestUpdate,
recordAutoUpdateImageCheck,
} from '../helpers/autoUpdateDigestGate';
import { ImageUpdateService } from '../services/ImageUpdateService';
import { ImageUpdateService, UPDATE_VERIFICATION_INCOMPLETE_WARNING } from '../services/ImageUpdateService';
import { FileSystemService } from '../services/FileSystemService';
import { StackUpdateOrchestrator } from '../services/StackUpdateOrchestrator';
import { StackOpLockService, stackOpSkipMessage } from '../services/StackOpLockService';
@@ -21,6 +21,8 @@ import { HealthGateService } from '../services/HealthGateService';
import { authMiddleware } from '../middleware/auth';
import { requireAdmin } from '../middleware/tierGates';
import { buildPolicyGateOptions } from '../helpers/policyGate';
import { FLEET_UPDATE_CACHE_KEY, invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
import { summarizeBlockReasons } from '../utils/policy-risk';
import { isValidStackName } from '../utils/validation';
import { sanitizeForLog } from '../utils/safeLog';
@@ -311,7 +313,7 @@ imageUpdatesRouter.post('/fleet/refresh', authMiddleware, async (_req: Request,
}
}
CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY);
invalidateFleetUpdateCache();
res.json({ triggered, rateLimited, failed });
});
@@ -349,7 +351,6 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
const docker = DockerController.getInstance(req.nodeId);
const imageUpdateService = ImageUpdateService.getInstance();
const db = DatabaseService.getInstance();
const atomic = true;
const results: string[] = [];
@@ -388,6 +389,11 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
results.push(messageWhenNoDigestUpdate(stackName, gate, imageRefs.length));
continue;
}
const checkErrorBlock = messageWhenDigestApplyBlockedByCheckErrors(stackName, gate);
if (checkErrorBlock) {
results.push(checkErrorBlock);
continue;
}
const { updatedImages } = gate;
@@ -421,7 +427,9 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
results.push(stackOpSkipMessage(stackName, lock.existing.action));
continue;
}
db.clearStackUpdateStatus(req.nodeId, stackName);
// Health observation starts immediately after Compose; registry recheck is
// isolated so a verification failure cannot turn Compose success into a failure.
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`);
const orchResult = lock.result;
const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
@@ -430,6 +438,21 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
}
// Recheck persists digest-cleared / tag-advisory state. Do not blind-clear.
let recheckWarning: string | undefined;
try {
const recheck = await imageUpdateService.recheckStack(req.nodeId, stackName);
if (recheck.warning) recheckWarning = recheck.warning;
} catch (recheckErr) {
console.warn(
'[AutoUpdate] Post-update recheck failed for %s: %s',
sanitizeForLog(stackName),
sanitizeForLog(getErrorMessage(recheckErr, 'unknown')),
);
recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING;
}
invalidateNodeCaches(req.nodeId);
NotificationService.getInstance().broadcastEvent({
type: 'state-invalidate',
scope: 'image-updates',
@@ -446,7 +469,8 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
{ stackName, actor: 'system:image-update' },
);
results.push(`Stack "${stackName}": updated (${updatedImages.join(', ')}).`);
const base = `Stack "${stackName}": updated (${updatedImages.join(', ')}).`;
results.push(recheckWarning ? `${base} ${recheckWarning}` : base);
} catch (e) {
const msg = getErrorMessage(e, String(e));
results.push(`Stack "${stackName}" failed: ${msg}`);
@@ -454,7 +478,7 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
}
}
CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY);
invalidateFleetUpdateCache();
res.json({ result: results.join('\n') });
} catch (error) {
const msg = getErrorMessage(error, 'Auto-update execution failed');
+30 -8
View File
@@ -18,8 +18,6 @@ import DockerController, { type BulkStackInfo } from '../services/DockerControll
import { DatabaseService, type StackDossierFields } from '../services/DatabaseService';
import { CacheService, type CacheFetchOutcome } from '../services/CacheService';
import { UpdatePreviewService, isAuthoritativeNegativePreview } from '../services/UpdatePreviewService';
import { ImageUpdateService } from '../services/ImageUpdateService';
import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService';
import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement';
import { buildStackDriftReport, type DriftFindingKind, type StackDriftReport } from '../services/DriftDetectionService';
@@ -58,6 +56,11 @@ import { buildPolicyGateOptions, runPolicyGate, triggerPostDeployScan, describeP
import { parseComposePreview, type ComposePreview } from '../helpers/composePreview';
import { filterContainersByComposeService } from '../helpers/composeServiceMatch';
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
import {
ImageUpdateService,
UPDATE_VERIFICATION_INCOMPLETE_WARNING,
} from '../services/ImageUpdateService';
import { parseComposeSelection, defaultEnvPath } from '../helpers/gitSourceSelection';
import { resolveStackEnvSources, discoverStackLocalEnvFiles } from '../helpers/envFileResolution';
import { STACK_STATUSES_CACHE_TTL_MS } from '../helpers/constants';
@@ -2313,7 +2316,26 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
{ nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'manual', actor: req.user?.username ?? null },
{ atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) },
);
DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName);
// Health observation starts immediately after Compose; registry recheck is
// isolated so a verification failure cannot turn Compose success into 500.
ok = true;
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
linkStackUpdateRecoveryGate(recoveryId, healthGateId);
let recheckWarning: string | undefined;
try {
const recheck = await ImageUpdateService.getInstance().recheckStack(req.nodeId, stackName);
if (recheck.warning) recheckWarning = recheck.warning;
} catch (recheckErr) {
console.warn(
'[Stacks] Post-update recheck failed for %s: %s',
sanitizeForLog(stackName),
sanitizeForLog(getErrorMessage(recheckErr, 'unknown')),
);
recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING;
}
invalidateFleetUpdateCache();
invalidateNodeCaches(req.nodeId);
NotificationService.getInstance().broadcastEvent({
@@ -2326,11 +2348,11 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
});
dlog(`[Stacks] Update completed: ${sanitizeForLog(stackName)}`);
if (debug) console.debug(`[Stacks:debug] Update finished in ${Date.now() - t0}ms`);
ok = true;
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
linkStackUpdateRecoveryGate(recoveryId, healthGateId);
res.json({ status: 'Update completed', healthGateId });
res.json({
status: 'Update completed',
healthGateId,
...(recheckWarning ? { recheckWarning } : {}),
});
notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, req.user?.username ?? 'system');
if (!skipScan) {
triggerPostDeployScan(stackName, req.nodeId).catch(err =>
+55 -7
View File
@@ -17,6 +17,25 @@ import { buildEffectiveServiceModel } from './effectiveServiceModel';
const BACKFILL_KEY = 'image_update_notifications_backfilled';
/** Post-update scanner reconciliation outcome for a single stack. */
export type StackRecheckOutcome =
| 'cleared'
| 'still_present'
| 'verification_incomplete'
| 'verification_failed';
export interface StackRecheckResult {
outcome: StackRecheckOutcome;
/** Present when the update condition remains or could not be verified. */
warning: string | null;
}
export const UPDATE_STILL_PRESENT_WARNING =
'The update command completed, but Sencho still detects an available image update.';
export const UPDATE_VERIFICATION_INCOMPLETE_WARNING =
'The update command completed, but Sencho could not fully verify whether an image update remains.';
export interface ImageCheckResult {
hasUpdate: boolean;
/** Same-tag registry digest drift; Compose pull can apply without pin change. */
@@ -884,19 +903,23 @@ export class ImageUpdateService {
}
/**
* Re-check a single stack after a service-scoped update or restore. On a
* render failure the prior row is left untouched and a warning is returned.
* Re-check a single stack after a service-scoped update or restore, or
* after a manual full-stack update. On a render failure the prior row is
* left untouched and a verification_failed result is returned.
*/
public async recheckStack(nodeId: number, stackName: string): Promise<{ warning: string | null }> {
public async recheckStack(nodeId: number, stackName: string): Promise<StackRecheckResult> {
const generation = this.reserveStackWriteGeneration(nodeId, stackName);
const db = DatabaseService.getInstance();
const docker = DockerController.getInstance(nodeId);
const model = await buildEffectiveServiceModel(nodeId, stackName);
if (!model.renderable) {
return { warning: model.error };
return {
outcome: 'verification_failed',
warning: model.error || UPDATE_VERIFICATION_INCOMPLETE_WARNING,
};
}
let containers: Array<{ Image?: string; Labels?: Record<string, string> }> = [];
let containers: Array<{ Image?: string; Labels?: Record<string, string> }>;
try {
containers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers');
} catch (e) {
@@ -905,6 +928,12 @@ export class ImageUpdateService {
sanitizeForLog(stackName),
sanitizeForLog(getErrorMessage(e, 'unknown')),
);
// Do not clear or upsert from declared-image-only checks: runtime
// digests were never observed, so "cleared" would be a false negative.
return {
outcome: 'verification_incomplete',
warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING,
};
}
const refs = new Set<string>();
@@ -942,15 +971,34 @@ export class ImageUpdateService {
const lastError = stackStatusLastError(services);
const now = Date.now();
await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => {
const committed = await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => {
if (checkStatus === 'failed') {
db.recordStackCheckFailure(nodeId, stackName, lastError ?? 'Update check failed', now, services, gen);
} else {
db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now, checkStatus, lastError, services, gen);
}
});
// A newer scanner reservation dropped this write; do not report cleared.
if (!committed) {
return {
outcome: 'verification_incomplete',
warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING,
};
}
return { warning: null };
if (checkStatus === 'partial' || checkStatus === 'failed') {
return {
outcome: 'verification_incomplete',
warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING,
};
}
if (hasUpdate) {
return {
outcome: 'still_present',
warning: UPDATE_STILL_PRESENT_WARNING,
};
}
return { outcome: 'cleared', warning: null };
}
private stackWriteKey(nodeId: number, stackName: string): string {
+36 -6
View File
@@ -10,12 +10,15 @@ import { StackOpLockService, stackOpSkipMessage as skipMessage } from './StackOp
import { FileSystemService } from './FileSystemService';
import { HealthGateService } from './HealthGateService';
import { ServiceUpdateRecoveryService } from './ServiceUpdateRecoveryService';
import { ImageUpdateService } from './ImageUpdateService';
import {
createAutoUpdateDigestGateState,
messageWhenDigestApplyBlockedByCheckErrors,
messageWhenNoDigestUpdate,
recordAutoUpdateImageCheck,
} from '../helpers/autoUpdateDigestGate';
import { ImageUpdateService, UPDATE_VERIFICATION_INCOMPLETE_WARNING } from './ImageUpdateService';
import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
import { formatNoTargetError } from '../utils/remoteTarget';
@@ -770,14 +773,13 @@ export class SchedulerService {
console.log(`[SchedulerService] executeUpdate: ${stackNames.length} stack(s) to check, fleet=${isFleet}, wildcard=${isWildcard}`);
}
const db = DatabaseService.getInstance();
const docker = DockerController.getInstance(task.node_id);
const imageUpdateService = ImageUpdateService.getInstance();
const results: string[] = [];
for (const stackName of stackNames) {
try {
const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, db, isFleet || isWildcard);
const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, isFleet || isWildcard);
results.push(output);
} catch (e) {
const msg = getErrorMessage(e, String(e));
@@ -1034,7 +1036,6 @@ export class SchedulerService {
nodeId: number,
docker: DockerController,
imageUpdateService: ImageUpdateService,
db: DatabaseService,
isWildcard = false
): Promise<string> {
const containers = await docker.getContainersByStack(stackName);
@@ -1076,6 +1077,8 @@ export class SchedulerService {
if (!gate.hasDigestUpdate) {
return messageWhenNoDigestUpdate(stackName, gate, imageRefs.length);
}
const checkErrorBlock = messageWhenDigestApplyBlockedByCheckErrors(stackName, gate);
if (checkErrorBlock) return checkErrorBlock;
const { updatedImages } = gate;
@@ -1096,7 +1099,9 @@ export class SchedulerService {
),
);
if (!lock.ran) return skipMessage(stackName, lock.existing.action);
db.clearStackUpdateStatus(nodeId, stackName);
// Health observation starts immediately after Compose; registry recheck is
// isolated so a verification failure cannot turn Compose success into a failure.
const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler');
const orchResult = lock.result;
const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
@@ -1105,6 +1110,30 @@ export class SchedulerService {
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
}
// Recheck persists digest-cleared / tag-advisory state. Do not blind-clear.
let recheckWarning: string | undefined;
try {
const recheck = await imageUpdateService.recheckStack(nodeId, stackName);
if (recheck.warning) recheckWarning = recheck.warning;
} catch (recheckErr) {
console.warn(
`[SchedulerService] Post-update recheck failed for ${sanitizeForLog(stackName)}:`,
sanitizeForLog(getErrorMessage(recheckErr, 'unknown')),
);
recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING;
}
invalidateFleetUpdateCache();
invalidateNodeCaches(nodeId);
NotificationService.getInstance().broadcastEvent({
type: 'state-invalidate',
scope: 'image-updates',
nodeId,
stackName,
action: 'stack-updated',
ts: Date.now(),
});
this.safeDispatch(
'info',
'image_update_applied',
@@ -1112,7 +1141,8 @@ export class SchedulerService {
stackName
);
return `Stack "${stackName}": updated (${updatedImages.join(', ')}).`;
const base = `Stack "${stackName}": updated (${updatedImages.join(', ')}).`;
return recheckWarning ? `${base} ${recheckWarning}` : base;
}
private async executeScan(task: ScheduledTask): Promise<{ output: string; failed: number }> {