mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-25 17:57:06 +00:00
fix(fleet): verify update status before removing readiness cards (#1697)
* fix(fleet): verify update status before removing readiness cards Full-stack Apply now rechecks persisted status after the health gate starts, reloads the live preview before dropping a card, and invalidates the hub fleet aggregation so cleared updates cannot resurrect from a stale cache. Closes #1686 * fix(fleet): align persisted update status with preview semver detection Share digest-plus-tag detection so post-Apply sidebar status matches Fleet and Anatomy. * fix(fleet): keep tag-only updates advisory for Compose automation Expose digestUpdate vs tagUpdate from checkImage so scheduled and API auto-update only apply same-tag digest drift Compose can pull. * docs: clarify scheduled auto-update applies digest drift only Document that higher pinned tags stay advisory until Compose is changed, matching schedule and Run Now behavior. * docs: require Compose pin edits for higher-tag advisories Stop recommending Apply now or Update as remedies that cannot rewrite a pinned image tag. * docs: clarify Apply now pulls pinned tags only Align the detection-cadence bullet with digest-rebuild vs higher-tag guidance. * fix(fleet): keep tag advisories after apply and scheduled updates Tag-only previews were treated as cleared on Fleet reload, and scheduled/ Run Now paths wiped status without rechecking. Align post-update verification with the manual Apply path (health gate first, recheck, no blind clear) and block digest apply when sibling image checks failed. * fix(fleet): clear eslint unused-arg and containers assignment
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
createAutoUpdateDigestGateState,
|
||||
messageWhenDigestApplyBlockedByCheckErrors,
|
||||
recordAutoUpdateImageCheck,
|
||||
} from '../helpers/autoUpdateDigestGate';
|
||||
|
||||
describe('autoUpdateDigestGate', () => {
|
||||
it('blocks digest apply when sibling check errors exist', () => {
|
||||
const state = createAutoUpdateDigestGateState();
|
||||
recordAutoUpdateImageCheck(state, 'nginx:latest', {
|
||||
hasUpdate: true,
|
||||
digestUpdate: true,
|
||||
tagUpdate: false,
|
||||
});
|
||||
recordAutoUpdateImageCheck(state, 'redis:latest', {
|
||||
hasUpdate: false,
|
||||
digestUpdate: false,
|
||||
tagUpdate: false,
|
||||
checkStatus: 'failed',
|
||||
error: 'registry timeout',
|
||||
});
|
||||
|
||||
const msg = messageWhenDigestApplyBlockedByCheckErrors('web', state);
|
||||
expect(msg).toContain('image check(s) failed');
|
||||
expect(msg).toContain('registry timeout');
|
||||
});
|
||||
|
||||
it('does not block when digest update exists without check errors', () => {
|
||||
const state = createAutoUpdateDigestGateState();
|
||||
recordAutoUpdateImageCheck(state, 'nginx:latest', {
|
||||
hasUpdate: true,
|
||||
digestUpdate: true,
|
||||
tagUpdate: false,
|
||||
});
|
||||
expect(messageWhenDigestApplyBlockedByCheckErrors('web', state)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,97 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { detectImageUpdate } from '../services/imageUpdateDetect';
|
||||
import { computeImagePreview } from '../services/UpdatePreviewService';
|
||||
import type { DigestComparisonResult } from '../services/registry-api';
|
||||
|
||||
const PLATFORM = { os: 'linux', architecture: 'amd64' };
|
||||
const LOCAL_DIGEST = `sha256:${'a'.repeat(64)}`;
|
||||
const CREDENTIALS = { username: 'u', password: 'p' };
|
||||
const IMAGE = 'nginx:1.2.3';
|
||||
|
||||
/**
|
||||
* COR-1 regression: persisted sidebar status (via detectImageUpdate /
|
||||
* checkImage) and Fleet/Anatomy preview (computeImagePreview) must agree when
|
||||
* the declared-tag digest matches but a higher semantic tag exists.
|
||||
*/
|
||||
describe('cross-surface update detection (digest match + higher semver)', () => {
|
||||
async function runDetectionAndPreview(
|
||||
comparison: DigestComparisonResult,
|
||||
tags: string[],
|
||||
localDigests: string[] = [LOCAL_DIGEST],
|
||||
) {
|
||||
const compareDigest = vi.fn().mockResolvedValue(comparison);
|
||||
const listRegistryTagsResult = vi.fn().mockResolvedValue({ ok: true, tags });
|
||||
|
||||
const detection = await detectImageUpdate({
|
||||
localDigests,
|
||||
platform: PLATFORM,
|
||||
registry: 'registry-1.docker.io',
|
||||
repo: 'library/nginx',
|
||||
tag: '1.2.3',
|
||||
credentials: CREDENTIALS,
|
||||
deps: { compareDigest, listRegistryTagsResult },
|
||||
});
|
||||
|
||||
const preview = await computeImagePreview('app', IMAGE, {
|
||||
getCredentials: vi.fn().mockResolvedValue(CREDENTIALS),
|
||||
getLocalDigest: vi.fn().mockResolvedValue({ digests: localDigests, platform: PLATFORM, emptyReason: null }),
|
||||
compareDigest,
|
||||
listRegistryTagsResult,
|
||||
});
|
||||
|
||||
return { detection, preview, compareDigest };
|
||||
}
|
||||
|
||||
it('shared detector and preview both report hasUpdate for app:1.2.3 when 1.2.4 exists', async () => {
|
||||
const { detection, preview } = await runDetectionAndPreview(
|
||||
{ kind: 'match' },
|
||||
['1.2.3', '1.2.4'],
|
||||
);
|
||||
|
||||
expect(detection.hasUpdate).toBe(true);
|
||||
expect(detection.nextTag).toBe('1.2.4');
|
||||
expect(detection.digestUpdate).toBe(false);
|
||||
expect(preview.has_update).toBe(true);
|
||||
expect(preview.next_tag).toBe('1.2.4');
|
||||
expect(preview.has_update).toBe(detection.hasUpdate);
|
||||
});
|
||||
|
||||
it('shared detector and preview both clear when digest matches and no higher tag exists', async () => {
|
||||
const { detection, preview } = await runDetectionAndPreview({ kind: 'match' }, ['1.2.3']);
|
||||
|
||||
expect(detection.hasUpdate).toBe(false);
|
||||
expect(preview.has_update).toBe(false);
|
||||
expect(preview.has_update).toBe(detection.hasUpdate);
|
||||
});
|
||||
|
||||
it('shared detector and preview both report hasUpdate when digest errors but 1.2.4 exists', async () => {
|
||||
const { detection, preview } = await runDetectionAndPreview(
|
||||
{ kind: 'error', reason: 'Registry unreachable' },
|
||||
['1.2.3', '1.2.4'],
|
||||
);
|
||||
|
||||
expect(detection.hasUpdate).toBe(true);
|
||||
expect(detection.nextTag).toBe('1.2.4');
|
||||
expect(detection.digestUpdate).toBe(false);
|
||||
expect(detection.digestError).toBe('Registry unreachable');
|
||||
expect(preview.has_update).toBe(true);
|
||||
expect(preview.next_tag).toBe('1.2.4');
|
||||
expect(preview.has_update).toBe(detection.hasUpdate);
|
||||
});
|
||||
|
||||
it('shared detector and preview both report hasUpdate with no local digests when 1.2.4 exists', async () => {
|
||||
const { detection, preview, compareDigest } = await runDetectionAndPreview(
|
||||
{ kind: 'update' },
|
||||
['1.2.3', '1.2.4'],
|
||||
[],
|
||||
);
|
||||
|
||||
expect(compareDigest).not.toHaveBeenCalled();
|
||||
expect(detection.hasUpdate).toBe(true);
|
||||
expect(detection.digestUpdate).toBe(false);
|
||||
expect(detection.nextTag).toBe('1.2.4');
|
||||
expect(preview.has_update).toBe(true);
|
||||
expect(preview.next_tag).toBe('1.2.4');
|
||||
expect(preview.has_update).toBe(detection.hasUpdate);
|
||||
});
|
||||
});
|
||||
@@ -239,6 +239,7 @@ describe('ImageUpdateService - checkImage surfaces the comparison resolver outco
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: [] });
|
||||
(ImageUpdateService as any).instance = undefined;
|
||||
service = ImageUpdateService.getInstance();
|
||||
});
|
||||
@@ -255,6 +256,16 @@ describe('ImageUpdateService - checkImage surfaces the comparison resolver outco
|
||||
}),
|
||||
} as any);
|
||||
|
||||
const dockerWithNginxSemver = (repoDigests: string[] = [`registry-1.docker.io/library/nginx@${LOCAL_DIGEST}`]) => ({
|
||||
getDocker: () => ({
|
||||
getImage: () => ({ inspect: vi.fn().mockResolvedValue({
|
||||
RepoDigests: repoDigests,
|
||||
Os: 'linux',
|
||||
Architecture: 'amd64',
|
||||
}) }),
|
||||
}),
|
||||
} as any);
|
||||
|
||||
it('surfaces the specific failure reason (not a generic "unreachable") as the check error', async () => {
|
||||
mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'error', reason: 'Authentication failed for ghcr.io/linuxserver/radarr:latest' });
|
||||
const result = await service.checkImage(dockerWithLocalDigest(LOCAL_DIGEST), 'ghcr.io/linuxserver/radarr:latest');
|
||||
@@ -286,6 +297,20 @@ describe('ImageUpdateService - checkImage surfaces the comparison resolver outco
|
||||
);
|
||||
});
|
||||
|
||||
it('reports an update when the declared tag digest matches but a higher semver tag exists', async () => {
|
||||
mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'match' });
|
||||
mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: ['1.2.3', '1.2.4'] });
|
||||
const result = await service.checkImage(dockerWithNginxSemver(), 'nginx:1.2.3');
|
||||
expect(result).toMatchObject({ hasUpdate: true, digestUpdate: false, tagUpdate: true, checkStatus: 'ok' });
|
||||
});
|
||||
|
||||
it('reports an update when digest comparison errors but a higher semver tag exists', async () => {
|
||||
mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'error', reason: 'Registry unreachable' });
|
||||
mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: ['1.2.3', '1.2.4'] });
|
||||
const result = await service.checkImage(dockerWithNginxSemver(), 'nginx:1.2.3');
|
||||
expect(result).toMatchObject({ hasUpdate: true, digestUpdate: false, tagUpdate: true, checkStatus: 'ok' });
|
||||
});
|
||||
|
||||
it('forwards every matching RepoDigest (stale index ahead of current) to the comparison resolver', async () => {
|
||||
const STALE = `sha256:${'f'.repeat(64)}`;
|
||||
const CURRENT = `sha256:${'e'.repeat(64)}`;
|
||||
@@ -339,6 +364,7 @@ services:
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: [] });
|
||||
(ImageUpdateService as any).instance = undefined;
|
||||
mockGetSystemState.mockReturnValue('1');
|
||||
mockGetStacks.mockResolvedValue(['stackA']);
|
||||
@@ -1667,7 +1693,7 @@ services:
|
||||
});
|
||||
|
||||
describe('recheckStack', () => {
|
||||
it('persists a fresh per-service reduction and returns no warning on success', async () => {
|
||||
it('returns still_present when a checkable service still has an update', async () => {
|
||||
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
|
||||
renderable: true,
|
||||
services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')],
|
||||
@@ -1680,7 +1706,10 @@ services:
|
||||
|
||||
const result = await service.recheckStack(1, 'stackA');
|
||||
|
||||
expect(result).toEqual({ warning: null });
|
||||
expect(result).toEqual({
|
||||
outcome: 'still_present',
|
||||
warning: 'The update command completed, but Sencho still detects an available image update.',
|
||||
});
|
||||
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(
|
||||
1, 'stackA', true, expect.any(Number), 'ok', null,
|
||||
[
|
||||
@@ -1691,13 +1720,103 @@ services:
|
||||
);
|
||||
});
|
||||
|
||||
it('returns a warning and leaves the prior row untouched when the model cannot render', async () => {
|
||||
it('returns cleared when every checkable service is up to date', async () => {
|
||||
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
|
||||
renderable: true,
|
||||
services: [specFor('web', 'web:latest')],
|
||||
});
|
||||
mockGetAllContainers.mockResolvedValue([
|
||||
{ Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } },
|
||||
]);
|
||||
const service = ImageUpdateService.getInstance();
|
||||
(service as any).checkImage = vi.fn().mockResolvedValue({ hasUpdate: false });
|
||||
|
||||
const result = await service.recheckStack(1, 'stackA');
|
||||
|
||||
expect(result).toEqual({ outcome: 'cleared', warning: null });
|
||||
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(
|
||||
1, 'stackA', false, expect.any(Number), 'ok', null,
|
||||
expect.any(Array),
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns verification_failed and leaves the prior row untouched when the model cannot render', async () => {
|
||||
mockBuildEffectiveServiceModel.mockResolvedValueOnce({ renderable: false, code: 'effective_model_render_failed', error: 'no model in test' });
|
||||
const service = ImageUpdateService.getInstance();
|
||||
|
||||
const result = await service.recheckStack(1, 'stackA');
|
||||
|
||||
expect(result).toEqual({ warning: 'no model in test' });
|
||||
expect(result).toEqual({ outcome: 'verification_failed', warning: 'no model in test' });
|
||||
expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled();
|
||||
expect(mockRecordStackCheckFailure).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns verification_incomplete and preserves prior hasUpdate on a fully failed check', async () => {
|
||||
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
|
||||
renderable: true,
|
||||
services: [specFor('web', 'web:latest')],
|
||||
});
|
||||
mockGetAllContainers.mockResolvedValue([
|
||||
{ Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } },
|
||||
]);
|
||||
mockGetStackServicesJson.mockReturnValueOnce([
|
||||
{ service: 'web', image: 'web:latest', hasUpdate: true, checkStatus: 'ok', lastError: null },
|
||||
]);
|
||||
const service = ImageUpdateService.getInstance();
|
||||
(service as any).checkImage = vi.fn().mockResolvedValue({
|
||||
hasUpdate: false,
|
||||
error: 'registry timeout',
|
||||
});
|
||||
|
||||
const result = await service.recheckStack(1, 'stackA');
|
||||
|
||||
expect(result).toEqual({
|
||||
outcome: 'verification_incomplete',
|
||||
warning: 'The update command completed, but Sencho could not fully verify whether an image update remains.',
|
||||
});
|
||||
expect(mockRecordStackCheckFailure).toHaveBeenCalled();
|
||||
expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns verification_incomplete when the write lock discards a stale commit', async () => {
|
||||
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
|
||||
renderable: true,
|
||||
services: [specFor('web', 'web:latest')],
|
||||
});
|
||||
mockGetAllContainers.mockResolvedValue([
|
||||
{ Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } },
|
||||
]);
|
||||
const service = ImageUpdateService.getInstance();
|
||||
(service as any).checkImage = vi.fn().mockResolvedValue({ hasUpdate: false });
|
||||
(service as any).withStackWriteLock = vi.fn().mockResolvedValue(false);
|
||||
|
||||
const result = await service.recheckStack(1, 'stackA');
|
||||
|
||||
expect(result).toEqual({
|
||||
outcome: 'verification_incomplete',
|
||||
warning: 'The update command completed, but Sencho could not fully verify whether an image update remains.',
|
||||
});
|
||||
expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled();
|
||||
expect(mockRecordStackCheckFailure).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns verification_incomplete when container listing fails', async () => {
|
||||
mockBuildEffectiveServiceModel.mockResolvedValueOnce({
|
||||
renderable: true,
|
||||
services: [specFor('web', 'web:latest')],
|
||||
});
|
||||
mockGetAllContainers.mockRejectedValueOnce(new Error('docker socket down'));
|
||||
const service = ImageUpdateService.getInstance();
|
||||
(service as any).checkImage = vi.fn().mockResolvedValue({ hasUpdate: false });
|
||||
|
||||
const result = await service.recheckStack(1, 'stackA');
|
||||
|
||||
expect(result).toEqual({
|
||||
outcome: 'verification_incomplete',
|
||||
warning: 'The update command completed, but Sencho could not fully verify whether an image update remains.',
|
||||
});
|
||||
expect((service as any).checkImage).not.toHaveBeenCalled();
|
||||
expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -428,13 +428,22 @@ describe('POST /api/auto-update/execute', () => {
|
||||
const { ComposeService } = await import('../services/ComposeService');
|
||||
const { HealthGateService } = await import('../services/HealthGateService');
|
||||
const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!;
|
||||
const callOrder: string[] = [];
|
||||
|
||||
const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack')
|
||||
.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never);
|
||||
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
|
||||
.mockResolvedValue({ hasUpdate: true, digestUpdate: true } as never);
|
||||
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-au');
|
||||
const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack')
|
||||
.mockImplementation(async () => {
|
||||
callOrder.push('recheckStack');
|
||||
return { outcome: 'cleared', warning: null } as never;
|
||||
});
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockImplementation(() => {
|
||||
callOrder.push('beginStack');
|
||||
return 'gate-au';
|
||||
});
|
||||
try {
|
||||
const res = await request(app)
|
||||
.post('/api/auto-update/execute')
|
||||
@@ -442,12 +451,113 @@ describe('POST /api/auto-update/execute', () => {
|
||||
.send({ target: 'auto-upd-gate' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(updateSpy).toHaveBeenCalledWith('auto-upd-gate', undefined, true);
|
||||
expect(recheckSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-gate');
|
||||
expect(beginSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-gate', 'update', `auto-update:${TEST_USERNAME}`);
|
||||
expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack'));
|
||||
} finally {
|
||||
containersSpy.mockRestore();
|
||||
checkSpy.mockRestore();
|
||||
updateSpy.mockRestore();
|
||||
recheckSpy.mockRestore();
|
||||
beginSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('skips Compose apply for tag-only availability without clearing status', async () => {
|
||||
const DockerController = (await import('../services/DockerController')).default;
|
||||
const { ImageUpdateService } = await import('../services/ImageUpdateService');
|
||||
const { ComposeService } = await import('../services/ComposeService');
|
||||
const { DatabaseService } = await import('../services/DatabaseService');
|
||||
const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!;
|
||||
|
||||
const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack')
|
||||
.mockResolvedValue([{ Id: 'c1', Image: 'nginx:1.2.3' }] as never);
|
||||
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
|
||||
.mockResolvedValue({ hasUpdate: true, digestUpdate: false, tagUpdate: true } as never);
|
||||
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
|
||||
const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack');
|
||||
const clearSpy = vi.spyOn(DatabaseService.getInstance(), 'clearStackUpdateStatus');
|
||||
try {
|
||||
const res = await request(app)
|
||||
.post('/api/auto-update/execute')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ target: 'auto-upd-tag-only' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.result).toContain('Compose pin unchanged');
|
||||
expect(updateSpy).not.toHaveBeenCalled();
|
||||
expect(recheckSpy).not.toHaveBeenCalled();
|
||||
expect(clearSpy).not.toHaveBeenCalledWith(nodeId, 'auto-upd-tag-only');
|
||||
} finally {
|
||||
containersSpy.mockRestore();
|
||||
checkSpy.mockRestore();
|
||||
updateSpy.mockRestore();
|
||||
recheckSpy.mockRestore();
|
||||
clearSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('skips digest apply when a sibling image check failed', async () => {
|
||||
const DockerController = (await import('../services/DockerController')).default;
|
||||
const { ImageUpdateService } = await import('../services/ImageUpdateService');
|
||||
const { ComposeService } = await import('../services/ComposeService');
|
||||
|
||||
const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack')
|
||||
.mockResolvedValue([
|
||||
{ Id: 'c1', Image: 'nginx:latest' },
|
||||
{ Id: 'c2', Image: 'redis:latest' },
|
||||
] as never);
|
||||
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
|
||||
.mockResolvedValueOnce({ hasUpdate: true, digestUpdate: true, tagUpdate: false } as never)
|
||||
.mockResolvedValueOnce({ hasUpdate: false, error: 'registry timeout', checkStatus: 'failed' } as never);
|
||||
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
|
||||
const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack');
|
||||
try {
|
||||
const res = await request(app)
|
||||
.post('/api/auto-update/execute')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ target: 'auto-upd-check-err' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.result).toContain('image check(s) failed');
|
||||
expect(updateSpy).not.toHaveBeenCalled();
|
||||
expect(recheckSpy).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
containersSpy.mockRestore();
|
||||
checkSpy.mockRestore();
|
||||
updateSpy.mockRestore();
|
||||
recheckSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('still applies when checkImage reports same-tag digestUpdate', async () => {
|
||||
const DockerController = (await import('../services/DockerController')).default;
|
||||
const { ImageUpdateService } = await import('../services/ImageUpdateService');
|
||||
const { ComposeService } = await import('../services/ComposeService');
|
||||
const { DatabaseService } = await import('../services/DatabaseService');
|
||||
const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!;
|
||||
|
||||
const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack')
|
||||
.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never);
|
||||
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
|
||||
.mockResolvedValue({ hasUpdate: true, digestUpdate: true, tagUpdate: false } as never);
|
||||
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
|
||||
const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack')
|
||||
.mockResolvedValue({ outcome: 'still_present', warning: null } as never);
|
||||
const clearSpy = vi.spyOn(DatabaseService.getInstance(), 'clearStackUpdateStatus');
|
||||
try {
|
||||
const res = await request(app)
|
||||
.post('/api/auto-update/execute')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ target: 'auto-upd-digest' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(updateSpy).toHaveBeenCalledWith('auto-upd-digest', undefined, true);
|
||||
expect(recheckSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-digest');
|
||||
expect(clearSpy).not.toHaveBeenCalledWith(nodeId, 'auto-upd-digest');
|
||||
} finally {
|
||||
containersSpy.mockRestore();
|
||||
checkSpy.mockRestore();
|
||||
updateSpy.mockRestore();
|
||||
recheckSpy.mockRestore();
|
||||
clearSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -18,8 +18,8 @@ const {
|
||||
mockStartContainer, mockStopContainer, mockPruneSystem,
|
||||
mockUpdateStack,
|
||||
mockGetStacks, mockGetStackContent, mockGetEnvContent,
|
||||
mockCheckImage,
|
||||
mockDispatchAlert,
|
||||
mockCheckImage, mockRecheckStack,
|
||||
mockDispatchAlert, mockBroadcastEvent,
|
||||
mockGetProxyTarget,
|
||||
mockIsTrivyAvailable,
|
||||
mockScanAllNodeImages,
|
||||
@@ -59,7 +59,9 @@ const {
|
||||
mockGetStackContent: vi.fn().mockResolvedValue(''),
|
||||
mockGetEnvContent: vi.fn().mockResolvedValue(''),
|
||||
mockCheckImage: vi.fn().mockResolvedValue({ hasUpdate: false }),
|
||||
mockRecheckStack: vi.fn().mockResolvedValue({ outcome: 'cleared', warning: null }),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
|
||||
mockBroadcastEvent: vi.fn(),
|
||||
mockGetProxyTarget: vi.fn().mockReturnValue(null),
|
||||
mockIsTrivyAvailable: vi.fn().mockReturnValue(true),
|
||||
mockScanAllNodeImages: vi.fn().mockResolvedValue({
|
||||
@@ -167,14 +169,18 @@ vi.mock('../services/ImageUpdateService', () => ({
|
||||
ImageUpdateService: {
|
||||
getInstance: () => ({
|
||||
checkImage: mockCheckImage,
|
||||
recheckStack: mockRecheckStack,
|
||||
}),
|
||||
},
|
||||
UPDATE_VERIFICATION_INCOMPLETE_WARNING:
|
||||
'The update command completed, but Sencho could not fully verify whether an image update remains.',
|
||||
}));
|
||||
|
||||
vi.mock('../services/NotificationService', () => ({
|
||||
NotificationService: {
|
||||
getInstance: () => ({
|
||||
dispatchAlert: mockDispatchAlert,
|
||||
broadcastEvent: mockBroadcastEvent,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
@@ -795,7 +801,47 @@ describe('SchedulerService - executeUpdate', () => {
|
||||
await svc.triggerTask(80);
|
||||
|
||||
expect(mockUpdateStack).toHaveBeenCalledWith('web-app', undefined, true);
|
||||
expect(mockClearStackUpdateStatus).toHaveBeenCalledWith(1, 'web-app');
|
||||
expect(mockRecheckStack).toHaveBeenCalledWith(1, 'web-app');
|
||||
expect(mockClearStackUpdateStatus).not.toHaveBeenCalled();
|
||||
expect(mockBroadcastEvent).toHaveBeenCalledWith(expect.objectContaining({
|
||||
type: 'state-invalidate',
|
||||
scope: 'image-updates',
|
||||
nodeId: 1,
|
||||
stackName: 'web-app',
|
||||
}));
|
||||
});
|
||||
|
||||
it('blocks scheduled Compose apply when a sibling image check failed', async () => {
|
||||
mockGetScheduledTask.mockReturnValue({
|
||||
id: 186,
|
||||
name: 'update-check-errors',
|
||||
action: 'update',
|
||||
cron_expression: '0 4 * * *',
|
||||
enabled: true,
|
||||
target_id: 'web-app',
|
||||
node_id: 1,
|
||||
created_by: 'admin',
|
||||
last_status: null,
|
||||
});
|
||||
mockGetContainersByStack.mockResolvedValue([
|
||||
{ Id: 'c1', Image: 'nginx:latest' },
|
||||
{ Id: 'c2', Image: 'redis:latest' },
|
||||
]);
|
||||
mockCheckImage
|
||||
.mockResolvedValueOnce({ hasUpdate: true, digestUpdate: true, tagUpdate: false })
|
||||
.mockResolvedValueOnce({ hasUpdate: false, error: 'registry timeout', checkStatus: 'failed' });
|
||||
|
||||
await SchedulerService.getInstance().triggerTask(186);
|
||||
|
||||
expect(mockUpdateStack).not.toHaveBeenCalled();
|
||||
expect(mockRecheckStack).not.toHaveBeenCalled();
|
||||
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
status: 'success',
|
||||
output: expect.stringContaining('image check(s) failed'),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('runs a scheduled update on the community tier (no paid gate)', async () => {
|
||||
@@ -828,7 +874,15 @@ describe('SchedulerService - executeUpdate', () => {
|
||||
|
||||
it('begins a health gate after a scheduled update succeeds', async () => {
|
||||
const { HealthGateService } = await import('../services/HealthGateService');
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-1');
|
||||
const callOrder: string[] = [];
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockImplementation(() => {
|
||||
callOrder.push('beginStack');
|
||||
return 'gate-1';
|
||||
});
|
||||
mockRecheckStack.mockImplementation(async () => {
|
||||
callOrder.push('recheckStack');
|
||||
return { outcome: 'cleared', warning: null };
|
||||
});
|
||||
try {
|
||||
mockGetScheduledTask.mockReturnValue({
|
||||
id: 83,
|
||||
@@ -847,6 +901,8 @@ describe('SchedulerService - executeUpdate', () => {
|
||||
await SchedulerService.getInstance().triggerTask(83);
|
||||
|
||||
expect(beginSpy).toHaveBeenCalledWith(1, 'web-app', 'update', 'system:scheduler');
|
||||
expect(mockRecheckStack).toHaveBeenCalledWith(1, 'web-app');
|
||||
expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack'));
|
||||
} finally {
|
||||
beginSpy.mockRestore();
|
||||
}
|
||||
@@ -875,6 +931,45 @@ describe('SchedulerService - executeUpdate', () => {
|
||||
expect(mockUpdateStack).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
||||
it('skips Compose apply for tag-only availability (pinned semver cannot be rewritten)', async () => {
|
||||
mockGetScheduledTask.mockReturnValue({
|
||||
id: 185,
|
||||
name: 'update-tag-only',
|
||||
action: 'update',
|
||||
cron_expression: '0 4 * * *',
|
||||
enabled: true,
|
||||
target_id: 'web-app',
|
||||
node_id: 1,
|
||||
created_by: 'admin',
|
||||
last_status: null,
|
||||
});
|
||||
mockGetContainersByStack.mockResolvedValue([
|
||||
{ Id: 'c1', Image: 'nginx:1.2.3' },
|
||||
]);
|
||||
// Higher tag is visible (hasUpdate) but not actionable via Compose pull.
|
||||
mockCheckImage.mockResolvedValue({ hasUpdate: true, digestUpdate: false, tagUpdate: true });
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(84);
|
||||
|
||||
expect(mockUpdateStack).not.toHaveBeenCalled();
|
||||
expect(mockClearStackUpdateStatus).not.toHaveBeenCalled();
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith(
|
||||
'info',
|
||||
'image_update_applied',
|
||||
expect.any(String),
|
||||
expect.anything(),
|
||||
);
|
||||
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
status: 'success',
|
||||
output: expect.stringContaining('Compose pin unchanged'),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('handles wildcard target (*) by updating all stacks', async () => {
|
||||
mockGetScheduledTask.mockReturnValue({
|
||||
id: 82,
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
/**
|
||||
* Manual POST /api/stacks/:name/update must start the health gate before
|
||||
* registry recheck, never blind-clear update status, and keep Compose success
|
||||
* as HTTP 200 even when recheck throws.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
|
||||
|
||||
const {
|
||||
mockExecute,
|
||||
mockRecheckStack,
|
||||
mockBeginStack,
|
||||
mockClearStackUpdateStatus,
|
||||
mockBroadcastEvent,
|
||||
mockDispatchAlert,
|
||||
} = vi.hoisted(() => ({
|
||||
mockExecute: vi.fn(),
|
||||
mockRecheckStack: vi.fn(),
|
||||
mockBeginStack: vi.fn(),
|
||||
mockClearStackUpdateStatus: vi.fn(),
|
||||
mockBroadcastEvent: vi.fn(),
|
||||
mockDispatchAlert: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../services/StackUpdateOrchestrator', () => ({
|
||||
StackUpdateOrchestrator: {
|
||||
getInstance: () => ({ execute: mockExecute }),
|
||||
},
|
||||
shortImageId: (id: string) => id.slice(0, 12),
|
||||
}));
|
||||
|
||||
vi.mock('../services/ImageUpdateService', async () => {
|
||||
const actual = await vi.importActual<typeof import('../services/ImageUpdateService')>(
|
||||
'../services/ImageUpdateService',
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
ImageUpdateService: {
|
||||
getInstance: () => ({ recheckStack: mockRecheckStack }),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('../services/HealthGateService', () => ({
|
||||
HealthGateService: {
|
||||
getInstance: () => ({
|
||||
beginStack: mockBeginStack,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/FileSystemService', () => ({
|
||||
FileSystemService: {
|
||||
getInstance: () => ({
|
||||
getBaseDir: () => '/tmp/compose',
|
||||
hasComposeFile: vi.fn().mockResolvedValue(true),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../helpers/policyGate', async () => {
|
||||
const actual = await vi.importActual<typeof import('../helpers/policyGate')>(
|
||||
'../helpers/policyGate',
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
runPolicyGate: vi.fn().mockResolvedValue(true),
|
||||
triggerPostDeployScan: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
});
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
let authCookie: string;
|
||||
let clearSpy: ReturnType<typeof vi.spyOn> | undefined;
|
||||
let broadcastSpy: ReturnType<typeof vi.spyOn> | undefined;
|
||||
const callOrder: string[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ app } = await import('../index'));
|
||||
authCookie = await loginAsTestAdmin(app);
|
||||
|
||||
const { DatabaseService } = await import('../services/DatabaseService');
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
clearSpy = vi.spyOn(DatabaseService.getInstance(), 'clearStackUpdateStatus').mockImplementation((...args) => {
|
||||
callOrder.push('clearStackUpdateStatus');
|
||||
return mockClearStackUpdateStatus(...args);
|
||||
});
|
||||
broadcastSpy = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent').mockImplementation((...args) => {
|
||||
callOrder.push('broadcastEvent');
|
||||
return mockBroadcastEvent(...args);
|
||||
});
|
||||
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockImplementation((...args) => {
|
||||
callOrder.push(`dispatchAlert:${String(args[2] ?? args[0])}`);
|
||||
return mockDispatchAlert(...args) ?? Promise.resolve({ persisted: true });
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
clearSpy?.mockRestore();
|
||||
broadcastSpy?.mockRestore();
|
||||
vi.restoreAllMocks();
|
||||
cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
callOrder.length = 0;
|
||||
mockExecute.mockReset();
|
||||
mockRecheckStack.mockReset();
|
||||
mockBeginStack.mockReset();
|
||||
mockClearStackUpdateStatus.mockReset();
|
||||
mockBroadcastEvent.mockReset();
|
||||
mockDispatchAlert.mockReset().mockResolvedValue({ persisted: true });
|
||||
|
||||
mockExecute.mockImplementation(async () => {
|
||||
callOrder.push('execute');
|
||||
return { kind: 'stack_compose_done', recoveryId: null };
|
||||
});
|
||||
mockBeginStack.mockImplementation(() => {
|
||||
callOrder.push('beginStack');
|
||||
return 'gate-1';
|
||||
});
|
||||
mockRecheckStack.mockImplementation(async () => {
|
||||
callOrder.push('recheckStack');
|
||||
return { outcome: 'cleared', warning: null };
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /api/stacks/:name/update post-compose verification', () => {
|
||||
it('starts the health gate before recheck, skips clear, and broadcasts after recheck', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/web/update')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ skip_scan: true });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toMatchObject({ status: 'Update completed', healthGateId: 'gate-1' });
|
||||
expect(res.body.recheckWarning).toBeUndefined();
|
||||
expect(callOrder.indexOf('execute')).toBeLessThan(callOrder.indexOf('beginStack'));
|
||||
expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack'));
|
||||
expect(callOrder.indexOf('recheckStack')).toBeLessThan(callOrder.indexOf('broadcastEvent'));
|
||||
expect(callOrder).not.toContain('clearStackUpdateStatus');
|
||||
expect(mockClearStackUpdateStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns recheckWarning when the update condition remains', async () => {
|
||||
mockRecheckStack.mockImplementation(async () => {
|
||||
callOrder.push('recheckStack');
|
||||
return {
|
||||
outcome: 'still_present',
|
||||
warning: 'The update command completed, but Sencho still detects an available image update.',
|
||||
};
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/web/update')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ skip_scan: true });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.recheckWarning).toBe(
|
||||
'The update command completed, but Sencho still detects an available image update.',
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps HTTP 200 and success notification when recheck throws after Compose', async () => {
|
||||
mockRecheckStack.mockImplementation(async () => {
|
||||
callOrder.push('recheckStack');
|
||||
throw new Error('registry blew up');
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/web/update')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ skip_scan: true });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.healthGateId).toBe('gate-1');
|
||||
expect(res.body.recheckWarning).toMatch(/could not fully verify/i);
|
||||
expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack'));
|
||||
expect(callOrder).toContain('broadcastEvent');
|
||||
// Success path still notifies; failure notification must not fire.
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'info',
|
||||
'image_update_applied',
|
||||
expect.any(String),
|
||||
expect.objectContaining({ stackName: 'web' }),
|
||||
);
|
||||
expect(mockDispatchAlert.mock.calls.some((c) => c[0] === 'error' && c[1] === 'deploy_failure')).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -40,6 +40,19 @@ export function recordAutoUpdateImageCheck(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Operator message when a sibling image check failed and a full-stack Compose
|
||||
* update must not run (it would pull/recreate the unverified image as
|
||||
* collateral). Null when digest apply may proceed.
|
||||
*/
|
||||
export function messageWhenDigestApplyBlockedByCheckErrors(
|
||||
stackName: string,
|
||||
state: Pick<AutoUpdateDigestGateState, 'hasDigestUpdate' | 'checkErrors'>,
|
||||
): string | null {
|
||||
if (!state.hasDigestUpdate || state.checkErrors.length === 0) return null;
|
||||
return `Stack "${stackName}": WARNING - digest update available but ${state.checkErrors.length} image check(s) failed; skipped auto-update (${state.checkErrors.join('; ')}).`;
|
||||
}
|
||||
|
||||
/** Operator message when no digest-actionable update was found. */
|
||||
export function messageWhenNoDigestUpdate(
|
||||
stackName: string,
|
||||
|
||||
@@ -5,13 +5,13 @@ import DockerController from '../services/DockerController';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import { CacheService } from '../services/CacheService';
|
||||
import { FLEET_UPDATE_CACHE_KEY } from '../helpers/fleetUpdateCache';
|
||||
import {
|
||||
createAutoUpdateDigestGateState,
|
||||
messageWhenDigestApplyBlockedByCheckErrors,
|
||||
messageWhenNoDigestUpdate,
|
||||
recordAutoUpdateImageCheck,
|
||||
} from '../helpers/autoUpdateDigestGate';
|
||||
import { ImageUpdateService } from '../services/ImageUpdateService';
|
||||
import { ImageUpdateService, UPDATE_VERIFICATION_INCOMPLETE_WARNING } from '../services/ImageUpdateService';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { StackUpdateOrchestrator } from '../services/StackUpdateOrchestrator';
|
||||
import { StackOpLockService, stackOpSkipMessage } from '../services/StackOpLockService';
|
||||
@@ -21,6 +21,8 @@ import { HealthGateService } from '../services/HealthGateService';
|
||||
import { authMiddleware } from '../middleware/auth';
|
||||
import { requireAdmin } from '../middleware/tierGates';
|
||||
import { buildPolicyGateOptions } from '../helpers/policyGate';
|
||||
import { FLEET_UPDATE_CACHE_KEY, invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
|
||||
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
|
||||
import { summarizeBlockReasons } from '../utils/policy-risk';
|
||||
import { isValidStackName } from '../utils/validation';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
@@ -311,7 +313,7 @@ imageUpdatesRouter.post('/fleet/refresh', authMiddleware, async (_req: Request,
|
||||
}
|
||||
}
|
||||
|
||||
CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY);
|
||||
invalidateFleetUpdateCache();
|
||||
res.json({ triggered, rateLimited, failed });
|
||||
});
|
||||
|
||||
@@ -349,7 +351,6 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
|
||||
|
||||
const docker = DockerController.getInstance(req.nodeId);
|
||||
const imageUpdateService = ImageUpdateService.getInstance();
|
||||
const db = DatabaseService.getInstance();
|
||||
const atomic = true;
|
||||
const results: string[] = [];
|
||||
|
||||
@@ -388,6 +389,11 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
|
||||
results.push(messageWhenNoDigestUpdate(stackName, gate, imageRefs.length));
|
||||
continue;
|
||||
}
|
||||
const checkErrorBlock = messageWhenDigestApplyBlockedByCheckErrors(stackName, gate);
|
||||
if (checkErrorBlock) {
|
||||
results.push(checkErrorBlock);
|
||||
continue;
|
||||
}
|
||||
|
||||
const { updatedImages } = gate;
|
||||
|
||||
@@ -421,7 +427,9 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
|
||||
results.push(stackOpSkipMessage(stackName, lock.existing.action));
|
||||
continue;
|
||||
}
|
||||
db.clearStackUpdateStatus(req.nodeId, stackName);
|
||||
|
||||
// Health observation starts immediately after Compose; registry recheck is
|
||||
// isolated so a verification failure cannot turn Compose success into a failure.
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`);
|
||||
const orchResult = lock.result;
|
||||
const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
|
||||
@@ -430,6 +438,21 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
|
||||
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
|
||||
}
|
||||
|
||||
// Recheck persists digest-cleared / tag-advisory state. Do not blind-clear.
|
||||
let recheckWarning: string | undefined;
|
||||
try {
|
||||
const recheck = await imageUpdateService.recheckStack(req.nodeId, stackName);
|
||||
if (recheck.warning) recheckWarning = recheck.warning;
|
||||
} catch (recheckErr) {
|
||||
console.warn(
|
||||
'[AutoUpdate] Post-update recheck failed for %s: %s',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(recheckErr, 'unknown')),
|
||||
);
|
||||
recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING;
|
||||
}
|
||||
|
||||
invalidateNodeCaches(req.nodeId);
|
||||
NotificationService.getInstance().broadcastEvent({
|
||||
type: 'state-invalidate',
|
||||
scope: 'image-updates',
|
||||
@@ -446,7 +469,8 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
|
||||
{ stackName, actor: 'system:image-update' },
|
||||
);
|
||||
|
||||
results.push(`Stack "${stackName}": updated (${updatedImages.join(', ')}).`);
|
||||
const base = `Stack "${stackName}": updated (${updatedImages.join(', ')}).`;
|
||||
results.push(recheckWarning ? `${base} ${recheckWarning}` : base);
|
||||
} catch (e) {
|
||||
const msg = getErrorMessage(e, String(e));
|
||||
results.push(`Stack "${stackName}" failed: ${msg}`);
|
||||
@@ -454,7 +478,7 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
|
||||
}
|
||||
}
|
||||
|
||||
CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY);
|
||||
invalidateFleetUpdateCache();
|
||||
res.json({ result: results.join('\n') });
|
||||
} catch (error) {
|
||||
const msg = getErrorMessage(error, 'Auto-update execution failed');
|
||||
|
||||
@@ -18,8 +18,6 @@ import DockerController, { type BulkStackInfo } from '../services/DockerControll
|
||||
import { DatabaseService, type StackDossierFields } from '../services/DatabaseService';
|
||||
import { CacheService, type CacheFetchOutcome } from '../services/CacheService';
|
||||
import { UpdatePreviewService, isAuthoritativeNegativePreview } from '../services/UpdatePreviewService';
|
||||
import { ImageUpdateService } from '../services/ImageUpdateService';
|
||||
import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
|
||||
import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService';
|
||||
import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement';
|
||||
import { buildStackDriftReport, type DriftFindingKind, type StackDriftReport } from '../services/DriftDetectionService';
|
||||
@@ -58,6 +56,11 @@ import { buildPolicyGateOptions, runPolicyGate, triggerPostDeployScan, describeP
|
||||
import { parseComposePreview, type ComposePreview } from '../helpers/composePreview';
|
||||
import { filterContainersByComposeService } from '../helpers/composeServiceMatch';
|
||||
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
|
||||
import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
|
||||
import {
|
||||
ImageUpdateService,
|
||||
UPDATE_VERIFICATION_INCOMPLETE_WARNING,
|
||||
} from '../services/ImageUpdateService';
|
||||
import { parseComposeSelection, defaultEnvPath } from '../helpers/gitSourceSelection';
|
||||
import { resolveStackEnvSources, discoverStackLocalEnvFiles } from '../helpers/envFileResolution';
|
||||
import { STACK_STATUSES_CACHE_TTL_MS } from '../helpers/constants';
|
||||
@@ -2313,7 +2316,26 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
|
||||
{ nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'manual', actor: req.user?.username ?? null },
|
||||
{ atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) },
|
||||
);
|
||||
DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName);
|
||||
// Health observation starts immediately after Compose; registry recheck is
|
||||
// isolated so a verification failure cannot turn Compose success into 500.
|
||||
ok = true;
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
|
||||
const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
|
||||
linkStackUpdateRecoveryGate(recoveryId, healthGateId);
|
||||
|
||||
let recheckWarning: string | undefined;
|
||||
try {
|
||||
const recheck = await ImageUpdateService.getInstance().recheckStack(req.nodeId, stackName);
|
||||
if (recheck.warning) recheckWarning = recheck.warning;
|
||||
} catch (recheckErr) {
|
||||
console.warn(
|
||||
'[Stacks] Post-update recheck failed for %s: %s',
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(recheckErr, 'unknown')),
|
||||
);
|
||||
recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING;
|
||||
}
|
||||
|
||||
invalidateFleetUpdateCache();
|
||||
invalidateNodeCaches(req.nodeId);
|
||||
NotificationService.getInstance().broadcastEvent({
|
||||
@@ -2326,11 +2348,11 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
|
||||
});
|
||||
dlog(`[Stacks] Update completed: ${sanitizeForLog(stackName)}`);
|
||||
if (debug) console.debug(`[Stacks:debug] Update finished in ${Date.now() - t0}ms`);
|
||||
ok = true;
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
|
||||
const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
|
||||
linkStackUpdateRecoveryGate(recoveryId, healthGateId);
|
||||
res.json({ status: 'Update completed', healthGateId });
|
||||
res.json({
|
||||
status: 'Update completed',
|
||||
healthGateId,
|
||||
...(recheckWarning ? { recheckWarning } : {}),
|
||||
});
|
||||
notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, req.user?.username ?? 'system');
|
||||
if (!skipScan) {
|
||||
triggerPostDeployScan(stackName, req.nodeId).catch(err =>
|
||||
|
||||
@@ -17,6 +17,25 @@ import { buildEffectiveServiceModel } from './effectiveServiceModel';
|
||||
|
||||
const BACKFILL_KEY = 'image_update_notifications_backfilled';
|
||||
|
||||
/** Post-update scanner reconciliation outcome for a single stack. */
|
||||
export type StackRecheckOutcome =
|
||||
| 'cleared'
|
||||
| 'still_present'
|
||||
| 'verification_incomplete'
|
||||
| 'verification_failed';
|
||||
|
||||
export interface StackRecheckResult {
|
||||
outcome: StackRecheckOutcome;
|
||||
/** Present when the update condition remains or could not be verified. */
|
||||
warning: string | null;
|
||||
}
|
||||
|
||||
export const UPDATE_STILL_PRESENT_WARNING =
|
||||
'The update command completed, but Sencho still detects an available image update.';
|
||||
|
||||
export const UPDATE_VERIFICATION_INCOMPLETE_WARNING =
|
||||
'The update command completed, but Sencho could not fully verify whether an image update remains.';
|
||||
|
||||
export interface ImageCheckResult {
|
||||
hasUpdate: boolean;
|
||||
/** Same-tag registry digest drift; Compose pull can apply without pin change. */
|
||||
@@ -884,19 +903,23 @@ export class ImageUpdateService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-check a single stack after a service-scoped update or restore. On a
|
||||
* render failure the prior row is left untouched and a warning is returned.
|
||||
* Re-check a single stack after a service-scoped update or restore, or
|
||||
* after a manual full-stack update. On a render failure the prior row is
|
||||
* left untouched and a verification_failed result is returned.
|
||||
*/
|
||||
public async recheckStack(nodeId: number, stackName: string): Promise<{ warning: string | null }> {
|
||||
public async recheckStack(nodeId: number, stackName: string): Promise<StackRecheckResult> {
|
||||
const generation = this.reserveStackWriteGeneration(nodeId, stackName);
|
||||
const db = DatabaseService.getInstance();
|
||||
const docker = DockerController.getInstance(nodeId);
|
||||
const model = await buildEffectiveServiceModel(nodeId, stackName);
|
||||
if (!model.renderable) {
|
||||
return { warning: model.error };
|
||||
return {
|
||||
outcome: 'verification_failed',
|
||||
warning: model.error || UPDATE_VERIFICATION_INCOMPLETE_WARNING,
|
||||
};
|
||||
}
|
||||
|
||||
let containers: Array<{ Image?: string; Labels?: Record<string, string> }> = [];
|
||||
let containers: Array<{ Image?: string; Labels?: Record<string, string> }>;
|
||||
try {
|
||||
containers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers');
|
||||
} catch (e) {
|
||||
@@ -905,6 +928,12 @@ export class ImageUpdateService {
|
||||
sanitizeForLog(stackName),
|
||||
sanitizeForLog(getErrorMessage(e, 'unknown')),
|
||||
);
|
||||
// Do not clear or upsert from declared-image-only checks: runtime
|
||||
// digests were never observed, so "cleared" would be a false negative.
|
||||
return {
|
||||
outcome: 'verification_incomplete',
|
||||
warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING,
|
||||
};
|
||||
}
|
||||
|
||||
const refs = new Set<string>();
|
||||
@@ -942,15 +971,34 @@ export class ImageUpdateService {
|
||||
const lastError = stackStatusLastError(services);
|
||||
const now = Date.now();
|
||||
|
||||
await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => {
|
||||
const committed = await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => {
|
||||
if (checkStatus === 'failed') {
|
||||
db.recordStackCheckFailure(nodeId, stackName, lastError ?? 'Update check failed', now, services, gen);
|
||||
} else {
|
||||
db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now, checkStatus, lastError, services, gen);
|
||||
}
|
||||
});
|
||||
// A newer scanner reservation dropped this write; do not report cleared.
|
||||
if (!committed) {
|
||||
return {
|
||||
outcome: 'verification_incomplete',
|
||||
warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING,
|
||||
};
|
||||
}
|
||||
|
||||
return { warning: null };
|
||||
if (checkStatus === 'partial' || checkStatus === 'failed') {
|
||||
return {
|
||||
outcome: 'verification_incomplete',
|
||||
warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING,
|
||||
};
|
||||
}
|
||||
if (hasUpdate) {
|
||||
return {
|
||||
outcome: 'still_present',
|
||||
warning: UPDATE_STILL_PRESENT_WARNING,
|
||||
};
|
||||
}
|
||||
return { outcome: 'cleared', warning: null };
|
||||
}
|
||||
|
||||
private stackWriteKey(nodeId: number, stackName: string): string {
|
||||
|
||||
@@ -10,12 +10,15 @@ import { StackOpLockService, stackOpSkipMessage as skipMessage } from './StackOp
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { HealthGateService } from './HealthGateService';
|
||||
import { ServiceUpdateRecoveryService } from './ServiceUpdateRecoveryService';
|
||||
import { ImageUpdateService } from './ImageUpdateService';
|
||||
import {
|
||||
createAutoUpdateDigestGateState,
|
||||
messageWhenDigestApplyBlockedByCheckErrors,
|
||||
messageWhenNoDigestUpdate,
|
||||
recordAutoUpdateImageCheck,
|
||||
} from '../helpers/autoUpdateDigestGate';
|
||||
import { ImageUpdateService, UPDATE_VERIFICATION_INCOMPLETE_WARNING } from './ImageUpdateService';
|
||||
import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
|
||||
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { formatNoTargetError } from '../utils/remoteTarget';
|
||||
@@ -770,14 +773,13 @@ export class SchedulerService {
|
||||
console.log(`[SchedulerService] executeUpdate: ${stackNames.length} stack(s) to check, fleet=${isFleet}, wildcard=${isWildcard}`);
|
||||
}
|
||||
|
||||
const db = DatabaseService.getInstance();
|
||||
const docker = DockerController.getInstance(task.node_id);
|
||||
const imageUpdateService = ImageUpdateService.getInstance();
|
||||
const results: string[] = [];
|
||||
|
||||
for (const stackName of stackNames) {
|
||||
try {
|
||||
const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, db, isFleet || isWildcard);
|
||||
const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, isFleet || isWildcard);
|
||||
results.push(output);
|
||||
} catch (e) {
|
||||
const msg = getErrorMessage(e, String(e));
|
||||
@@ -1034,7 +1036,6 @@ export class SchedulerService {
|
||||
nodeId: number,
|
||||
docker: DockerController,
|
||||
imageUpdateService: ImageUpdateService,
|
||||
db: DatabaseService,
|
||||
isWildcard = false
|
||||
): Promise<string> {
|
||||
const containers = await docker.getContainersByStack(stackName);
|
||||
@@ -1076,6 +1077,8 @@ export class SchedulerService {
|
||||
if (!gate.hasDigestUpdate) {
|
||||
return messageWhenNoDigestUpdate(stackName, gate, imageRefs.length);
|
||||
}
|
||||
const checkErrorBlock = messageWhenDigestApplyBlockedByCheckErrors(stackName, gate);
|
||||
if (checkErrorBlock) return checkErrorBlock;
|
||||
|
||||
const { updatedImages } = gate;
|
||||
|
||||
@@ -1096,7 +1099,9 @@ export class SchedulerService {
|
||||
),
|
||||
);
|
||||
if (!lock.ran) return skipMessage(stackName, lock.existing.action);
|
||||
db.clearStackUpdateStatus(nodeId, stackName);
|
||||
|
||||
// Health observation starts immediately after Compose; registry recheck is
|
||||
// isolated so a verification failure cannot turn Compose success into a failure.
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler');
|
||||
const orchResult = lock.result;
|
||||
const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
|
||||
@@ -1105,6 +1110,30 @@ export class SchedulerService {
|
||||
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
|
||||
}
|
||||
|
||||
// Recheck persists digest-cleared / tag-advisory state. Do not blind-clear.
|
||||
let recheckWarning: string | undefined;
|
||||
try {
|
||||
const recheck = await imageUpdateService.recheckStack(nodeId, stackName);
|
||||
if (recheck.warning) recheckWarning = recheck.warning;
|
||||
} catch (recheckErr) {
|
||||
console.warn(
|
||||
`[SchedulerService] Post-update recheck failed for ${sanitizeForLog(stackName)}:`,
|
||||
sanitizeForLog(getErrorMessage(recheckErr, 'unknown')),
|
||||
);
|
||||
recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING;
|
||||
}
|
||||
|
||||
invalidateFleetUpdateCache();
|
||||
invalidateNodeCaches(nodeId);
|
||||
NotificationService.getInstance().broadcastEvent({
|
||||
type: 'state-invalidate',
|
||||
scope: 'image-updates',
|
||||
nodeId,
|
||||
stackName,
|
||||
action: 'stack-updated',
|
||||
ts: Date.now(),
|
||||
});
|
||||
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
'image_update_applied',
|
||||
@@ -1112,7 +1141,8 @@ export class SchedulerService {
|
||||
stackName
|
||||
);
|
||||
|
||||
return `Stack "${stackName}": updated (${updatedImages.join(', ')}).`;
|
||||
const base = `Stack "${stackName}": updated (${updatedImages.join(', ')}).`;
|
||||
return recheckWarning ? `${base} ${recheckWarning}` : base;
|
||||
}
|
||||
|
||||
private async executeScan(task: ScheduledTask): Promise<{ output: string; failed: number }> {
|
||||
|
||||
Reference in New Issue
Block a user