fix(fleet): verify update status before removing readiness cards (#1697)

* fix(fleet): verify update status before removing readiness cards

Full-stack Apply now rechecks persisted status after the health gate starts, reloads the live preview before dropping a card, and invalidates the hub fleet aggregation so cleared updates cannot resurrect from a stale cache.

Closes #1686

* fix(fleet): align persisted update status with preview semver detection

Share digest-plus-tag detection so post-Apply sidebar status matches Fleet and Anatomy.

* fix(fleet): keep tag-only updates advisory for Compose automation

Expose digestUpdate vs tagUpdate from checkImage so scheduled and API auto-update only apply same-tag digest drift Compose can pull.

* docs: clarify scheduled auto-update applies digest drift only

Document that higher pinned tags stay advisory until Compose is changed, matching schedule and Run Now behavior.

* docs: require Compose pin edits for higher-tag advisories

Stop recommending Apply now or Update as remedies that cannot rewrite a pinned image tag.

* docs: clarify Apply now pulls pinned tags only

Align the detection-cadence bullet with digest-rebuild vs higher-tag guidance.

* fix(fleet): keep tag advisories after apply and scheduled updates

Tag-only previews were treated as cleared on Fleet reload, and scheduled/
Run Now paths wiped status without rechecking. Align post-update verification
with the manual Apply path (health gate first, recheck, no blind clear) and
block digest apply when sibling image checks failed.

* fix(fleet): clear eslint unused-arg and containers assignment
This commit is contained in:
Anso
2026-07-26 03:09:21 -04:00
committed by GitHub
parent bb7c76ba46
commit 719180f156
16 changed files with 1320 additions and 74 deletions
@@ -100,6 +100,9 @@ export interface StackCard {
// Name of the service currently applying a per-service update on this card,
// or null when none is in flight. Distinct from `applying` (full-stack).
applyingService: string | null;
// Post-Apply verification note when Compose succeeded but clearance could
// not be confirmed (distinct from a failed preview fetch).
verificationNote: string | null;
}
interface NodeGroup {
@@ -275,9 +278,10 @@ function StackReadinessCard({
onApply: (stack: string, nodeId: number) => void;
onApplyService?: (stack: string, nodeId: number, serviceName: string) => void;
}) {
const { stack, nodeId, preview, previewLoaded, scheduledTask, applying, applyingService, autoUpdateEnabled } = card;
const { stack, nodeId, preview, previewLoaded, scheduledTask, applying, applyingService, autoUpdateEnabled, verificationNote } = card;
const loading = !previewLoaded;
const failed = previewLoaded && preview === null;
const uncertain = previewLoaded && !!verificationNote;
const failed = previewLoaded && preview === null && !verificationNote;
const blocked = preview?.summary.blocked ?? false;
const bump = preview?.summary.semver_bump ?? 'none';
const updatingImages = preview?.images.filter(i => i.has_update) ?? [];
@@ -328,6 +332,10 @@ function StackReadinessCard({
{loading ? (
<div className="font-mono text-xs text-stat-subtitle/80">Checking registry...</div>
) : uncertain ? (
<div className="font-mono text-xs text-warning">
{verificationNote}
</div>
) : failed ? (
<div className="font-mono text-xs text-destructive/80">
Preview failed. Registry may be unreachable.
@@ -584,8 +592,9 @@ export function MobileReadinessCard({
onApply: (stack: string, nodeId: number) => void;
onApplyService?: (stack: string, nodeId: number, serviceName: string) => void;
}) {
const { stack, nodeId, preview, previewLoaded, scheduledTask, applying, applyingService, autoUpdateEnabled } = card;
const failed = previewLoaded && preview === null;
const { stack, nodeId, preview, previewLoaded, scheduledTask, applying, applyingService, autoUpdateEnabled, verificationNote } = card;
const uncertain = previewLoaded && !!verificationNote;
const failed = previewLoaded && preview === null && !verificationNote;
const blocked = preview?.summary.blocked ?? false;
const bump = preview?.summary.semver_bump ?? 'none';
const updatingImages = preview?.images.filter(i => i.has_update) ?? [];
@@ -628,6 +637,8 @@ export function MobileReadinessCard({
{!previewLoaded ? (
<div className="font-mono text-xs text-stat-subtitle/80">Checking registry...</div>
) : uncertain ? (
<div className="font-mono text-xs text-warning">{verificationNote}</div>
) : failed ? (
<div className="font-mono text-xs text-destructive/80">Preview failed. Registry may be unreachable.</div>
) : (() => {
@@ -901,6 +912,7 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps)
applying: false,
applyingService: null,
autoUpdateEnabled: scheduledTask !== null,
verificationNote: null,
};
});
initialGroups.push({
@@ -1148,8 +1160,26 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps)
...g,
cards: g.cards.map(c => predicate(c) ? { ...c, ...patch } : c),
})));
const matchCard = (c: StackCard) => c.stack === stack && c.nodeId === nodeId;
const removeCard = () => setGroups(prev => prev
.map(g => g.nodeId === nodeId
? { ...g, cards: g.cards.filter(c => c.stack !== stack) }
: g)
.filter(g => g.cards.length > 0));
const retainPreviewFailed = () => setCardField(matchCard, {
applying: false,
preview: null,
previewLoaded: true,
verificationNote: null,
});
const retainUncertain = (note: string) => setCardField(matchCard, {
applying: false,
preview: null,
previewLoaded: true,
verificationNote: note,
});
setCardField(c => c.stack === stack && c.nodeId === nodeId, { applying: true });
setCardField(matchCard, { applying: true, verificationNote: null });
const loadingId = toast.loading(`Applying update to ${stack}...`);
try {
const res = await fetchForNode(
@@ -1161,15 +1191,57 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps)
const data = await res.json().catch(() => ({ error: 'Update failed' }));
throw new Error(data.error ?? 'Update failed');
}
toast.success(`${stack} updated successfully`);
setGroups(prev => prev
.map(g => g.nodeId === nodeId
? { ...g, cards: g.cards.filter(c => c.stack !== stack) }
: g)
.filter(g => g.cards.length > 0));
const body = await res.json().catch(() => ({})) as { recheckWarning?: unknown };
const recheckWarning = typeof body.recheckWarning === 'string' ? body.recheckWarning : undefined;
if (recheckWarning) toast.info(recheckWarning);
else toast.success(`${stack} updated successfully`);
// Authoritative live preview decides card removal. When it disagrees with
// a backend recheckWarning (preview cleared, persisted check uncertain),
// keep an uncertain card so Fleet does not diverge from the sidebar.
try {
const previewRes = await fetchForNode(
`/stacks/${encodeURIComponent(stack)}/update-preview`,
nodeId,
);
if (!previewRes.ok) {
retainPreviewFailed();
return;
}
const next = await previewRes.json() as UpdatePreview;
if (typeof next?.summary?.has_update !== 'boolean') {
retainPreviewFailed();
return;
}
// Drop only when the live preview proves nothing remains (tag-only
// advisories stay pending via isClearedUpdatePreview).
const cleared = isAuthoritativeNegativePreview(next) || isClearedUpdatePreview(next);
if (!cleared) {
if (next.summary.has_update && !recheckWarning) {
toast.info(
'The update command completed, but Sencho still detects an available image update.',
);
}
setCardField(matchCard, {
applying: false,
preview: next,
previewLoaded: true,
verificationNote: recheckWarning ?? null,
});
return;
}
if (recheckWarning) {
retainUncertain(recheckWarning);
return;
}
removeCard();
} catch (previewErr) {
console.error('[AutoUpdate] post-Apply preview reconciliation failed', previewErr);
retainPreviewFailed();
}
} catch (err) {
toast.error((err as Error)?.message || 'Update failed');
setCardField(c => c.stack === stack && c.nodeId === nodeId, { applying: false });
setCardField(matchCard, { applying: false });
} finally {
toast.dismiss(loadingId);
}
@@ -40,6 +40,7 @@ vi.mock('@/context/NodeContext', () => ({
import { apiFetch, fetchForNode } from '@/lib/api';
import { requestServiceUpdate } from '@/lib/serviceUpdate';
import { toast } from '@/components/ui/toast-store';
import AutoUpdateReadinessView, {
MobileReadinessCard,
CadenceStrip,
@@ -49,6 +50,7 @@ import {
isActionableUpdatePreview,
isClearedUpdatePreview,
isReviewRequiredUpdatePreview,
isTagOnlyAdvisory,
isVerificationOnlyPreview,
} from '@/lib/updatePreviewActionability';
import { isAuthoritativeNegativePreview } from '@/types/imageUpdates';
@@ -62,6 +64,7 @@ function card(over: Partial<StackCard> = {}): StackCard {
applyingService: null,
autoUpdateEnabled: true,
scheduledTask: null,
verificationNote: null,
preview: {
stack_name: 'nextcloud',
images: [{
@@ -288,6 +291,29 @@ describe('verification preview helpers', () => {
expect(isClearedUpdatePreview(preview)).toBe(false);
expect(isActionableUpdatePreview(preview)).toBe(false);
});
it('does not treat a tag-only advisory as cleared (Fleet must keep the card)', () => {
const preview = {
...previewSummary({
has_update: true,
update_kind: 'tag',
semver_bump: 'patch',
next_tag: '1.31.3',
current_tag: '1.25.3',
}),
images: [{
service: 'web',
image: 'nginx:1.25.3',
has_update: true,
digest_update: false,
tag_update: true,
check_status: 'ok',
}],
};
expect(isTagOnlyAdvisory(preview)).toBe(true);
expect(isActionableUpdatePreview(preview)).toBe(false);
expect(isClearedUpdatePreview(preview)).toBe(false);
});
});
it('enables Apply for a safe, non-blocked update', () => {
@@ -509,6 +535,12 @@ describe('AutoUpdateReadinessView desktop Apply now', () => {
mockedFetchForNode.mockReset();
mockNodeMeta.clear();
vi.mocked(requestServiceUpdate).mockReset();
vi.mocked(toast.info).mockReset();
vi.mocked(toast.success).mockReset();
vi.mocked(toast.error).mockReset();
vi.mocked(toast.loading).mockReset();
vi.mocked(toast.dismiss).mockReset();
mockNodes.splice(0, mockNodes.length, { id: 1, name: 'Local', type: 'local', status: 'online' });
});
it('enables Apply for a safe update with no covering schedule', async () => {
@@ -633,6 +665,274 @@ describe('AutoUpdateReadinessView desktop Apply now', () => {
});
});
const basePreview = {
stack_name: 'nextcloud',
images: [{
service: 'app',
image: 'nextcloud:27',
current_tag: '27.1.4',
next_tag: '27.1.4',
has_update: true,
digest_update: true,
tag_update: false,
semver_bump: 'patch' as const,
check_status: 'ok' as const,
check_error: null,
digest_error: null,
}],
summary: {
has_update: true,
primary_image: 'nextcloud',
current_tag: '27.1.4',
next_tag: '27.1.4',
semver_bump: 'patch' as const,
update_kind: 'digest' as const,
blocked: false,
blocked_reason: null,
rebuild_available: false,
check_status: 'ok' as const,
verification_failed: false,
verification_error: null,
},
rollback_target: null,
changelog: 'Fixes.',
};
function mockFleetLoad(fleetMap: Record<string, Record<string, boolean>>) {
mockedFetch.mockImplementation((url: string) => {
if (url === '/image-updates/fleet') {
return Promise.resolve({ ok: true, json: async () => fleetMap });
}
if (url.startsWith('/scheduled-tasks')) {
return Promise.resolve({ ok: true, json: async () => [] });
}
return Promise.resolve({ ok: true, json: async () => ({}) });
});
}
it('removes the card only after a cleared preview with no recheckWarning', async () => {
mockFleetLoad({ '1': { nextcloud: true } });
const cleared = {
...basePreview,
images: basePreview.images.map((img) => ({ ...img, has_update: false, digest_update: false })),
summary: { ...basePreview.summary, has_update: false },
};
mockedFetchForNode.mockImplementation((url: string, _nodeId?: number, init?: { method?: string }) => {
if (String(url).includes('/update') && !String(url).includes('update-preview') && init?.method === 'POST') {
return Promise.resolve({ ok: true, json: async () => ({ status: 'Update completed' }) });
}
if (String(url).includes('/update-preview')) {
const previewCalls = mockedFetchForNode.mock.calls.filter((c) => String(c[0]).includes('/update-preview')).length;
return Promise.resolve({
ok: true,
json: async () => (previewCalls <= 1 ? basePreview : cleared),
});
}
return Promise.resolve({ ok: true, json: async () => ({}) });
});
render(<AutoUpdateReadinessView />);
const applyBtn = await screen.findByRole('button', { name: /Apply now/i });
await act(async () => { fireEvent.click(applyBtn); });
await waitFor(() => {
expect(screen.queryByRole('button', { name: /Apply now/i })).not.toBeInTheDocument();
});
const postIdx = mockedFetchForNode.mock.calls.findIndex(
(c) => String(c[0]).includes('/stacks/nextcloud/update') && !String(c[0]).includes('update-preview'),
);
const previewAfter = mockedFetchForNode.mock.calls.findIndex(
(c, i) => i > postIdx && String(c[0]).includes('/update-preview'),
);
expect(postIdx).toBeGreaterThanOrEqual(0);
expect(previewAfter).toBeGreaterThan(postIdx);
expect(mockedFetchForNode.mock.calls[postIdx][1]).toBe(1);
expect(mockedFetchForNode.mock.calls[previewAfter][1]).toBe(1);
});
it('retains the card and warns when the preview still reports an update', async () => {
mockFleetLoad({ '1': { nextcloud: true } });
mockedFetchForNode.mockImplementation((url: string, _nodeId?: number, init?: { method?: string }) => {
if (String(url).includes('/update') && !String(url).includes('update-preview') && init?.method === 'POST') {
return Promise.resolve({ ok: true, json: async () => ({ status: 'Update completed' }) });
}
if (String(url).includes('/update-preview')) {
return Promise.resolve({ ok: true, json: async () => basePreview });
}
return Promise.resolve({ ok: true, json: async () => ({}) });
});
render(<AutoUpdateReadinessView />);
const applyBtn = await screen.findByRole('button', { name: /Apply now/i });
await act(async () => { fireEvent.click(applyBtn); });
await waitFor(() => {
expect(screen.getByRole('button', { name: /Apply now/i })).toBeEnabled();
});
expect(toast.info).toHaveBeenCalledWith(
'The update command completed, but Sencho still detects an available image update.',
);
});
it('retains the card when post-Apply preview is partial (not an authoritative clear)', async () => {
mockFleetLoad({ '1': { nextcloud: true } });
const partialNegative = {
...basePreview,
images: basePreview.images.map((img) => ({
...img,
has_update: false,
digest_update: false,
tag_update: false,
check_status: 'partial' as const,
check_error: 'registry timeout',
})),
summary: {
...basePreview.summary,
has_update: false,
check_status: 'partial' as const,
},
};
mockedFetchForNode.mockImplementation((url: string, _nodeId?: number, init?: { method?: string }) => {
if (String(url).includes('/update') && !String(url).includes('update-preview') && init?.method === 'POST') {
return Promise.resolve({ ok: true, json: async () => ({ status: 'Update completed' }) });
}
if (String(url).includes('/update-preview')) {
const previewCalls = mockedFetchForNode.mock.calls.filter((c) => String(c[0]).includes('/update-preview')).length;
return Promise.resolve({
ok: true,
json: async () => (previewCalls <= 1 ? basePreview : partialNegative),
});
}
return Promise.resolve({ ok: true, json: async () => ({}) });
});
render(<AutoUpdateReadinessView />);
const applyBtn = await screen.findByRole('button', { name: /Apply now/i });
await act(async () => { fireEvent.click(applyBtn); });
await waitFor(() => {
expect(screen.getAllByText('nextcloud').length).toBeGreaterThan(0);
});
expect(screen.queryByText(/Everything is up to date/)).toBeNull();
expect(screen.getByText(/1 update pending/)).toBeInTheDocument();
expect(isAuthoritativeNegativePreview(partialNegative)).toBe(false);
expect(isClearedUpdatePreview(partialNegative)).toBe(false);
});
it('retains an unknown card when recheckWarning disagrees with a cleared preview', async () => {
mockFleetLoad({ '1': { nextcloud: true } });
const cleared = {
...basePreview,
images: basePreview.images.map((img) => ({ ...img, has_update: false, digest_update: false })),
summary: { ...basePreview.summary, has_update: false },
};
mockedFetchForNode.mockImplementation((url: string, _nodeId?: number, init?: { method?: string }) => {
if (String(url).includes('/update') && !String(url).includes('update-preview') && init?.method === 'POST') {
return Promise.resolve({
ok: true,
json: async () => ({
status: 'Update completed',
recheckWarning: 'The update command completed, but Sencho still detects an available image update.',
}),
});
}
if (String(url).includes('/update-preview')) {
const previewCalls = mockedFetchForNode.mock.calls.filter((c) => String(c[0]).includes('/update-preview')).length;
return Promise.resolve({
ok: true,
json: async () => (previewCalls <= 1 ? basePreview : cleared),
});
}
return Promise.resolve({ ok: true, json: async () => ({}) });
});
render(<AutoUpdateReadinessView />);
const applyBtn = await screen.findByRole('button', { name: /Apply now/i });
await act(async () => { fireEvent.click(applyBtn); });
await waitFor(() => {
expect(screen.getByText(
'The update command completed, but Sencho still detects an available image update.',
)).toBeInTheDocument();
});
expect(screen.getByText('nextcloud')).toBeInTheDocument();
expect(screen.queryByText(/Preview failed/i)).not.toBeInTheDocument();
expect(screen.queryByRole('button', { name: /Apply now/i })).not.toBeInTheDocument();
expect(toast.info).toHaveBeenCalledWith(
'The update command completed, but Sencho still detects an available image update.',
);
});
it('retains an unknown card when the post-Apply preview request fails', async () => {
mockFleetLoad({ '1': { nextcloud: true } });
mockedFetchForNode.mockImplementation((url: string, _nodeId?: number, init?: { method?: string }) => {
if (String(url).includes('/update') && !String(url).includes('update-preview') && init?.method === 'POST') {
return Promise.resolve({ ok: true, json: async () => ({ status: 'Update completed' }) });
}
if (String(url).includes('/update-preview')) {
const previewCalls = mockedFetchForNode.mock.calls.filter((c) => String(c[0]).includes('/update-preview')).length;
if (previewCalls <= 1) {
return Promise.resolve({ ok: true, json: async () => basePreview });
}
return Promise.resolve({ ok: false, status: 500, json: async () => ({ error: 'boom' }) });
}
return Promise.resolve({ ok: true, json: async () => ({}) });
});
render(<AutoUpdateReadinessView />);
const applyBtn = await screen.findByRole('button', { name: /Apply now/i });
await act(async () => { fireEvent.click(applyBtn); });
await waitFor(() => {
expect(screen.getByText(/Preview failed/i)).toBeInTheDocument();
});
expect(screen.getByText('nextcloud')).toBeInTheDocument();
expect(screen.queryByRole('button', { name: /Apply now/i })).not.toBeInTheDocument();
});
it('pins full-stack Apply POST and preview to a remote card nodeId', async () => {
mockNodes.splice(0, mockNodes.length,
{ id: 1, name: 'Local', type: 'local', status: 'online' },
{ id: 2, name: 'Remote', type: 'remote', status: 'online' },
);
mockFleetLoad({ '2': { nextcloud: true } });
const cleared = {
...basePreview,
images: basePreview.images.map((img) => ({ ...img, has_update: false, digest_update: false })),
summary: { ...basePreview.summary, has_update: false },
};
mockedFetchForNode.mockImplementation((url: string, _nodeId?: number, init?: { method?: string }) => {
if (String(url).includes('/update') && !String(url).includes('update-preview') && init?.method === 'POST') {
return Promise.resolve({ ok: true, json: async () => ({ status: 'Update completed' }) });
}
if (String(url).includes('/update-preview')) {
const previewCalls = mockedFetchForNode.mock.calls.filter((c) => String(c[0]).includes('/update-preview')).length;
return Promise.resolve({
ok: true,
json: async () => (previewCalls <= 1 ? basePreview : cleared),
});
}
return Promise.resolve({ ok: true, json: async () => ({}) });
});
render(<AutoUpdateReadinessView />);
const applyBtn = await screen.findByRole('button', { name: /Apply now/i });
await act(async () => { fireEvent.click(applyBtn); });
await waitFor(() => {
expect(screen.queryByRole('button', { name: /Apply now/i })).not.toBeInTheDocument();
});
const postCall = mockedFetchForNode.mock.calls.find(
(c) => String(c[0]).includes('/stacks/nextcloud/update') && !String(c[0]).includes('update-preview'),
);
const postApplyPreview = mockedFetchForNode.mock.calls.filter(
(c) => String(c[0]).includes('/update-preview') && c[1] === 2,
);
expect(postCall?.[1]).toBe(2);
expect(postApplyPreview.length).toBeGreaterThanOrEqual(2);
mockNodes.splice(0, mockNodes.length, { id: 1, name: 'Local', type: 'local', status: 'online' });
});
it('holds the desktop full-stack Apply for review, but keeps per-service Apply enabled, when a confirmed update sits alongside another image failing verification', async () => {
mockNodeMeta.set(1, {
version: '1.0.0',
@@ -1026,6 +1326,73 @@ describe('AutoUpdateReadinessView check-failed advisory', () => {
expect(screen.queryByText(/ready to apply automatically/)).toBeNull();
});
it('keeps a tag-only advisory card on load instead of treating it as cleared', async () => {
mockedFetch.mockImplementation((url: string) => {
if (url === '/image-updates/fleet') {
return Promise.resolve({ ok: true, json: async () => ({ '1': { nginx: true } }) });
}
if (url.startsWith('/scheduled-tasks')) {
return Promise.resolve({ ok: true, json: async () => [] });
}
if (url === '/image-updates/detail') {
return Promise.resolve({
ok: true,
json: async () => ({
nginx: { hasUpdate: true, checkStatus: 'ok', lastError: null, checkedAt: 1 },
}),
});
}
return Promise.resolve({ ok: true, json: async () => ({}) });
});
mockedFetchForNode.mockImplementation((url: string) => {
if (String(url).includes('/update-preview')) {
return Promise.resolve({
ok: true,
json: async () => ({
stack_name: 'nginx',
images: [{
service: 'web',
image: 'nginx:1.25.3',
current_tag: '1.25.3',
next_tag: '1.31.3',
has_update: true,
digest_update: false,
tag_update: true,
semver_bump: 'minor',
check_status: 'ok',
check_error: null,
digest_error: null,
}],
summary: {
has_update: true,
primary_image: 'nginx',
current_tag: '1.25.3',
next_tag: '1.31.3',
semver_bump: 'minor',
update_kind: 'tag',
blocked: false,
blocked_reason: null,
rebuild_available: false,
check_status: 'ok',
verification_failed: false,
verification_error: null,
},
rollback_target: null,
changelog: null,
}),
});
}
return Promise.resolve({ ok: true, json: async () => null });
});
render(<AutoUpdateReadinessView />);
expect(await screen.findByText(/1 update pending/)).toBeInTheDocument();
expect(screen.getAllByText('nginx').length).toBeGreaterThan(0);
expect(screen.queryByText(/Everything is up to date/)).toBeNull();
expect(screen.getByRole('button', { name: /Apply now/i })).toBeDisabled();
});
it('keeps a sticky card instead of silently clearing it when a remote sends a legacy preview missing verification_failed entirely', async () => {
// Same sticky-fleet shape as the cleared-preview test above, but the
// fresh preview response is missing verification_failed/rebuild_available
@@ -171,9 +171,9 @@ export function isServiceApplyActionable(
}
/**
* Fresh preview successfully proved there is nothing to apply, and the stack
* is not held for major-bump review. Sticky fleet booleans must not keep these
* cards pending.
* Fresh preview successfully proved there is nothing pending (no digest
* rebuild, no higher-tag advisory). Tag-only advisories set has_update and
* must not clear: Fleet keeps showing them even though Apply is disabled.
*/
export function isClearedUpdatePreview(
preview: UpdatePreviewActionInput | null | undefined,
@@ -184,5 +184,6 @@ export function isClearedUpdatePreview(
if (isPreviewUncertain(preview)) return false;
if (isVerificationOnlyPreview(preview)) return false;
if (isReviewRequiredUpdatePreview(preview)) return false;
return !isActionableUpdatePreview(preview);
if (preview.summary.has_update || preview.summary.rebuild_available) return false;
return summaryCheckOk(preview.summary);
}