fix: distinguish failed image-update checks from "up to date" (#1470)

* fix: distinguish failed image-update checks from "up to date"

The image-update detector collapsed every failure (registry unreachable,
missing auth, rate limit, unresolved local digest) into hasUpdate:false and
dropped the captured reason, so a failed check was indistinguishable from a
current image and never raised a notification, even while a manual stack
update still pulled a newer image.

Detection now records a tri-state per stack (ok / partial / failed) with the
failure reason, exposed via a new GET /api/image-updates/detail (the boolean
GET / is unchanged so fleet aggregation is unaffected). A fully-failed check
preserves the last known has_update, so a transient outage neither erases a
real update nor flaps the notification state. The sidebar shows a muted
"couldn't check" indicator with the reason on hover, and the Update board
lists stacks whose check failed in a "could not be checked" advisory.

Detector hardening: the manifest digest lookup issues HEAD first (falling back
to GET) so it no longer draws down Docker Hub's anonymous pull-rate budget, and
local RepoDigest matching is normalized so official library/* images resolve
their digest instead of falling through to a silent "no update".

* fix: preserve confirmed updates through partial checks; tighten failure surfacing

Address review findings on the tri-state image-update detection:

- A partial check (some images errored) no longer erases a previously
  confirmed update; only a fully-ok check can lower has_update, so a single
  image's registry blip cannot drop the stack's update and re-fire the
  notification on recovery. Adds a regression test.
- The image-level catch stores getErrorMessage(e) rather than raw String(e),
  since that value surfaces verbatim in the sidebar tooltip and readiness
  advisory.
- useImageUpdates and the readiness detail fetch now log unexpected non-ok
  responses instead of silently leaving stale state.
- Remove an unused checkFailedCount derivation (the row indicator is driven by
  the checkStatus prop).
- Reword the recordStackCheckFailure docstring and the HEAD-first comment.
This commit is contained in:
Anso
2026-06-26 16:16:34 -04:00
committed by GitHub
parent e3b3c3b857
commit d9b7911f12
19 changed files with 820 additions and 38 deletions
@@ -6,7 +6,7 @@ import { RefreshCw, Shield, AlertTriangle, ShieldAlert, CircleSlash, Clock, Play
import { toast } from '@/components/ui/toast-store';
import { apiFetch, fetchForNode } from '@/lib/api';
import { formatTimeAgo } from '@/lib/relativeTime';
import type { ImageUpdateStatus } from '@/types/imageUpdates';
import type { ImageUpdateStatus, StackUpdateInfo } from '@/types/imageUpdates';
import { useNodes } from '@/context/NodeContext';
import { useIsMobile } from '@/hooks/use-is-mobile';
import { Masthead, Kicker } from '@/components/mobile/mobile-ui';
@@ -496,6 +496,30 @@ function MobileNodeSection({ group, onApply }: { group: NodeGroup; onApply: (sta
);
}
/**
* Advisory for local-node stacks whose latest image-update check could not
* determine status. These never appear in the card grid (which lists only
* confirmed updates), so without this they would be invisible here.
*/
function CheckFailuresNotice({ failures }: { failures: { stack: string; reason: string | null }[] }) {
if (failures.length === 0) return null;
return (
<div className="rounded-lg border border-warning/30 bg-warning/5 p-3">
<div className="flex items-center gap-2 font-mono text-[11px] text-warning">
<AlertTriangle className="h-3.5 w-3.5" strokeWidth={1.5} aria-hidden="true" />
{failures.length} stack{failures.length !== 1 ? 's' : ''} could not be checked
</div>
<ul className="mt-1.5 space-y-0.5 pl-5">
{failures.map(f => (
<li key={f.stack} className="font-mono text-[11px] text-stat-subtitle">
<span className="text-stat-value">{f.stack}</span>{f.reason ? `: ${f.reason}` : ''}
</li>
))}
</ul>
</div>
);
}
interface AutoUpdateReadinessProps {
/** Notifications + more-menu cluster for the mobile masthead, rehomed from the dropped TopBar. */
headerActions?: ReactNode;
@@ -509,6 +533,10 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps)
const [loading, setLoading] = useState(true);
const [refreshing, setRefreshing] = useState(false);
const [cadence, setCadence] = useState<ImageUpdateStatus | null>(null);
// Local-node stacks whose latest check could not determine status. The fleet
// list only shows stacks with a confirmed update, so without this a stack
// whose checks all fail would silently vanish from this view.
const [checkFailures, setCheckFailures] = useState<{ stack: string; reason: string | null }[]>([]);
const refreshTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
// Monotonic token guards against stale setGroups from older fetches.
const loadTokenRef = useRef(0);
@@ -535,9 +563,10 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps)
const token = ++loadTokenRef.current;
setLoading(true);
try {
const [statusRes, tasksRes] = await Promise.all([
const [statusRes, tasksRes, detailRes] = await Promise.all([
apiFetch('/image-updates/fleet', { localOnly: true }),
apiFetch('/scheduled-tasks?action=update', { localOnly: true }),
apiFetch('/image-updates/detail', { localOnly: true }),
]);
if (token !== loadTokenRef.current) return;
@@ -547,6 +576,23 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps)
const fleetStatus = await statusRes.json() as FleetUpdateResponse;
setReachableNodeCount(Object.keys(fleetStatus).length);
// Local-node check failures: surfaced separately because the fleet map is
// boolean and the card grid only lists stacks with a confirmed update.
if (detailRes.ok) {
const detail = await detailRes.json() as Record<string, StackUpdateInfo>;
setCheckFailures(
Object.entries(detail)
.filter(([, info]) => info.checkStatus === 'failed')
.map(([stack, info]) => ({ stack, reason: info.lastError }))
.sort((a, b) => a.stack.localeCompare(b.stack)),
);
} else {
// Clear stale failures rather than persist them across a load, but log:
// an empty advisory must not silently stand in for "detail unavailable".
console.error('[AutoUpdateReadiness] /image-updates/detail failed:', detailRes.status);
setCheckFailures([]);
}
const tasks: ScheduledTask[] = tasksRes.ok ? await tasksRes.json() : [];
// A stack is "covered" by an enabled action='update' row when either
// a per-stack row targets it or a fleet row targets its node. We pick
@@ -805,6 +851,7 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps)
{reachableNodeCount} of {onlineNodeCount} nodes reachable. Unreachable nodes are not shown.
</div>
)}
<CheckFailuresNotice failures={checkFailures} />
{loading && groups.length === 0 ? (
<div className="flex items-center justify-center py-16 font-mono text-xs text-stat-subtitle">Loading readiness...</div>
) : groups.length === 0 ? (
@@ -839,6 +886,8 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps)
</div>
)}
<CheckFailuresNotice failures={checkFailures} />
{loading && groups.length === 0 ? (
<div className="flex items-center justify-center py-16 font-mono text-xs text-stat-subtitle">
Loading readiness...