fix(fleet): verify update status before removing readiness cards (#1697)

* fix(fleet): verify update status before removing readiness cards

Full-stack Apply now rechecks persisted status after the health gate starts, reloads the live preview before dropping a card, and invalidates the hub fleet aggregation so cleared updates cannot resurrect from a stale cache.

Closes #1686

* fix(fleet): align persisted update status with preview semver detection

Share digest-plus-tag detection so post-Apply sidebar status matches Fleet and Anatomy.

* fix(fleet): keep tag-only updates advisory for Compose automation

Expose digestUpdate vs tagUpdate from checkImage so scheduled and API auto-update only apply same-tag digest drift Compose can pull.

* docs: clarify scheduled auto-update applies digest drift only

Document that higher pinned tags stay advisory until Compose is changed, matching schedule and Run Now behavior.

* docs: require Compose pin edits for higher-tag advisories

Stop recommending Apply now or Update as remedies that cannot rewrite a pinned image tag.

* docs: clarify Apply now pulls pinned tags only

Align the detection-cadence bullet with digest-rebuild vs higher-tag guidance.

* fix(fleet): keep tag advisories after apply and scheduled updates

Tag-only previews were treated as cleared on Fleet reload, and scheduled/
Run Now paths wiped status without rechecking. Align post-update verification
with the manual Apply path (health gate first, recheck, no blind clear) and
block digest apply when sibling image checks failed.

* fix(fleet): clear eslint unused-arg and containers assignment

backend/src/services/ImageUpdateService.ts

@@ -17,6 +17,25 @@ import { buildEffectiveServiceModel } from './effectiveServiceModel';
 
 const BACKFILL_KEY = 'image_update_notifications_backfilled';
 
+/** Post-update scanner reconciliation outcome for a single stack. */
+export type StackRecheckOutcome =
+    | 'cleared'
+    | 'still_present'
+    | 'verification_incomplete'
+    | 'verification_failed';
+
+export interface StackRecheckResult {
+    outcome: StackRecheckOutcome;
+    /** Present when the update condition remains or could not be verified. */
+    warning: string | null;
+}
+
+export const UPDATE_STILL_PRESENT_WARNING =
+    'The update command completed, but Sencho still detects an available image update.';
+
+export const UPDATE_VERIFICATION_INCOMPLETE_WARNING =
+    'The update command completed, but Sencho could not fully verify whether an image update remains.';
+
 export interface ImageCheckResult {
     hasUpdate: boolean;
     /** Same-tag registry digest drift; Compose pull can apply without pin change. */
@@ -884,19 +903,23 @@ export class ImageUpdateService {
     }
 
     /**
-     * Re-check a single stack after a service-scoped update or restore. On a
-     * render failure the prior row is left untouched and a warning is returned.
+     * Re-check a single stack after a service-scoped update or restore, or
+     * after a manual full-stack update. On a render failure the prior row is
+     * left untouched and a verification_failed result is returned.
      */
-    public async recheckStack(nodeId: number, stackName: string): Promise<{ warning: string | null }> {
+    public async recheckStack(nodeId: number, stackName: string): Promise<StackRecheckResult> {
         const generation = this.reserveStackWriteGeneration(nodeId, stackName);
         const db = DatabaseService.getInstance();
         const docker = DockerController.getInstance(nodeId);
         const model = await buildEffectiveServiceModel(nodeId, stackName);
         if (!model.renderable) {
-            return { warning: model.error };
+            return {
+                outcome: 'verification_failed',
+                warning: model.error || UPDATE_VERIFICATION_INCOMPLETE_WARNING,
+            };
         }
 
-        let containers: Array<{ Image?: string; Labels?: Record<string, string> }> = [];
+        let containers: Array<{ Image?: string; Labels?: Record<string, string> }>;
         try {
             containers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers');
         } catch (e) {
@@ -905,6 +928,12 @@ export class ImageUpdateService {
                 sanitizeForLog(stackName),
                 sanitizeForLog(getErrorMessage(e, 'unknown')),
             );
+            // Do not clear or upsert from declared-image-only checks: runtime
+            // digests were never observed, so "cleared" would be a false negative.
+            return {
+                outcome: 'verification_incomplete',
+                warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING,
+            };
         }
 
         const refs = new Set<string>();
@@ -942,15 +971,34 @@ export class ImageUpdateService {
         const lastError = stackStatusLastError(services);
         const now = Date.now();
 
-        await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => {
+        const committed = await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => {
             if (checkStatus === 'failed') {
                 db.recordStackCheckFailure(nodeId, stackName, lastError ?? 'Update check failed', now, services, gen);
             } else {
                 db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now, checkStatus, lastError, services, gen);
             }
         });
+        // A newer scanner reservation dropped this write; do not report cleared.
+        if (!committed) {
+            return {
+                outcome: 'verification_incomplete',
+                warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING,
+            };
+        }
 
-        return { warning: null };
+        if (checkStatus === 'partial' || checkStatus === 'failed') {
+            return {
+                outcome: 'verification_incomplete',
+                warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING,
+            };
+        }
+        if (hasUpdate) {
+            return {
+                outcome: 'still_present',
+                warning: UPDATE_STILL_PRESENT_WARNING,
+            };
+        }
+        return { outcome: 'cleared', warning: null };
     }
 
     private stackWriteKey(nodeId: number, stackName: string): string {

backend/src/services/SchedulerService.ts

@@ -10,12 +10,15 @@ import { StackOpLockService, stackOpSkipMessage as skipMessage } from './StackOp
 import { FileSystemService } from './FileSystemService';
 import { HealthGateService } from './HealthGateService';
 import { ServiceUpdateRecoveryService } from './ServiceUpdateRecoveryService';
-import { ImageUpdateService } from './ImageUpdateService';
 import {
     createAutoUpdateDigestGateState,
+    messageWhenDigestApplyBlockedByCheckErrors,
     messageWhenNoDigestUpdate,
     recordAutoUpdateImageCheck,
 } from '../helpers/autoUpdateDigestGate';
+import { ImageUpdateService, UPDATE_VERIFICATION_INCOMPLETE_WARNING } from './ImageUpdateService';
+import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
+import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
 import { isDebugEnabled } from '../utils/debug';
 import { getErrorMessage } from '../utils/errors';
 import { formatNoTargetError } from '../utils/remoteTarget';
@@ -770,14 +773,13 @@ export class SchedulerService {
             console.log(`[SchedulerService] executeUpdate: ${stackNames.length} stack(s) to check, fleet=${isFleet}, wildcard=${isWildcard}`);
         }
 
-        const db = DatabaseService.getInstance();
         const docker = DockerController.getInstance(task.node_id);
         const imageUpdateService = ImageUpdateService.getInstance();
         const results: string[] = [];
 
         for (const stackName of stackNames) {
             try {
-                const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, db, isFleet || isWildcard);
+                const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, isFleet || isWildcard);
                 results.push(output);
             } catch (e) {
                 const msg = getErrorMessage(e, String(e));
@@ -1034,7 +1036,6 @@ export class SchedulerService {
         nodeId: number,
         docker: DockerController,
         imageUpdateService: ImageUpdateService,
-        db: DatabaseService,
         isWildcard = false
     ): Promise<string> {
         const containers = await docker.getContainersByStack(stackName);
@@ -1076,6 +1077,8 @@ export class SchedulerService {
         if (!gate.hasDigestUpdate) {
             return messageWhenNoDigestUpdate(stackName, gate, imageRefs.length);
         }
+        const checkErrorBlock = messageWhenDigestApplyBlockedByCheckErrors(stackName, gate);
+        if (checkErrorBlock) return checkErrorBlock;
 
         const { updatedImages } = gate;
 
@@ -1096,7 +1099,9 @@ export class SchedulerService {
             ),
         );
         if (!lock.ran) return skipMessage(stackName, lock.existing.action);
-        db.clearStackUpdateStatus(nodeId, stackName);
+
+        // Health observation starts immediately after Compose; registry recheck is
+        // isolated so a verification failure cannot turn Compose success into a failure.
         const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler');
         const orchResult = lock.result;
         const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
@@ -1105,6 +1110,30 @@ export class SchedulerService {
             StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
         }
 
+        // Recheck persists digest-cleared / tag-advisory state. Do not blind-clear.
+        let recheckWarning: string | undefined;
+        try {
+            const recheck = await imageUpdateService.recheckStack(nodeId, stackName);
+            if (recheck.warning) recheckWarning = recheck.warning;
+        } catch (recheckErr) {
+            console.warn(
+                `[SchedulerService] Post-update recheck failed for ${sanitizeForLog(stackName)}:`,
+                sanitizeForLog(getErrorMessage(recheckErr, 'unknown')),
+            );
+            recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING;
+        }
+
+        invalidateFleetUpdateCache();
+        invalidateNodeCaches(nodeId);
+        NotificationService.getInstance().broadcastEvent({
+            type: 'state-invalidate',
+            scope: 'image-updates',
+            nodeId,
+            stackName,
+            action: 'stack-updated',
+            ts: Date.now(),
+        });
+
         this.safeDispatch(
             'info',
             'image_update_applied',
@@ -1112,7 +1141,8 @@ export class SchedulerService {
             stackName
         );
 
-        return `Stack "${stackName}": updated (${updatedImages.join(', ')}).`;
+        const base = `Stack "${stackName}": updated (${updatedImages.join(', ')}).`;
+        return recheckWarning ? `${base} ${recheckWarning}` : base;
     }
 
     private async executeScan(task: ScheduledTask): Promise<{ output: string; failed: number }> {