gsadmin/sencho
1
0
Fork 0
mirror of https://github.com/Studio-Saelix/sencho.git synced 2026-08-31 20:58:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(fleet): verify update status before removing readiness cards (#1697)

Browse Source 
* fix(fleet): verify update status before removing readiness cards

Full-stack Apply now rechecks persisted status after the health gate starts, reloads the live preview before dropping a card, and invalidates the hub fleet aggregation so cleared updates cannot resurrect from a stale cache.

Closes #1686

* fix(fleet): align persisted update status with preview semver detection

Share digest-plus-tag detection so post-Apply sidebar status matches Fleet and Anatomy.

* fix(fleet): keep tag-only updates advisory for Compose automation

Expose digestUpdate vs tagUpdate from checkImage so scheduled and API auto-update only apply same-tag digest drift Compose can pull.

* docs: clarify scheduled auto-update applies digest drift only

Document that higher pinned tags stay advisory until Compose is changed, matching schedule and Run Now behavior.

* docs: require Compose pin edits for higher-tag advisories

Stop recommending Apply now or Update as remedies that cannot rewrite a pinned image tag.

* docs: clarify Apply now pulls pinned tags only

Align the detection-cadence bullet with digest-rebuild vs higher-tag guidance.

* fix(fleet): keep tag advisories after apply and scheduled updates

Tag-only previews were treated as cleared on Fleet reload, and scheduled/
Run Now paths wiped status without rechecking. Align post-update verification
with the manual Apply path (health gate first, recheck, no blind clear) and
block digest apply when sibling image checks failed.

* fix(fleet): clear eslint unused-arg and containers assignment
This commit is contained in:
Anso
2026-07-26 03:09:21 -04:00
committed by GitHub
parent bb7c76ba46
commit 719180f156
16 changed files with 1320 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/routes/imageUpdates.ts
+31 -7
View File
@@ -5,13 +5,13 @@ import DockerController from '../services/DockerController';
import { DatabaseService } from '../services/DatabaseService';
import { NodeRegistry } from '../services/NodeRegistry';
import { CacheService } from '../services/CacheService';
import { FLEET_UPDATE_CACHE_KEY } from '../helpers/fleetUpdateCache';
import {
createAutoUpdateDigestGateState,
messageWhenDigestApplyBlockedByCheckErrors,
messageWhenNoDigestUpdate,
recordAutoUpdateImageCheck,
} from '../helpers/autoUpdateDigestGate';
import { ImageUpdateService } from '../services/ImageUpdateService';
import { ImageUpdateService, UPDATE_VERIFICATION_INCOMPLETE_WARNING } from '../services/ImageUpdateService';
import { FileSystemService } from '../services/FileSystemService';
import { StackUpdateOrchestrator } from '../services/StackUpdateOrchestrator';
import { StackOpLockService, stackOpSkipMessage } from '../services/StackOpLockService';
@@ -21,6 +21,8 @@ import { HealthGateService } from '../services/HealthGateService';
import { authMiddleware } from '../middleware/auth';
import { requireAdmin } from '../middleware/tierGates';
import { buildPolicyGateOptions } from '../helpers/policyGate';
import { FLEET_UPDATE_CACHE_KEY, invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
import { summarizeBlockReasons } from '../utils/policy-risk';
import { isValidStackName } from '../utils/validation';
import { sanitizeForLog } from '../utils/safeLog';
@@ -311,7 +313,7 @@ imageUpdatesRouter.post('/fleet/refresh', authMiddleware, async (_req: Request,
}
}
CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY);
invalidateFleetUpdateCache();
res.json({ triggered, rateLimited, failed });
});
@@ -349,7 +351,6 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
const docker = DockerController.getInstance(req.nodeId);
const imageUpdateService = ImageUpdateService.getInstance();
const db = DatabaseService.getInstance();
const atomic = true;
const results: string[] = [];
@@ -388,6 +389,11 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
results.push(messageWhenNoDigestUpdate(stackName, gate, imageRefs.length));
continue;
}
const checkErrorBlock = messageWhenDigestApplyBlockedByCheckErrors(stackName, gate);
if (checkErrorBlock) {
results.push(checkErrorBlock);
continue;
}
const { updatedImages } = gate;
@@ -421,7 +427,9 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
results.push(stackOpSkipMessage(stackName, lock.existing.action));
continue;
}
db.clearStackUpdateStatus(req.nodeId, stackName);
// Health observation starts immediately after Compose; registry recheck is
// isolated so a verification failure cannot turn Compose success into a failure.
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`);
const orchResult = lock.result;
const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
@@ -430,6 +438,21 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
}
// Recheck persists digest-cleared / tag-advisory state. Do not blind-clear.
let recheckWarning: string | undefined;
try {
const recheck = await imageUpdateService.recheckStack(req.nodeId, stackName);
if (recheck.warning) recheckWarning = recheck.warning;
} catch (recheckErr) {
console.warn(
'[AutoUpdate] Post-update recheck failed for %s: %s',
sanitizeForLog(stackName),
sanitizeForLog(getErrorMessage(recheckErr, 'unknown')),
);
recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING;
}
invalidateNodeCaches(req.nodeId);
NotificationService.getInstance().broadcastEvent({
type: 'state-invalidate',
scope: 'image-updates',
@@ -446,7 +469,8 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
{ stackName, actor: 'system:image-update' },
);
results.push(`Stack "${stackName}": updated (${updatedImages.join(', ')}).`);
const base = `Stack "${stackName}": updated (${updatedImages.join(', ')}).`;
results.push(recheckWarning ? `${base} ${recheckWarning}` : base);
} catch (e) {
const msg = getErrorMessage(e, String(e));
results.push(`Stack "${stackName}" failed: ${msg}`);
@@ -454,7 +478,7 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
}
}
CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY);
invalidateFleetUpdateCache();
res.json({ result: results.join('\n') });
} catch (error) {
const msg = getErrorMessage(error, 'Auto-update execution failed');
backend/src/routes/stacks.ts
+30 -8
View File
@@ -18,8 +18,6 @@ import DockerController, { type BulkStackInfo } from '../services/DockerControll
import { DatabaseService, type StackDossierFields } from '../services/DatabaseService';
import { CacheService, type CacheFetchOutcome } from '../services/CacheService';
import { UpdatePreviewService, isAuthoritativeNegativePreview } from '../services/UpdatePreviewService';
import { ImageUpdateService } from '../services/ImageUpdateService';
import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService';
import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement';
import { buildStackDriftReport, type DriftFindingKind, type StackDriftReport } from '../services/DriftDetectionService';
@@ -58,6 +56,11 @@ import { buildPolicyGateOptions, runPolicyGate, triggerPostDeployScan, describeP
import { parseComposePreview, type ComposePreview } from '../helpers/composePreview';
import { filterContainersByComposeService } from '../helpers/composeServiceMatch';
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
import {
ImageUpdateService,
UPDATE_VERIFICATION_INCOMPLETE_WARNING,
} from '../services/ImageUpdateService';
import { parseComposeSelection, defaultEnvPath } from '../helpers/gitSourceSelection';
import { resolveStackEnvSources, discoverStackLocalEnvFiles } from '../helpers/envFileResolution';
import { STACK_STATUSES_CACHE_TTL_MS } from '../helpers/constants';
@@ -2313,7 +2316,26 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
{ nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'manual', actor: req.user?.username ?? null },
{ atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) },
);
DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName);
// Health observation starts immediately after Compose; registry recheck is
// isolated so a verification failure cannot turn Compose success into 500.
ok = true;
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
linkStackUpdateRecoveryGate(recoveryId, healthGateId);
let recheckWarning: string | undefined;
try {
const recheck = await ImageUpdateService.getInstance().recheckStack(req.nodeId, stackName);
if (recheck.warning) recheckWarning = recheck.warning;
} catch (recheckErr) {
console.warn(
'[Stacks] Post-update recheck failed for %s: %s',
sanitizeForLog(stackName),
sanitizeForLog(getErrorMessage(recheckErr, 'unknown')),
);
recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING;
}
invalidateFleetUpdateCache();
invalidateNodeCaches(req.nodeId);
NotificationService.getInstance().broadcastEvent({
@@ -2326,11 +2348,11 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
});
dlog(`[Stacks] Update completed: ${sanitizeForLog(stackName)}`);
if (debug) console.debug(`[Stacks:debug] Update finished in ${Date.now() - t0}ms`);
ok = true;
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
linkStackUpdateRecoveryGate(recoveryId, healthGateId);
res.json({ status: 'Update completed', healthGateId });
res.json({
status: 'Update completed',
healthGateId,
...(recheckWarning ? { recheckWarning } : {}),
});
notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, req.user?.username ?? 'system');
if (!skipScan) {
triggerPostDeployScan(stackName, req.nodeId).catch(err =>