Files
sencho/backend/src/services/updateGuard/readiness.ts
T
Anso 3f1f15a6f4 fix: keep running containers until stack pull/build succeeds (#1657)
* fix: keep running containers until stack pull/build succeeds

Acquire images before reconcile, capture a recovery generation for
compensation, and only remove classified orphans after handoff.

* fix: address recovery audit blockers for safe stack updates

Retire abandoned and expired recovery artifacts, probe compensated
runtimes before reporting rollback success, preserve local Docker when
deleting a node, validate the exact Compose invocation before capture,
and repair updateStack return-contract fixtures.

* fix: resolve ESLint errors blocking CI on this branch

Unused-import and unused-variable errors left over from the stack
deletion refactor: MeshService in stacks.ts (its opt-out cascade moved
into DeployedStackDeletionService), a redundant pruneVolumes
destructure in deleteDeployedStack (the real one is re-derived from
the same input object inside runDeletionBody), and an unused beforeAll
import in a Docker-integration test stub. Also scopes the webhook
pull-action case body in a block to satisfy no-case-declarations;
purely syntactic, no behavior change.

* fix: harden recovery probe, cleanup retry, and failed-pull Docker test

Reject absent or unhealthy expected replicas before reporting rollback
success, keep cleanup records until artifacts are actually removed, fail
closed when a mesh override cannot be generated, and assert a real
failed pull leaves the original container running.

* fix: verify recovery probe image identity and stack-scoped override paths

Reject recovered runtimes that use the wrong image or leave scale-zero
services running, and confine tombstone override deletion to the intent
stack directory so forged cross-stack paths cannot be swept.

* test: batch notification cap fixtures in a SQLite transaction

Unbatched 1200-row inserts were timing out at the default 30s under
CI load even though the same assertions pass in under 2s when green.
2026-07-21 12:18:01 -04:00

359 lines
19 KiB
TypeScript

import type { PreflightStatus } from '../preflight/types';
import type { UpdatePreviewSummary } from '../UpdatePreviewService';
import type {
ContainerProbe,
ReadinessSignal,
ReadinessVerdict,
RollbackOverall,
RollbackReadinessItem,
SignalStatus,
} from './types';
/**
* Pure readiness scoring. UpdateGuardService gathers the inputs (each of which
* degrades independently to the 'error' sentinel) and these functions map them
* to signals and a verdict, so every grading rule is synchronously testable.
*/
/** Sentinel for an input whose collection failed. */
export type Errored = 'error';
const formatAge = (timestamp: number, now: number): string => {
const minutes = Math.max(0, Math.round((now - timestamp) / 60_000));
if (minutes < 60) return `${minutes}m ago`;
const hours = Math.round(minutes / 60);
if (hours < 48) return `${hours}h ago`;
return `${Math.round(hours / 24)}d ago`;
};
export function preflightSignal(
input: { activeStatus: PreflightStatus } | Errored,
): ReadinessSignal {
const base = { id: 'preflight' as const, title: 'Compose Doctor' };
if (input === 'error') {
return { ...base, status: 'unknown', affectsVerdict: false, detail: 'The stored preflight report could not be read.' };
}
switch (input.activeStatus) {
case 'never-run':
return { ...base, status: 'unknown', affectsVerdict: false, detail: 'Compose Doctor has not been run for this stack yet. Run it for a deeper pre-update check.' };
case 'blocker':
return { ...base, status: 'blocked', affectsVerdict: true, detail: 'The last preflight found a blocker. Resolve it before updating.' };
case 'unrenderable':
return { ...base, status: 'attention', affectsVerdict: true, detail: 'The compose file did not render in the last preflight; the update is likely to fail the same way.' };
case 'high':
return { ...base, status: 'attention', affectsVerdict: true, detail: 'The last preflight found high-risk findings. Review them before updating.' };
case 'warning':
return { ...base, status: 'warning', affectsVerdict: true, detail: 'The last preflight found warnings.' };
case 'pass':
case 'info':
return { ...base, status: 'ok', affectsVerdict: true, detail: 'The last preflight passed.' };
}
}
export function driftSignal(input: number | Errored): ReadinessSignal {
const base = { id: 'drift' as const, title: 'Drift' };
if (input === 'error') {
return { ...base, status: 'unknown', affectsVerdict: false, detail: 'Drift findings could not be read.' };
}
if (input > 0) {
const plural = input === 1 ? 'finding' : 'findings';
return {
...base,
status: 'warning',
affectsVerdict: true,
detail: `${input} open drift ${plural}: the running state has diverged from the compose file, so the rollback target may not match what is running.`,
};
}
return { ...base, status: 'ok', affectsVerdict: true, detail: 'No open drift findings.' };
}
/** A container already unhealthy, restarting, or crash-exited before any update runs. */
export function isTroubledContainer(c: ContainerProbe): boolean {
return c.health === 'unhealthy' || c.state === 'restarting' || (c.state === 'exited' && (c.exitCode ?? 0) !== 0);
}
export function containersSignal(input: ContainerProbe[] | Errored): ReadinessSignal {
const base = { id: 'containers' as const, title: 'Current containers' };
if (input === 'error') {
return { ...base, status: 'unknown', affectsVerdict: true, detail: 'Container state could not be read from Docker.' };
}
if (input.length === 0) {
return { ...base, status: 'warning', affectsVerdict: true, detail: 'The stack is not running; updating will start it.' };
}
const troubled = input.filter(isTroubledContainer);
if (troubled.length > 0) {
const names = troubled.map(c => c.name).join(', ');
return {
...base,
status: 'attention',
affectsVerdict: true,
detail: `Already unhealthy before the update: ${names}. An update on top of a failing stack is hard to evaluate; consider fixing or stopping it first.`,
};
}
return { ...base, status: 'ok', affectsVerdict: true, detail: `${input.length} container${input.length === 1 ? '' : 's'} running normally.` };
}
export function healthchecksSignal(input: ContainerProbe[] | Errored): ReadinessSignal {
const base = { id: 'healthchecks' as const, title: 'Healthcheck coverage', status: 'ok' as SignalStatus, affectsVerdict: false };
if (input === 'error' || input.length === 0) {
return { ...base, detail: 'Coverage is unknown until the stack runs. Containers without healthchecks are verified by run state only after an update.' };
}
const withCheck = input.filter(c => c.hasHealthcheck).length;
const withoutRestart = input.filter(c => !c.restartPolicy || c.restartPolicy === 'no').length;
const parts = [
`${withCheck} of ${input.length} container${input.length === 1 ? '' : 's'} define a healthcheck; the rest are verified by run state only after an update.`,
];
if (withoutRestart > 0) {
parts.push(`${withoutRestart} ha${withoutRestart === 1 ? 's' : 've'} no restart policy.`);
}
return { ...base, detail: parts.join(' ') };
}
export function updatePreviewSignal(input: UpdatePreviewSummary | Errored): ReadinessSignal {
const base = { id: 'update_preview' as const, title: 'Pending update' };
if (input === 'error') {
return { ...base, status: 'unknown', affectsVerdict: false, detail: 'The update preview is unavailable.' };
}
if (input.blocked) {
return {
...base,
status: 'blocked',
affectsVerdict: true,
detail: input.blocked_reason ?? 'A scan policy blocks this update.',
};
}
if (input.has_update && input.semver_bump === 'major') {
const change = input.current_tag && input.next_tag ? ` (${input.current_tag} to ${input.next_tag})` : '';
return { ...base, status: 'attention', affectsVerdict: true, detail: `A major version bump is pending${change}. Review the upstream changelog for breaking changes.` };
}
if (input.has_update && input.semver_bump === 'unknown') {
return { ...base, status: 'warning', affectsVerdict: true, detail: 'An image update is pending but the version change could not be classified.' };
}
if (input.has_update) {
const kind = input.update_kind === 'digest' ? 'a same-tag image refresh' : `a ${input.semver_bump} update`;
const buildNote = input.has_build_services
? ' Local build services will also be rebuilt from source.'
: '';
return { ...base, status: 'ok', affectsVerdict: true, detail: `Pending: ${kind}.${buildNote}` };
}
if (input.rebuild_available) {
const n = input.has_build_services ? 'Local build service(s)' : 'Build';
return {
...base,
status: 'warning',
affectsVerdict: true,
detail: `${n} require a rebuild from source; the update rebuilds images and recreates containers.`,
};
}
return { ...base, status: 'ok', affectsVerdict: true, detail: 'No pending image update detected; the update re-pulls and recreates with current tags.' };
}
export function buildServicesSignal(buildServices: string[] | Errored): ReadinessSignal {
const base = { id: 'build_services' as const, title: 'Local build services', affectsVerdict: false };
if (buildServices === 'error') {
return { ...base, status: 'unknown', detail: 'Build services could not be detected from the compose model.' };
}
if (buildServices.length === 0) {
return { ...base, status: 'ok', detail: 'No services declare a local build; the update pulls registry images only.' };
}
const plural = buildServices.length === 1 ? 'service' : 'services';
const names = buildServices.join(', ');
return {
...base,
status: 'warning',
detail: `${buildServices.length} ${plural} (${names}) rebuild from source. This may take longer and depends on the local Dockerfile context, network access, and base-image availability.`,
};
}
/** Model-membership facts for the selected service, gathered from EffectiveServiceModel. */
export type ServiceMembership =
| { found: false }
| { found: true; hasBuild: boolean; declaredImage: string | null; expectedReplicas: number };
/** Matches `ServiceUpdateRecoveryService`'s digest-pin check (a recovery snapshot is never eligible for a digest-pinned declared ref). */
const DIGEST_PIN_RE = /@sha256:[a-f0-9]{64}$/i;
/**
* Selected-service signal for a service-scoped readiness check: model
* membership and whether the update would leave a rollback recovery snapshot
* behind. Render failure or a missing service fails closed (blocked), never
* unknown, so a service-scoped check cannot silently proceed against a stale
* or absent model.
*/
export function serviceSignal(input: ServiceMembership | Errored): ReadinessSignal {
const base = { id: 'service' as const, title: 'Selected service' };
if (input === 'error') {
return { ...base, status: 'blocked', affectsVerdict: true, detail: 'The compose model could not be rendered, so the selected service cannot be validated. Resolve the compose error before updating.' };
}
if (!input.found) {
return { ...base, status: 'blocked', affectsVerdict: true, detail: 'The selected service is not declared in this stack\u2019s compose model.' };
}
if (!input.hasBuild && !input.declaredImage) {
return { ...base, status: 'blocked', affectsVerdict: true, detail: 'The selected service has neither an image nor a build, so it cannot be updated.' };
}
const digestPinned = input.declaredImage !== null && DIGEST_PIN_RE.test(input.declaredImage);
if (!input.declaredImage || digestPinned) {
const reason = !input.declaredImage ? 'it builds from source with no declared image' : 'its declared image is pinned to a digest';
return {
...base,
status: 'warning',
affectsVerdict: true,
detail: `Declared with ${input.expectedReplicas} expected replica${input.expectedReplicas === 1 ? '' : 's'}. No rollback recovery snapshot will be available for this update because ${reason}.`,
};
}
return {
...base,
status: 'ok',
affectsVerdict: true,
detail: `Declared with ${input.expectedReplicas} expected replica${input.expectedReplicas === 1 ? '' : 's'}; a rollback recovery snapshot will be captured before the update.`,
};
}
export function backupSlotSignal(
input: { exists: boolean; timestamp: number | null } | Errored,
now: number,
): ReadinessSignal {
const base = { id: 'backup_slot' as const, title: 'Rollback backup' };
if (input === 'error') {
return { ...base, status: 'unknown', affectsVerdict: false, detail: 'The backup slot could not be read.' };
}
if (!input.exists) {
return { ...base, status: 'warning', affectsVerdict: true, detail: 'No rollback backup exists yet; one is created automatically when the update starts.' };
}
const age = input.timestamp ? ` (from ${formatAge(input.timestamp, now)})` : '';
return { ...base, status: 'ok', affectsVerdict: true, detail: `A compose and env file backup exists${age} and is refreshed when the update starts.` };
}
export function diskSignal(
input: { usePercent: number; limitPercent: number } | null | Errored,
): ReadinessSignal {
const base = { id: 'disk' as const, title: 'Node disk' };
if (input === 'error' || input === null) {
return { ...base, status: 'unknown', affectsVerdict: false, detail: 'Disk usage could not be read.' };
}
const use = Math.round(input.usePercent);
if (input.usePercent >= input.limitPercent) {
return { ...base, status: 'attention', affectsVerdict: true, detail: `Disk usage is at ${use}%, at or above the ${input.limitPercent}% alert threshold. Image pulls may fail; free space first.` };
}
if (input.usePercent >= input.limitPercent - 5) {
return { ...base, status: 'warning', affectsVerdict: true, detail: `Disk usage is at ${use}%, close to the ${input.limitPercent}% alert threshold.` };
}
return { ...base, status: 'ok', affectsVerdict: true, detail: `Disk usage is at ${use}%.` };
}
/**
* Severity precedence: blocked > attention (review required) > verdict-affecting
* unknown > warning > ready. Informational unknowns never affect the verdict.
*/
export function aggregateVerdict(signals: ReadinessSignal[]): ReadinessVerdict {
const affecting = signals.filter(s => s.affectsVerdict);
if (affecting.some(s => s.status === 'blocked')) return 'blocked';
if (affecting.some(s => s.status === 'attention')) return 'review_required';
if (affecting.some(s => s.status === 'unknown')) return 'unknown';
if (affecting.some(s => s.status === 'warning')) return 'ready_with_warnings';
return 'ready';
}
// ── Rollback readiness ───────────────────────────────────────────────────────
export interface RollbackInputs {
backup: { exists: boolean; timestamp: number | null } | Errored;
envSummary: { exists: boolean; envPresent: boolean; keys: string[] } | Errored;
/** Whether the stack currently has an env file (distinguishes "no env to cover"). */
stackHasEnv: boolean | Errored;
/**
* UpdatePreview.rollback_target wrapped in an object so the Errored sentinel
* cannot be absorbed into the string domain (an image literally named
* "error" must not read as a failed preview). `moving` is true when any image
* in the stack uses a moving tag (latest, a branch, an unpinned major/minor),
* in which case restoring files does not revert the image because the local
* tag already resolves to the newer digest.
*/
rollbackTarget: { target: string | null; moving: boolean } | Errored;
/** Timestamp of the most recent deploy_success activity event, if any. */
lastDeployAt: number | null | Errored;
containers: ContainerProbe[] | Errored;
}
export function buildRollbackItems(inputs: RollbackInputs, now: number): RollbackReadinessItem[] {
const items: RollbackReadinessItem[] = [];
const backupExists = inputs.backup !== 'error' && inputs.backup.exists;
if (inputs.backup === 'error') {
items.push({ id: 'compose_source', state: 'unknown', label: 'Previous compose file', detail: 'The backup slot could not be read.' });
} else if (backupExists) {
const age = inputs.backup.timestamp ? ` from ${formatAge(inputs.backup.timestamp, now)}` : '';
items.push({ id: 'compose_source', state: 'ready', label: 'Previous compose file', detail: `A backup${age} is available to restore.` });
} else {
items.push({ id: 'compose_source', state: 'missing', label: 'Previous compose file', detail: 'No backup exists yet. One is created automatically by the next update or deploy.' });
}
if (inputs.envSummary === 'error') {
items.push({ id: 'env_keys', state: 'unknown', label: 'Previous env file', detail: 'The backed-up env file could not be read.' });
} else if (inputs.envSummary.envPresent) {
const n = inputs.envSummary.keys.length;
items.push({ id: 'env_keys', state: 'ready', label: 'Previous env file', detail: `${n} variable name${n === 1 ? '' : 's'} captured in the backup (values are restored with the file, never shown here).` });
} else if (inputs.stackHasEnv === true && backupExists) {
items.push({ id: 'env_keys', state: 'missing', label: 'Previous env file', detail: 'The stack has an env file but the backup does not contain one; a rollback would not restore env changes.' });
} else if (!backupExists) {
items.push({ id: 'env_keys', state: 'missing', label: 'Previous env file', detail: 'No backup exists yet.' });
} else {
items.push({ id: 'env_keys', state: 'ready', label: 'Previous env file', detail: 'The stack uses no env file, so there is nothing to restore.' });
}
if (inputs.rollbackTarget === 'error') {
items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The update preview is unavailable.' });
} else if (inputs.rollbackTarget.target && inputs.rollbackTarget.moving) {
items.push({ id: 'previous_images', state: 'ready', label: 'Previous image tag', detail: `Rollback target ${inputs.rollbackTarget.target}. This stack uses a moving image tag. Full-stack updates capture the running image ID before pull/build and retain it through the recovery window so an immediate restore can retarget the exact prior image, not only the tag string.` });
} else if (inputs.rollbackTarget.target) {
items.push({ id: 'previous_images', state: 'ready', label: 'Previous image tag', detail: `Known rollback target: ${inputs.rollbackTarget.target}. The compose file pins an immutable tag, so restoring files also restores the image.` });
} else {
items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The previous image tag could not be determined. When no prior image is recorded, a restore may fall back to compose and env files alone; a moving tag can still resolve to a newer digest.' });
}
if (inputs.lastDeployAt === 'error') {
items.push({ id: 'last_deploy', state: 'unknown', label: 'Last successful deploy', detail: 'The activity history could not be read.' });
} else if (inputs.lastDeployAt) {
items.push({ id: 'last_deploy', state: 'ready', label: 'Last successful deploy', detail: `Recorded ${formatAge(inputs.lastDeployAt, now)}; the backup reflects a configuration that deployed successfully.` });
} else {
items.push({ id: 'last_deploy', state: 'missing', label: 'Last successful deploy', detail: 'No successful deploy is recorded in the recent activity history.' });
}
if (inputs.containers === 'error') {
items.push({ id: 'healthchecks', state: 'unknown', label: 'Healthchecks', detail: 'Container state could not be read from Docker.' });
} else if (inputs.containers.some(c => c.hasHealthcheck)) {
items.push({ id: 'healthchecks', state: 'ready', label: 'Healthchecks', detail: 'At least one service defines a healthcheck, so a rollback can be verified beyond run state.' });
} else {
items.push({ id: 'healthchecks', state: 'missing', label: 'Healthchecks', detail: 'No service defines a healthcheck; rollback verification relies on run state only.' });
}
const mounts = inputs.containers === 'error'
? []
: [...new Set(inputs.containers.flatMap(c => c.mounts))];
const mountDetail = mounts.length > 0 ? ` This stack mounts: ${mounts.join(', ')}.` : '';
items.push({
id: 'volume_data',
state: 'not_covered',
label: 'Application data',
detail: `Named volumes and bind-mounted data are not included in file backups. Rolling back restores compose and env files only; application data keeps its current state.${mountDetail}`,
});
return items;
}
/**
* compose_source gates not_ready; ready additionally requires env coverage and
* a known previous image tag. volume_data, healthchecks, and last_deploy are
* disclosures and never gate the overall state.
*/
export function aggregateRollbackOverall(items: RollbackReadinessItem[]): RollbackOverall {
const byId = new Map(items.map(i => [i.id, i.state]));
if (byId.get('compose_source') !== 'ready') {
return byId.get('compose_source') === 'unknown' ? 'partial' : 'not_ready';
}
if (byId.get('env_keys') === 'ready' && byId.get('previous_images') === 'ready') {
return 'ready';
}
return 'partial';
}