Files
sencho/backend/src/services/UpdateGuardService.ts
T
Anso 78475d96ef fix(compose-doctor): resolve effective healthcheck coverage (#1713)
* fix(compose-doctor): resolve effective healthcheck coverage

Compose Doctor now classifies healthcheck coverage from the Compose model, running containers, and local images so image-provided HEALTHCHECKs are not false positives. Update Guard shares the same presence helper so test NONE is not treated as active.

* fix(compose-doctor): fix healthcheck project label and empty compose HC

Use the Compose project name for runtime container listing so stacks whose name: differs from the directory still get runtime evidence. Treat empty or timing-only healthcheck objects as absent rather than active.

* fix(compose-doctor): treat inherited healthcheck as All Clear note

Inherited image healthchecks no longer block All Clear; they surface under a notes section and cannot be acknowledged.
2026-07-28 14:26:00 -04:00

288 lines
13 KiB
TypeScript

import si from 'systeminformation';
import DockerController from './DockerController';
import { DatabaseService } from './DatabaseService';
import { FileSystemService } from './FileSystemService';
import { ComposeDoctorService } from './ComposeDoctorService';
import { UpdatePreviewService, isMovingTag, filterPreviewForService, buildDetectionDisabledPreview } from './UpdatePreviewService';
import { ImageUpdateService } from './ImageUpdateService';
import { buildEffectiveServiceModel, type EffectiveServiceModelResult } from './effectiveServiceModel';
import { filterContainersByComposeService } from '../helpers/composeServiceMatch';
import { isDockerHealthcheckActive } from '../helpers/healthcheckPresence';
import { withTimeout } from '../utils/withTimeout';
import { getErrorMessage } from '../utils/errors';
import { sanitizeForLog } from '../utils/safeLog';
import {
aggregateRollbackOverall,
aggregateVerdict,
backupSlotSignal,
buildRollbackItems,
buildServicesSignal,
containersSignal,
diskSignal,
driftSignal,
healthchecksSignal,
isTroubledContainer,
preflightSignal,
serviceSignal,
updatePreviewSignal,
type Errored,
type ServiceMembership,
} from './updateGuard/readiness';
import type { ContainerProbe, RollbackReadinessReport, UpdateReadinessReport } from './updateGuard/types';
// Bound on the network-and-socket-backed inputs (container probe, update
// preview, disk stats) so a hung registry or Docker socket cannot stall the
// report past the dialog's own fetch timeout; the remaining inputs are local
// DB/file reads. A timed-out input degrades to its 'unknown' signal instead
// of failing the report.
const INPUT_TIMEOUT_MS = 3_000;
/**
* Thrown by `computeUpdateReadiness` when a `serviceName` is given for a
* stack that has one (or zero) declared services: service-scoped readiness
* requires a real choice among siblings. The route maps this to a 400,
* distinct from the 500 a genuine computation failure gets.
*/
export class SingleServiceUpdateReadinessError extends Error {
constructor() {
super('Service-scoped readiness requires a stack with more than one service.');
this.name = 'SingleServiceUpdateReadinessError';
}
}
/**
* Computes update readiness and rollback readiness for a stack, on demand,
* from existing per-feature stores (preflight runs, drift findings, the atomic
* backup slot, the update preview, live Docker state). Derived data only;
* nothing here is persisted.
*/
export class UpdateGuardService {
private static instance: UpdateGuardService;
public static getInstance(): UpdateGuardService {
if (!UpdateGuardService.instance) {
UpdateGuardService.instance = new UpdateGuardService();
}
return UpdateGuardService.instance;
}
/**
* Probe the stack's containers via the compose project label, normalized for
* the pure scoring functions. Throws on Docker errors; callers map that to
* the 'error' sentinel. When `serviceName` is given, narrows to that
* service's containers before inspecting (labeled containers, per §6).
*/
async probeContainers(nodeId: number, stackName: string, serviceName?: string): Promise<ContainerProbe[]> {
const docker = DockerController.getInstance(nodeId).getDocker();
const listed = await docker.listContainers({
all: true,
filters: { label: [`com.docker.compose.project=${stackName}`] },
});
const scoped = serviceName ? filterContainersByComposeService(listed, serviceName) : listed;
const probes = await Promise.all(
scoped.map(async (info): Promise<ContainerProbe | null> => {
const name = info.Names?.[0]?.replace(/^\//, '') ?? info.Id.slice(0, 12);
let inspect: Awaited<ReturnType<ReturnType<typeof docker.getContainer>['inspect']>>;
try {
inspect = await docker.getContainer(info.Id).inspect();
} catch (e: unknown) {
// A container removed between list and inspect (auto-heal or update
// churn) should not collapse the whole probe; skip just that one.
if ((e as { statusCode?: number })?.statusCode === 404) return null;
throw e;
}
const mounts = (inspect.Mounts ?? []).map(m =>
m.Type === 'volume' ? `volume ${m.Name ?? 'unnamed'}` : `${m.Type} ${m.Source ?? ''}`.trim(),
);
return {
name,
state: inspect.State?.Status ?? info.State ?? 'unknown',
health: inspect.State?.Health?.Status ?? null,
exitCode: typeof inspect.State?.ExitCode === 'number' ? inspect.State.ExitCode : null,
hasHealthcheck: isDockerHealthcheckActive(inspect.Config?.Healthcheck?.Test),
restartPolicy: inspect.HostConfig?.RestartPolicy?.Name || null,
mounts,
};
}),
);
return probes.filter((p): p is ContainerProbe => p !== null);
}
/**
* When `serviceName` is set, scopes the verdict-affecting inputs (labeled
* containers, healthchecks, the update preview, and drift) to that service
* only; the stack guardrails (preflight, disk, backup, Docker reachability)
* stay unchanged (§6). A render failure or missing service fails closed via
* the added `service` signal; a single-service stack throws instead of
* scoping (the caller/route maps that to a 400).
*/
async computeUpdateReadiness(nodeId: number, stackName: string, serviceName?: string): Promise<UpdateReadinessReport> {
const db = DatabaseService.getInstance();
const now = Date.now();
let model: EffectiveServiceModelResult | null = null;
if (serviceName) {
model = await buildEffectiveServiceModel(nodeId, stackName);
if (model.renderable && model.services.length <= 1) {
throw new SingleServiceUpdateReadinessError();
}
}
const [preflight, drift, containers, siblings, preview, backup, disk] = await Promise.all([
this.collect('preflight', stackName, async () => ComposeDoctorService.getInstance().getLatest(nodeId, stackName)),
this.collect('drift', stackName, async () =>
db.getOpenDriftFindings(nodeId, stackName).filter(f => !serviceName || f.service === serviceName).length),
this.collect('containers', stackName, () =>
withTimeout(this.probeContainers(nodeId, stackName, serviceName), INPUT_TIMEOUT_MS, 'readiness container probe')),
serviceName
? this.collect('sibling containers', stackName, () =>
withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness sibling probe'))
: Promise.resolve<ContainerProbe[] | Errored>([]),
this.collect('update preview', stackName, async () => {
// Check inside the thunk so the read stays with the getPreview call.
// Stack GET/POST update-preview use the same isChecksEnabled gate.
if (!ImageUpdateService.isChecksEnabled()) {
const disabled = buildDetectionDisabledPreview(stackName);
return serviceName ? filterPreviewForService(disabled, serviceName) : disabled;
}
const full = await withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness update preview');
return serviceName ? filterPreviewForService(full, serviceName) : full;
}),
this.collect('backup info', stackName, () => FileSystemService.getInstance(nodeId).getBackupInfo(stackName)),
this.collect('disk', stackName, () => this.readDiskUsage()),
]);
const settings = db.getGlobalSettings();
const limitPercent = parseInt(settings['host_disk_limit'] ?? '90', 10) || 90;
const signals = [
preflightSignal(preflight),
driftSignal(drift),
containersSignal(containers),
healthchecksSignal(containers),
updatePreviewSignal(preview === 'error' ? 'error' : preview.summary, preview === 'error' ? undefined : preview.images),
buildServicesSignal(preview === 'error' ? 'error' : preview.build_services),
backupSlotSignal(backup, now),
diskSignal(typeof disk === 'number' ? { usePercent: disk, limitPercent } : 'error'),
];
if (serviceName) {
signals.push(serviceSignal(this.resolveServiceMembership(model, serviceName)));
}
const advisories = serviceName
? [
this.siblingHealthAdvisory(containers, siblings),
this.dependencyAdvisory(model, serviceName),
].filter((note): note is string => note !== null)
: [];
return {
stack: stackName,
computedAt: now,
verdict: aggregateVerdict(signals),
signals,
serviceName: serviceName ?? null,
advisories,
};
}
/** Model-membership facts for the selected service; 'error' when the model failed to render. */
private resolveServiceMembership(model: EffectiveServiceModelResult | null, serviceName: string): ServiceMembership | Errored {
if (!model || !model.renderable) return 'error';
const spec = model.services.find(s => s.name === serviceName);
if (!spec) return { found: false };
return { found: true, hasBuild: spec.hasBuild, declaredImage: spec.declaredImage, expectedReplicas: spec.expectedReplicas };
}
/** Advisory only (§6): a sibling already unhealthy before the update never blocks the selected service's verdict. */
private siblingHealthAdvisory(selected: ContainerProbe[] | Errored, all: ContainerProbe[] | Errored): string | null {
if (selected === 'error' || all === 'error') return null;
const selectedNames = new Set(selected.map(c => c.name));
const troubled = all.filter(c => !selectedNames.has(c.name) && isTroubledContainer(c));
if (troubled.length === 0) return null;
const names = troubled.map(c => c.name).join(', ');
return `Other containers in this stack are already unhealthy: ${names}. This does not block the selected service.`;
}
/** Advisory only (§6): dependsOn relationships are informational; this update never recreates siblings. */
private dependencyAdvisory(model: EffectiveServiceModelResult | null, serviceName: string): string | null {
if (!model || !model.renderable) return null;
const spec = model.services.find(s => s.name === serviceName);
const dependsOn = spec?.dependsOn ?? [];
const dependents = model.services.filter(s => s.dependsOn.includes(serviceName)).map(s => s.name);
const parts: string[] = [];
if (dependsOn.length > 0) parts.push(`depends on ${dependsOn.join(', ')}`);
if (dependents.length > 0) parts.push(`is a dependency of ${dependents.join(', ')}`);
if (parts.length === 0) return null;
return `This service ${parts.join(' and ')}. Those services are not restarted by this update.`;
}
async computeRollbackReadiness(nodeId: number, stackName: string): Promise<RollbackReadinessReport> {
const db = DatabaseService.getInstance();
const fsSvc = FileSystemService.getInstance(nodeId);
const now = Date.now();
const [backup, envSummary, stackHasEnv, preview, lastDeployAt, containers] = await Promise.all([
this.collect('backup info', stackName, () => fsSvc.getBackupInfo(stackName)),
this.collect('backup env summary', stackName, () => fsSvc.getBackupEnvSummary(stackName)),
this.collect('stack env presence', stackName, () => fsSvc.envExists(stackName)),
this.collect('update preview', stackName, async () => {
if (!ImageUpdateService.isChecksEnabled()) {
return buildDetectionDisabledPreview(stackName);
}
return withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'rollback readiness update preview');
}),
this.collect('activity history', stackName, async () => {
const events = db.getStackActivity(nodeId, stackName, { limit: 50 });
// A successful update is as good a known-good marker as a deploy.
return events.find(e => e.category === 'deploy_success' || e.category === 'image_update_applied')?.timestamp ?? null;
}),
this.collect('containers', stackName, () =>
withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'rollback readiness container probe')),
]);
const items = buildRollbackItems({
backup,
envSummary,
stackHasEnv,
rollbackTarget: preview === 'error'
? 'error'
: {
target: preview.rollback_target,
// Any image on a moving tag means restoring files cannot guarantee
// the image reverts, so the rollback target is not a true revert.
moving: preview.images.some(img => isMovingTag(img.current_tag)),
},
lastDeployAt,
containers,
}, now);
return { stack: stackName, computedAt: now, overall: aggregateRollbackOverall(items), items };
}
/** Host disk use percent for the main filesystem, or null when unavailable. */
private async readDiskUsage(): Promise<number | null> {
const fsSize = await withTimeout(si.fsSize(), INPUT_TIMEOUT_MS, 'readiness disk stats');
const mainDisk = fsSize.find(fs => fs.mount === '/' || fs.mount === 'C:') || fsSize[0];
if (typeof mainDisk?.use !== 'number') {
console.warn('[UpdateGuard] disk stats returned no usable mount; disk signal degrades to unknown');
return null;
}
return mainDisk.use;
}
/** Run one input collector; a failure degrades to the 'error' sentinel. */
private async collect<T>(label: string, stackName: string, fn: () => Promise<T>): Promise<T | Errored> {
try {
return await fn();
} catch (error) {
console.warn(
'[UpdateGuard] %s unavailable for %s:',
label, sanitizeForLog(stackName),
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
return 'error';
}
}
}