mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-18 06:23:18 +00:00
feat(recovery): complete authored-project atomic rollback generations (#1819)
* feat(recovery): capture complete authored Compose project for atomic rollback Replace the root-compose-only backup slot with staged recovery generations that record the managed inventory, exact Compose invocation, and prior image identity, and wire the same engine through deploy, update, manual rollback, and Git apply. * fix(recovery): satisfy CodeQL path barriers and update-guard mock Inline resolve+startsWith checks at generation/inventory fs sinks and stub getCurrentStackUpdateRecovery in UpdateGuardService tests. * fix(recovery): drop unused FileSystemService import in generation store test * fix(recovery): harden authored-project rollback for upgrade and restore safety Preserve legacy UUID backup rows, restore Git deploy state with files, make multi-file restore recoverable, evaluate policy on the restored target, and fail closed when Git capture cannot cover an apply. * fix(recovery): unblock Git apply unit tests and CodeQL pre-restore TOCTOU Mock recovery capture in git-source-service tests after fail-closed apply capture, and re-resolve live paths immediately before pre-restore snapshot reads. * fix(recovery): fall back to authored inventory when Git manifesto is missing First Git apply captures before promote, so a missing managed-project manifesto must not block rollback capture when the live stack already has authored files. * fix(recovery): make authored-project rollback atomic across Git state Restore the managed-project manifesto with files, keep nullable Git identity on first-apply captures, persist Git side-state in restore intents for startup reconcile, compensate legacy materialize failures, and refuse directory collisions before mutation. * fix(recovery): satisfy CodeQL path and TOCTOU barriers on manifesto restore Add inline resolve barriers for manifesto read/clear sinks and remove the access-then-read race when restoring a generation manifesto snapshot. * fix(recovery): close third-audit rollback generation blockers Fail closed on incomplete Git inventory fallbacks, execute captured Compose invocation during recovery, refuse startup and mutations while restore intents remain unresolved, propagate legacy stale-delete failures, and add Docker-level exact prior-image coverage plus regression tests. * fix(recovery): mark acquired before handoff in prior-image Docker test Match the production updateStack CAS sequence so the exact prior-image integration test does not fail handoff from the captured phase. * fix(recovery): close fourth-audit rollback safety blockers Evaluate policy against held images, use index-based pre-restore snapshots, hold the shared stack lock across Git apply, replay Mesh and empty captured invocations exactly, restore POSIX modes with fail-closed sensitive permissions, keep case-sensitive paths, and link Git auto-deploy health gates. Add regression coverage for these cases. * test(recovery): fix mocks for health-gate link and authored compose args Add linkGateOrRetain to the Git apply recovery mock, and mock authoredComposeArgs so the case-collision inventory test is not masked by a missing getComposeDir stub. * fix(recovery): close fifth-audit rollback safety blockers Share git_apply locking for webhook auto-apply, fail closed on malformed recovery service records, refuse mixed-image capture, and require exact probe counts with hold-tag eligibility checks. * fix(recovery): close sixth-audit rollback safety blockers Preserve the legacy backup slot during generation capture, encrypt sensitive pre-restore snapshots, revert files on a failed health probe without committing Git, fail closed when an absent-file revert would delete a directory, skip Compose one-offs, route manual and scheduled backup through the current generation, and persist runtime image platform identity. * fix(recovery): close seventh-audit rollback safety blockers Fleet snapshot restore and restore-all now capture a recovery generation under the stack lock before any authored file write, including on remote nodes. * fix(recovery): keep pre-deploy generations during health-gate observe Link deploy recovery generations to the observing gate so backup cannot replace them mid-observe. Distinguish missing hold tags from probe failures, refuse generation release when services metadata is corrupt, classify mixed-replica and coverage refusals, and toast the backend rollback message. * fix(recovery): wrap webhook deploy case for eslint const bindings in an unbraced switch case trip no-case-declarations. Match the pull case block.
This commit is contained in:
Anso
+1631
-1412
File diff suppressed because it is too large
Load Diff
@@ -18,8 +18,11 @@ import {
|
||||
import { readSnapshotFileRow, type SnapshotFileReadResult, type SnapshotFileRow } from '../helpers/snapshotFileDecrypt';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import type { GitSourceManifestState } from '../types/gitProjectManifest';
|
||||
import type { RollbackOperationKind } from '../types/rollbackGeneration';
|
||||
import { collectImageIds, parseServicesJsonStrict } from './recoveryServicesJson';
|
||||
|
||||
export type { SnapshotFileReadResult } from '../helpers/snapshotFileDecrypt';
|
||||
export type { RollbackOperationKind } from '../types/rollbackGeneration';
|
||||
|
||||
function isPilotMode(): boolean {
|
||||
return process.env.SENCHO_MODE === 'pilot';
|
||||
@@ -249,6 +252,10 @@ export interface StackUpdateRecoveryGenerationRow {
|
||||
phase: 'captured' | 'acquired' | 'handoff_committed' | 'reconciling' | 'immediate_verified';
|
||||
is_current: number;
|
||||
backup_slot_id: string | null;
|
||||
/** Generation content key (often equal to backup_slot_id / generation id). */
|
||||
content_path: string | null;
|
||||
/** Capture trigger: update | deployment | git_apply | manual_backup | unknown. */
|
||||
operation_kind: RollbackOperationKind | null;
|
||||
override_path: string | null;
|
||||
services_json: string;
|
||||
health_gate_id: string | null;
|
||||
@@ -1920,6 +1927,9 @@ export class DatabaseService {
|
||||
// pattern used for health_gate_runs below.
|
||||
maybeAddCol('stack_update_recovery_generations', 'released_at', 'INTEGER');
|
||||
maybeAddCol('stack_update_recovery_generations', 'released_by', 'TEXT');
|
||||
// Authored-project generation content key + capture trigger kind.
|
||||
maybeAddCol('stack_update_recovery_generations', 'content_path', 'TEXT');
|
||||
maybeAddCol('stack_update_recovery_generations', 'operation_kind', 'TEXT');
|
||||
maybeAddCol('stack_update_cleanup_pending', 'required_blueprint_id', 'INTEGER');
|
||||
|
||||
// Distributed API model columns
|
||||
@@ -4203,13 +4213,15 @@ export class DatabaseService {
|
||||
public insertStackUpdateRecoveryGeneration(row: StackUpdateRecoveryGenerationRow): void {
|
||||
this.db.prepare(
|
||||
`INSERT INTO stack_update_recovery_generations (
|
||||
id, node_id, stack_name, status, phase, is_current, backup_slot_id, override_path,
|
||||
services_json, health_gate_id, gate_retain_until, artifact_expires_at,
|
||||
operation_lease_expires_at, created_at, updated_at, created_by, artifacts_retired
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
id, node_id, stack_name, status, phase, is_current, backup_slot_id, content_path,
|
||||
operation_kind, override_path, services_json, health_gate_id, gate_retain_until,
|
||||
artifact_expires_at, operation_lease_expires_at, created_at, updated_at,
|
||||
created_by, artifacts_retired
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
).run(
|
||||
row.id, row.node_id, row.stack_name, row.status, row.phase, row.is_current,
|
||||
row.backup_slot_id, row.override_path, row.services_json, row.health_gate_id,
|
||||
row.backup_slot_id, row.content_path ?? null, row.operation_kind ?? null,
|
||||
row.override_path, row.services_json, row.health_gate_id,
|
||||
row.gate_retain_until, row.artifact_expires_at, row.operation_lease_expires_at,
|
||||
row.created_at, row.updated_at, row.created_by, row.artifacts_retired ?? 0,
|
||||
);
|
||||
@@ -4241,7 +4253,8 @@ export class DatabaseService {
|
||||
id: string,
|
||||
patch: Partial<Pick<StackUpdateRecoveryGenerationRow,
|
||||
'status' | 'phase' | 'is_current' | 'override_path' | 'health_gate_id' |
|
||||
'gate_retain_until' | 'artifact_expires_at' | 'operation_lease_expires_at' | 'services_json'>>,
|
||||
'gate_retain_until' | 'artifact_expires_at' | 'operation_lease_expires_at' | 'services_json' |
|
||||
'content_path' | 'operation_kind'>>,
|
||||
): void {
|
||||
const keys = Object.keys(patch) as Array<keyof typeof patch>;
|
||||
if (keys.length === 0) return;
|
||||
@@ -4432,26 +4445,13 @@ export class DatabaseService {
|
||||
).all(nodeId, now, now) as Array<{ services_json: string }>;
|
||||
const ids = new Set<string>();
|
||||
for (const row of rows) {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(row.services_json);
|
||||
if (!Array.isArray(parsed)) continue;
|
||||
for (const item of parsed) {
|
||||
if (!item || typeof item !== 'object') continue;
|
||||
const replicas = (item as { replicas?: unknown }).replicas;
|
||||
if (Array.isArray(replicas)) {
|
||||
for (const replica of replicas) {
|
||||
if (replica && typeof replica === 'object'
|
||||
&& typeof (replica as { imageId?: unknown }).imageId === 'string'
|
||||
&& (replica as { imageId: string }).imageId.trim()) {
|
||||
ids.add((replica as { imageId: string }).imageId);
|
||||
}
|
||||
}
|
||||
} else if (typeof (item as { imageId?: unknown }).imageId === 'string') {
|
||||
ids.add((item as { imageId: string }).imageId);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Corrupt JSON: skip.
|
||||
const parsed = parseServicesJsonStrict(row.services_json);
|
||||
if (!parsed.ok) {
|
||||
// Fail closed: corrupt hold metadata must not look like "nothing held".
|
||||
throw new Error('Malformed stack recovery services_json while listing held images');
|
||||
}
|
||||
for (const id of collectImageIds(parsed.services)) {
|
||||
ids.add(id);
|
||||
}
|
||||
}
|
||||
return [...ids];
|
||||
@@ -6323,6 +6323,25 @@ export class DatabaseService {
|
||||
).run(commitSha, contentHash, Date.now(), stackName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the last-applied revision identity without removing the Git source
|
||||
* row. Used when compensating to a capture that had a null commit SHA
|
||||
* (first-apply preimage).
|
||||
*/
|
||||
public clearGitSourceAppliedRevision(stackName: string): void {
|
||||
this.db.prepare(
|
||||
`UPDATE stack_git_sources SET
|
||||
last_applied_commit_sha = NULL,
|
||||
last_applied_content_hash = NULL,
|
||||
pending_commit_sha = NULL,
|
||||
pending_compose_content = NULL,
|
||||
pending_env_content = NULL,
|
||||
pending_fetched_at = NULL,
|
||||
updated_at = ?
|
||||
WHERE stack_name = ?`
|
||||
).run(Date.now(), stackName);
|
||||
}
|
||||
|
||||
public touchGitSourceDebounce(stackName: string): void {
|
||||
this.db.prepare('UPDATE stack_git_sources SET last_debounce_at = ? WHERE stack_name = ?')
|
||||
.run(Date.now(), stackName);
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
BLUEPRINT_MARKER_FILENAME,
|
||||
parseBlueprintMarker,
|
||||
} from '../helpers/blueprintMarker';
|
||||
import { scrapeRollbackTagsLenient } from './recoveryServicesJson';
|
||||
|
||||
/**
|
||||
* Directory that may contain recovery override files for a tombstone sweep.
|
||||
@@ -84,23 +85,7 @@ function collectArtifactsFromGenerations(
|
||||
|
||||
for (const gen of generations) {
|
||||
if (gen.override_path) overridePaths.add(gen.override_path);
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(gen.services_json);
|
||||
if (!Array.isArray(parsed)) continue;
|
||||
for (const svc of parsed) {
|
||||
if (!svc || typeof svc !== 'object') continue;
|
||||
const replicas = (svc as { replicas?: unknown }).replicas;
|
||||
if (!Array.isArray(replicas)) continue;
|
||||
for (const replica of replicas) {
|
||||
const tag = replica && typeof replica === 'object'
|
||||
? (replica as { rollbackTag?: unknown }).rollbackTag
|
||||
: null;
|
||||
if (typeof tag === 'string' && tag.trim()) tags.add(tag);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Corrupt capture JSON: skip tags for this generation.
|
||||
}
|
||||
for (const tag of scrapeRollbackTagsLenient(gen.services_json)) tags.add(tag);
|
||||
}
|
||||
|
||||
return { tags: [...tags], overridePaths: [...overridePaths] };
|
||||
|
||||
@@ -24,6 +24,7 @@ import { StackFileRootsService } from './StackFileRootsService';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import { ComposeInputDiscoveryService, type ContextCopyPlan, type CopyEntry } from './ComposeInputDiscoveryService';
|
||||
import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs';
|
||||
import { collectManifestFilePaths } from '../helpers/manifestFilePaths';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { isPathWithinBase, isValidStackName } from '../utils/validation';
|
||||
import type {
|
||||
@@ -411,6 +412,34 @@ export class GitProjectManifestService {
|
||||
await fs.promises.rename(tmp, target);
|
||||
}
|
||||
|
||||
/** Raw on-disk manifesto JSON, or null when the file is absent. */
|
||||
async readRawManifestText(stackName: string): Promise<string | null> {
|
||||
// Inline barrier at the readFile sink (CodeQL path-injection).
|
||||
const root = path.resolve(this.managedRoot(stackName));
|
||||
const target = path.resolve(root, MANIFEST_FILENAME);
|
||||
if (!target.startsWith(root + path.sep)) {
|
||||
throw Object.assign(new Error('Path escapes managed project directory'), { code: 'INVALID_PATH' });
|
||||
}
|
||||
try {
|
||||
return await fs.promises.readFile(target, 'utf8');
|
||||
} catch (e) {
|
||||
if ((e as NodeJS.ErrnoException).code === 'ENOENT') return null;
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove the managed manifesto file (first-apply preimage restore). */
|
||||
async clearManifestFile(stackName: string): Promise<void> {
|
||||
// Inline barrier at the rm sink (CodeQL path-injection).
|
||||
const root = path.resolve(this.managedRoot(stackName));
|
||||
const target = path.resolve(root, MANIFEST_FILENAME);
|
||||
if (!target.startsWith(root + path.sep)) {
|
||||
throw Object.assign(new Error('Path escapes managed project directory'), { code: 'INVALID_PATH' });
|
||||
}
|
||||
await fs.promises.rm(target, { force: true });
|
||||
StackFileRootsService.invalidate(NodeRegistry.getInstance().getDefaultNodeId(), stackName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Public projection for the manifest read endpoint: hashes, size metadata,
|
||||
* provenance, and deletion authority are internal-only, and for
|
||||
@@ -700,19 +729,7 @@ export class GitProjectManifestService {
|
||||
|
||||
/** Exact file paths owned by one manifest, excluding directory inventory entries. */
|
||||
private manifestFilePaths(manifest: Pick<GitProjectManifest, 'inputs' | 'buildContexts'>): string[] {
|
||||
const paths = new Map<string, string>();
|
||||
for (const entry of manifest.inputs) {
|
||||
if (entry.ownership !== 'managed' || entry.state !== 'present' || entry.materializedPath === null) continue;
|
||||
if (entry.dependencyKind === 'build-context' || entry.dependencyKind === 'build-additional-context') continue;
|
||||
paths.set(entry.materializedPath.toLowerCase(), entry.materializedPath);
|
||||
}
|
||||
for (const context of manifest.buildContexts) {
|
||||
for (const file of context.files) {
|
||||
const rel = context.repoPath ? `${context.repoPath}/${file.path}` : file.path;
|
||||
paths.set(rel.toLowerCase(), rel);
|
||||
}
|
||||
}
|
||||
return [...paths.values()].sort((a, b) => a.localeCompare(b));
|
||||
return collectManifestFilePaths(manifest);
|
||||
}
|
||||
|
||||
/** Hash one snapshot file, preserving the distinction between missing and unreadable. */
|
||||
|
||||
@@ -19,6 +19,7 @@ import { isPathWithinBase, isValidRelativeStackPath } from '../utils/validation'
|
||||
import { gitSourceLocalComposeFiles, PRIMARY_COMPOSE_FILENAME } from '../utils/gitComposeFiles';
|
||||
import { ComposeInputDiscoveryService, type ContextCopyPlan } from './ComposeInputDiscoveryService';
|
||||
import { GitProjectManifestService } from './GitProjectManifestService';
|
||||
import { StackUpdateRecoveryService } from './StackUpdateRecoveryService';
|
||||
import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs';
|
||||
import type { ComposeInputEntry, GitProjectManifest, GitSourceManifestState, InventoryResult, ManifestSummary, RefusalInfo } from '../types/gitProjectManifest';
|
||||
import type { GitHttpRequest, GitHttpResponse, HttpClient } from 'isomorphic-git/http/node';
|
||||
@@ -1618,11 +1619,7 @@ export class GitSourceService {
|
||||
for (const old of prevSpec.files) {
|
||||
if (old === PRIMARY_COMPOSE_FILENAME || keep.has(old)) continue;
|
||||
if (!isValidRelativeStackPath(old) || old === '') continue;
|
||||
try {
|
||||
await fsSvc.deleteStackPath(stackName, old);
|
||||
} catch (e) {
|
||||
console.warn(`[GitSource] stale file cleanup skipped ${sanitizeForLog(old)} for ${sanitizeForLog(stackName)}:`, (e as Error).message);
|
||||
}
|
||||
await fsSvc.deleteStackPath(stackName, old);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1743,16 +1740,44 @@ export class GitSourceService {
|
||||
stackName: string,
|
||||
commitSha: string,
|
||||
opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean } = {},
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string }> {
|
||||
return this.withStackLock(stackName, () => this.applyLocked(stackName, commitSha, opts));
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> {
|
||||
return this.withStackLock(stackName, () => this.applyWithSharedLock(stackName, commitSha, opts));
|
||||
}
|
||||
|
||||
/** Body of apply(); assumes the caller already holds the per-stack lock. */
|
||||
/**
|
||||
* Acquire the shared stack-operation lock then run applyLocked.
|
||||
* Callers that already hold the Git per-stack mutex (public apply, webhook
|
||||
* auto-apply) use this so capture/promote/handoff/deploy cannot race other
|
||||
* lifecycle ops. Do not nest withStackLock here.
|
||||
*/
|
||||
private async applyWithSharedLock(
|
||||
stackName: string,
|
||||
commitSha: string,
|
||||
opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean },
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> {
|
||||
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
nodeId,
|
||||
stackName,
|
||||
'git_apply',
|
||||
opts.actor ?? 'system:git-source',
|
||||
() => this.applyLocked(stackName, commitSha, opts),
|
||||
);
|
||||
if (!lock.ran) {
|
||||
throw new GitSourceError(
|
||||
'GIT_ERROR',
|
||||
`Another operation (${lock.existing.action}) is already in progress for ${stackName}.`,
|
||||
);
|
||||
}
|
||||
return lock.result;
|
||||
}
|
||||
|
||||
/** Body of apply(); assumes the caller already holds Git mutex + shared stack lock. */
|
||||
private async applyLocked(
|
||||
stackName: string,
|
||||
commitSha: string,
|
||||
opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean },
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string }> {
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> {
|
||||
const diag = isDebugEnabled();
|
||||
const db = DatabaseService.getInstance();
|
||||
const src = db.getGitSource(stackName);
|
||||
@@ -1775,6 +1800,9 @@ export class GitSourceService {
|
||||
? this.crypto.decrypt(src.pending_env_content)
|
||||
: null;
|
||||
const manifestSvc = GitProjectManifestService.getInstance();
|
||||
const recoverySvc = StackUpdateRecoveryService.getInstance();
|
||||
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
let recoveryId: string | undefined;
|
||||
|
||||
let appliedSpec: GitSourceAppliedSpec | null;
|
||||
if (pending.candidateRelPath !== null && pending.inventory !== null) {
|
||||
@@ -1797,7 +1825,7 @@ export class GitSourceService {
|
||||
// The staged candidate must still exist and be complete; a deleted
|
||||
// candidate (or a node restart that swept it) invalidates the pull.
|
||||
const dataDir = process.env.DATA_DIR || path.join(process.cwd(), 'data');
|
||||
const candidateAbs = path.join(dataDir, 'git-managed', String(NodeRegistry.getInstance().getDefaultNodeId()), stackName, pending.candidateRelPath);
|
||||
const candidateAbs = path.join(dataDir, 'git-managed', String(nodeId), stackName, pending.candidateRelPath);
|
||||
try {
|
||||
await fsPromises.access(candidateAbs);
|
||||
} catch {
|
||||
@@ -1872,8 +1900,8 @@ export class GitSourceService {
|
||||
: null;
|
||||
const invocation: string[] = [];
|
||||
try {
|
||||
invocation.push(...(await authoredComposeFileArgs(stackName, NodeRegistry.getInstance().getDefaultNodeId())));
|
||||
invocation.push(...(await authoredComposeEnvFileArgs(stackName, NodeRegistry.getInstance().getDefaultNodeId())));
|
||||
invocation.push(...(await authoredComposeFileArgs(stackName, nodeId)));
|
||||
invocation.push(...(await authoredComposeEnvFileArgs(stackName, nodeId)));
|
||||
} catch (e) {
|
||||
console.warn(`[GitSource] invocation build failed for ${stackName}:`, (e as Error).message);
|
||||
}
|
||||
@@ -1903,6 +1931,25 @@ export class GitSourceService {
|
||||
...(src.applied_deploy_spec?.files ?? [PRIMARY_COMPOSE_FILENAME]),
|
||||
...(src.sync_env ? ['.env'] : []),
|
||||
];
|
||||
try {
|
||||
const candidate = await recoverySvc.captureCandidate({
|
||||
nodeId,
|
||||
stackName,
|
||||
createdBy: opts.actor ?? 'git-source',
|
||||
operationKind: 'git_apply',
|
||||
});
|
||||
recoveryId = candidate.id;
|
||||
} catch (captureError) {
|
||||
const detail = captureError instanceof Error ? captureError.message : String(captureError);
|
||||
console.error(
|
||||
`[GitSource] Recovery capture failed before apply of ${sanitizeForLog(stackName)}:`,
|
||||
detail,
|
||||
);
|
||||
throw new GitSourceError(
|
||||
'GIT_ERROR',
|
||||
`Rollback capture failed before apply; refusing to promote without recovery coverage: ${scrubCredentials(detail)}`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
await manifestSvc.promoteGeneration(stackName, {
|
||||
sha: commitSha,
|
||||
@@ -1917,6 +1964,16 @@ export class GitSourceService {
|
||||
// The original error is logged with its stack for diagnosis,
|
||||
// and the message is scrubbed of credentials and of the
|
||||
// incoming manifest's high-sensitivity paths.
|
||||
if (recoveryId) {
|
||||
try {
|
||||
await recoverySvc.abandon(recoveryId);
|
||||
} catch (abandonError) {
|
||||
console.warn(
|
||||
`[GitSource] Failed to abandon recovery after promote failure for ${sanitizeForLog(stackName)}:`,
|
||||
abandonError instanceof Error ? abandonError.message : String(abandonError),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (e instanceof GitSourceError) throw e;
|
||||
const raw = e instanceof Error ? e.message : String(e);
|
||||
console.error(`[GitSource] promotion failed for ${sanitizeForLog(stackName)}:`, e instanceof Error ? e.stack ?? e.message : raw);
|
||||
@@ -1937,9 +1994,51 @@ export class GitSourceService {
|
||||
if (diag) console.log(`[GitSource:diag] apply validation fail stack=${stackName}`);
|
||||
throw new GitSourceError('GIT_ERROR', `Compose validation failed: ${validation.error}`);
|
||||
}
|
||||
// Capture the true pre-apply project BEFORE materialize writes new files.
|
||||
try {
|
||||
const captured = await recoverySvc.captureCandidate({
|
||||
nodeId,
|
||||
stackName,
|
||||
createdBy: opts.actor ?? 'git-source',
|
||||
operationKind: 'git_apply',
|
||||
});
|
||||
recoveryId = captured.id;
|
||||
} catch (captureError) {
|
||||
const detail = captureError instanceof Error ? captureError.message : String(captureError);
|
||||
console.error(
|
||||
`[GitSource] Recovery capture failed before legacy apply of ${sanitizeForLog(stackName)}:`,
|
||||
detail,
|
||||
);
|
||||
throw new GitSourceError(
|
||||
'GIT_ERROR',
|
||||
`Rollback capture failed before apply; refusing to materialize without recovery coverage: ${scrubCredentials(detail)}`,
|
||||
);
|
||||
}
|
||||
appliedSpec = await this.materialize(
|
||||
stackName, pending.files, pending.contextDir, src.sync_env, envContent, src.applied_deploy_spec,
|
||||
);
|
||||
).catch(async (materializeError: unknown) => {
|
||||
if (recoveryId) {
|
||||
const reverted = await recoverySvc.revertToGenerationContent(recoveryId);
|
||||
if (!reverted) {
|
||||
throw new GitSourceError(
|
||||
'GIT_ERROR',
|
||||
`Legacy materialize failed and pre-apply generation restore also failed: ${scrubCredentials(
|
||||
materializeError instanceof Error ? materializeError.message : String(materializeError),
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
await recoverySvc.abandon(recoveryId);
|
||||
} catch (abandonError) {
|
||||
console.warn(
|
||||
`[GitSource] Failed to abandon recovery after legacy materialize failure for ${sanitizeForLog(stackName)}:`,
|
||||
abandonError instanceof Error ? abandonError.message : String(abandonError),
|
||||
);
|
||||
}
|
||||
recoveryId = undefined;
|
||||
}
|
||||
throw materializeError;
|
||||
});
|
||||
// Migration: build the conservative manifest from spec + disk.
|
||||
const migrated = await manifestSvc.buildMigratedManifest(stackName, {
|
||||
repo_url: src.repo_url,
|
||||
@@ -1958,9 +2057,25 @@ export class GitSourceService {
|
||||
const shouldDeploy = opts.deploy ?? src.auto_deploy_on_apply;
|
||||
if (diag) console.log('[GitSource:diag] apply wrote stack=%s sha=%s deploy=%s', sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), sanitizeForLog(shouldDeploy));
|
||||
|
||||
const finalizeRecoveryCurrent = async (id: string, immediateVerified: boolean): Promise<void> => {
|
||||
if (!recoverySvc.markAcquired(id)) {
|
||||
await recoverySvc.abandon(id);
|
||||
throw new Error('Failed to mark recovery generation as acquired');
|
||||
}
|
||||
if (!recoverySvc.handoff(id, nodeId, stackName)) {
|
||||
await recoverySvc.abandon(id);
|
||||
throw new Error('Failed to hand off recovery generation');
|
||||
}
|
||||
if (!recoverySvc.markReconciling(id)) {
|
||||
throw new Error('Failed to mark recovery generation as reconciling');
|
||||
}
|
||||
if (immediateVerified && !recoverySvc.markImmediateVerified(id)) {
|
||||
console.warn(`[GitSource] Could not CAS immediate_verified for recovery ${sanitizeForLog(id)}`);
|
||||
}
|
||||
};
|
||||
|
||||
if (shouldDeploy) {
|
||||
try {
|
||||
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
await assertPolicyGateAllows(
|
||||
stackName,
|
||||
nodeId,
|
||||
@@ -1969,34 +2084,73 @@ export class GitSourceService {
|
||||
auditPath: `/api/stacks/${stackName}/git-source/apply`,
|
||||
}),
|
||||
);
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
nodeId, stackName, 'deploy', 'system',
|
||||
() => ComposeService.getInstance(nodeId).deployStack(
|
||||
stackName,
|
||||
undefined,
|
||||
undefined,
|
||||
{ source: 'git_apply', actor: opts.actor ?? 'system:git-source' },
|
||||
),
|
||||
);
|
||||
if (!lock.ran) {
|
||||
const busy = `Auto-deploy skipped: another operation (${lock.existing.action}) is already in progress for ${stackName}.`;
|
||||
console.warn(`[GitSource] ${busy}`);
|
||||
return { applied: true, deployed: false, deployError: busy };
|
||||
if (recoveryId) {
|
||||
await finalizeRecoveryCurrent(recoveryId, false);
|
||||
}
|
||||
// Shared stack lock already held as git_apply for capture→deploy.
|
||||
await ComposeService.getInstance(nodeId).deployStack(
|
||||
stackName,
|
||||
undefined,
|
||||
undefined,
|
||||
{ source: 'git_apply', actor: opts.actor ?? 'system:git-source' },
|
||||
);
|
||||
if (recoveryId) {
|
||||
if (!recoverySvc.markImmediateVerified(recoveryId)) {
|
||||
console.warn(`[GitSource] Could not CAS immediate_verified for recovery ${sanitizeForLog(recoveryId)}`);
|
||||
}
|
||||
}
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(
|
||||
nodeId,
|
||||
stackName,
|
||||
'deploy',
|
||||
'system:git-source',
|
||||
);
|
||||
if (recoveryId) {
|
||||
recoverySvc.linkGateOrRetain(recoveryId, healthGateId);
|
||||
}
|
||||
HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:git-source');
|
||||
console.log(`[GitSource] Applied and deployed ${stackName} at ${commitSha.slice(0, 7)}`);
|
||||
return { applied: true, deployed: true };
|
||||
return { applied: true, deployed: true, recoveryId };
|
||||
} catch (e) {
|
||||
// File is on disk, DB is marked applied. Returning the
|
||||
// error separately lets the UI flag it as a partial
|
||||
// success rather than rolling back the disk.
|
||||
// R1: do not auto-compensate. Keep applied files and leave the
|
||||
// pre-promote generation is_current for manual rollback.
|
||||
if (recoveryId) {
|
||||
const row = recoverySvc.get(recoveryId);
|
||||
if (row && row.is_current !== 1) {
|
||||
try {
|
||||
await finalizeRecoveryCurrent(recoveryId, false);
|
||||
} catch (handoffError) {
|
||||
console.warn(
|
||||
`[GitSource] Failed to hand off recovery after deploy failure for ${sanitizeForLog(stackName)}:`,
|
||||
handoffError instanceof Error ? handoffError.message : String(handoffError),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
const scrubbed = scrubCredentials((e as Error).message || String(e));
|
||||
console.error(`[GitSource] Auto-deploy failed for ${stackName}: ${scrubbed}`);
|
||||
return { applied: true, deployed: false, deployError: scrubbed };
|
||||
return { applied: true, deployed: false, deployError: scrubbed, recoveryId };
|
||||
}
|
||||
}
|
||||
|
||||
if (recoveryId) {
|
||||
try {
|
||||
await finalizeRecoveryCurrent(recoveryId, true);
|
||||
} catch (finalizeError) {
|
||||
const detail = finalizeError instanceof Error ? finalizeError.message : String(finalizeError);
|
||||
console.error(
|
||||
`[GitSource] Failed to finalize recovery for apply-only ${sanitizeForLog(stackName)}:`,
|
||||
detail,
|
||||
);
|
||||
return {
|
||||
applied: true,
|
||||
deployed: false,
|
||||
deployError: `Recovery finalization failed after apply: ${scrubCredentials(detail)}`,
|
||||
recoveryId,
|
||||
};
|
||||
}
|
||||
}
|
||||
console.log(`[GitSource] Applied ${stackName} at ${commitSha.slice(0, 7)}`);
|
||||
return { applied: true, deployed: false };
|
||||
return { applied: true, deployed: false, recoveryId };
|
||||
}
|
||||
|
||||
public dismissPending(stackName: string): void {
|
||||
@@ -2369,7 +2523,10 @@ export class GitSourceService {
|
||||
return { status: 'success', message: `Pending update ready at ${pullResult.commitSha.slice(0, 7)}.` };
|
||||
}
|
||||
|
||||
const applied = await this.applyLocked(stackName, pullResult.commitSha, { deploy: src.auto_deploy_on_apply });
|
||||
const applied = await this.applyWithSharedLock(stackName, pullResult.commitSha, {
|
||||
deploy: src.auto_deploy_on_apply,
|
||||
actor: 'system:webhook',
|
||||
});
|
||||
if (applied.deployError) {
|
||||
// Apply wrote to disk but deploy failed. Surface it so the
|
||||
// webhook_executions row records a degraded outcome instead
|
||||
|
||||
@@ -20,6 +20,7 @@ import { applySuppressions } from '../utils/suppression-filter';
|
||||
import { validateImageRef } from '../utils/image-ref';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import type { RollbackInvocationRecord } from '../types/rollbackGeneration';
|
||||
import {
|
||||
evaluatePolicyRisk,
|
||||
describePolicyInputs,
|
||||
@@ -58,6 +59,11 @@ export interface PolicyEnforcementOptions {
|
||||
auditMethod?: string;
|
||||
/** Request path of the originating route; used for audit attribution. */
|
||||
auditPath?: string;
|
||||
/**
|
||||
* Rollback compensation: list images via this captured Compose invocation
|
||||
* instead of the live database-derived args.
|
||||
*/
|
||||
composeInvocation?: RollbackInvocationRecord | null;
|
||||
}
|
||||
|
||||
export interface PolicyEnforcementResult {
|
||||
@@ -286,7 +292,10 @@ export async function enforcePolicyPreDeploy(
|
||||
|
||||
let imageRefs: string[] = [];
|
||||
try {
|
||||
imageRefs = await ComposeService.getInstance(nodeId).listStackImages(stackName);
|
||||
imageRefs = await ComposeService.getInstance(nodeId).listStackImages(
|
||||
stackName,
|
||||
opts.composeInvocation ?? null,
|
||||
);
|
||||
} catch (err) {
|
||||
const message = getErrorMessage(err, 'compose parse failed');
|
||||
console.error('[Policy] listStackImages failed for %s:', sanitizeForLog(stackName), sanitizeForLog(message));
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -8,6 +8,7 @@ import { ComposeService } from './ComposeService';
|
||||
import { StackUpdateOrchestrator } from './StackUpdateOrchestrator';
|
||||
import { StackOpLockService, stackOpSkipMessage as skipMessage } from './StackOpLockService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { StackUpdateRecoveryService } from './StackUpdateRecoveryService';
|
||||
import { HealthGateService } from './HealthGateService';
|
||||
import { ServiceUpdateRecoveryService } from './ServiceUpdateRecoveryService';
|
||||
import {
|
||||
@@ -559,17 +560,21 @@ export class SchedulerService {
|
||||
this.assertStackTarget(task, 'Auto-backup');
|
||||
if (this.isRemoteNode(task.node_id)) {
|
||||
await this.postToRemoteStack(task.node_id, `${encodeURIComponent(task.target_id)}/backup`);
|
||||
return `Backed up stack "${task.target_id}" files on remote node`;
|
||||
return `Captured a recovery generation for stack "${task.target_id}" on remote node`;
|
||||
}
|
||||
const localNodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
localNodeId, task.target_id, 'backup', 'system',
|
||||
() => FileSystemService.getInstance(localNodeId).backupStackFiles(task.target_id),
|
||||
() => StackUpdateRecoveryService.getInstance().captureCurrentBackup({
|
||||
nodeId: localNodeId,
|
||||
stackName: task.target_id,
|
||||
createdBy: 'system:scheduler',
|
||||
}),
|
||||
);
|
||||
// Throw (not return) so the skip records as a failed run instead of a
|
||||
// silent success; the next scheduled tick retries once the lock frees.
|
||||
if (!lock.ran) throw new Error(skipMessage(task.target_id, lock.existing.action));
|
||||
return `Backed up stack "${task.target_id}" files`;
|
||||
return `Captured a recovery generation for stack "${task.target_id}"`;
|
||||
}
|
||||
|
||||
private async executeAutoStop(task: ScheduledTask): Promise<string> {
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
/**
|
||||
* Tracks in-flight stack lifecycle operations (deploy, down, restart, stop,
|
||||
* start, update, rollback, backup) per (nodeId, stackName). A second request to
|
||||
* start, update, rollback, backup, delete, git_apply) per (nodeId, stackName). A second request to
|
||||
* the same stack while the first is still running returns 409 instead of racing
|
||||
* the first. Backup is included because it rewrites the shared rollback slot, so
|
||||
* it must not interleave with a deploy/update/rollback on the same stack.
|
||||
* Git apply holds this lock across capture, promote, handoff, and optional deploy.
|
||||
*
|
||||
* State is intentionally process-local: a Sencho restart clears all locks,
|
||||
* which matches the lifecycle of any in-flight `docker compose` child process.
|
||||
*/
|
||||
|
||||
export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup' | 'delete';
|
||||
export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup' | 'delete' | 'git_apply';
|
||||
|
||||
/**
|
||||
* Note returned by a background path that skipped its operation because a manual
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -241,6 +241,35 @@ export class UpdateGuardService {
|
||||
withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'rollback readiness container probe')),
|
||||
]);
|
||||
|
||||
const { StackUpdateRecoveryService, shortGenerationId } = await import('./StackUpdateRecoveryService');
|
||||
const { assessGenerationEligibility } = await import('./rollbackEligibility');
|
||||
const recoverySvc = StackUpdateRecoveryService.getInstance();
|
||||
const currentGen = recoverySvc.getCurrent(nodeId, stackName);
|
||||
|
||||
const recoveryGeneration = currentGen
|
||||
? { exists: true as const, shortId: shortGenerationId(currentGen.id) }
|
||||
: { exists: false as const };
|
||||
|
||||
const policyEligibility = currentGen
|
||||
? await this.collect('rollback eligibility', stackName, () => assessGenerationEligibility(currentGen))
|
||||
: null;
|
||||
|
||||
const managedInputs = await this.collect('managed inputs', stackName, async () => {
|
||||
const { resolveRollbackInventory } = await import('./rollbackInventory');
|
||||
const inventory = await resolveRollbackInventory(nodeId, stackName);
|
||||
if (inventory.exactCoverage) {
|
||||
return {
|
||||
covered: true,
|
||||
detail: `Exact authored-project coverage includes ${inventory.entries.length} managed path(s).`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
covered: false,
|
||||
detail: inventory.coverageRefusal
|
||||
|| 'Exact authored-project coverage is incomplete for this stack.',
|
||||
};
|
||||
});
|
||||
|
||||
const items = buildRollbackItems({
|
||||
backup,
|
||||
envSummary,
|
||||
@@ -255,15 +284,18 @@ export class UpdateGuardService {
|
||||
},
|
||||
lastDeployAt,
|
||||
containers,
|
||||
recoveryGeneration,
|
||||
policyEligibility: policyEligibility === 'error' ? 'error' : policyEligibility,
|
||||
managedInputs: managedInputs === 'error' ? 'error' : managedInputs,
|
||||
}, now);
|
||||
|
||||
// Partial-revert disclosure for Git-managed stacks: rollback restores only
|
||||
// compose files and .env; the rest of the materialized project is not
|
||||
// reverted by the backup slot. State the scope rather than imply a
|
||||
// complete revert.
|
||||
// Partial-revert disclosure for Git-managed stacks when exact generation
|
||||
// coverage is not available. Prefer generation-backed wording when present.
|
||||
let note: string | undefined;
|
||||
const gitSource = db.getGitSource(stackName);
|
||||
if (gitSource && (gitSource.manifest_state === 'active' || gitSource.manifest_state === 'partial' || gitSource.manifest_state === 'migrated')) {
|
||||
if (currentGen) {
|
||||
note = undefined;
|
||||
} else if (gitSource && (gitSource.manifest_state === 'active' || gitSource.manifest_state === 'partial' || gitSource.manifest_state === 'migrated')) {
|
||||
note = 'This stack is Git-managed. Rollback restores compose files and .env; other materialized inputs are not reverted. Re-apply the previous revision from Git to restore them.';
|
||||
}
|
||||
|
||||
|
||||
@@ -151,20 +151,25 @@ export class WebhookService {
|
||||
nodeId, stackName, lockAction, 'system',
|
||||
async () => {
|
||||
switch (action) {
|
||||
case 'deploy':
|
||||
case 'deploy': {
|
||||
await assertPolicyGateAllows(
|
||||
stackName,
|
||||
nodeId,
|
||||
buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }),
|
||||
);
|
||||
await compose.deployStack(
|
||||
const deployResult = await compose.deployStack(
|
||||
stackName,
|
||||
undefined,
|
||||
atomic,
|
||||
{ source: 'webhook', actor: 'system:webhook' },
|
||||
);
|
||||
HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:webhook');
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:webhook');
|
||||
if (deployResult.recoveryId) {
|
||||
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
||||
StackUpdateRecoveryService.getInstance().linkGateOrRetain(deployResult.recoveryId, healthGateId);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'restart':
|
||||
await compose.runCommand(stackName, 'restart');
|
||||
break;
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
/**
|
||||
* Thin Compose project context for safe full-stack updates.
|
||||
* Shared Compose project context for safe full-stack mutations.
|
||||
*
|
||||
* Wraps the current authored compose argument path and atomic file backup/restore.
|
||||
* When a richer shared Compose project context lands, migrate callers to that type;
|
||||
* this module must not become a competing full-manifest resolver.
|
||||
* Resolves the authored inventory (Git managed-project manifest or live stack
|
||||
* discovery), captures/restores generation content through RollbackGenerationStore,
|
||||
* and builds the exact Compose invocation used for deploy/update/rollback.
|
||||
*/
|
||||
import path from 'path';
|
||||
import { randomUUID } from 'crypto';
|
||||
@@ -11,9 +11,18 @@ import { ComposeService } from './ComposeService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { buildEffectiveServiceModel } from './effectiveServiceModel';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { resolveRollbackInventory } from './rollbackInventory';
|
||||
import { RollbackGenerationStore } from './RollbackGenerationStore';
|
||||
import type {
|
||||
RollbackGenerationManifest,
|
||||
RollbackOperationKind,
|
||||
RollbackRestoreTransactionMeta,
|
||||
} from '../types/rollbackGeneration';
|
||||
|
||||
export type ImageReferenceKind = 'moving_tag' | 'digest_pinned' | 'none';
|
||||
|
||||
export type BackupOperation = RollbackOperationKind;
|
||||
|
||||
const DIGEST_PIN_PATTERN = /@sha256:[a-f0-9]{64}$/i;
|
||||
|
||||
export function classifyReferenceKind(declaredImageRef: string | null): ImageReferenceKind {
|
||||
@@ -34,11 +43,18 @@ export interface ComposeProjectContext {
|
||||
readonly nodeId: number;
|
||||
readonly stackName: string;
|
||||
readonly stackDir: string;
|
||||
/** Generation id used as content-store key and backup_slot_id on the DB row. */
|
||||
backupSlotId: string | null;
|
||||
toComposeArgs(action: string[]): Promise<string[]>;
|
||||
validateForMutation(): Promise<void>;
|
||||
backupFromContext(operation: 'update' | 'deployment'): Promise<string>;
|
||||
restoreFromContext(): Promise<void>;
|
||||
/**
|
||||
* Capture a staged generation. When exactCoverage is required and inventory
|
||||
* refuses it, throws before writing any generation content.
|
||||
*/
|
||||
backupFromContext(operation: BackupOperation): Promise<string>;
|
||||
restoreFromContext(
|
||||
transactionMeta?: RollbackRestoreTransactionMeta,
|
||||
): Promise<RollbackGenerationManifest | void>;
|
||||
resolveServiceImageMap(): Promise<Map<string, string | null>>;
|
||||
}
|
||||
|
||||
@@ -60,15 +76,63 @@ class AuthoredComposeProjectContext implements ComposeProjectContext {
|
||||
await requireRenderableModel(this.nodeId, this.stackName);
|
||||
}
|
||||
|
||||
async backupFromContext(_operation: 'update' | 'deployment'): Promise<string> {
|
||||
await FileSystemService.getInstance(this.nodeId).backupStackFiles(this.stackName);
|
||||
const slotId = randomUUID();
|
||||
this.backupSlotId = slotId;
|
||||
return slotId;
|
||||
async backupFromContext(operation: BackupOperation): Promise<string> {
|
||||
const inventory = await resolveRollbackInventory(this.nodeId, this.stackName);
|
||||
if (!inventory.exactCoverage) {
|
||||
throw Object.assign(
|
||||
new Error(
|
||||
inventory.coverageRefusal
|
||||
|| 'Exact authored-project rollback coverage is unavailable for this stack',
|
||||
),
|
||||
{ code: 'ROLLBACK_COVERAGE_UNAVAILABLE' },
|
||||
);
|
||||
}
|
||||
|
||||
// Do not refresh the legacy single-slot backup. Clearing that reused slot
|
||||
// would destroy a pre-migration recovery point if a later capture step fails.
|
||||
|
||||
const generationId = randomUUID();
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: this.nodeId,
|
||||
stackName: this.stackName,
|
||||
generationId,
|
||||
inventory,
|
||||
operationKind: operation,
|
||||
});
|
||||
this.backupSlotId = generationId;
|
||||
return generationId;
|
||||
}
|
||||
|
||||
async restoreFromContext(): Promise<void> {
|
||||
await FileSystemService.getInstance(this.nodeId).restoreStackFiles(this.stackName);
|
||||
async restoreFromContext(
|
||||
transactionMeta?: RollbackRestoreTransactionMeta,
|
||||
): Promise<RollbackGenerationManifest | void> {
|
||||
const generationId = this.backupSlotId;
|
||||
if (!generationId) {
|
||||
// Legacy pre-migration restore: only when no generation id is bound.
|
||||
await FileSystemService.getInstance(this.nodeId).restoreStackFiles(this.stackName);
|
||||
return;
|
||||
}
|
||||
|
||||
const present = await RollbackGenerationStore.verifyGenerationContent(
|
||||
this.nodeId,
|
||||
this.stackName,
|
||||
generationId,
|
||||
);
|
||||
if (!present) {
|
||||
throw Object.assign(
|
||||
new Error('Recovery generation content is missing or incomplete'),
|
||||
{ code: 'GENERATION_CONTENT_MISSING' },
|
||||
);
|
||||
}
|
||||
|
||||
const inventory = await resolveRollbackInventory(this.nodeId, this.stackName);
|
||||
return RollbackGenerationStore.restoreGeneration(
|
||||
this.nodeId,
|
||||
this.stackName,
|
||||
generationId,
|
||||
inventory.entries.map((e) => e.relativePath),
|
||||
transactionMeta,
|
||||
);
|
||||
}
|
||||
|
||||
async resolveServiceImageMap(): Promise<Map<string, string | null>> {
|
||||
@@ -89,6 +153,17 @@ export async function resolveComposeProjectContext(
|
||||
return new AuthoredComposeProjectContext(nodeId, stackName, stackDir);
|
||||
}
|
||||
|
||||
/** Bind an existing generation id onto a fresh context for restore. */
|
||||
export async function resolveComposeProjectContextForGeneration(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
generationId: string,
|
||||
): Promise<ComposeProjectContext> {
|
||||
const ctx = await resolveComposeProjectContext(nodeId, stackName);
|
||||
ctx.backupSlotId = generationId;
|
||||
return ctx;
|
||||
}
|
||||
|
||||
export function describeContextError(error: unknown): string {
|
||||
return getErrorMessage(error, 'Compose project context failed');
|
||||
}
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
/**
|
||||
* Structural validation for stack-update recovery services_json payloads.
|
||||
* Fail closed on any unexpected shape so eligibility, compensate, and probe
|
||||
* share one authoritative parser.
|
||||
*/
|
||||
import type { ImageReferenceKind } from './composeProjectContext';
|
||||
|
||||
export interface StackRecoveryReplicaCapture {
|
||||
containerId: string | null;
|
||||
imageId: string | null;
|
||||
repoDigest: string | null;
|
||||
state: 'running' | 'stopped' | 'none';
|
||||
rollbackTag: string | null;
|
||||
}
|
||||
|
||||
export interface StackRecoveryServiceCapture {
|
||||
serviceName: string;
|
||||
/** Observed running replica count at capture (supported restore scale). */
|
||||
scale: number;
|
||||
hasBuild: boolean;
|
||||
declaredImageRef: string | null;
|
||||
referenceKind: ImageReferenceKind;
|
||||
replicas: StackRecoveryReplicaCapture[];
|
||||
}
|
||||
|
||||
export type ParsedServicesJson =
|
||||
| { ok: true; services: StackRecoveryServiceCapture[] }
|
||||
| { ok: false };
|
||||
|
||||
const REPLICA_STATES = new Set(['running', 'stopped', 'none']);
|
||||
const REFERENCE_KINDS = new Set<ImageReferenceKind>(['moving_tag', 'digest_pinned', 'none']);
|
||||
|
||||
function isNonEmptyString(v: unknown): v is string {
|
||||
return typeof v === 'string' && v.trim().length > 0;
|
||||
}
|
||||
|
||||
/** Accepts null/undefined/string; rejects any other type. */
|
||||
function isNullableString(v: unknown): v is string | null | undefined {
|
||||
return v === null || v === undefined || typeof v === 'string';
|
||||
}
|
||||
|
||||
function asNullableString(v: unknown): string | null {
|
||||
return typeof v === 'string' ? v : null;
|
||||
}
|
||||
|
||||
function parseReplicaCapture(raw: unknown): StackRecoveryReplicaCapture | null {
|
||||
if (!raw || typeof raw !== 'object') return null;
|
||||
const r = raw as Record<string, unknown>;
|
||||
if (!isNullableString(r.containerId)
|
||||
|| !isNullableString(r.imageId)
|
||||
|| !isNullableString(r.repoDigest)
|
||||
|| !isNullableString(r.rollbackTag)) {
|
||||
return null;
|
||||
}
|
||||
if (typeof r.state !== 'string' || !REPLICA_STATES.has(r.state)) return null;
|
||||
|
||||
const state = r.state as StackRecoveryReplicaCapture['state'];
|
||||
const imageId = asNullableString(r.imageId);
|
||||
// Running/stopped replicas must carry a protectable image identity.
|
||||
if ((state === 'running' || state === 'stopped') && !imageId?.trim()) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
containerId: asNullableString(r.containerId),
|
||||
imageId,
|
||||
repoDigest: asNullableString(r.repoDigest),
|
||||
state,
|
||||
rollbackTag: asNullableString(r.rollbackTag),
|
||||
};
|
||||
}
|
||||
|
||||
function parseServiceCapture(raw: unknown): StackRecoveryServiceCapture | null {
|
||||
if (!raw || typeof raw !== 'object') return null;
|
||||
const s = raw as Record<string, unknown>;
|
||||
if (!isNonEmptyString(s.serviceName)) return null;
|
||||
if (typeof s.scale !== 'number' || !Number.isInteger(s.scale) || s.scale < 0) return null;
|
||||
if (typeof s.hasBuild !== 'boolean') return null;
|
||||
if (!isNullableString(s.declaredImageRef)) return null;
|
||||
if (typeof s.referenceKind !== 'string' || !REFERENCE_KINDS.has(s.referenceKind as ImageReferenceKind)) {
|
||||
return null;
|
||||
}
|
||||
if (!Array.isArray(s.replicas)) return null;
|
||||
|
||||
const replicas: StackRecoveryReplicaCapture[] = [];
|
||||
for (const item of s.replicas) {
|
||||
const replica = parseReplicaCapture(item);
|
||||
if (!replica) return null;
|
||||
replicas.push(replica);
|
||||
}
|
||||
return {
|
||||
serviceName: s.serviceName,
|
||||
scale: s.scale,
|
||||
hasBuild: s.hasBuild,
|
||||
declaredImageRef: asNullableString(s.declaredImageRef),
|
||||
referenceKind: s.referenceKind as ImageReferenceKind,
|
||||
replicas,
|
||||
};
|
||||
}
|
||||
|
||||
/** Strict structural validation for recovery services_json (fail closed). */
|
||||
export function parseServicesJsonStrict(raw: string): ParsedServicesJson {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) return { ok: false };
|
||||
const services: StackRecoveryServiceCapture[] = [];
|
||||
for (const item of parsed) {
|
||||
const svc = parseServiceCapture(item);
|
||||
if (!svc) return { ok: false };
|
||||
services.push(svc);
|
||||
}
|
||||
return { ok: true, services };
|
||||
} catch {
|
||||
return { ok: false };
|
||||
}
|
||||
}
|
||||
|
||||
/** Lenient parse for callers that treat empty as no services (legacy). Prefer strict. */
|
||||
export function parseServicesJson(raw: string): StackRecoveryServiceCapture[] {
|
||||
const parsed = parseServicesJsonStrict(raw);
|
||||
return parsed.ok ? parsed.services : [];
|
||||
}
|
||||
|
||||
export function collectImageIds(services: StackRecoveryServiceCapture[]): string[] {
|
||||
const ids = new Set<string>();
|
||||
for (const svc of services) {
|
||||
for (const replica of svc.replicas) {
|
||||
if (replica.imageId?.trim()) ids.add(replica.imageId);
|
||||
}
|
||||
}
|
||||
return [...ids];
|
||||
}
|
||||
|
||||
export function collectImageIdsFromServicesJson(servicesJson: string): string[] {
|
||||
return collectImageIds(parseServicesJson(servicesJson));
|
||||
}
|
||||
|
||||
export function collectRollbackTags(services: StackRecoveryServiceCapture[]): string[] {
|
||||
const tags = new Set<string>();
|
||||
for (const svc of services) {
|
||||
for (const replica of svc.replicas) {
|
||||
if (replica.rollbackTag?.trim()) tags.add(replica.rollbackTag);
|
||||
}
|
||||
}
|
||||
return [...tags];
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort rollbackTag scrape for cleanup paths. Prefer strict parse; on
|
||||
* structural failure walk nested objects for string rollbackTag fields so
|
||||
* opaque holds are still removed when possible.
|
||||
*/
|
||||
export function scrapeRollbackTagsLenient(raw: string): string[] {
|
||||
const strict = parseServicesJsonStrict(raw);
|
||||
if (strict.ok) return collectRollbackTags(strict.services);
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
const tags = new Set<string>();
|
||||
const walk = (value: unknown): void => {
|
||||
if (!value || typeof value !== 'object') return;
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) walk(item);
|
||||
return;
|
||||
}
|
||||
const obj = value as Record<string, unknown>;
|
||||
if (typeof obj.rollbackTag === 'string' && obj.rollbackTag.trim()) {
|
||||
tags.add(obj.rollbackTag);
|
||||
}
|
||||
for (const nested of Object.values(obj)) walk(nested);
|
||||
};
|
||||
walk(parsed);
|
||||
return [...tags];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
/**
|
||||
* Rollback restore eligibility (fail closed on known-bad evidence).
|
||||
*
|
||||
* Pure evaluateRollbackEligibility maps known signals to a verdict.
|
||||
* assessGenerationEligibility gathers best-effort evidence for a recovery row.
|
||||
*/
|
||||
import type { StackUpdateRecoveryGenerationRow } from './DatabaseService';
|
||||
import DockerController from './DockerController';
|
||||
import { enforcePolicyForImageRefs } from './PolicyEnforcement';
|
||||
import {
|
||||
collectImageIds,
|
||||
collectRollbackTags,
|
||||
parseServicesJsonStrict,
|
||||
} from './recoveryServicesJson';
|
||||
import { RollbackGenerationStore } from './RollbackGenerationStore';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
|
||||
type HeldImagesParse =
|
||||
| { ok: true; ids: string[]; rollbackTags: string[] }
|
||||
| { ok: false };
|
||||
|
||||
function parseHeldImageState(servicesJson: string): HeldImagesParse {
|
||||
const parsed = parseServicesJsonStrict(servicesJson);
|
||||
if (!parsed.ok) return { ok: false };
|
||||
return {
|
||||
ok: true,
|
||||
ids: collectImageIds(parsed.services),
|
||||
rollbackTags: collectRollbackTags(parsed.services),
|
||||
};
|
||||
}
|
||||
|
||||
export type RollbackEligibilityVerdict =
|
||||
| 'eligible'
|
||||
| 'eligible_with_warning'
|
||||
| 'prohibited'
|
||||
| 'unknown';
|
||||
|
||||
export interface RollbackEligibilityInput {
|
||||
/** null = unknown */
|
||||
generationIntegrityOk: boolean | null;
|
||||
heldImagesPresent: boolean | null;
|
||||
/** true when known blocked; null = unknown */
|
||||
securityPostureBlocked: boolean | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rules (fail closed on known bad):
|
||||
* - securityPostureBlocked === true → prohibited
|
||||
* - generationIntegrityOk === false → prohibited
|
||||
* - heldImagesPresent === false → eligible_with_warning
|
||||
* - any remaining null → unknown (unless already prohibited)
|
||||
* - else eligible
|
||||
*/
|
||||
export function evaluateRollbackEligibility(
|
||||
input: RollbackEligibilityInput,
|
||||
): RollbackEligibilityVerdict {
|
||||
if (input.securityPostureBlocked === true || input.generationIntegrityOk === false) {
|
||||
return 'prohibited';
|
||||
}
|
||||
if (input.heldImagesPresent === false) return 'eligible_with_warning';
|
||||
if (
|
||||
input.generationIntegrityOk === null
|
||||
|| input.heldImagesPresent === null
|
||||
|| input.securityPostureBlocked === null
|
||||
) {
|
||||
return 'unknown';
|
||||
}
|
||||
return 'eligible';
|
||||
}
|
||||
|
||||
async function checkGenerationIntegrity(
|
||||
row: StackUpdateRecoveryGenerationRow,
|
||||
): Promise<boolean | null> {
|
||||
// Only explicit content_path generations use the content store.
|
||||
const contentKey = row.content_path;
|
||||
if (!contentKey) return null;
|
||||
try {
|
||||
return await RollbackGenerationStore.verifyGenerationContent(
|
||||
row.node_id,
|
||||
row.stack_name,
|
||||
contentKey,
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[RollbackEligibility] Integrity check failed for %s: %s',
|
||||
sanitizeForLog(row.id),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function inspectImagePresent(
|
||||
docker: ReturnType<ReturnType<typeof DockerController.getInstance>['getDocker']>,
|
||||
ref: string,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
await docker.getImage(ref).inspect();
|
||||
return true;
|
||||
} catch (error) {
|
||||
const status = (error as { statusCode?: number }).statusCode;
|
||||
const message = getErrorMessage(error, '').toLowerCase();
|
||||
if (status === 404 || message.includes('no such image') || message.includes('not found')) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Held recovery launch requires both underlying image ids and opaque rollback
|
||||
* tags used by the recovery override to still resolve locally.
|
||||
*/
|
||||
async function checkHeldImagesPresent(
|
||||
row: StackUpdateRecoveryGenerationRow,
|
||||
held: HeldImagesParse,
|
||||
): Promise<boolean | null> {
|
||||
if (!held.ok) return null;
|
||||
const refs = [...new Set([...held.ids, ...held.rollbackTags])];
|
||||
if (refs.length === 0) return true;
|
||||
try {
|
||||
const docker = DockerController.getInstance(row.node_id).getDocker();
|
||||
for (const ref of refs) {
|
||||
if (!(await inspectImagePresent(docker, ref))) return false;
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[RollbackEligibility] Docker check failed for %s: %s',
|
||||
sanitizeForLog(row.id),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function checkSecurityPostureBlocked(
|
||||
row: StackUpdateRecoveryGenerationRow,
|
||||
held: HeldImagesParse,
|
||||
): Promise<boolean | null> {
|
||||
if (!held.ok) return null;
|
||||
const refs = [...new Set([...held.ids, ...held.rollbackTags])];
|
||||
if (refs.length === 0) return false;
|
||||
try {
|
||||
const gate = await enforcePolicyForImageRefs(row.stack_name, row.node_id, refs, {
|
||||
bypass: false,
|
||||
actor: 'rollback-eligibility',
|
||||
auditMethod: 'GET',
|
||||
auditPath: '/api/stacks/rollback-eligibility',
|
||||
});
|
||||
return !gate.ok;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[RollbackEligibility] Security posture check failed for %s: %s',
|
||||
sanitizeForLog(row.id),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Best-effort eligibility for a recovery generation row. */
|
||||
export async function assessGenerationEligibility(
|
||||
row: StackUpdateRecoveryGenerationRow,
|
||||
): Promise<RollbackEligibilityVerdict> {
|
||||
const held = parseHeldImageState(row.services_json);
|
||||
// Malformed recovery state cannot be assessed safely; refuse restore.
|
||||
if (!held.ok) return 'prohibited';
|
||||
const generationIntegrityOk = await checkGenerationIntegrity(row);
|
||||
const heldImagesPresent = await checkHeldImagesPresent(row, held);
|
||||
const securityPostureBlocked = await checkSecurityPostureBlocked(row, held);
|
||||
return evaluateRollbackEligibility({
|
||||
generationIntegrityOk,
|
||||
heldImagesPresent,
|
||||
securityPostureBlocked,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,565 @@
|
||||
/**
|
||||
* Resolve the authored-project file set that an atomic rollback generation
|
||||
* must capture. Git-managed stacks consume the managed-project manifest;
|
||||
* authored stacks rediscover against the live stack directory.
|
||||
*/
|
||||
import { promises as fsPromises, readFileSync } from 'fs';
|
||||
import path from 'path';
|
||||
import { DatabaseService, type StackGitSource } from './DatabaseService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { GitProjectManifestService } from './GitProjectManifestService';
|
||||
import { collectManifestFilePaths } from '../helpers/manifestFilePaths';
|
||||
import { isHostAbsolutePath, parseDeclaredInputs } from '../helpers/composeInputParse';
|
||||
import { isValidRelativeStackPath, isValidStackName } from '../utils/validation';
|
||||
import { authoredComposeEnvFileArgs, authoredComposeFileArgs } from '../utils/authoredComposeArgs';
|
||||
import type {
|
||||
ComposeInputEntry,
|
||||
GitProjectManifest,
|
||||
InputSensitivity,
|
||||
} from '../types/gitProjectManifest';
|
||||
import type {
|
||||
ResolvedRollbackInventory,
|
||||
RollbackEntryKind,
|
||||
RollbackEntryProvenance,
|
||||
RollbackInvocationRecord,
|
||||
} from '../types/rollbackGeneration';
|
||||
|
||||
const ROOT_COMPOSE_FILENAMES = [
|
||||
'compose.yaml',
|
||||
'compose.yml',
|
||||
'docker-compose.yaml',
|
||||
'docker-compose.yml',
|
||||
] as const;
|
||||
|
||||
const DOT_ENV = '.' + 'env';
|
||||
|
||||
const COMPOSE_INVOCATION_KINDS = new Set<RollbackEntryKind>([
|
||||
'compose-root',
|
||||
'implicit-override',
|
||||
'explicit',
|
||||
'include',
|
||||
'extends',
|
||||
]);
|
||||
|
||||
type InventoryAccum = {
|
||||
relativePath: string;
|
||||
dependencyKind: RollbackEntryKind;
|
||||
provenance: RollbackEntryProvenance;
|
||||
sensitivity: InputSensitivity;
|
||||
absolutePath: string | null;
|
||||
};
|
||||
|
||||
function posixRel(rel: string): string {
|
||||
return rel.replace(/\\/g, '/').replace(/^\.\//, '');
|
||||
}
|
||||
|
||||
function foldedKey(rel: string): string {
|
||||
return posixRel(rel).toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Prefer exact POSIX paths as map keys so case-distinct Linux paths are kept.
|
||||
* When two different paths collide under case-folding, record a refusal note
|
||||
* instead of silently merging them.
|
||||
*/
|
||||
function upsertEntry(
|
||||
map: Map<string, InventoryAccum>,
|
||||
foldedOwners: Map<string, string>,
|
||||
entry: InventoryAccum,
|
||||
caseCollisions: string[],
|
||||
): void {
|
||||
const exact = posixRel(entry.relativePath);
|
||||
const folded = foldedKey(exact);
|
||||
const owner = foldedOwners.get(folded);
|
||||
if (owner !== undefined && owner !== exact) {
|
||||
caseCollisions.push(`Case-colliding managed paths "${owner}" and "${exact}"`);
|
||||
return;
|
||||
}
|
||||
const prev = map.get(exact);
|
||||
if (!prev) {
|
||||
map.set(exact, { ...entry, relativePath: exact });
|
||||
foldedOwners.set(folded, exact);
|
||||
return;
|
||||
}
|
||||
if (prev.absolutePath !== null && entry.absolutePath === null) return;
|
||||
map.set(exact, {
|
||||
...entry,
|
||||
relativePath: exact,
|
||||
absolutePath: entry.absolutePath ?? prev.absolutePath,
|
||||
dependencyKind: prev.dependencyKind === 'compose-root' && entry.dependencyKind !== 'compose-root'
|
||||
? entry.dependencyKind
|
||||
: entry.dependencyKind,
|
||||
sensitivity: higherSensitivity(entry.sensitivity, prev.sensitivity),
|
||||
});
|
||||
}
|
||||
|
||||
function resolveStackRoot(fsSvc: FileSystemService, stackName: string): string {
|
||||
if (!isValidStackName(stackName)) {
|
||||
throw Object.assign(new Error('Invalid stack name'), { code: 'INVALID_STACK_NAME' });
|
||||
}
|
||||
// Canonical js/path-injection barrier: resolve + startsWith.
|
||||
// CodeQL does not credit isPathWithinBase helpers at later sinks.
|
||||
const base = path.resolve(fsSvc.getBaseDir());
|
||||
const stackRoot = path.resolve(base, stackName);
|
||||
if (!stackRoot.startsWith(base + path.sep)) {
|
||||
throw Object.assign(new Error('Stack name escapes compose directory'), { code: 'INVALID_PATH' });
|
||||
}
|
||||
return stackRoot;
|
||||
}
|
||||
|
||||
function resolveStackRel(
|
||||
stackRoot: string,
|
||||
relRaw: string,
|
||||
): { relativePath: string; absolutePath: string } | null {
|
||||
const relativePath = posixRel(relRaw);
|
||||
if (!relativePath || !isValidRelativeStackPath(relativePath)) return null;
|
||||
// Join-time containment; callers still re-check at each fs sink.
|
||||
const baseResolved = path.resolve(stackRoot);
|
||||
const absolutePath = path.resolve(baseResolved, relativePath);
|
||||
if (!absolutePath.startsWith(baseResolved + path.sep)) return null;
|
||||
return { relativePath, absolutePath };
|
||||
}
|
||||
|
||||
async function pathExistsAsFile(stackRoot: string, relativePath: string): Promise<boolean> {
|
||||
// Inline barrier at the lstat sink.
|
||||
const baseResolved = path.resolve(stackRoot);
|
||||
const abs = path.resolve(baseResolved, relativePath);
|
||||
if (!abs.startsWith(baseResolved + path.sep)) return false;
|
||||
try {
|
||||
const st = await fsPromises.lstat(abs);
|
||||
return st.isFile() || st.isSymbolicLink();
|
||||
} catch (e) {
|
||||
if ((e as NodeJS.ErrnoException).code === 'ENOENT') return false;
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
function authoredSensitivity(kind: RollbackEntryKind): InputSensitivity {
|
||||
if (kind === 'secret' || kind === 'config' || kind === 'build-secret') return 'high';
|
||||
if (
|
||||
kind === 'env_file'
|
||||
|| kind === 'include-env'
|
||||
|| kind === 'interpolation-env'
|
||||
|| kind === 'sync-env'
|
||||
|| kind === 'project-env'
|
||||
|| kind === 'label_file'
|
||||
) {
|
||||
return 'medium';
|
||||
}
|
||||
return 'low';
|
||||
}
|
||||
|
||||
function higherSensitivity(a: InputSensitivity, b: InputSensitivity): InputSensitivity {
|
||||
if (a === 'high' || b === 'high') return 'high';
|
||||
if (a === 'medium' || b === 'medium') return 'medium';
|
||||
return 'low';
|
||||
}
|
||||
|
||||
function appliedDeploySpecString(
|
||||
spec: { files: string[]; contextDir: string | null } | null | undefined,
|
||||
): string | null {
|
||||
if (!spec) return null;
|
||||
return JSON.stringify(spec);
|
||||
}
|
||||
|
||||
function refusedGitInventory(
|
||||
gitSource: StackGitSource,
|
||||
emptyInvocation: RollbackInvocationRecord,
|
||||
coverageRefusal: string,
|
||||
manifestVersion: number | null = gitSource.manifest_version,
|
||||
): ResolvedRollbackInventory {
|
||||
return {
|
||||
entries: [],
|
||||
invocation: emptyInvocation,
|
||||
git: {
|
||||
repoUrl: gitSource.repo_url,
|
||||
branch: gitSource.branch,
|
||||
commitSha: gitSource.last_applied_commit_sha || '',
|
||||
manifestVersion,
|
||||
},
|
||||
appliedDeploySpec: appliedDeploySpecString(gitSource.applied_deploy_spec),
|
||||
lastAppliedContentHash: gitSource.last_applied_content_hash,
|
||||
manifestState: gitSource.manifest_state,
|
||||
manifestGeneration: gitSource.manifest_generation,
|
||||
exactCoverage: false,
|
||||
coverageRefusal,
|
||||
};
|
||||
}
|
||||
|
||||
function isGitManifest(
|
||||
value: GitProjectManifest | { corrupt: string } | null,
|
||||
): value is GitProjectManifest {
|
||||
return value !== null && !('corrupt' in value);
|
||||
}
|
||||
|
||||
function sensitivityForManifestPath(
|
||||
manifest: GitProjectManifest,
|
||||
rel: string,
|
||||
): { kind: RollbackEntryKind; sensitivity: InputSensitivity; provenance: RollbackEntryProvenance } {
|
||||
const key = foldedKey(rel);
|
||||
const input = manifest.inputs.find(
|
||||
(i: ComposeInputEntry) => i.materializedPath !== null && foldedKey(i.materializedPath) === key,
|
||||
);
|
||||
if (input) {
|
||||
return {
|
||||
kind: input.dependencyKind,
|
||||
sensitivity: input.sensitivity,
|
||||
provenance: input.provenance,
|
||||
};
|
||||
}
|
||||
return { kind: 'other', sensitivity: 'low', provenance: 'fetch' };
|
||||
}
|
||||
|
||||
async function resolveGitInventory(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
stackRoot: string,
|
||||
): Promise<ResolvedRollbackInventory | null> {
|
||||
const gitSource = DatabaseService.getInstance().getGitSource(stackName);
|
||||
if (!gitSource) return null;
|
||||
|
||||
const emptyInvocation: RollbackInvocationRecord = {
|
||||
composeArgsPrefix: [],
|
||||
projectDirectory: null,
|
||||
projectName: stackName,
|
||||
explicitComposeFiles: [],
|
||||
meshOverrideRelativePath: null,
|
||||
meshEnabled: false,
|
||||
};
|
||||
|
||||
const read = await GitProjectManifestService.getInstance().readManifest(
|
||||
stackName,
|
||||
gitSource.repo_url,
|
||||
gitSource.branch,
|
||||
);
|
||||
if (!isGitManifest(read)) {
|
||||
const corrupt = Boolean(read && 'corrupt' in read);
|
||||
const reason = corrupt
|
||||
? `Managed-project manifest is unreadable (${(read as { corrupt: string }).corrupt}). Fix or re-link the Git source before capturing rollback coverage.`
|
||||
: 'Managed-project manifest is missing. Pull or re-link the Git source before capturing rollback coverage.';
|
||||
|
||||
const established = Boolean(
|
||||
gitSource.applied_deploy_spec
|
||||
|| gitSource.last_applied_content_hash
|
||||
|| gitSource.last_applied_commit_sha,
|
||||
);
|
||||
|
||||
// Established missing/corrupt manifesto: fail closed. applied_deploy_spec
|
||||
// alone cannot claim exact managed-input coverage (includes, extends, env,
|
||||
// labels, configs, secrets, and build inputs are omitted).
|
||||
if (established || corrupt) {
|
||||
return refusedGitInventory(gitSource, emptyInvocation, reason);
|
||||
}
|
||||
|
||||
// First apply (no applied revision yet): signal incomplete Git coverage so
|
||||
// resolveRollbackInventory can merge authored disk files with this identity.
|
||||
return refusedGitInventory(gitSource, emptyInvocation, reason, null);
|
||||
}
|
||||
|
||||
const map = new Map<string, InventoryAccum>();
|
||||
const foldedOwners = new Map<string, string>();
|
||||
const caseCollisions: string[] = [];
|
||||
for (const rel of collectManifestFilePaths(read)) {
|
||||
const resolved = resolveStackRel(stackRoot, rel);
|
||||
if (!resolved) continue;
|
||||
const meta = sensitivityForManifestPath(read, resolved.relativePath);
|
||||
const exists = await pathExistsAsFile(stackRoot, resolved.relativePath);
|
||||
upsertEntry(map, foldedOwners, {
|
||||
relativePath: resolved.relativePath,
|
||||
dependencyKind: meta.kind,
|
||||
provenance: meta.provenance,
|
||||
sensitivity: meta.sensitivity,
|
||||
absolutePath: exists ? resolved.absolutePath : null,
|
||||
}, caseCollisions);
|
||||
}
|
||||
|
||||
const refused = read.refusals.length > 0
|
||||
|| read.counts.refused > 0
|
||||
|| read.state === 'unsupported'
|
||||
|| read.state === 'partial'
|
||||
|| caseCollisions.length > 0;
|
||||
const coverageRefusal = refused
|
||||
? (caseCollisions[0]
|
||||
?? read.refusals[0]?.reason
|
||||
?? `Managed-project manifest state "${read.state}" does not claim exact coverage`)
|
||||
: null;
|
||||
|
||||
let meshEnabled = false;
|
||||
let meshReadFailed: string | null = null;
|
||||
try {
|
||||
meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(nodeId, stackName);
|
||||
} catch (e) {
|
||||
meshReadFailed = `Could not read Mesh enablement: ${(e as Error).message}`;
|
||||
}
|
||||
|
||||
const invocation: RollbackInvocationRecord = {
|
||||
composeArgsPrefix: [...read.project.invocation],
|
||||
projectDirectory: read.project.effectiveProjectDir,
|
||||
projectName: read.project.projectName || stackName,
|
||||
explicitComposeFiles: [...read.project.composeFiles],
|
||||
meshOverrideRelativePath: null,
|
||||
meshEnabled,
|
||||
};
|
||||
|
||||
const meshRefused = meshReadFailed !== null;
|
||||
return {
|
||||
entries: [...map.values()].sort((a, b) => a.relativePath.localeCompare(b.relativePath)),
|
||||
invocation,
|
||||
git: {
|
||||
repoUrl: read.repo.url,
|
||||
branch: read.repo.branch,
|
||||
commitSha: read.resolvedRevision.commitSha || gitSource.last_applied_commit_sha || '',
|
||||
manifestVersion: read.manifestVersion,
|
||||
},
|
||||
appliedDeploySpec: appliedDeploySpecString(gitSource.applied_deploy_spec),
|
||||
lastAppliedContentHash: gitSource.last_applied_content_hash,
|
||||
manifestState: gitSource.manifest_state,
|
||||
manifestGeneration: gitSource.manifest_generation,
|
||||
exactCoverage: !refused && !meshRefused,
|
||||
coverageRefusal: coverageRefusal ?? meshReadFailed,
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveAuthoredInventory(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
stackRoot: string,
|
||||
fsSvc: FileSystemService,
|
||||
): Promise<ResolvedRollbackInventory> {
|
||||
const map = new Map<string, InventoryAccum>();
|
||||
const foldedOwners = new Map<string, string>();
|
||||
const coverageNotes: string[] = [];
|
||||
const caseCollisions: string[] = [];
|
||||
|
||||
const composePaths: string[] = [];
|
||||
for (const name of ROOT_COMPOSE_FILENAMES) {
|
||||
const resolved = resolveStackRel(stackRoot, name);
|
||||
if (!resolved) continue;
|
||||
if (!(await pathExistsAsFile(stackRoot, resolved.relativePath))) continue;
|
||||
composePaths.push(resolved.relativePath);
|
||||
upsertEntry(map, foldedOwners, {
|
||||
relativePath: resolved.relativePath,
|
||||
dependencyKind: 'compose-root',
|
||||
provenance: 'authored',
|
||||
sensitivity: 'low',
|
||||
absolutePath: resolved.absolutePath,
|
||||
}, caseCollisions);
|
||||
}
|
||||
|
||||
const overrideName = await fsSvc.getOverrideFilename(stackName);
|
||||
if (overrideName) {
|
||||
const resolved = resolveStackRel(stackRoot, overrideName);
|
||||
if (resolved && await pathExistsAsFile(stackRoot, resolved.relativePath)) {
|
||||
if (!composePaths.includes(resolved.relativePath)) {
|
||||
composePaths.push(resolved.relativePath);
|
||||
}
|
||||
upsertEntry(map, foldedOwners, {
|
||||
relativePath: resolved.relativePath,
|
||||
dependencyKind: 'implicit-override',
|
||||
provenance: 'authored',
|
||||
sensitivity: 'low',
|
||||
absolutePath: resolved.absolutePath,
|
||||
}, caseCollisions);
|
||||
}
|
||||
}
|
||||
|
||||
const envCandidates = new Set<string>([DOT_ENV]);
|
||||
for (const f of DatabaseService.getInstance().getStackProjectEnvFiles(nodeId, stackName)) {
|
||||
envCandidates.add(posixRel(f));
|
||||
}
|
||||
for (const envFile of envCandidates) {
|
||||
const resolved = resolveStackRel(stackRoot, envFile);
|
||||
if (!resolved) continue;
|
||||
if (!(await pathExistsAsFile(stackRoot, resolved.relativePath))) continue;
|
||||
upsertEntry(map, foldedOwners, {
|
||||
relativePath: resolved.relativePath,
|
||||
dependencyKind: envFile === DOT_ENV ? 'interpolation-env' : 'project-env',
|
||||
provenance: 'authored',
|
||||
sensitivity: authoredSensitivity(envFile === DOT_ENV ? 'interpolation-env' : 'project-env'),
|
||||
absolutePath: resolved.absolutePath,
|
||||
}, caseCollisions);
|
||||
}
|
||||
|
||||
const readCallback = (repoPath: string): string | null => {
|
||||
const relativePath = posixRel(repoPath);
|
||||
if (!relativePath || !isValidRelativeStackPath(relativePath)) return null;
|
||||
// Inline barrier at the readFileSync sink.
|
||||
const baseResolved = path.resolve(stackRoot);
|
||||
const abs = path.resolve(baseResolved, relativePath);
|
||||
if (!abs.startsWith(baseResolved + path.sep)) return null;
|
||||
try {
|
||||
return readFileSync(abs, 'utf8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
if (composePaths.length > 0) {
|
||||
const orderedContents: Array<{ path: string; content: string }> = [];
|
||||
for (const rel of composePaths) {
|
||||
if (!isValidRelativeStackPath(rel)) continue;
|
||||
// Inline barrier at the readFile sink (same form as readCallback).
|
||||
const baseResolved = path.resolve(stackRoot);
|
||||
const abs = path.resolve(baseResolved, rel);
|
||||
if (!abs.startsWith(baseResolved + path.sep)) continue;
|
||||
try {
|
||||
const content = await fsPromises.readFile(abs, 'utf8');
|
||||
orderedContents.push({ path: rel, content });
|
||||
} catch (e) {
|
||||
coverageNotes.push(`Could not read compose file ${rel}: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (orderedContents.length > 0) {
|
||||
const parsed = parseDeclaredInputs(orderedContents, {
|
||||
projectRoot: null,
|
||||
read: readCallback,
|
||||
});
|
||||
|
||||
if (parsed.parseErrors.length > 0) {
|
||||
coverageNotes.push(...parsed.parseErrors);
|
||||
}
|
||||
if (parsed.dynamic.length > 0) {
|
||||
coverageNotes.push(
|
||||
`${parsed.dynamic.length} dynamic path declaration(s) cannot be captured exactly`,
|
||||
);
|
||||
}
|
||||
|
||||
for (const input of parsed.inputs) {
|
||||
const hostAbs = (input.sourcePath !== null && isHostAbsolutePath(input.sourcePath))
|
||||
|| input.baseDir === 'host'
|
||||
|| input.materializedPath === null;
|
||||
|
||||
if (hostAbs) {
|
||||
if (input.kind === 'include' || input.kind === 'extends') {
|
||||
coverageNotes.push(
|
||||
`Host-absolute ${input.kind} path cannot be captured for exact rollback`,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const candidate = input.materializedPath ?? input.sourcePath;
|
||||
if (!candidate) continue;
|
||||
const resolved = resolveStackRel(stackRoot, candidate);
|
||||
if (!resolved) continue;
|
||||
if (!(await pathExistsAsFile(stackRoot, resolved.relativePath))) continue;
|
||||
|
||||
const kind = input.kind;
|
||||
upsertEntry(map, foldedOwners, {
|
||||
relativePath: resolved.relativePath,
|
||||
dependencyKind: kind,
|
||||
provenance: 'authored',
|
||||
sensitivity: authoredSensitivity(kind),
|
||||
absolutePath: resolved.absolutePath,
|
||||
}, caseCollisions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let composeArgsPrefix: string[] = [];
|
||||
try {
|
||||
composeArgsPrefix = [
|
||||
...authoredComposeFileArgs(stackName, nodeId),
|
||||
...(await authoredComposeEnvFileArgs(stackName, nodeId)),
|
||||
];
|
||||
} catch (e) {
|
||||
coverageNotes.push(`Could not build compose invocation args: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
const explicitComposeFiles = [...map.values()]
|
||||
.filter((e) => COMPOSE_INVOCATION_KINDS.has(e.dependencyKind))
|
||||
.map((e) => e.relativePath)
|
||||
.sort((a, b) => a.localeCompare(b));
|
||||
|
||||
if (caseCollisions.length > 0) {
|
||||
coverageNotes.push(caseCollisions[0]);
|
||||
}
|
||||
|
||||
let meshEnabled = false;
|
||||
try {
|
||||
meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(nodeId, stackName);
|
||||
} catch (e) {
|
||||
coverageNotes.push(`Could not read Mesh enablement: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
const exactCoverage = coverageNotes.length === 0 && composePaths.length > 0;
|
||||
const coverageRefusal = exactCoverage
|
||||
? null
|
||||
: (coverageNotes[0]
|
||||
?? (composePaths.length === 0
|
||||
? 'No compose file found in the stack directory'
|
||||
: 'Exact coverage unavailable'));
|
||||
|
||||
return {
|
||||
entries: [...map.values()].sort((a, b) => a.relativePath.localeCompare(b.relativePath)),
|
||||
invocation: {
|
||||
composeArgsPrefix,
|
||||
projectDirectory: null,
|
||||
projectName: stackName,
|
||||
explicitComposeFiles: explicitComposeFiles.length > 0 ? explicitComposeFiles : composePaths,
|
||||
meshOverrideRelativePath: null,
|
||||
meshEnabled,
|
||||
},
|
||||
git: null,
|
||||
appliedDeploySpec: null,
|
||||
lastAppliedContentHash: null,
|
||||
manifestState: null,
|
||||
manifestGeneration: null,
|
||||
exactCoverage,
|
||||
coverageRefusal,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Prefer an exact Git-managed inventory when available. When the manifesto is
|
||||
* missing on a true first apply (no applied revision yet), merge authored disk
|
||||
* discovery with the Git identity fields so capture preserves nullable Git
|
||||
* state. Established missing/corrupt manifesto cases fail closed via
|
||||
* resolveGitInventory and are not overwritten by authored exactCoverage.
|
||||
*/
|
||||
export async function resolveRollbackInventory(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
): Promise<ResolvedRollbackInventory> {
|
||||
const fsSvc = FileSystemService.getInstance(nodeId);
|
||||
const stackRoot = resolveStackRoot(fsSvc, stackName);
|
||||
|
||||
try {
|
||||
const gitInventory = await resolveGitInventory(nodeId, stackName, stackRoot);
|
||||
if (gitInventory?.exactCoverage) return gitInventory;
|
||||
|
||||
const authored = await resolveAuthoredInventory(nodeId, stackName, stackRoot, fsSvc);
|
||||
|
||||
if (gitInventory && !gitInventory.exactCoverage) {
|
||||
const established = Boolean(
|
||||
gitInventory.appliedDeploySpec
|
||||
|| gitInventory.lastAppliedContentHash
|
||||
|| gitInventory.git?.commitSha,
|
||||
);
|
||||
const firstApplyCorrupt = Boolean(gitInventory.coverageRefusal?.includes('unreadable'));
|
||||
// Established or corrupt first-apply: fail closed. Otherwise merge Git
|
||||
// identity onto authored exact coverage for a true first apply.
|
||||
if (established || firstApplyCorrupt || !authored.exactCoverage) {
|
||||
return gitInventory;
|
||||
}
|
||||
return {
|
||||
...authored,
|
||||
git: gitInventory.git,
|
||||
appliedDeploySpec: gitInventory.appliedDeploySpec,
|
||||
lastAppliedContentHash: gitInventory.lastAppliedContentHash,
|
||||
manifestState: gitInventory.manifestState,
|
||||
manifestGeneration: gitInventory.manifestGeneration,
|
||||
};
|
||||
}
|
||||
|
||||
if (authored.exactCoverage) return authored;
|
||||
return gitInventory ?? authored;
|
||||
} catch (e) {
|
||||
console.error(
|
||||
'[rollbackInventory] Failed to resolve inventory:',
|
||||
(e as Error).message,
|
||||
);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
@@ -91,6 +91,18 @@ const RULES: ClassifierRule[] = [
|
||||
suggestion: 'Review the compose file syntax (Compose Doctor can pinpoint the issue), then retry.',
|
||||
pattern: /yaml:|mapping values are not allowed|cannot unmarshal|additional propert|undefined volume|undefined network|invalid compose/i,
|
||||
},
|
||||
{
|
||||
reason: 'mixed_replica_images',
|
||||
label: 'Mixed replica images',
|
||||
suggestion: 'Bring every replica of the service onto the same image, then retry.',
|
||||
pattern: /mixed replica images/i,
|
||||
},
|
||||
{
|
||||
reason: 'rollback_coverage_unavailable',
|
||||
label: 'Exact rollback coverage unavailable',
|
||||
suggestion: 'Remove host-absolute include or extends paths, or capture from a project Sencho can enumerate completely, then retry.',
|
||||
pattern: /cannot be captured for exact rollback|rollback coverage is unavailable/i,
|
||||
},
|
||||
];
|
||||
|
||||
const UNKNOWN_FAILURE: FailureClassification = {
|
||||
|
||||
@@ -331,13 +331,55 @@ export interface RollbackInputs {
|
||||
/** Timestamp of the most recent deploy_success activity event, if any. */
|
||||
lastDeployAt: number | null | Errored;
|
||||
containers: ContainerProbe[] | Errored;
|
||||
/**
|
||||
* Current recovery generation, when one exists. When present, compose_source
|
||||
* readiness is driven by this generation rather than the legacy backup slot.
|
||||
*/
|
||||
recoveryGeneration: { exists: boolean; shortId?: string } | Errored | null;
|
||||
/** Eligibility verdict for the current generation, when assessed. */
|
||||
policyEligibility: 'eligible' | 'eligible_with_warning' | 'prohibited' | 'unknown' | Errored | null;
|
||||
/** Whether managed authored inputs are covered by exact inventory. */
|
||||
managedInputs: { covered: boolean; detail: string } | Errored | null;
|
||||
}
|
||||
|
||||
export function buildRollbackItems(inputs: RollbackInputs, now: number): RollbackReadinessItem[] {
|
||||
const items: RollbackReadinessItem[] = [];
|
||||
|
||||
const recoveryRaw = inputs.recoveryGeneration;
|
||||
const recoveryInfo = recoveryRaw !== null && recoveryRaw !== 'error' ? recoveryRaw : null;
|
||||
const recoveryExists = !!recoveryInfo?.exists;
|
||||
const backupExists = inputs.backup !== 'error' && inputs.backup.exists;
|
||||
if (inputs.backup === 'error') {
|
||||
|
||||
if (recoveryRaw === 'error') {
|
||||
items.push({ id: 'recovery_generation', state: 'unknown', label: 'Recovery generation', detail: 'The recovery generation could not be read.' });
|
||||
} else if (recoveryInfo?.exists) {
|
||||
const short = recoveryInfo.shortId;
|
||||
items.push({
|
||||
id: 'recovery_generation',
|
||||
state: 'ready',
|
||||
label: 'Recovery generation',
|
||||
detail: short
|
||||
? `Current recovery generation ${short} is available for exact restore.`
|
||||
: 'A current recovery generation is available for exact restore.',
|
||||
});
|
||||
} else {
|
||||
items.push({
|
||||
id: 'recovery_generation',
|
||||
state: 'missing',
|
||||
label: 'Recovery generation',
|
||||
detail: 'No recovery generation is current yet. One is created by the next update, atomic deploy, or Git apply.',
|
||||
});
|
||||
}
|
||||
|
||||
// Supersede rule: when a recovery generation exists, compose_source tracks it.
|
||||
if (recoveryExists) {
|
||||
items.push({
|
||||
id: 'compose_source',
|
||||
state: 'ready',
|
||||
label: 'Previous compose file',
|
||||
detail: 'Authored project files are covered by the current recovery generation.',
|
||||
});
|
||||
} else if (inputs.backup === 'error') {
|
||||
items.push({ id: 'compose_source', state: 'unknown', label: 'Previous compose file', detail: 'The backup slot could not be read.' });
|
||||
} else if (backupExists) {
|
||||
const age = inputs.backup.timestamp ? ` from ${formatAge(inputs.backup.timestamp, now)}` : '';
|
||||
@@ -385,6 +427,30 @@ export function buildRollbackItems(inputs: RollbackInputs, now: number): Rollbac
|
||||
items.push({ id: 'healthchecks', state: 'missing', label: 'Healthchecks', detail: 'No service defines a healthcheck; rollback verification relies on run state only.' });
|
||||
}
|
||||
|
||||
if (inputs.policyEligibility === 'error') {
|
||||
items.push({ id: 'policy_eligibility', state: 'unknown', label: 'Rollback eligibility', detail: 'Eligibility could not be assessed.' });
|
||||
} else if (inputs.policyEligibility === null) {
|
||||
items.push({ id: 'policy_eligibility', state: 'ready', label: 'Rollback eligibility', detail: 'No recovery generation to assess yet.' });
|
||||
} else if (inputs.policyEligibility === 'eligible') {
|
||||
items.push({ id: 'policy_eligibility', state: 'ready', label: 'Rollback eligibility', detail: 'Restore is eligible with known-good generation integrity and held images.' });
|
||||
} else if (inputs.policyEligibility === 'eligible_with_warning') {
|
||||
items.push({ id: 'policy_eligibility', state: 'warning', label: 'Rollback eligibility', detail: 'Restore is possible but held images may be missing; moving-tag recoverability is weak.' });
|
||||
} else if (inputs.policyEligibility === 'prohibited') {
|
||||
items.push({ id: 'policy_eligibility', state: 'blocked', label: 'Rollback eligibility', detail: 'Restore is prohibited until generation integrity or security posture is repaired.' });
|
||||
} else {
|
||||
items.push({ id: 'policy_eligibility', state: 'unknown', label: 'Rollback eligibility', detail: 'Eligibility is not fully known yet.' });
|
||||
}
|
||||
|
||||
if (inputs.managedInputs === 'error') {
|
||||
items.push({ id: 'managed_inputs', state: 'unknown', label: 'Managed inputs', detail: 'Managed input coverage could not be read.' });
|
||||
} else if (inputs.managedInputs === null) {
|
||||
items.push({ id: 'managed_inputs', state: 'unknown', label: 'Managed inputs', detail: 'Managed input coverage has not been assessed.' });
|
||||
} else if (inputs.managedInputs.covered) {
|
||||
items.push({ id: 'managed_inputs', state: 'ready', label: 'Managed inputs', detail: inputs.managedInputs.detail });
|
||||
} else {
|
||||
items.push({ id: 'managed_inputs', state: 'warning', label: 'Managed inputs', detail: inputs.managedInputs.detail });
|
||||
}
|
||||
|
||||
const mounts = inputs.containers === 'error'
|
||||
? []
|
||||
: [...new Set(inputs.containers.flatMap(c => c.mounts))];
|
||||
@@ -393,7 +459,7 @@ export function buildRollbackItems(inputs: RollbackInputs, now: number): Rollbac
|
||||
id: 'volume_data',
|
||||
state: 'not_covered',
|
||||
label: 'Application data',
|
||||
detail: `Named volumes and bind-mounted data are not included in file backups. Rolling back restores compose and env files only; application data keeps its current state.${mountDetail}`,
|
||||
detail: `Named volumes and bind-mounted data are not included in recovery generations. Rolling back restores the managed authored project (compose files, overrides, includes/extends, env and related inputs), Git identity when captured, and prior image holds; application data keeps its current state.${mountDetail}`,
|
||||
});
|
||||
|
||||
return items;
|
||||
@@ -406,9 +472,16 @@ export function buildRollbackItems(inputs: RollbackInputs, now: number): Rollbac
|
||||
*/
|
||||
export function aggregateRollbackOverall(items: RollbackReadinessItem[]): RollbackOverall {
|
||||
const byId = new Map(items.map(i => [i.id, i.state]));
|
||||
if (byId.get('policy_eligibility') === 'blocked') {
|
||||
return 'not_ready';
|
||||
}
|
||||
if (byId.get('compose_source') !== 'ready') {
|
||||
return byId.get('compose_source') === 'unknown' ? 'partial' : 'not_ready';
|
||||
}
|
||||
const policy = byId.get('policy_eligibility');
|
||||
if (policy === 'unknown' || policy === 'warning') {
|
||||
return 'partial';
|
||||
}
|
||||
if (byId.get('env_keys') === 'ready' && byId.get('previous_images') === 'ready') {
|
||||
return 'ready';
|
||||
}
|
||||
|
||||
@@ -35,10 +35,10 @@ export interface UpdateReadinessReport {
|
||||
}
|
||||
|
||||
/** State of one rollback readiness item. */
|
||||
export type RollbackItemState = 'ready' | 'missing' | 'unknown' | 'not_covered';
|
||||
export type RollbackItemState = 'ready' | 'missing' | 'unknown' | 'not_covered' | 'blocked' | 'warning';
|
||||
|
||||
export interface RollbackReadinessItem {
|
||||
id: 'compose_source' | 'env_keys' | 'previous_images' | 'last_deploy' | 'healthchecks' | 'volume_data';
|
||||
id: 'compose_source' | 'env_keys' | 'previous_images' | 'last_deploy' | 'healthchecks' | 'volume_data' | 'policy_eligibility' | 'managed_inputs' | 'recovery_generation';
|
||||
state: RollbackItemState;
|
||||
label: string;
|
||||
/** Names only for env coverage; values never appear here. */
|
||||
@@ -117,6 +117,8 @@ export type FailureReason =
|
||||
| 'healthcheck_failed'
|
||||
| 'dependency_unavailable'
|
||||
| 'node_unreachable'
|
||||
| 'mixed_replica_images'
|
||||
| 'rollback_coverage_unavailable'
|
||||
| 'unknown';
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user