mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-09-02 21:58:06 +00:00
feat(recovery): complete authored-project atomic rollback generations (#1819)
* feat(recovery): capture complete authored Compose project for atomic rollback Replace the root-compose-only backup slot with staged recovery generations that record the managed inventory, exact Compose invocation, and prior image identity, and wire the same engine through deploy, update, manual rollback, and Git apply. * fix(recovery): satisfy CodeQL path barriers and update-guard mock Inline resolve+startsWith checks at generation/inventory fs sinks and stub getCurrentStackUpdateRecovery in UpdateGuardService tests. * fix(recovery): drop unused FileSystemService import in generation store test * fix(recovery): harden authored-project rollback for upgrade and restore safety Preserve legacy UUID backup rows, restore Git deploy state with files, make multi-file restore recoverable, evaluate policy on the restored target, and fail closed when Git capture cannot cover an apply. * fix(recovery): unblock Git apply unit tests and CodeQL pre-restore TOCTOU Mock recovery capture in git-source-service tests after fail-closed apply capture, and re-resolve live paths immediately before pre-restore snapshot reads. * fix(recovery): fall back to authored inventory when Git manifesto is missing First Git apply captures before promote, so a missing managed-project manifesto must not block rollback capture when the live stack already has authored files. * fix(recovery): make authored-project rollback atomic across Git state Restore the managed-project manifesto with files, keep nullable Git identity on first-apply captures, persist Git side-state in restore intents for startup reconcile, compensate legacy materialize failures, and refuse directory collisions before mutation. * fix(recovery): satisfy CodeQL path and TOCTOU barriers on manifesto restore Add inline resolve barriers for manifesto read/clear sinks and remove the access-then-read race when restoring a generation manifesto snapshot. * fix(recovery): close third-audit rollback generation blockers Fail closed on incomplete Git inventory fallbacks, execute captured Compose invocation during recovery, refuse startup and mutations while restore intents remain unresolved, propagate legacy stale-delete failures, and add Docker-level exact prior-image coverage plus regression tests. * fix(recovery): mark acquired before handoff in prior-image Docker test Match the production updateStack CAS sequence so the exact prior-image integration test does not fail handoff from the captured phase. * fix(recovery): close fourth-audit rollback safety blockers Evaluate policy against held images, use index-based pre-restore snapshots, hold the shared stack lock across Git apply, replay Mesh and empty captured invocations exactly, restore POSIX modes with fail-closed sensitive permissions, keep case-sensitive paths, and link Git auto-deploy health gates. Add regression coverage for these cases. * test(recovery): fix mocks for health-gate link and authored compose args Add linkGateOrRetain to the Git apply recovery mock, and mock authoredComposeArgs so the case-collision inventory test is not masked by a missing getComposeDir stub. * fix(recovery): close fifth-audit rollback safety blockers Share git_apply locking for webhook auto-apply, fail closed on malformed recovery service records, refuse mixed-image capture, and require exact probe counts with hold-tag eligibility checks. * fix(recovery): close sixth-audit rollback safety blockers Preserve the legacy backup slot during generation capture, encrypt sensitive pre-restore snapshots, revert files on a failed health probe without committing Git, fail closed when an absent-file revert would delete a directory, skip Compose one-offs, route manual and scheduled backup through the current generation, and persist runtime image platform identity. * fix(recovery): close seventh-audit rollback safety blockers Fleet snapshot restore and restore-all now capture a recovery generation under the stack lock before any authored file write, including on remote nodes. * fix(recovery): keep pre-deploy generations during health-gate observe Link deploy recovery generations to the observing gate so backup cannot replace them mid-observe. Distinguish missing hold tags from probe failures, refuse generation release when services metadata is corrupt, classify mixed-replica and coverage refusals, and toast the backend rollback message. * fix(recovery): wrap webhook deploy case for eslint const bindings in an unbraced switch case trip no-case-declarations. Match the pull case block.
This commit is contained in:
Anso
@@ -19,6 +19,7 @@ import { isPathWithinBase, isValidRelativeStackPath } from '../utils/validation'
|
||||
import { gitSourceLocalComposeFiles, PRIMARY_COMPOSE_FILENAME } from '../utils/gitComposeFiles';
|
||||
import { ComposeInputDiscoveryService, type ContextCopyPlan } from './ComposeInputDiscoveryService';
|
||||
import { GitProjectManifestService } from './GitProjectManifestService';
|
||||
import { StackUpdateRecoveryService } from './StackUpdateRecoveryService';
|
||||
import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs';
|
||||
import type { ComposeInputEntry, GitProjectManifest, GitSourceManifestState, InventoryResult, ManifestSummary, RefusalInfo } from '../types/gitProjectManifest';
|
||||
import type { GitHttpRequest, GitHttpResponse, HttpClient } from 'isomorphic-git/http/node';
|
||||
@@ -1618,11 +1619,7 @@ export class GitSourceService {
|
||||
for (const old of prevSpec.files) {
|
||||
if (old === PRIMARY_COMPOSE_FILENAME || keep.has(old)) continue;
|
||||
if (!isValidRelativeStackPath(old) || old === '') continue;
|
||||
try {
|
||||
await fsSvc.deleteStackPath(stackName, old);
|
||||
} catch (e) {
|
||||
console.warn(`[GitSource] stale file cleanup skipped ${sanitizeForLog(old)} for ${sanitizeForLog(stackName)}:`, (e as Error).message);
|
||||
}
|
||||
await fsSvc.deleteStackPath(stackName, old);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1743,16 +1740,44 @@ export class GitSourceService {
|
||||
stackName: string,
|
||||
commitSha: string,
|
||||
opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean } = {},
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string }> {
|
||||
return this.withStackLock(stackName, () => this.applyLocked(stackName, commitSha, opts));
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> {
|
||||
return this.withStackLock(stackName, () => this.applyWithSharedLock(stackName, commitSha, opts));
|
||||
}
|
||||
|
||||
/** Body of apply(); assumes the caller already holds the per-stack lock. */
|
||||
/**
|
||||
* Acquire the shared stack-operation lock then run applyLocked.
|
||||
* Callers that already hold the Git per-stack mutex (public apply, webhook
|
||||
* auto-apply) use this so capture/promote/handoff/deploy cannot race other
|
||||
* lifecycle ops. Do not nest withStackLock here.
|
||||
*/
|
||||
private async applyWithSharedLock(
|
||||
stackName: string,
|
||||
commitSha: string,
|
||||
opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean },
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> {
|
||||
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
nodeId,
|
||||
stackName,
|
||||
'git_apply',
|
||||
opts.actor ?? 'system:git-source',
|
||||
() => this.applyLocked(stackName, commitSha, opts),
|
||||
);
|
||||
if (!lock.ran) {
|
||||
throw new GitSourceError(
|
||||
'GIT_ERROR',
|
||||
`Another operation (${lock.existing.action}) is already in progress for ${stackName}.`,
|
||||
);
|
||||
}
|
||||
return lock.result;
|
||||
}
|
||||
|
||||
/** Body of apply(); assumes the caller already holds Git mutex + shared stack lock. */
|
||||
private async applyLocked(
|
||||
stackName: string,
|
||||
commitSha: string,
|
||||
opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean },
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string }> {
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> {
|
||||
const diag = isDebugEnabled();
|
||||
const db = DatabaseService.getInstance();
|
||||
const src = db.getGitSource(stackName);
|
||||
@@ -1775,6 +1800,9 @@ export class GitSourceService {
|
||||
? this.crypto.decrypt(src.pending_env_content)
|
||||
: null;
|
||||
const manifestSvc = GitProjectManifestService.getInstance();
|
||||
const recoverySvc = StackUpdateRecoveryService.getInstance();
|
||||
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
let recoveryId: string | undefined;
|
||||
|
||||
let appliedSpec: GitSourceAppliedSpec | null;
|
||||
if (pending.candidateRelPath !== null && pending.inventory !== null) {
|
||||
@@ -1797,7 +1825,7 @@ export class GitSourceService {
|
||||
// The staged candidate must still exist and be complete; a deleted
|
||||
// candidate (or a node restart that swept it) invalidates the pull.
|
||||
const dataDir = process.env.DATA_DIR || path.join(process.cwd(), 'data');
|
||||
const candidateAbs = path.join(dataDir, 'git-managed', String(NodeRegistry.getInstance().getDefaultNodeId()), stackName, pending.candidateRelPath);
|
||||
const candidateAbs = path.join(dataDir, 'git-managed', String(nodeId), stackName, pending.candidateRelPath);
|
||||
try {
|
||||
await fsPromises.access(candidateAbs);
|
||||
} catch {
|
||||
@@ -1872,8 +1900,8 @@ export class GitSourceService {
|
||||
: null;
|
||||
const invocation: string[] = [];
|
||||
try {
|
||||
invocation.push(...(await authoredComposeFileArgs(stackName, NodeRegistry.getInstance().getDefaultNodeId())));
|
||||
invocation.push(...(await authoredComposeEnvFileArgs(stackName, NodeRegistry.getInstance().getDefaultNodeId())));
|
||||
invocation.push(...(await authoredComposeFileArgs(stackName, nodeId)));
|
||||
invocation.push(...(await authoredComposeEnvFileArgs(stackName, nodeId)));
|
||||
} catch (e) {
|
||||
console.warn(`[GitSource] invocation build failed for ${stackName}:`, (e as Error).message);
|
||||
}
|
||||
@@ -1903,6 +1931,25 @@ export class GitSourceService {
|
||||
...(src.applied_deploy_spec?.files ?? [PRIMARY_COMPOSE_FILENAME]),
|
||||
...(src.sync_env ? ['.env'] : []),
|
||||
];
|
||||
try {
|
||||
const candidate = await recoverySvc.captureCandidate({
|
||||
nodeId,
|
||||
stackName,
|
||||
createdBy: opts.actor ?? 'git-source',
|
||||
operationKind: 'git_apply',
|
||||
});
|
||||
recoveryId = candidate.id;
|
||||
} catch (captureError) {
|
||||
const detail = captureError instanceof Error ? captureError.message : String(captureError);
|
||||
console.error(
|
||||
`[GitSource] Recovery capture failed before apply of ${sanitizeForLog(stackName)}:`,
|
||||
detail,
|
||||
);
|
||||
throw new GitSourceError(
|
||||
'GIT_ERROR',
|
||||
`Rollback capture failed before apply; refusing to promote without recovery coverage: ${scrubCredentials(detail)}`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
await manifestSvc.promoteGeneration(stackName, {
|
||||
sha: commitSha,
|
||||
@@ -1917,6 +1964,16 @@ export class GitSourceService {
|
||||
// The original error is logged with its stack for diagnosis,
|
||||
// and the message is scrubbed of credentials and of the
|
||||
// incoming manifest's high-sensitivity paths.
|
||||
if (recoveryId) {
|
||||
try {
|
||||
await recoverySvc.abandon(recoveryId);
|
||||
} catch (abandonError) {
|
||||
console.warn(
|
||||
`[GitSource] Failed to abandon recovery after promote failure for ${sanitizeForLog(stackName)}:`,
|
||||
abandonError instanceof Error ? abandonError.message : String(abandonError),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (e instanceof GitSourceError) throw e;
|
||||
const raw = e instanceof Error ? e.message : String(e);
|
||||
console.error(`[GitSource] promotion failed for ${sanitizeForLog(stackName)}:`, e instanceof Error ? e.stack ?? e.message : raw);
|
||||
@@ -1937,9 +1994,51 @@ export class GitSourceService {
|
||||
if (diag) console.log(`[GitSource:diag] apply validation fail stack=${stackName}`);
|
||||
throw new GitSourceError('GIT_ERROR', `Compose validation failed: ${validation.error}`);
|
||||
}
|
||||
// Capture the true pre-apply project BEFORE materialize writes new files.
|
||||
try {
|
||||
const captured = await recoverySvc.captureCandidate({
|
||||
nodeId,
|
||||
stackName,
|
||||
createdBy: opts.actor ?? 'git-source',
|
||||
operationKind: 'git_apply',
|
||||
});
|
||||
recoveryId = captured.id;
|
||||
} catch (captureError) {
|
||||
const detail = captureError instanceof Error ? captureError.message : String(captureError);
|
||||
console.error(
|
||||
`[GitSource] Recovery capture failed before legacy apply of ${sanitizeForLog(stackName)}:`,
|
||||
detail,
|
||||
);
|
||||
throw new GitSourceError(
|
||||
'GIT_ERROR',
|
||||
`Rollback capture failed before apply; refusing to materialize without recovery coverage: ${scrubCredentials(detail)}`,
|
||||
);
|
||||
}
|
||||
appliedSpec = await this.materialize(
|
||||
stackName, pending.files, pending.contextDir, src.sync_env, envContent, src.applied_deploy_spec,
|
||||
);
|
||||
).catch(async (materializeError: unknown) => {
|
||||
if (recoveryId) {
|
||||
const reverted = await recoverySvc.revertToGenerationContent(recoveryId);
|
||||
if (!reverted) {
|
||||
throw new GitSourceError(
|
||||
'GIT_ERROR',
|
||||
`Legacy materialize failed and pre-apply generation restore also failed: ${scrubCredentials(
|
||||
materializeError instanceof Error ? materializeError.message : String(materializeError),
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
await recoverySvc.abandon(recoveryId);
|
||||
} catch (abandonError) {
|
||||
console.warn(
|
||||
`[GitSource] Failed to abandon recovery after legacy materialize failure for ${sanitizeForLog(stackName)}:`,
|
||||
abandonError instanceof Error ? abandonError.message : String(abandonError),
|
||||
);
|
||||
}
|
||||
recoveryId = undefined;
|
||||
}
|
||||
throw materializeError;
|
||||
});
|
||||
// Migration: build the conservative manifest from spec + disk.
|
||||
const migrated = await manifestSvc.buildMigratedManifest(stackName, {
|
||||
repo_url: src.repo_url,
|
||||
@@ -1958,9 +2057,25 @@ export class GitSourceService {
|
||||
const shouldDeploy = opts.deploy ?? src.auto_deploy_on_apply;
|
||||
if (diag) console.log('[GitSource:diag] apply wrote stack=%s sha=%s deploy=%s', sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), sanitizeForLog(shouldDeploy));
|
||||
|
||||
const finalizeRecoveryCurrent = async (id: string, immediateVerified: boolean): Promise<void> => {
|
||||
if (!recoverySvc.markAcquired(id)) {
|
||||
await recoverySvc.abandon(id);
|
||||
throw new Error('Failed to mark recovery generation as acquired');
|
||||
}
|
||||
if (!recoverySvc.handoff(id, nodeId, stackName)) {
|
||||
await recoverySvc.abandon(id);
|
||||
throw new Error('Failed to hand off recovery generation');
|
||||
}
|
||||
if (!recoverySvc.markReconciling(id)) {
|
||||
throw new Error('Failed to mark recovery generation as reconciling');
|
||||
}
|
||||
if (immediateVerified && !recoverySvc.markImmediateVerified(id)) {
|
||||
console.warn(`[GitSource] Could not CAS immediate_verified for recovery ${sanitizeForLog(id)}`);
|
||||
}
|
||||
};
|
||||
|
||||
if (shouldDeploy) {
|
||||
try {
|
||||
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
await assertPolicyGateAllows(
|
||||
stackName,
|
||||
nodeId,
|
||||
@@ -1969,34 +2084,73 @@ export class GitSourceService {
|
||||
auditPath: `/api/stacks/${stackName}/git-source/apply`,
|
||||
}),
|
||||
);
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
nodeId, stackName, 'deploy', 'system',
|
||||
() => ComposeService.getInstance(nodeId).deployStack(
|
||||
stackName,
|
||||
undefined,
|
||||
undefined,
|
||||
{ source: 'git_apply', actor: opts.actor ?? 'system:git-source' },
|
||||
),
|
||||
);
|
||||
if (!lock.ran) {
|
||||
const busy = `Auto-deploy skipped: another operation (${lock.existing.action}) is already in progress for ${stackName}.`;
|
||||
console.warn(`[GitSource] ${busy}`);
|
||||
return { applied: true, deployed: false, deployError: busy };
|
||||
if (recoveryId) {
|
||||
await finalizeRecoveryCurrent(recoveryId, false);
|
||||
}
|
||||
// Shared stack lock already held as git_apply for capture→deploy.
|
||||
await ComposeService.getInstance(nodeId).deployStack(
|
||||
stackName,
|
||||
undefined,
|
||||
undefined,
|
||||
{ source: 'git_apply', actor: opts.actor ?? 'system:git-source' },
|
||||
);
|
||||
if (recoveryId) {
|
||||
if (!recoverySvc.markImmediateVerified(recoveryId)) {
|
||||
console.warn(`[GitSource] Could not CAS immediate_verified for recovery ${sanitizeForLog(recoveryId)}`);
|
||||
}
|
||||
}
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(
|
||||
nodeId,
|
||||
stackName,
|
||||
'deploy',
|
||||
'system:git-source',
|
||||
);
|
||||
if (recoveryId) {
|
||||
recoverySvc.linkGateOrRetain(recoveryId, healthGateId);
|
||||
}
|
||||
HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:git-source');
|
||||
console.log(`[GitSource] Applied and deployed ${stackName} at ${commitSha.slice(0, 7)}`);
|
||||
return { applied: true, deployed: true };
|
||||
return { applied: true, deployed: true, recoveryId };
|
||||
} catch (e) {
|
||||
// File is on disk, DB is marked applied. Returning the
|
||||
// error separately lets the UI flag it as a partial
|
||||
// success rather than rolling back the disk.
|
||||
// R1: do not auto-compensate. Keep applied files and leave the
|
||||
// pre-promote generation is_current for manual rollback.
|
||||
if (recoveryId) {
|
||||
const row = recoverySvc.get(recoveryId);
|
||||
if (row && row.is_current !== 1) {
|
||||
try {
|
||||
await finalizeRecoveryCurrent(recoveryId, false);
|
||||
} catch (handoffError) {
|
||||
console.warn(
|
||||
`[GitSource] Failed to hand off recovery after deploy failure for ${sanitizeForLog(stackName)}:`,
|
||||
handoffError instanceof Error ? handoffError.message : String(handoffError),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
const scrubbed = scrubCredentials((e as Error).message || String(e));
|
||||
console.error(`[GitSource] Auto-deploy failed for ${stackName}: ${scrubbed}`);
|
||||
return { applied: true, deployed: false, deployError: scrubbed };
|
||||
return { applied: true, deployed: false, deployError: scrubbed, recoveryId };
|
||||
}
|
||||
}
|
||||
|
||||
if (recoveryId) {
|
||||
try {
|
||||
await finalizeRecoveryCurrent(recoveryId, true);
|
||||
} catch (finalizeError) {
|
||||
const detail = finalizeError instanceof Error ? finalizeError.message : String(finalizeError);
|
||||
console.error(
|
||||
`[GitSource] Failed to finalize recovery for apply-only ${sanitizeForLog(stackName)}:`,
|
||||
detail,
|
||||
);
|
||||
return {
|
||||
applied: true,
|
||||
deployed: false,
|
||||
deployError: `Recovery finalization failed after apply: ${scrubCredentials(detail)}`,
|
||||
recoveryId,
|
||||
};
|
||||
}
|
||||
}
|
||||
console.log(`[GitSource] Applied ${stackName} at ${commitSha.slice(0, 7)}`);
|
||||
return { applied: true, deployed: false };
|
||||
return { applied: true, deployed: false, recoveryId };
|
||||
}
|
||||
|
||||
public dismissPending(stackName: string): void {
|
||||
@@ -2369,7 +2523,10 @@ export class GitSourceService {
|
||||
return { status: 'success', message: `Pending update ready at ${pullResult.commitSha.slice(0, 7)}.` };
|
||||
}
|
||||
|
||||
const applied = await this.applyLocked(stackName, pullResult.commitSha, { deploy: src.auto_deploy_on_apply });
|
||||
const applied = await this.applyWithSharedLock(stackName, pullResult.commitSha, {
|
||||
deploy: src.auto_deploy_on_apply,
|
||||
actor: 'system:webhook',
|
||||
});
|
||||
if (applied.deployError) {
|
||||
// Apply wrote to disk but deploy failed. Surface it so the
|
||||
// webhook_executions row records a degraded outcome instead
|
||||
|
||||
Reference in New Issue
Block a user