mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-09-03 14:18:02 +00:00
feat(recovery): complete authored-project atomic rollback generations (#1819)
* feat(recovery): capture complete authored Compose project for atomic rollback Replace the root-compose-only backup slot with staged recovery generations that record the managed inventory, exact Compose invocation, and prior image identity, and wire the same engine through deploy, update, manual rollback, and Git apply. * fix(recovery): satisfy CodeQL path barriers and update-guard mock Inline resolve+startsWith checks at generation/inventory fs sinks and stub getCurrentStackUpdateRecovery in UpdateGuardService tests. * fix(recovery): drop unused FileSystemService import in generation store test * fix(recovery): harden authored-project rollback for upgrade and restore safety Preserve legacy UUID backup rows, restore Git deploy state with files, make multi-file restore recoverable, evaluate policy on the restored target, and fail closed when Git capture cannot cover an apply. * fix(recovery): unblock Git apply unit tests and CodeQL pre-restore TOCTOU Mock recovery capture in git-source-service tests after fail-closed apply capture, and re-resolve live paths immediately before pre-restore snapshot reads. * fix(recovery): fall back to authored inventory when Git manifesto is missing First Git apply captures before promote, so a missing managed-project manifesto must not block rollback capture when the live stack already has authored files. * fix(recovery): make authored-project rollback atomic across Git state Restore the managed-project manifesto with files, keep nullable Git identity on first-apply captures, persist Git side-state in restore intents for startup reconcile, compensate legacy materialize failures, and refuse directory collisions before mutation. * fix(recovery): satisfy CodeQL path and TOCTOU barriers on manifesto restore Add inline resolve barriers for manifesto read/clear sinks and remove the access-then-read race when restoring a generation manifesto snapshot. * fix(recovery): close third-audit rollback generation blockers Fail closed on incomplete Git inventory fallbacks, execute captured Compose invocation during recovery, refuse startup and mutations while restore intents remain unresolved, propagate legacy stale-delete failures, and add Docker-level exact prior-image coverage plus regression tests. * fix(recovery): mark acquired before handoff in prior-image Docker test Match the production updateStack CAS sequence so the exact prior-image integration test does not fail handoff from the captured phase. * fix(recovery): close fourth-audit rollback safety blockers Evaluate policy against held images, use index-based pre-restore snapshots, hold the shared stack lock across Git apply, replay Mesh and empty captured invocations exactly, restore POSIX modes with fail-closed sensitive permissions, keep case-sensitive paths, and link Git auto-deploy health gates. Add regression coverage for these cases. * test(recovery): fix mocks for health-gate link and authored compose args Add linkGateOrRetain to the Git apply recovery mock, and mock authoredComposeArgs so the case-collision inventory test is not masked by a missing getComposeDir stub. * fix(recovery): close fifth-audit rollback safety blockers Share git_apply locking for webhook auto-apply, fail closed on malformed recovery service records, refuse mixed-image capture, and require exact probe counts with hold-tag eligibility checks. * fix(recovery): close sixth-audit rollback safety blockers Preserve the legacy backup slot during generation capture, encrypt sensitive pre-restore snapshots, revert files on a failed health probe without committing Git, fail closed when an absent-file revert would delete a directory, skip Compose one-offs, route manual and scheduled backup through the current generation, and persist runtime image platform identity. * fix(recovery): close seventh-audit rollback safety blockers Fleet snapshot restore and restore-all now capture a recovery generation under the stack lock before any authored file write, including on remote nodes. * fix(recovery): keep pre-deploy generations during health-gate observe Link deploy recovery generations to the observing gate so backup cannot replace them mid-observe. Distinguish missing hold tags from probe failures, refuse generation release when services metadata is corrupt, classify mixed-replica and coverage refusals, and toast the backend rollback message. * fix(recovery): wrap webhook deploy case for eslint const bindings in an unbraced switch case trip no-case-declarations. Match the pull case block.
This commit is contained in:
Anso
@@ -18,8 +18,11 @@ import {
|
||||
import { readSnapshotFileRow, type SnapshotFileReadResult, type SnapshotFileRow } from '../helpers/snapshotFileDecrypt';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import type { GitSourceManifestState } from '../types/gitProjectManifest';
|
||||
import type { RollbackOperationKind } from '../types/rollbackGeneration';
|
||||
import { collectImageIds, parseServicesJsonStrict } from './recoveryServicesJson';
|
||||
|
||||
export type { SnapshotFileReadResult } from '../helpers/snapshotFileDecrypt';
|
||||
export type { RollbackOperationKind } from '../types/rollbackGeneration';
|
||||
|
||||
function isPilotMode(): boolean {
|
||||
return process.env.SENCHO_MODE === 'pilot';
|
||||
@@ -249,6 +252,10 @@ export interface StackUpdateRecoveryGenerationRow {
|
||||
phase: 'captured' | 'acquired' | 'handoff_committed' | 'reconciling' | 'immediate_verified';
|
||||
is_current: number;
|
||||
backup_slot_id: string | null;
|
||||
/** Generation content key (often equal to backup_slot_id / generation id). */
|
||||
content_path: string | null;
|
||||
/** Capture trigger: update | deployment | git_apply | manual_backup | unknown. */
|
||||
operation_kind: RollbackOperationKind | null;
|
||||
override_path: string | null;
|
||||
services_json: string;
|
||||
health_gate_id: string | null;
|
||||
@@ -1920,6 +1927,9 @@ export class DatabaseService {
|
||||
// pattern used for health_gate_runs below.
|
||||
maybeAddCol('stack_update_recovery_generations', 'released_at', 'INTEGER');
|
||||
maybeAddCol('stack_update_recovery_generations', 'released_by', 'TEXT');
|
||||
// Authored-project generation content key + capture trigger kind.
|
||||
maybeAddCol('stack_update_recovery_generations', 'content_path', 'TEXT');
|
||||
maybeAddCol('stack_update_recovery_generations', 'operation_kind', 'TEXT');
|
||||
maybeAddCol('stack_update_cleanup_pending', 'required_blueprint_id', 'INTEGER');
|
||||
|
||||
// Distributed API model columns
|
||||
@@ -4203,13 +4213,15 @@ export class DatabaseService {
|
||||
public insertStackUpdateRecoveryGeneration(row: StackUpdateRecoveryGenerationRow): void {
|
||||
this.db.prepare(
|
||||
`INSERT INTO stack_update_recovery_generations (
|
||||
id, node_id, stack_name, status, phase, is_current, backup_slot_id, override_path,
|
||||
services_json, health_gate_id, gate_retain_until, artifact_expires_at,
|
||||
operation_lease_expires_at, created_at, updated_at, created_by, artifacts_retired
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
id, node_id, stack_name, status, phase, is_current, backup_slot_id, content_path,
|
||||
operation_kind, override_path, services_json, health_gate_id, gate_retain_until,
|
||||
artifact_expires_at, operation_lease_expires_at, created_at, updated_at,
|
||||
created_by, artifacts_retired
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
).run(
|
||||
row.id, row.node_id, row.stack_name, row.status, row.phase, row.is_current,
|
||||
row.backup_slot_id, row.override_path, row.services_json, row.health_gate_id,
|
||||
row.backup_slot_id, row.content_path ?? null, row.operation_kind ?? null,
|
||||
row.override_path, row.services_json, row.health_gate_id,
|
||||
row.gate_retain_until, row.artifact_expires_at, row.operation_lease_expires_at,
|
||||
row.created_at, row.updated_at, row.created_by, row.artifacts_retired ?? 0,
|
||||
);
|
||||
@@ -4241,7 +4253,8 @@ export class DatabaseService {
|
||||
id: string,
|
||||
patch: Partial<Pick<StackUpdateRecoveryGenerationRow,
|
||||
'status' | 'phase' | 'is_current' | 'override_path' | 'health_gate_id' |
|
||||
'gate_retain_until' | 'artifact_expires_at' | 'operation_lease_expires_at' | 'services_json'>>,
|
||||
'gate_retain_until' | 'artifact_expires_at' | 'operation_lease_expires_at' | 'services_json' |
|
||||
'content_path' | 'operation_kind'>>,
|
||||
): void {
|
||||
const keys = Object.keys(patch) as Array<keyof typeof patch>;
|
||||
if (keys.length === 0) return;
|
||||
@@ -4432,26 +4445,13 @@ export class DatabaseService {
|
||||
).all(nodeId, now, now) as Array<{ services_json: string }>;
|
||||
const ids = new Set<string>();
|
||||
for (const row of rows) {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(row.services_json);
|
||||
if (!Array.isArray(parsed)) continue;
|
||||
for (const item of parsed) {
|
||||
if (!item || typeof item !== 'object') continue;
|
||||
const replicas = (item as { replicas?: unknown }).replicas;
|
||||
if (Array.isArray(replicas)) {
|
||||
for (const replica of replicas) {
|
||||
if (replica && typeof replica === 'object'
|
||||
&& typeof (replica as { imageId?: unknown }).imageId === 'string'
|
||||
&& (replica as { imageId: string }).imageId.trim()) {
|
||||
ids.add((replica as { imageId: string }).imageId);
|
||||
}
|
||||
}
|
||||
} else if (typeof (item as { imageId?: unknown }).imageId === 'string') {
|
||||
ids.add((item as { imageId: string }).imageId);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Corrupt JSON: skip.
|
||||
const parsed = parseServicesJsonStrict(row.services_json);
|
||||
if (!parsed.ok) {
|
||||
// Fail closed: corrupt hold metadata must not look like "nothing held".
|
||||
throw new Error('Malformed stack recovery services_json while listing held images');
|
||||
}
|
||||
for (const id of collectImageIds(parsed.services)) {
|
||||
ids.add(id);
|
||||
}
|
||||
}
|
||||
return [...ids];
|
||||
@@ -6323,6 +6323,25 @@ export class DatabaseService {
|
||||
).run(commitSha, contentHash, Date.now(), stackName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the last-applied revision identity without removing the Git source
|
||||
* row. Used when compensating to a capture that had a null commit SHA
|
||||
* (first-apply preimage).
|
||||
*/
|
||||
public clearGitSourceAppliedRevision(stackName: string): void {
|
||||
this.db.prepare(
|
||||
`UPDATE stack_git_sources SET
|
||||
last_applied_commit_sha = NULL,
|
||||
last_applied_content_hash = NULL,
|
||||
pending_commit_sha = NULL,
|
||||
pending_compose_content = NULL,
|
||||
pending_env_content = NULL,
|
||||
pending_fetched_at = NULL,
|
||||
updated_at = ?
|
||||
WHERE stack_name = ?`
|
||||
).run(Date.now(), stackName);
|
||||
}
|
||||
|
||||
public touchGitSourceDebounce(stackName: string): void {
|
||||
this.db.prepare('UPDATE stack_git_sources SET last_debounce_at = ? WHERE stack_name = ?')
|
||||
.run(Date.now(), stackName);
|
||||
|
||||
Reference in New Issue
Block a user