mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-16 13:38:33 +00:00
feat(recovery): complete authored-project atomic rollback generations (#1819)
* feat(recovery): capture complete authored Compose project for atomic rollback Replace the root-compose-only backup slot with staged recovery generations that record the managed inventory, exact Compose invocation, and prior image identity, and wire the same engine through deploy, update, manual rollback, and Git apply. * fix(recovery): satisfy CodeQL path barriers and update-guard mock Inline resolve+startsWith checks at generation/inventory fs sinks and stub getCurrentStackUpdateRecovery in UpdateGuardService tests. * fix(recovery): drop unused FileSystemService import in generation store test * fix(recovery): harden authored-project rollback for upgrade and restore safety Preserve legacy UUID backup rows, restore Git deploy state with files, make multi-file restore recoverable, evaluate policy on the restored target, and fail closed when Git capture cannot cover an apply. * fix(recovery): unblock Git apply unit tests and CodeQL pre-restore TOCTOU Mock recovery capture in git-source-service tests after fail-closed apply capture, and re-resolve live paths immediately before pre-restore snapshot reads. * fix(recovery): fall back to authored inventory when Git manifesto is missing First Git apply captures before promote, so a missing managed-project manifesto must not block rollback capture when the live stack already has authored files. * fix(recovery): make authored-project rollback atomic across Git state Restore the managed-project manifesto with files, keep nullable Git identity on first-apply captures, persist Git side-state in restore intents for startup reconcile, compensate legacy materialize failures, and refuse directory collisions before mutation. * fix(recovery): satisfy CodeQL path and TOCTOU barriers on manifesto restore Add inline resolve barriers for manifesto read/clear sinks and remove the access-then-read race when restoring a generation manifesto snapshot. * fix(recovery): close third-audit rollback generation blockers Fail closed on incomplete Git inventory fallbacks, execute captured Compose invocation during recovery, refuse startup and mutations while restore intents remain unresolved, propagate legacy stale-delete failures, and add Docker-level exact prior-image coverage plus regression tests. * fix(recovery): mark acquired before handoff in prior-image Docker test Match the production updateStack CAS sequence so the exact prior-image integration test does not fail handoff from the captured phase. * fix(recovery): close fourth-audit rollback safety blockers Evaluate policy against held images, use index-based pre-restore snapshots, hold the shared stack lock across Git apply, replay Mesh and empty captured invocations exactly, restore POSIX modes with fail-closed sensitive permissions, keep case-sensitive paths, and link Git auto-deploy health gates. Add regression coverage for these cases. * test(recovery): fix mocks for health-gate link and authored compose args Add linkGateOrRetain to the Git apply recovery mock, and mock authoredComposeArgs so the case-collision inventory test is not masked by a missing getComposeDir stub. * fix(recovery): close fifth-audit rollback safety blockers Share git_apply locking for webhook auto-apply, fail closed on malformed recovery service records, refuse mixed-image capture, and require exact probe counts with hold-tag eligibility checks. * fix(recovery): close sixth-audit rollback safety blockers Preserve the legacy backup slot during generation capture, encrypt sensitive pre-restore snapshots, revert files on a failed health probe without committing Git, fail closed when an absent-file revert would delete a directory, skip Compose one-offs, route manual and scheduled backup through the current generation, and persist runtime image platform identity. * fix(recovery): close seventh-audit rollback safety blockers Fleet snapshot restore and restore-all now capture a recovery generation under the stack lock before any authored file write, including on remote nodes. * fix(recovery): keep pre-deploy generations during health-gate observe Link deploy recovery generations to the observing gate so backup cannot replace them mid-observe. Distinguish missing hold tags from probe failures, refuse generation release when services metadata is corrupt, classify mixed-replica and coverage refusals, and toast the backend rollback message. * fix(recovery): wrap webhook deploy case for eslint const bindings in an unbraced switch case trip no-case-declarations. Match the pull case block.
This commit is contained in:
Anso
@@ -0,0 +1,159 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import { applyFleetSnapshotFiles, selectFleetSnapshotApplyFiles, type FleetSnapshotApplyFile } from '../helpers/applyFleetSnapshotFiles';
|
||||
import { StackOpLockService } from '../services/StackOpLockService';
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
hasComposeFile: vi.fn(),
|
||||
saveStackContent: vi.fn(),
|
||||
saveEnvContent: vi.fn(),
|
||||
getBaseDir: vi.fn(() => '/tmp/compose'),
|
||||
captureCurrentBackup: vi.fn(),
|
||||
invalidateNodeCaches: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../services/FileSystemService', () => ({
|
||||
FileSystemService: {
|
||||
getInstance: () => ({
|
||||
getBaseDir: mocks.getBaseDir,
|
||||
hasComposeFile: mocks.hasComposeFile,
|
||||
saveStackContent: mocks.saveStackContent,
|
||||
saveEnvContent: mocks.saveEnvContent,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../helpers/cacheInvalidation', () => ({
|
||||
invalidateNodeCaches: mocks.invalidateNodeCaches,
|
||||
}));
|
||||
|
||||
vi.mock('../services/StackUpdateRecoveryService', () => ({
|
||||
StackUpdateRecoveryService: {
|
||||
getInstance: () => ({
|
||||
captureCurrentBackup: mocks.captureCurrentBackup,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
const FILES: FleetSnapshotApplyFile[] = [
|
||||
{ filename: 'compose.yaml', content: 'services:\n snap: {}\n' },
|
||||
{ filename: '.env', content: 'SNAP=1\n' },
|
||||
];
|
||||
|
||||
function applyWeb(overrides: Partial<{
|
||||
stackName: string;
|
||||
files: FleetSnapshotApplyFile[];
|
||||
actor: string;
|
||||
}> = {}) {
|
||||
return applyFleetSnapshotFiles({
|
||||
nodeId: 1,
|
||||
stackName: 'web',
|
||||
files: FILES,
|
||||
actor: 'system:fleet-snapshot',
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
describe('applyFleetSnapshotFiles', () => {
|
||||
beforeEach(() => {
|
||||
StackOpLockService.resetForTests();
|
||||
mocks.hasComposeFile.mockReset().mockResolvedValue(true);
|
||||
mocks.saveStackContent.mockReset().mockResolvedValue(undefined);
|
||||
mocks.saveEnvContent.mockReset().mockResolvedValue(undefined);
|
||||
mocks.captureCurrentBackup.mockReset().mockResolvedValue({ id: 'gen-pre' });
|
||||
mocks.invalidateNodeCaches.mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
StackOpLockService.resetForTests();
|
||||
});
|
||||
|
||||
it('captures a recovery generation before writing when the stack already exists', async () => {
|
||||
const order: string[] = [];
|
||||
mocks.captureCurrentBackup.mockImplementation(async () => {
|
||||
order.push('capture');
|
||||
return { id: 'gen-pre' };
|
||||
});
|
||||
mocks.saveStackContent.mockImplementation(async () => {
|
||||
order.push('compose');
|
||||
});
|
||||
mocks.saveEnvContent.mockImplementation(async () => {
|
||||
order.push('env');
|
||||
});
|
||||
|
||||
const result = await applyWeb();
|
||||
|
||||
expect(result.capturedGenerationId).toBe('gen-pre');
|
||||
expect(order).toEqual(['capture', 'compose', 'env']);
|
||||
expect(mocks.invalidateNodeCaches).toHaveBeenCalledWith(1);
|
||||
expect(mocks.captureCurrentBackup).toHaveBeenCalledWith({
|
||||
nodeId: 1,
|
||||
stackName: 'web',
|
||||
createdBy: 'system:fleet-snapshot',
|
||||
});
|
||||
});
|
||||
|
||||
it('skips capture for a new stack with no compose file', async () => {
|
||||
mocks.hasComposeFile.mockResolvedValue(false);
|
||||
|
||||
const result = await applyWeb({ stackName: 'fresh' });
|
||||
|
||||
expect(result.capturedGenerationId).toBeNull();
|
||||
expect(mocks.captureCurrentBackup).not.toHaveBeenCalled();
|
||||
expect(mocks.saveStackContent).toHaveBeenCalledWith('fresh', FILES[0].content);
|
||||
expect(mocks.saveEnvContent).toHaveBeenCalledWith('fresh', FILES[1].content);
|
||||
});
|
||||
|
||||
it('aborts without writing when capture fails', async () => {
|
||||
mocks.captureCurrentBackup.mockRejectedValue(Object.assign(new Error('capture failed'), { code: 'CAPTURE_FAILED' }));
|
||||
|
||||
await expect(applyWeb()).rejects.toThrow('capture failed');
|
||||
|
||||
expect(mocks.saveStackContent).not.toHaveBeenCalled();
|
||||
expect(mocks.saveEnvContent).not.toHaveBeenCalled();
|
||||
expect(mocks.invalidateNodeCaches).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('refuses to mutate when another stack operation holds the lock', async () => {
|
||||
StackOpLockService.getInstance().tryAcquire(1, 'web', 'deploy', 'other');
|
||||
|
||||
await expect(applyWeb()).rejects.toMatchObject({ code: 'stack_op_in_progress' });
|
||||
|
||||
expect(mocks.captureCurrentBackup).not.toHaveBeenCalled();
|
||||
expect(mocks.saveStackContent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('leaves the captured generation in place when a later file write fails', async () => {
|
||||
mocks.saveEnvContent.mockRejectedValue(new Error('disk full'));
|
||||
|
||||
await expect(applyWeb()).rejects.toThrow('disk full');
|
||||
|
||||
expect(mocks.captureCurrentBackup).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.saveStackContent).toHaveBeenCalledWith('web', FILES[0].content);
|
||||
expect(mocks.invalidateNodeCaches).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects an empty apply file list before locking', async () => {
|
||||
await expect(applyWeb({ files: [] })).rejects.toMatchObject({ code: 'INVALID_SNAPSHOT_FILES' });
|
||||
|
||||
expect(mocks.captureCurrentBackup).not.toHaveBeenCalled();
|
||||
expect(mocks.saveStackContent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects an invalid stack name before locking or writing', async () => {
|
||||
await expect(applyWeb({ stackName: '../escape' })).rejects.toMatchObject({ code: 'INVALID_STACK_NAME' });
|
||||
|
||||
expect(mocks.captureCurrentBackup).not.toHaveBeenCalled();
|
||||
expect(mocks.saveStackContent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('drops snapshot filenames other than compose.yaml and .env', () => {
|
||||
expect(selectFleetSnapshotApplyFiles([
|
||||
{ filename: 'compose.yaml', content: 'a' },
|
||||
{ filename: 'notes.txt', content: 'nope' },
|
||||
{ filename: '.env', content: 'b' },
|
||||
])).toEqual([
|
||||
{ filename: 'compose.yaml', content: 'a' },
|
||||
{ filename: '.env', content: 'b' },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -83,7 +83,7 @@ afterAll(() => {
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
mockDeployStack.mockReset();
|
||||
mockDeployStack.mockReset().mockResolvedValue({ recoveryId: null });
|
||||
mockGetBackupInfo.mockReset().mockResolvedValue({ exists: true, timestamp: Date.now() });
|
||||
mockRestoreStackFiles.mockReset().mockResolvedValue(undefined);
|
||||
mockSnapshotStackFiles.mockReset().mockResolvedValue(async () => {});
|
||||
@@ -96,7 +96,7 @@ afterEach(() => vi.restoreAllMocks());
|
||||
describe('Rollback holds the stack lifecycle lock (H-1)', () => {
|
||||
it('blocks deploy while a rollback is in flight on the same stack', async () => {
|
||||
mockTier('paid');
|
||||
const gate = deferred<void>();
|
||||
const gate = deferred<{ recoveryId: string | null }>();
|
||||
mockDeployStack.mockImplementationOnce(() => gate.promise);
|
||||
|
||||
const rollback = request(app)
|
||||
@@ -113,14 +113,14 @@ describe('Rollback holds the stack lifecycle lock (H-1)', () => {
|
||||
expect(deploy.body.code).toBe('stack_op_in_progress');
|
||||
expect(deploy.body.inProgress.action).toBe('rollback');
|
||||
|
||||
gate.resolve();
|
||||
gate.resolve({ recoveryId: null });
|
||||
const rollbackRes = await rollback;
|
||||
expect(rollbackRes.status).toBe(200);
|
||||
});
|
||||
|
||||
it('returns 409 when a rollback lands while a deploy is in flight', async () => {
|
||||
mockTier('paid');
|
||||
const gate = deferred<void>();
|
||||
const gate = deferred<{ recoveryId: string | null }>();
|
||||
mockDeployStack.mockImplementationOnce(() => gate.promise);
|
||||
|
||||
const deploy = request(app)
|
||||
@@ -136,13 +136,13 @@ describe('Rollback holds the stack lifecycle lock (H-1)', () => {
|
||||
expect(rollback.status).toBe(409);
|
||||
expect(rollback.body.inProgress.action).toBe('deploy');
|
||||
|
||||
gate.resolve();
|
||||
gate.resolve({ recoveryId: null });
|
||||
await deploy;
|
||||
});
|
||||
|
||||
it('releases the lock after a successful rollback', async () => {
|
||||
mockTier('paid');
|
||||
mockDeployStack.mockResolvedValue(undefined);
|
||||
mockDeployStack.mockResolvedValue({ recoveryId: null });
|
||||
|
||||
const first = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie);
|
||||
expect(first.status).toBe(200);
|
||||
@@ -157,7 +157,7 @@ describe('Rollback holds the stack lifecycle lock (H-1)', () => {
|
||||
const first = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie);
|
||||
expect(first.status).toBe(500);
|
||||
|
||||
mockDeployStack.mockResolvedValueOnce(undefined);
|
||||
mockDeployStack.mockResolvedValueOnce({ recoveryId: null });
|
||||
const second = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie);
|
||||
expect(second.status).toBe(200);
|
||||
});
|
||||
@@ -166,7 +166,7 @@ describe('Rollback holds the stack lifecycle lock (H-1)', () => {
|
||||
describe('Rollback notifications (M-2)', () => {
|
||||
it('dispatches a success notification when a rollback completes', async () => {
|
||||
mockTier('paid');
|
||||
mockDeployStack.mockResolvedValue(undefined);
|
||||
mockDeployStack.mockResolvedValue({ recoveryId: null });
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
|
||||
|
||||
@@ -214,7 +214,7 @@ describe('Rollback returns 404 when no backup exists', () => {
|
||||
// The 404 is an early return inside the try; the finally must still release
|
||||
// the lock so the stack is not wedged at 409 afterwards.
|
||||
mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: Date.now() });
|
||||
mockDeployStack.mockResolvedValue(undefined);
|
||||
mockDeployStack.mockResolvedValue({ recoveryId: null });
|
||||
const next = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie);
|
||||
expect(next.status).toBe(200);
|
||||
});
|
||||
@@ -223,7 +223,7 @@ describe('Rollback returns 404 when no backup exists', () => {
|
||||
describe('Developer Mode logging matrix', () => {
|
||||
it('only emits rollback diagnostic logs when Developer Mode is enabled', async () => {
|
||||
mockTier('paid');
|
||||
mockDeployStack.mockResolvedValue(undefined);
|
||||
mockDeployStack.mockResolvedValue({ recoveryId: null });
|
||||
const { DatabaseService } = await import('../services/DatabaseService');
|
||||
const db = DatabaseService.getInstance();
|
||||
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
@@ -246,7 +246,7 @@ describe('Deploy safety is available on every tier', () => {
|
||||
it('allows rollback on community', async () => {
|
||||
mockTier('community');
|
||||
mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: 1700000000000 });
|
||||
mockDeployStack.mockResolvedValue(undefined);
|
||||
mockDeployStack.mockResolvedValue({ recoveryId: null });
|
||||
const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(mockDeployStack).toHaveBeenCalled();
|
||||
@@ -257,7 +257,7 @@ describe('Deploy safety is available on every tier', () => {
|
||||
mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: 1700000000000 });
|
||||
const res = await request(app).get('/api/stacks/web/backup').set('Cookie', authCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({ exists: true, timestamp: 1700000000000 });
|
||||
expect(res.body).toMatchObject({ exists: true, timestamp: 1700000000000 });
|
||||
});
|
||||
|
||||
it('returns backup metadata on paid', async () => {
|
||||
@@ -265,6 +265,6 @@ describe('Deploy safety is available on every tier', () => {
|
||||
mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: 1700000000000 });
|
||||
const res = await request(app).get('/api/stacks/web/backup').set('Cookie', authCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({ exists: true, timestamp: 1700000000000 });
|
||||
expect(res.body).toMatchObject({ exists: true, timestamp: 1700000000000 });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -75,7 +75,7 @@ describe('Blueprint compose apply (real filesystem)', () => {
|
||||
const composeContent = 'services:\n web:\n image: traefik:v3\n';
|
||||
const markerContent = JSON.stringify({ blueprintId: 1, revision: 1, lastApplied: Date.now() }, null, 2);
|
||||
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined);
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null });
|
||||
|
||||
const outcome = await BlueprintService.getInstance().applyLocalUnderLock(
|
||||
nodeId,
|
||||
@@ -120,6 +120,7 @@ describe('Blueprint compose apply (real filesystem)', () => {
|
||||
path.join(stackDir, 'docker-compose.yaml'),
|
||||
path.join(stackDir, 'docker-compose.yml'),
|
||||
);
|
||||
return { recoveryId: null };
|
||||
});
|
||||
|
||||
const outcome = await BlueprintService.getInstance().applyLocalUnderLock(
|
||||
@@ -197,7 +198,7 @@ describe('Blueprint compose apply (real filesystem)', () => {
|
||||
const original = 'services:\n mine:\n image: nginx:alpine\n';
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), original);
|
||||
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined);
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null });
|
||||
|
||||
await expect(
|
||||
BlueprintService.getInstance().applyLocalUnderLock(
|
||||
|
||||
@@ -410,7 +410,7 @@ describe('BlueprintService per-stack lock', () => {
|
||||
vi.spyOn(FileSystemService.prototype, 'createStack').mockResolvedValue(undefined);
|
||||
const writeSpy = vi.spyOn(FileSystemService.prototype, 'writeStackFile').mockResolvedValue(undefined);
|
||||
const cleanupSpy = vi.spyOn(FileSystemService.prototype, 'removeAlternateRootComposeFiles').mockResolvedValue(undefined);
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined);
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null });
|
||||
|
||||
const outcome = await BlueprintService.getInstance().deployToNode(bp, node);
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ beforeAll(async () => {
|
||||
|
||||
const { ComposeService } = await import('../services/ComposeService');
|
||||
listImagesSpy = vi.spyOn(ComposeService.prototype, 'listStackImages').mockResolvedValue(['nginx:bad']);
|
||||
deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined);
|
||||
deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null });
|
||||
|
||||
const TrivyService = (await import('../services/TrivyService')).default;
|
||||
const trivy = TrivyService.getInstance();
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* Generation capture must not rewrite the legacy single-slot backup. A later
|
||||
* capture failure has to leave a pre-migration recovery point byte-identical.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
import { promises as fsPromises } from 'fs';
|
||||
|
||||
const mockState = { composeDir: '', composeDirs: new Map<number, string>() };
|
||||
|
||||
vi.mock('../services/NodeRegistry', () => ({
|
||||
NodeRegistry: {
|
||||
getInstance: () => ({
|
||||
getComposeDir: (nodeId?: number) => mockState.composeDirs.get(nodeId ?? 1) ?? mockState.composeDir,
|
||||
getDefaultNodeId: () => 1,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../utils/debug', () => ({
|
||||
isDebugEnabled: () => false,
|
||||
}));
|
||||
|
||||
vi.mock('../services/rollbackInventory', () => ({
|
||||
resolveRollbackInventory: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../services/DatabaseService', () => ({
|
||||
DatabaseService: {
|
||||
getInstance: () => ({
|
||||
getStackProjectEnvFiles: () => [],
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { RollbackGenerationStore } from '../services/RollbackGenerationStore';
|
||||
import { resolveRollbackInventory } from '../services/rollbackInventory';
|
||||
import { resolveComposeProjectContext } from '../services/composeProjectContext';
|
||||
|
||||
describe('composeProjectContext backupFromContext', () => {
|
||||
let composeDir: string;
|
||||
let dataDir: string;
|
||||
let originalDataDir: string | undefined;
|
||||
|
||||
beforeEach(async () => {
|
||||
composeDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-compose-'));
|
||||
dataDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-data-'));
|
||||
mockState.composeDir = composeDir;
|
||||
mockState.composeDirs = new Map([[1, composeDir]]);
|
||||
originalDataDir = process.env.DATA_DIR;
|
||||
process.env.DATA_DIR = dataDir;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (originalDataDir === undefined) delete process.env.DATA_DIR;
|
||||
else process.env.DATA_DIR = originalDataDir;
|
||||
await fsPromises.rm(composeDir, { recursive: true, force: true });
|
||||
await fsPromises.rm(dataDir, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('preserves a seeded legacy backup when generation capture fails', async () => {
|
||||
const stackName = 'legacy-slot';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'ORIGINAL\n', 'utf8');
|
||||
|
||||
const fsSvc = FileSystemService.getInstance(1);
|
||||
await fsSvc.backupStackFiles(stackName);
|
||||
const backupPath = path.join(dataDir, 'backups', '1', stackName, 'compose.yaml');
|
||||
const originalBackup = await fsPromises.readFile(backupPath);
|
||||
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'MUTATED\n', 'utf8');
|
||||
|
||||
vi.mocked(resolveRollbackInventory).mockResolvedValue({
|
||||
entries: [{
|
||||
relativePath: 'compose.yaml',
|
||||
dependencyKind: 'compose-root',
|
||||
provenance: 'authored',
|
||||
sensitivity: 'low',
|
||||
absolutePath: path.join(stackDir, 'compose.yaml'),
|
||||
}],
|
||||
invocation: {
|
||||
composeArgsPrefix: [],
|
||||
projectDirectory: null,
|
||||
projectName: stackName,
|
||||
explicitComposeFiles: ['compose.yaml'],
|
||||
meshEnabled: false,
|
||||
meshOverrideRelativePath: null,
|
||||
},
|
||||
git: null,
|
||||
appliedDeploySpec: null,
|
||||
lastAppliedContentHash: null,
|
||||
manifestState: null,
|
||||
manifestGeneration: null,
|
||||
exactCoverage: true,
|
||||
coverageRefusal: null,
|
||||
});
|
||||
vi.spyOn(RollbackGenerationStore, 'captureGeneration').mockRejectedValue(
|
||||
new Error('injected capture failure'),
|
||||
);
|
||||
|
||||
const ctx = await resolveComposeProjectContext(1, stackName);
|
||||
await expect(ctx.backupFromContext('update')).rejects.toThrow(/injected capture failure/);
|
||||
|
||||
expect(await fsPromises.readFile(backupPath)).toEqual(originalBackup);
|
||||
await fsSvc.restoreStackFiles(stackName);
|
||||
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf8')).toBe('ORIGINAL\n');
|
||||
});
|
||||
});
|
||||
@@ -735,7 +735,7 @@ describe('ComposeService - deployStack', () => {
|
||||
const promise = ComposeService.getInstance(1).deployStack('my-stack');
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
|
||||
await expect(promise).resolves.toBeUndefined();
|
||||
await expect(promise).resolves.toEqual({ recoveryId: null });
|
||||
expect(mockGetLegacyOrphanContainersByStack).toHaveBeenCalledWith('my-stack');
|
||||
});
|
||||
|
||||
@@ -829,7 +829,7 @@ describe('ComposeService - deployStack', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('creates backup when atomic=true', async () => {
|
||||
it('captures recovery generation when atomic=true', async () => {
|
||||
setupAutoCloseSpawn();
|
||||
mockListContainers.mockResolvedValue([]);
|
||||
|
||||
@@ -837,9 +837,16 @@ describe('ComposeService - deployStack', () => {
|
||||
const promise = svc.deployStack('my-stack', undefined, true);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(3100);
|
||||
await promise;
|
||||
const result = await promise;
|
||||
|
||||
expect(mockBackupStackFiles).toHaveBeenCalledWith('my-stack');
|
||||
expect(result).toEqual({ recoveryId: 'recovery-1' });
|
||||
expect(mockCaptureCandidate).toHaveBeenCalledWith(expect.objectContaining({
|
||||
stackName: 'my-stack',
|
||||
operationKind: 'deployment',
|
||||
createdBy: 'atomic-deploy',
|
||||
}));
|
||||
expect(mockHandoff).toHaveBeenCalled();
|
||||
expect(mockMarkImmediateVerified).toHaveBeenCalledWith('recovery-1');
|
||||
});
|
||||
|
||||
it('blocks deploy before backup when missing external networks need a prompt', async () => {
|
||||
@@ -861,7 +868,7 @@ describe('ComposeService - deployStack', () => {
|
||||
|
||||
const svc = ComposeService.getInstance(1);
|
||||
await expect(svc.deployStack('my-stack', undefined, true)).rejects.toBeInstanceOf(MissingExternalNetworksError);
|
||||
expect(mockBackupStackFiles).not.toHaveBeenCalled();
|
||||
expect(mockCaptureCandidate).not.toHaveBeenCalled();
|
||||
expect(mockSpawn).not.toHaveBeenCalled();
|
||||
expect(mockAddNotificationHistory).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -913,7 +920,7 @@ describe('ComposeService - deployStack', () => {
|
||||
stack_name: 'my-stack',
|
||||
}),
|
||||
);
|
||||
expect(mockBackupStackFiles).toHaveBeenCalled();
|
||||
expect(mockCaptureCandidate).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not wrap a missing-external prompt in ComposeRollbackError when atomic', async () => {
|
||||
@@ -938,14 +945,12 @@ describe('ComposeService - deployStack', () => {
|
||||
expect(error?.name).toBe('MissingExternalNetworksError');
|
||||
});
|
||||
|
||||
it('aborts atomic deploy before docker side effects when backup fails', async () => {
|
||||
mockBackupStackFiles.mockRejectedValueOnce(new Error('disk full'));
|
||||
it('aborts atomic deploy before docker side effects when capture fails', async () => {
|
||||
mockCaptureCandidate.mockRejectedValueOnce(new Error('disk full'));
|
||||
|
||||
const svc = ComposeService.getInstance(1);
|
||||
|
||||
await expect(svc.deployStack('my-stack', undefined, true)).rejects.toThrow(
|
||||
'Atomic deployment backup failed',
|
||||
);
|
||||
await expect(svc.deployStack('my-stack', undefined, true)).rejects.toThrow('disk full');
|
||||
expect(mockSpawn).not.toHaveBeenCalled();
|
||||
expect(mockGetLegacyOrphanContainersByStack).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -990,7 +995,7 @@ describe('ComposeService - deployStack', () => {
|
||||
expect(error).not.toBeNull();
|
||||
expect(error!.message).toContain('CONTAINER_CRASHED');
|
||||
expect(getComposeRollbackInfo(error)).toEqual({ attempted: true, rolledBack: true });
|
||||
expect(mockRestoreStackFiles).toHaveBeenCalledWith('my-stack');
|
||||
expect(mockCompensateWithCandidate).toHaveBeenCalledWith('recovery-1', expect.any(Function));
|
||||
});
|
||||
|
||||
it('reports rollback failure when atomic restore fails', async () => {
|
||||
@@ -1001,7 +1006,7 @@ describe('ComposeService - deployStack', () => {
|
||||
Labels: { 'com.docker.compose.project': 'my-stack' },
|
||||
}]);
|
||||
mockContainerInspect.mockResolvedValue({ State: { ExitCode: 1 } });
|
||||
mockRestoreStackFiles.mockRejectedValueOnce(new Error('restore denied'));
|
||||
mockCompensateWithCandidate.mockResolvedValueOnce(false);
|
||||
|
||||
const svc = ComposeService.getInstance(1);
|
||||
const result = svc.deployStack('my-stack', undefined, true).then(() => null, (e: Error) => e);
|
||||
@@ -1012,6 +1017,28 @@ describe('ComposeService - deployStack', () => {
|
||||
expect(getComposeRollbackInfo(error)).toEqual({ attempted: true, rolledBack: false });
|
||||
});
|
||||
|
||||
it('wraps a missing hold tag as rolledBack=false without replacing the original error', async () => {
|
||||
setupAutoCloseSpawn();
|
||||
mockListContainers.mockResolvedValue([{
|
||||
Id: 'crashed-c1',
|
||||
State: 'exited',
|
||||
Labels: { 'com.docker.compose.project': 'my-stack' },
|
||||
}]);
|
||||
mockContainerInspect.mockResolvedValue({ State: { ExitCode: 1 } });
|
||||
mockCompensateWithCandidate.mockRejectedValueOnce(
|
||||
Object.assign(new Error('Held recovery image is missing'), { code: 'HELD_IMAGE_MISSING' }),
|
||||
);
|
||||
|
||||
const svc = ComposeService.getInstance(1);
|
||||
const result = svc.deployStack('my-stack', undefined, true).then(() => null, (e: Error) => e);
|
||||
|
||||
await vi.runAllTimersAsync();
|
||||
const error = await result;
|
||||
expect(error).not.toBeNull();
|
||||
expect(error!.message).toContain('CONTAINER_CRASHED');
|
||||
expect(getComposeRollbackInfo(error)).toEqual({ attempted: true, rolledBack: false });
|
||||
});
|
||||
|
||||
it('does not roll back when atomic=false', async () => {
|
||||
setupAutoCloseSpawn();
|
||||
mockListContainers.mockResolvedValue([{
|
||||
|
||||
@@ -50,6 +50,8 @@ describe('DeployedStackDeletionService ready transaction', () => {
|
||||
phase: 'immediate_verified',
|
||||
is_current: 1,
|
||||
backup_slot_id: null,
|
||||
content_path: null,
|
||||
operation_kind: null,
|
||||
override_path: null,
|
||||
services_json: '[]',
|
||||
health_gate_id: null,
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
/**
|
||||
* Docker-backed: recovery override must recreate containers whose inspected
|
||||
* Image ID equals the captured prior image ID (moving-tag case).
|
||||
* Skipped when Docker is unavailable.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { execFileSync } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { setupTestDb, cleanupTestDb } from '../helpers/setupTestDb';
|
||||
|
||||
function dockerAvailable(): boolean {
|
||||
try {
|
||||
execFileSync('docker', ['info'], {
|
||||
stdio: 'ignore',
|
||||
timeout: 8_000,
|
||||
windowsHide: true,
|
||||
});
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const hasDocker = dockerAvailable();
|
||||
const STACK = 'exactimg';
|
||||
|
||||
function compose(args: string[], cwd: string): string {
|
||||
return execFileSync('docker', ['compose', '-p', STACK, ...args], {
|
||||
cwd,
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
}
|
||||
|
||||
function inspectImageId(containerId: string): string {
|
||||
const raw = execFileSync('docker', ['inspect', containerId, '--format', '{{.Image}}'], {
|
||||
encoding: 'utf8',
|
||||
}).trim();
|
||||
expect(raw.length).toBeGreaterThan(0);
|
||||
return raw;
|
||||
}
|
||||
|
||||
describe.skipIf(!hasDocker)('exact prior-image rollback after post-handoff failure', () => {
|
||||
let tmpDir: string;
|
||||
let composeDir: string;
|
||||
let stackDir: string;
|
||||
let nodeId: number;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
composeDir = process.env.COMPOSE_DIR!;
|
||||
stackDir = path.join(composeDir, STACK);
|
||||
fs.mkdirSync(stackDir, { recursive: true });
|
||||
|
||||
const { DatabaseService } = await import('../../services/DatabaseService');
|
||||
const db = DatabaseService.getInstance();
|
||||
const local = db.getDefaultNode();
|
||||
if (!local?.id) throw new Error('Test DB has no default local node');
|
||||
nodeId = local.id;
|
||||
db.updateGlobalSetting('prune_on_update', '0');
|
||||
|
||||
execFileSync('docker', ['pull', 'busybox:1.36.1'], { stdio: 'ignore' });
|
||||
execFileSync('docker', ['pull', 'busybox:1.36.0'], { stdio: 'ignore' });
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(stackDir, 'compose.yaml'),
|
||||
[
|
||||
'services:',
|
||||
' web:',
|
||||
' image: busybox:1.36.1',
|
||||
' command: ["sleep", "3600"]',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
compose(['up', '-d', '--pull', 'never'], stackDir);
|
||||
}, 300_000);
|
||||
|
||||
afterAll(async () => {
|
||||
try {
|
||||
if (stackDir && fs.existsSync(stackDir)) {
|
||||
compose(['down', '--remove-orphans'], stackDir);
|
||||
}
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
if (tmpDir) cleanupTestDb(tmpDir);
|
||||
}, 120_000);
|
||||
|
||||
it('restores a container whose inspected Image equals the captured prior image ID', async () => {
|
||||
const beforeId = compose(['ps', '-q'], stackDir).trim();
|
||||
expect(beforeId.length).toBeGreaterThan(0);
|
||||
const priorImageId = inspectImageId(beforeId);
|
||||
|
||||
const { StackUpdateRecoveryService } = await import('../../services/StackUpdateRecoveryService');
|
||||
StackUpdateRecoveryService.resetForTests();
|
||||
const recoverySvc = StackUpdateRecoveryService.getInstance();
|
||||
|
||||
// Capture while the prior runtime is still healthy.
|
||||
const candidate = await recoverySvc.captureCandidate({
|
||||
nodeId,
|
||||
stackName: STACK,
|
||||
createdBy: null,
|
||||
operationKind: 'update',
|
||||
});
|
||||
expect(candidate.override_path).toBeTruthy();
|
||||
expect(recoverySvc.markAcquired(candidate.id)).toBe(true);
|
||||
expect(recoverySvc.handoff(candidate.id, nodeId, STACK)).toBe(true);
|
||||
|
||||
// Simulate a post-handoff mutation that moves the tag / image identity.
|
||||
fs.writeFileSync(
|
||||
path.join(stackDir, 'compose.yaml'),
|
||||
[
|
||||
'services:',
|
||||
' web:',
|
||||
' image: busybox:1.36.0',
|
||||
' command: ["sleep", "3600"]',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf8',
|
||||
);
|
||||
compose(['up', '-d', '--pull', 'never', '--force-recreate'], stackDir);
|
||||
const midId = compose(['ps', '-q'], stackDir).trim();
|
||||
const midImageId = inspectImageId(midId);
|
||||
expect(midImageId).not.toBe(priorImageId);
|
||||
|
||||
const { ComposeService } = await import('../../services/ComposeService');
|
||||
const rolledBack = await recoverySvc.compensateWithCandidate(
|
||||
candidate.id,
|
||||
(overridePath, invocation) => ComposeService.getInstance(nodeId).composeUpWithRecoveryOverride(
|
||||
STACK,
|
||||
overridePath,
|
||||
undefined,
|
||||
invocation,
|
||||
),
|
||||
);
|
||||
expect(rolledBack).toBe(true);
|
||||
|
||||
const afterId = compose(['ps', '-q'], stackDir).trim();
|
||||
expect(afterId.length).toBeGreaterThan(0);
|
||||
expect(inspectImageId(afterId)).toBe(priorImageId);
|
||||
|
||||
const afterInspect = JSON.parse(
|
||||
execFileSync('docker', ['inspect', afterId], { encoding: 'utf8' }),
|
||||
) as Array<{ State: { Running: boolean } }>;
|
||||
expect(afterInspect[0].State.Running).toBe(true);
|
||||
}, 300_000);
|
||||
});
|
||||
@@ -10,7 +10,11 @@ import { setupTestDb, cleanupTestDb } from '../helpers/setupTestDb';
|
||||
|
||||
function dockerAvailable(): boolean {
|
||||
try {
|
||||
execFileSync('docker', ['info'], { stdio: 'ignore' });
|
||||
execFileSync('docker', ['info'], {
|
||||
stdio: 'ignore',
|
||||
timeout: 8_000,
|
||||
windowsHide: true,
|
||||
});
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
|
||||
@@ -109,6 +109,21 @@ describe('classifyFailure', () => {
|
||||
message: 'something completely unexpected happened',
|
||||
reason: 'unknown',
|
||||
},
|
||||
{
|
||||
name: 'mixed replica images capture refusal',
|
||||
message: 'Service "web" has mixed replica images; refusing recovery capture that cannot restore exact prior identity',
|
||||
reason: 'mixed_replica_images',
|
||||
},
|
||||
{
|
||||
name: 'host-absolute include coverage refusal',
|
||||
message: 'Host-absolute include path cannot be captured for exact rollback',
|
||||
reason: 'rollback_coverage_unavailable',
|
||||
},
|
||||
{
|
||||
name: 'generic rollback coverage unavailable',
|
||||
message: 'Exact authored-project rollback coverage is unavailable for this stack',
|
||||
reason: 'rollback_coverage_unavailable',
|
||||
},
|
||||
];
|
||||
|
||||
it.each(cases)('classifies $name as $reason', ({ message, reason }) => {
|
||||
|
||||
@@ -8,8 +8,11 @@ import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest
|
||||
import request from 'supertest';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { createHash, randomUUID } from 'crypto';
|
||||
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
|
||||
import * as policyGate from '../helpers/policyGate';
|
||||
import { RollbackGenerationStore } from '../services/RollbackGenerationStore';
|
||||
import type { ResolvedRollbackInventory, RollbackGenerationManifest } from '../types/rollbackGeneration';
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
@@ -18,6 +21,9 @@ let CryptoService: typeof import('../services/CryptoService').CryptoService;
|
||||
let ComposeService: typeof import('../services/ComposeService').ComposeService;
|
||||
let LicenseService: typeof import('../services/LicenseService').LicenseService;
|
||||
let NodeRegistry: typeof import('../services/NodeRegistry').NodeRegistry;
|
||||
let FileSystemService: typeof import('../services/FileSystemService').FileSystemService;
|
||||
let StackUpdateRecoveryService: typeof import('../services/StackUpdateRecoveryService').StackUpdateRecoveryService;
|
||||
let StackOpLockService: typeof import('../services/StackOpLockService').StackOpLockService;
|
||||
let adminCookie: string;
|
||||
let viewerCookie: string;
|
||||
let snapshotId: number;
|
||||
@@ -39,6 +45,9 @@ beforeAll(async () => {
|
||||
({ ComposeService } = await import('../services/ComposeService'));
|
||||
({ LicenseService } = await import('../services/LicenseService'));
|
||||
({ NodeRegistry } = await import('../services/NodeRegistry'));
|
||||
({ FileSystemService } = await import('../services/FileSystemService'));
|
||||
({ StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'));
|
||||
({ StackOpLockService } = await import('../services/StackOpLockService'));
|
||||
({ app } = await import('../index'));
|
||||
adminCookie = await loginAsTestAdmin(app);
|
||||
|
||||
@@ -289,7 +298,7 @@ describe('Single-stack snapshot restore (behavior lock)', () => {
|
||||
fs.writeFileSync(composePath('corrupt-web'), beforeCompose);
|
||||
fs.writeFileSync(envPath('corrupt-web'), beforeEnv);
|
||||
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null });
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
@@ -320,7 +329,7 @@ describe('Single-stack snapshot restore (behavior lock)', () => {
|
||||
fs.writeFileSync(composePath('mixed-web'), beforeCompose);
|
||||
fs.writeFileSync(envPath('mixed-web'), beforeEnv);
|
||||
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null });
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
@@ -346,7 +355,7 @@ describe('Single-stack snapshot restore (behavior lock)', () => {
|
||||
const beforeCompose = 'services:\n keep: {}\n';
|
||||
fs.writeFileSync(composePath('delim-web'), beforeCompose);
|
||||
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null });
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
@@ -387,7 +396,7 @@ describe('Single-stack snapshot restore (behavior lock)', () => {
|
||||
|
||||
it('redeploys after restore when requested', async () => {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null });
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-redeploy', 'admin', 1, 1, '[]', '[]');
|
||||
db.insertSnapshotFiles(id, [
|
||||
@@ -749,7 +758,7 @@ describe('Restore-all', () => {
|
||||
|
||||
it('isolates corrupt decrypt stacks before any mutation with notes and redeploy requested', async () => {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null });
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-all-corrupt', 'admin', 1, 2, '[]', '[]');
|
||||
const good = CryptoService.getInstance().encrypt('services:\n app: {}\n');
|
||||
@@ -791,7 +800,7 @@ describe('Restore-all', () => {
|
||||
|
||||
it('isolates delimiter-byte corruption before restore-all mutation', async () => {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null });
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-all-delim', 'admin', 1, 2, '[]', '[]');
|
||||
const good = CryptoService.getInstance().encrypt('services:\n app: {}\n');
|
||||
@@ -801,13 +810,13 @@ describe('Restore-all', () => {
|
||||
const damaged = `enc:${iv} ${tag}:${ct}`;
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(id, LOCAL_NODE_ID, 'local', 'healthy', 'compose.yaml', good);
|
||||
).run(id, LOCAL_NODE_ID, 'local', 'delim-healthy', 'compose.yaml', good);
|
||||
db.getDb().prepare(
|
||||
'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)',
|
||||
).run(id, LOCAL_NODE_ID, 'local', 'corrupt', 'compose.yaml', damaged);
|
||||
seedStackDir('healthy');
|
||||
seedStackDir('corrupt');
|
||||
fs.writeFileSync(composePath('corrupt'), 'services:\n keep: {}\n');
|
||||
).run(id, LOCAL_NODE_ID, 'local', 'delim-corrupt', 'compose.yaml', damaged);
|
||||
seedStackDir('delim-healthy');
|
||||
seedStackDir('delim-corrupt');
|
||||
fs.writeFileSync(composePath('delim-corrupt'), 'services:\n keep: {}\n');
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore-all`)
|
||||
@@ -817,16 +826,16 @@ describe('Restore-all', () => {
|
||||
expect(res.body.restored).toBe(1);
|
||||
expect(res.body.failed).toBe(1);
|
||||
const corrupt = (res.body.results as Array<{ stackName: string; success: boolean; error?: string }>)
|
||||
.find(r => r.stackName === 'corrupt');
|
||||
.find(r => r.stackName === 'delim-corrupt');
|
||||
expect(corrupt?.success).toBe(false);
|
||||
expect(fs.readFileSync(composePath('corrupt'), 'utf-8')).toContain('keep: {}');
|
||||
expect(fs.readFileSync(composePath('delim-corrupt'), 'utf-8')).toContain('keep: {}');
|
||||
expect(deploySpy).toHaveBeenCalledTimes(1);
|
||||
expect(deploySpy.mock.calls.every(call => call[0] !== 'corrupt')).toBe(true);
|
||||
expect(deploySpy.mock.calls.every(call => call[0] !== 'delim-corrupt')).toBe(true);
|
||||
});
|
||||
|
||||
it('redeploys each restored stack when requested', async () => {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null });
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot('restore-all-redeploy', 'admin', 1, 1, '[]', '[]');
|
||||
db.insertSnapshotFiles(id, [
|
||||
@@ -847,7 +856,7 @@ describe('Restore-all', () => {
|
||||
});
|
||||
|
||||
it('records a policy-blocked redeploy as a per-stack failure and still restores the rest', async () => {
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null });
|
||||
vi.spyOn(policyGate, 'assertPolicyGateAllows').mockImplementation(async (stackName: string) => {
|
||||
if (stackName === 'blocked-web') throw new Error('Policy "block-criticals" blocked deploy: 1 image(s) exceed high');
|
||||
});
|
||||
@@ -876,3 +885,456 @@ describe('Restore-all', () => {
|
||||
expect(deploySpy.mock.calls.map((c) => c[0])).not.toContain('blocked-web');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Snapshot restore: recovery generation contract', () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
StackOpLockService.resetForTests();
|
||||
DatabaseService.getInstance().getDb().prepare('DELETE FROM stack_update_recovery_generations').run();
|
||||
});
|
||||
|
||||
function seedExistingStack(stack: string, compose: string, extraName?: string, extraContent?: string): void {
|
||||
seedStackDir(stack);
|
||||
fs.writeFileSync(composePath(stack), compose);
|
||||
if (extraName && extraContent !== undefined) {
|
||||
fs.writeFileSync(path.join(process.env.COMPOSE_DIR as string, stack, extraName), extraContent);
|
||||
}
|
||||
}
|
||||
|
||||
function insertSnapshot(
|
||||
label: string,
|
||||
files: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }>,
|
||||
stackCount = 1,
|
||||
): number {
|
||||
const db = DatabaseService.getInstance();
|
||||
const id = db.createSnapshot(label, 'admin', 1, stackCount, '[]', '[]');
|
||||
db.insertSnapshotFiles(id, files);
|
||||
return id;
|
||||
}
|
||||
|
||||
function addRemoteNode(name: string): number {
|
||||
return DatabaseService.getInstance().addNode({
|
||||
name,
|
||||
type: 'remote',
|
||||
api_url: 'http://remote:1852',
|
||||
api_token: 'tok',
|
||||
compose_dir: '/app/compose',
|
||||
is_default: false,
|
||||
});
|
||||
}
|
||||
|
||||
function stubRemoteProxy(): void {
|
||||
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
|
||||
apiUrl: 'http://remote:1852',
|
||||
apiToken: 'tok',
|
||||
});
|
||||
}
|
||||
|
||||
function sha256Text(text: string): string {
|
||||
return createHash('sha256').update(text, 'utf8').digest('hex');
|
||||
}
|
||||
|
||||
function liveInventory(stack: string): ResolvedRollbackInventory {
|
||||
const stackDir = path.join(process.env.COMPOSE_DIR as string, stack);
|
||||
const entries: ResolvedRollbackInventory['entries'] = [{
|
||||
relativePath: 'compose.yaml',
|
||||
dependencyKind: 'compose-root',
|
||||
provenance: 'authored',
|
||||
sensitivity: 'low',
|
||||
absolutePath: path.join(stackDir, 'compose.yaml'),
|
||||
}];
|
||||
if (fs.existsSync(envPath(stack))) {
|
||||
entries.push({
|
||||
relativePath: '.env',
|
||||
dependencyKind: 'project-env',
|
||||
provenance: 'authored',
|
||||
sensitivity: 'low',
|
||||
absolutePath: envPath(stack),
|
||||
});
|
||||
}
|
||||
return {
|
||||
entries,
|
||||
invocation: {
|
||||
composeArgsPrefix: [],
|
||||
projectDirectory: null,
|
||||
projectName: stack,
|
||||
explicitComposeFiles: ['compose.yaml'],
|
||||
meshEnabled: false,
|
||||
meshOverrideRelativePath: null,
|
||||
},
|
||||
git: null,
|
||||
appliedDeploySpec: null,
|
||||
lastAppliedContentHash: null,
|
||||
manifestState: null,
|
||||
manifestGeneration: null,
|
||||
exactCoverage: true,
|
||||
coverageRefusal: null,
|
||||
};
|
||||
}
|
||||
|
||||
async function persistPreRestoreGeneration(stack: string): Promise<string> {
|
||||
const id = randomUUID();
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: LOCAL_NODE_ID,
|
||||
stackName: stack,
|
||||
generationId: id,
|
||||
inventory: liveInventory(stack),
|
||||
operationKind: 'manual_backup',
|
||||
});
|
||||
const now = Date.now();
|
||||
DatabaseService.getInstance().insertStackUpdateRecoveryGeneration({
|
||||
id,
|
||||
node_id: LOCAL_NODE_ID,
|
||||
stack_name: stack,
|
||||
status: 'active',
|
||||
phase: 'immediate_verified',
|
||||
is_current: 1,
|
||||
backup_slot_id: id,
|
||||
content_path: id,
|
||||
operation_kind: 'manual_backup',
|
||||
override_path: null,
|
||||
services_json: '[]',
|
||||
health_gate_id: null,
|
||||
gate_retain_until: null,
|
||||
artifact_expires_at: null,
|
||||
operation_lease_expires_at: null,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
created_by: 'system:fleet-snapshot',
|
||||
artifacts_retired: 0,
|
||||
released_at: null,
|
||||
released_by: null,
|
||||
});
|
||||
return id;
|
||||
}
|
||||
|
||||
function spyCaptureRecordingLive(stack: string): { composeAtCapture: string; envAtCapture: string } {
|
||||
const captured = { composeAtCapture: '', envAtCapture: '' };
|
||||
vi.spyOn(StackUpdateRecoveryService.getInstance(), 'captureCurrentBackup').mockImplementation(async () => {
|
||||
captured.composeAtCapture = fs.readFileSync(composePath(stack), 'utf-8');
|
||||
captured.envAtCapture = fs.existsSync(envPath(stack)) ? fs.readFileSync(envPath(stack), 'utf-8') : '';
|
||||
const id = await persistPreRestoreGeneration(stack);
|
||||
return { id } as never;
|
||||
});
|
||||
return captured;
|
||||
}
|
||||
|
||||
function expectCurrentGenerationMatches(stack: string, compose: string, env: string): void {
|
||||
const current = StackUpdateRecoveryService.getInstance().getCurrent(LOCAL_NODE_ID, stack);
|
||||
expect(current?.id).toBeTruthy();
|
||||
const genDir = RollbackGenerationStore.getGenerationDir(LOCAL_NODE_ID, stack, current!.id);
|
||||
const manifest = JSON.parse(
|
||||
fs.readFileSync(path.join(genDir, 'generation.json'), 'utf-8'),
|
||||
) as RollbackGenerationManifest;
|
||||
expect(manifest.entries.find((e) => e.relativePath === 'compose.yaml')?.contentSha256).toBe(sha256Text(compose));
|
||||
expect(manifest.entries.find((e) => e.relativePath === '.env')?.contentSha256).toBe(sha256Text(env));
|
||||
}
|
||||
|
||||
it('captures the pre-restore multi-file project before overwriting compose.yaml', async () => {
|
||||
const id = insertSnapshot('restore-gen-local', [
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'preimage-web', filename: 'compose.yaml', content: 'services:\n snap: {}\n' },
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'preimage-web', filename: '.env', content: 'SNAP=1\n' },
|
||||
]);
|
||||
const preCompose = 'services:\n old: {}\n';
|
||||
const preEnv = 'OLD=1\n';
|
||||
seedExistingStack('preimage-web', preCompose, 'extra.yml', 'x: 1\n');
|
||||
fs.writeFileSync(envPath('preimage-web'), preEnv);
|
||||
const captured = spyCaptureRecordingLive('preimage-web');
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: LOCAL_NODE_ID, stackName: 'preimage-web' });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(captured.composeAtCapture).toBe(preCompose);
|
||||
expect(captured.envAtCapture).toBe(preEnv);
|
||||
expect(fs.readFileSync(composePath('preimage-web'), 'utf-8')).toContain('snap: {}');
|
||||
expect(fs.readFileSync(envPath('preimage-web'), 'utf-8')).toContain('SNAP=1');
|
||||
expect(fs.readFileSync(path.join(process.env.COMPOSE_DIR as string, 'preimage-web', 'extra.yml'), 'utf-8')).toBe('x: 1\n');
|
||||
expectCurrentGenerationMatches('preimage-web', preCompose, preEnv);
|
||||
});
|
||||
|
||||
it('does not mutate live files when capture fails', async () => {
|
||||
const id = insertSnapshot('restore-gen-fail', [
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'fail-web', filename: 'compose.yaml', content: 'services:\n snap: {}\n' },
|
||||
]);
|
||||
const preCompose = 'services:\n keep: {}\n';
|
||||
seedExistingStack('fail-web', preCompose);
|
||||
vi.spyOn(StackUpdateRecoveryService.getInstance(), 'captureCurrentBackup')
|
||||
.mockRejectedValue(new Error('generation capture failed'));
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: LOCAL_NODE_ID, stackName: 'fail-web' });
|
||||
|
||||
expect(res.status).toBe(500);
|
||||
expect(fs.readFileSync(composePath('fail-web'), 'utf-8')).toBe(preCompose);
|
||||
});
|
||||
|
||||
it('returns 409 and does not write when the stack operation lock is held', async () => {
|
||||
const id = insertSnapshot('restore-gen-lock', [
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'lock-web', filename: 'compose.yaml', content: 'services:\n snap: {}\n' },
|
||||
]);
|
||||
const preCompose = 'services:\n keep: {}\n';
|
||||
seedExistingStack('lock-web', preCompose);
|
||||
const captureSpy = vi.spyOn(StackUpdateRecoveryService.getInstance(), 'captureCurrentBackup')
|
||||
.mockResolvedValue({ id: 'gen-lock-web' } as never);
|
||||
StackOpLockService.getInstance().tryAcquire(LOCAL_NODE_ID, 'lock-web', 'deploy', 'other');
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: LOCAL_NODE_ID, stackName: 'lock-web' });
|
||||
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.code).toBe('stack_op_in_progress');
|
||||
expect(captureSpy).not.toHaveBeenCalled();
|
||||
expect(fs.readFileSync(composePath('lock-web'), 'utf-8')).toBe(preCompose);
|
||||
});
|
||||
|
||||
it('keeps the captured generation when a later snapshot file write fails', async () => {
|
||||
const id = insertSnapshot('restore-gen-partial', [
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'partial-web', filename: 'compose.yaml', content: 'services:\n snap: {}\n' },
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'partial-web', filename: '.env', content: 'SNAP=1\n' },
|
||||
]);
|
||||
const preCompose = 'services:\n old: {}\n';
|
||||
const preEnv = 'OLD=1\n';
|
||||
seedExistingStack('partial-web', preCompose);
|
||||
fs.writeFileSync(envPath('partial-web'), preEnv);
|
||||
const captured = spyCaptureRecordingLive('partial-web');
|
||||
vi.spyOn(FileSystemService.prototype, 'saveEnvContent').mockRejectedValue(new Error('disk full'));
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: LOCAL_NODE_ID, stackName: 'partial-web' });
|
||||
|
||||
expect(res.status).toBe(500);
|
||||
expect(captured.composeAtCapture).toBe(preCompose);
|
||||
expect(captured.envAtCapture).toBe(preEnv);
|
||||
expect(fs.readFileSync(composePath('partial-web'), 'utf-8')).toContain('snap: {}');
|
||||
expect(fs.readFileSync(envPath('partial-web'), 'utf-8')).toBe(preEnv);
|
||||
expectCurrentGenerationMatches('partial-web', preCompose, preEnv);
|
||||
});
|
||||
|
||||
it('does not capture a generation when restoring a stack that does not exist yet', async () => {
|
||||
const id = insertSnapshot('restore-gen-new', [
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'brand-new', filename: 'compose.yaml', content: 'services:\n snap: {}\n' },
|
||||
]);
|
||||
seedStackDir('brand-new');
|
||||
const captureSpy = vi.spyOn(StackUpdateRecoveryService.getInstance(), 'captureCurrentBackup')
|
||||
.mockResolvedValue({ id: 'should-not-run' } as never);
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: LOCAL_NODE_ID, stackName: 'brand-new' });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(captureSpy).not.toHaveBeenCalled();
|
||||
expect(fs.readFileSync(composePath('brand-new'), 'utf-8')).toContain('snap: {}');
|
||||
});
|
||||
|
||||
it('restore-all captures existing stacks and still restores siblings when one capture fails', async () => {
|
||||
const id = insertSnapshot('restore-all-gen', [
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'all-keep', filename: 'compose.yaml', content: 'services:\n keep: {}\n' },
|
||||
{ nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'all-fail', filename: 'compose.yaml', content: 'services:\n fail: {}\n' },
|
||||
], 2);
|
||||
seedExistingStack('all-keep', 'services:\n old-keep: {}\n');
|
||||
fs.writeFileSync(envPath('all-keep'), 'KEEP=1\n');
|
||||
seedExistingStack('all-fail', 'services:\n old-fail: {}\n');
|
||||
fs.writeFileSync(envPath('all-fail'), 'FAIL=1\n');
|
||||
vi.spyOn(StackUpdateRecoveryService.getInstance(), 'captureCurrentBackup')
|
||||
.mockImplementation(async (input) => {
|
||||
if (input.stackName === 'all-fail') throw new Error('generation capture failed');
|
||||
const id = await persistPreRestoreGeneration(input.stackName);
|
||||
return { id } as never;
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore-all`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.restored).toBe(1);
|
||||
expect(res.body.failed).toBe(1);
|
||||
const failed = (res.body.results as Array<{ stackName: string; success: boolean }>).find(r => r.stackName === 'all-fail');
|
||||
expect(failed?.success).toBe(false);
|
||||
expect(fs.readFileSync(composePath('all-keep'), 'utf-8')).toContain('keep: {}');
|
||||
expect(fs.readFileSync(composePath('all-fail'), 'utf-8')).toContain('old-fail: {}');
|
||||
expect(fs.readFileSync(envPath('all-fail'), 'utf-8')).toBe('FAIL=1\n');
|
||||
expectCurrentGenerationMatches('all-keep', 'services:\n old-keep: {}\n', 'KEEP=1\n');
|
||||
expect(StackUpdateRecoveryService.getInstance().getCurrent(LOCAL_NODE_ID, 'all-fail')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('restores a remote stack through one node-local apply request', async () => {
|
||||
const remoteId = addRemoteNode('remote-gen');
|
||||
const id = insertSnapshot('restore-gen-remote', [
|
||||
{ nodeId: remoteId, nodeName: 'remote-gen', stackName: 'rweb', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
{ nodeId: remoteId, nodeName: 'remote-gen', stackName: 'rweb', filename: '.env', content: 'SNAP=1\n' },
|
||||
]);
|
||||
stubRemoteProxy();
|
||||
const calls: Array<{ url: string; method?: string; body?: string }> = [];
|
||||
vi.stubGlobal('fetch', vi.fn(async (url: string, opts?: { method?: string; body?: string }) => {
|
||||
calls.push({ url, method: opts?.method, body: opts?.body });
|
||||
return { ok: true, status: 200, text: async () => '' } as unknown as Response;
|
||||
}));
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: remoteId, stackName: 'rweb' });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const applyCalls = calls.filter(c => c.url.includes('/fleet-snapshot-apply'));
|
||||
expect(applyCalls).toHaveLength(1);
|
||||
expect(applyCalls[0].method).toBe('POST');
|
||||
expect(applyCalls[0].body).toContain('compose.yaml');
|
||||
expect(applyCalls[0].body).toContain('.env');
|
||||
expect(calls.some(c => c.method === 'PUT' && /\/api\/stacks\/[^/]+$/.test(c.url))).toBe(false);
|
||||
expect(calls.some(c => c.method === 'PUT' && /\/env$/.test(c.url))).toBe(false);
|
||||
});
|
||||
|
||||
it('fails a remote restore without writing when the node-local apply returns 409', async () => {
|
||||
const remoteId = addRemoteNode('remote-lock');
|
||||
const id = insertSnapshot('restore-gen-remote-lock', [
|
||||
{ nodeId: remoteId, nodeName: 'remote-lock', stackName: 'rlock', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
]);
|
||||
stubRemoteProxy();
|
||||
vi.stubGlobal('fetch', vi.fn(async () => ({
|
||||
ok: false,
|
||||
status: 409,
|
||||
text: async () => JSON.stringify({ error: 'locked', code: 'stack_op_in_progress' }),
|
||||
} as unknown as Response)));
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: remoteId, stackName: 'rlock' });
|
||||
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.code).toBe('stack_op_in_progress');
|
||||
});
|
||||
|
||||
it('does not invent stack_op_in_progress from a bare remote 409', async () => {
|
||||
const remoteId = addRemoteNode('remote-bare-409');
|
||||
const id = insertSnapshot('restore-gen-remote-bare-409', [
|
||||
{ nodeId: remoteId, nodeName: 'remote-bare-409', stackName: 'rbare', filename: 'compose.yaml', content: 'services: {}\n' },
|
||||
]);
|
||||
stubRemoteProxy();
|
||||
vi.stubGlobal('fetch', vi.fn(async () => ({
|
||||
ok: false,
|
||||
status: 409,
|
||||
text: async () => JSON.stringify({ error: 'conflict' }),
|
||||
} as unknown as Response)));
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ nodeId: remoteId, stackName: 'rbare' });
|
||||
|
||||
expect(res.status).toBe(500);
|
||||
expect(res.body.code).toBeUndefined();
|
||||
});
|
||||
|
||||
it('restore-all sends one node-local apply per remote stack', async () => {
|
||||
const remoteId = addRemoteNode('remote-all');
|
||||
const id = insertSnapshot('restore-all-gen-remote', [
|
||||
{ nodeId: remoteId, nodeName: 'remote-all', stackName: 'ra', filename: 'compose.yaml', content: 'services:\n a: {}\n' },
|
||||
{ nodeId: remoteId, nodeName: 'remote-all', stackName: 'rb', filename: 'compose.yaml', content: 'services:\n b: {}\n' },
|
||||
], 2);
|
||||
stubRemoteProxy();
|
||||
const urls: string[] = [];
|
||||
vi.stubGlobal('fetch', vi.fn(async (url: string) => {
|
||||
urls.push(url);
|
||||
return { ok: true, status: 200, text: async () => '' } as unknown as Response;
|
||||
}));
|
||||
|
||||
const res = await request(app)
|
||||
.post(`/api/fleet/snapshots/${id}/restore-all`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.restored).toBe(2);
|
||||
expect(urls.filter(u => u.includes('/fleet-snapshot-apply'))).toHaveLength(2);
|
||||
expect(urls.some(u => /\/api\/stacks\/[^/]+$/.test(u))).toBe(false);
|
||||
});
|
||||
|
||||
it('POST /api/stacks/:stackName/fleet-snapshot-apply returns 403 for a viewer', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/preimage-web/fleet-snapshot-apply')
|
||||
.set('Cookie', viewerCookie)
|
||||
.send({ files: [{ filename: 'compose.yaml', content: 'services: {}\n' }] });
|
||||
expect(res.status).toBe(403);
|
||||
});
|
||||
|
||||
it('POST /api/stacks/:stackName/fleet-snapshot-apply captures an existing stack before writing', async () => {
|
||||
const preCompose = 'services:\n old: {}\n';
|
||||
const preEnv = 'OLD=1\n';
|
||||
seedExistingStack('apply-existing', preCompose);
|
||||
fs.writeFileSync(envPath('apply-existing'), preEnv);
|
||||
spyCaptureRecordingLive('apply-existing');
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/apply-existing/fleet-snapshot-apply')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({
|
||||
files: [
|
||||
{ filename: 'compose.yaml', content: 'services:\n snap: {}\n' },
|
||||
{ filename: '.env', content: 'SNAP=1\n' },
|
||||
],
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.capturedGenerationId).toBe(
|
||||
StackUpdateRecoveryService.getInstance().getCurrent(LOCAL_NODE_ID, 'apply-existing')?.id,
|
||||
);
|
||||
expect(fs.readFileSync(composePath('apply-existing'), 'utf-8')).toContain('snap: {}');
|
||||
expect(fs.readFileSync(envPath('apply-existing'), 'utf-8')).toBe('SNAP=1\n');
|
||||
expectCurrentGenerationMatches('apply-existing', preCompose, preEnv);
|
||||
});
|
||||
|
||||
it('POST /api/stacks/:stackName/fleet-snapshot-apply returns 400 when no restoreable files remain', async () => {
|
||||
seedStackDir('apply-empty');
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/apply-empty/fleet-snapshot-apply')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ files: [{ filename: 'notes.txt', content: 'nope\n' }] });
|
||||
expect(res.status).toBe(400);
|
||||
expect(fs.existsSync(composePath('apply-empty'))).toBe(false);
|
||||
});
|
||||
|
||||
it('POST /api/stacks/:stackName/fleet-snapshot-apply ignores extra filenames', async () => {
|
||||
seedStackDir('apply-extra');
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/apply-extra/fleet-snapshot-apply')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({
|
||||
files: [
|
||||
{ filename: 'compose.yaml', content: 'services:\n snap: {}\n' },
|
||||
{ filename: 'notes.txt', content: 'should-not-write\n' },
|
||||
{ filename: '.env', content: 'SNAP=1\n' },
|
||||
],
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
expect(fs.readFileSync(composePath('apply-extra'), 'utf-8')).toContain('snap: {}');
|
||||
expect(fs.readFileSync(envPath('apply-extra'), 'utf-8')).toBe('SNAP=1\n');
|
||||
expect(fs.existsSync(path.join(process.env.COMPOSE_DIR as string, 'apply-extra', 'notes.txt'))).toBe(false);
|
||||
});
|
||||
|
||||
it('POST /api/stacks/:stackName/fleet-snapshot-apply accepts a combined body over 100 KB', async () => {
|
||||
seedStackDir('apply-large');
|
||||
const compose = `services:\n snap:\n image: nginx\n labels:\n note: "${'x'.repeat(150 * 1024)}"\n`;
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/apply-large/fleet-snapshot-apply')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ files: [{ filename: 'compose.yaml', content: compose }] });
|
||||
expect(res.status).toBe(200);
|
||||
expect(fs.readFileSync(composePath('apply-large'), 'utf-8')).toBe(compose);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
/**
|
||||
* R1: Git apply promote succeeds, deploy fails → applied true, generation current,
|
||||
* compensateWithCandidate is not called.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const mockCaptureCandidate = vi.fn();
|
||||
const mockAbandon = vi.fn();
|
||||
const mockMarkAcquired = vi.fn().mockReturnValue(true);
|
||||
const mockHandoff = vi.fn().mockReturnValue(true);
|
||||
const mockMarkReconciling = vi.fn().mockReturnValue(true);
|
||||
const mockMarkImmediateVerified = vi.fn().mockReturnValue(true);
|
||||
const mockGet = vi.fn();
|
||||
const mockCompensate = vi.fn();
|
||||
|
||||
vi.mock('../services/StackUpdateRecoveryService', () => ({
|
||||
StackUpdateRecoveryService: {
|
||||
getInstance: () => ({
|
||||
captureCandidate: mockCaptureCandidate,
|
||||
abandon: mockAbandon,
|
||||
markAcquired: mockMarkAcquired,
|
||||
handoff: mockHandoff,
|
||||
markReconciling: mockMarkReconciling,
|
||||
markImmediateVerified: mockMarkImmediateVerified,
|
||||
get: mockGet,
|
||||
compensateWithCandidate: mockCompensate,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
const mockDeployStack = vi.fn();
|
||||
vi.mock('../services/ComposeService', () => ({
|
||||
ComposeService: {
|
||||
getInstance: () => ({
|
||||
deployStack: mockDeployStack,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/StackOpLockService', () => ({
|
||||
StackOpLockService: {
|
||||
getInstance: () => ({
|
||||
runExclusive: async (
|
||||
_n: number,
|
||||
_s: string,
|
||||
_a: string,
|
||||
_who: string,
|
||||
fn: () => Promise<unknown>,
|
||||
) => {
|
||||
const result = await fn();
|
||||
return { ran: true, result };
|
||||
},
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../helpers/policyGate', () => ({
|
||||
assertPolicyGateAllows: vi.fn().mockResolvedValue(undefined),
|
||||
buildSystemPolicyGateOptions: vi.fn().mockReturnValue({}),
|
||||
}));
|
||||
|
||||
vi.mock('../services/HealthGateService', () => ({
|
||||
HealthGateService: {
|
||||
getInstance: () => ({
|
||||
beginStack: vi.fn(),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/NodeRegistry', () => ({
|
||||
NodeRegistry: {
|
||||
getInstance: () => ({
|
||||
getDefaultNodeId: () => 1,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
const mockPromoteGeneration = vi.fn().mockResolvedValue(undefined);
|
||||
vi.mock('../services/GitProjectManifestService', () => ({
|
||||
GitProjectManifestService: {
|
||||
getInstance: () => ({
|
||||
readManifest: vi.fn().mockResolvedValue(null),
|
||||
buildManifest: vi.fn().mockReturnValue({
|
||||
manifestVersion: 1,
|
||||
state: 'active',
|
||||
inputs: [],
|
||||
refusals: [],
|
||||
generation: { candidateDir: 'c', appliedDir: 'a', previousDir: null },
|
||||
}),
|
||||
promoteGeneration: mockPromoteGeneration,
|
||||
boundsConfig: vi.fn().mockReturnValue({}),
|
||||
hashStackFile: vi.fn(),
|
||||
verifyContextOnDisk: vi.fn().mockResolvedValue([]),
|
||||
writeManifest: vi.fn(),
|
||||
buildMigratedManifest: vi.fn(),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../utils/authoredComposeArgs', () => ({
|
||||
authoredComposeFileArgs: vi.fn().mockResolvedValue(['-f', 'compose.yaml']),
|
||||
authoredComposeEnvFileArgs: vi.fn().mockResolvedValue([]),
|
||||
}));
|
||||
|
||||
const mockGetGitSource = vi.fn();
|
||||
const mockMarkGitSourceApplied = vi.fn();
|
||||
const mockSetGitSourceAppliedSpec = vi.fn();
|
||||
const mockSetGitSourceManifestState = vi.fn();
|
||||
|
||||
vi.mock('../services/DatabaseService', () => ({
|
||||
DatabaseService: {
|
||||
getInstance: () => ({
|
||||
getGitSource: mockGetGitSource,
|
||||
markGitSourceApplied: mockMarkGitSourceApplied,
|
||||
setGitSourceAppliedSpec: mockSetGitSourceAppliedSpec,
|
||||
setGitSourceManifestState: mockSetGitSourceManifestState,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/CryptoService', () => ({
|
||||
CryptoService: {
|
||||
getInstance: () => ({
|
||||
decrypt: (v: string) => v,
|
||||
encrypt: (v: string) => v,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('fs', async () => {
|
||||
const actual = await vi.importActual<typeof import('fs')>('fs');
|
||||
return {
|
||||
...actual,
|
||||
promises: {
|
||||
...actual.promises,
|
||||
access: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
describe('git-source apply recovery (R1)', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockCaptureCandidate.mockResolvedValue({
|
||||
id: 'rec-1',
|
||||
node_id: 1,
|
||||
stack_name: 'app',
|
||||
status: 'candidate',
|
||||
phase: 'captured',
|
||||
is_current: 0,
|
||||
});
|
||||
mockGet.mockReturnValue({
|
||||
id: 'rec-1',
|
||||
is_current: 1,
|
||||
status: 'active',
|
||||
phase: 'reconciling',
|
||||
});
|
||||
mockDeployStack.mockRejectedValue(new Error('compose up failed'));
|
||||
mockGetGitSource.mockReturnValue({
|
||||
stack_name: 'app',
|
||||
repo_url: 'https://example.com/repo.git',
|
||||
branch: 'main',
|
||||
pending_commit_sha: 'abc1234deadbeef',
|
||||
pending_compose_content: JSON.stringify({
|
||||
v: 3,
|
||||
files: { 'compose.yaml': 'services:\n web:\n image: nginx\n' },
|
||||
contextDir: null,
|
||||
candidateRelPath: 'generations/cand',
|
||||
inventory: {
|
||||
inputs: [],
|
||||
refusals: [],
|
||||
buildContexts: [],
|
||||
},
|
||||
}),
|
||||
pending_env_content: null,
|
||||
sync_env: false,
|
||||
compose_paths: ['compose.yaml'],
|
||||
context_dir: null,
|
||||
auto_deploy_on_apply: true,
|
||||
applied_deploy_spec: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps applied=true and generation current without compensate when deploy fails', async () => {
|
||||
const { GitSourceService } = await import('../services/GitSourceService');
|
||||
// Avoid withStackLock contention by calling applyLocked through apply
|
||||
// after stubbing the lock if present.
|
||||
const svc = GitSourceService.getInstance();
|
||||
const withLock = vi.spyOn(
|
||||
svc as unknown as { withStackLock: (name: string, fn: () => Promise<unknown>) => Promise<unknown> },
|
||||
'withStackLock',
|
||||
);
|
||||
withLock.mockImplementation(async (_name, fn) => fn());
|
||||
|
||||
// validateCandidate is used on the v3 path; stub it open.
|
||||
vi.spyOn(
|
||||
svc as unknown as {
|
||||
validateCandidate: (...args: unknown[]) => Promise<{ ok: boolean }>;
|
||||
},
|
||||
'validateCandidate',
|
||||
).mockResolvedValue({ ok: true });
|
||||
|
||||
vi.spyOn(
|
||||
svc as unknown as {
|
||||
decodePendingCompose: (raw: string) => unknown;
|
||||
},
|
||||
'decodePendingCompose',
|
||||
).mockReturnValue({
|
||||
files: { 'compose.yaml': 'services:\n web:\n image: nginx\n' },
|
||||
contextDir: null,
|
||||
candidateRelPath: 'generations/cand',
|
||||
inventory: { inputs: [], refusals: [], buildContexts: [] },
|
||||
});
|
||||
|
||||
vi.spyOn(
|
||||
svc as unknown as {
|
||||
deriveAppliedSpec: (...args: unknown[]) => unknown;
|
||||
},
|
||||
'deriveAppliedSpec',
|
||||
).mockReturnValue({ files: ['compose.yaml'], contextDir: null });
|
||||
|
||||
vi.spyOn(
|
||||
svc as unknown as {
|
||||
hashContent: (...args: unknown[]) => string;
|
||||
},
|
||||
'hashContent',
|
||||
).mockReturnValue('hash');
|
||||
|
||||
const result = await svc.apply('app', 'abc1234deadbeef', { deploy: true, actor: 'tester' });
|
||||
|
||||
expect(result.applied).toBe(true);
|
||||
expect(result.deployed).toBe(false);
|
||||
expect(result.deployError).toBeTruthy();
|
||||
expect(result.recoveryId).toBe('rec-1');
|
||||
expect(mockPromoteGeneration).toHaveBeenCalled();
|
||||
expect(mockCaptureCandidate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ operationKind: 'git_apply', stackName: 'app' }),
|
||||
);
|
||||
expect(mockHandoff).toHaveBeenCalled();
|
||||
expect(mockCompensate).not.toHaveBeenCalled();
|
||||
expect(mockAbandon).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('refuses to promote when recovery capture fails', async () => {
|
||||
mockCaptureCandidate.mockRejectedValue(new Error('Exact authored-project rollback coverage is unavailable'));
|
||||
mockDeployStack.mockResolvedValue({ recoveryId: null });
|
||||
|
||||
const { GitSourceService, GitSourceError } = await import('../services/GitSourceService');
|
||||
const svc = GitSourceService.getInstance();
|
||||
vi.spyOn(
|
||||
svc as unknown as { withStackLock: (name: string, fn: () => Promise<unknown>) => Promise<unknown> },
|
||||
'withStackLock',
|
||||
).mockImplementation(async (_name, fn) => fn());
|
||||
vi.spyOn(
|
||||
svc as unknown as { validateCandidate: (...args: unknown[]) => Promise<{ ok: boolean }> },
|
||||
'validateCandidate',
|
||||
).mockResolvedValue({ ok: true });
|
||||
vi.spyOn(
|
||||
svc as unknown as { decodePendingCompose: (raw: string) => unknown },
|
||||
'decodePendingCompose',
|
||||
).mockReturnValue({
|
||||
files: { 'compose.yaml': 'services:\n web:\n image: nginx\n' },
|
||||
contextDir: null,
|
||||
candidateRelPath: 'generations/cand',
|
||||
inventory: { inputs: [], refusals: [], buildContexts: [] },
|
||||
});
|
||||
vi.spyOn(
|
||||
svc as unknown as { deriveAppliedSpec: (...args: unknown[]) => unknown },
|
||||
'deriveAppliedSpec',
|
||||
).mockReturnValue({ files: ['compose.yaml'], contextDir: null });
|
||||
vi.spyOn(
|
||||
svc as unknown as { hashContent: (...args: unknown[]) => string },
|
||||
'hashContent',
|
||||
).mockReturnValue('hash');
|
||||
|
||||
await expect(svc.apply('app', 'abc1234deadbeef', { deploy: true, actor: 'tester' })).rejects.toBeInstanceOf(GitSourceError);
|
||||
expect(mockPromoteGeneration).not.toHaveBeenCalled();
|
||||
expect(mockCaptureCandidate).toHaveBeenCalled();
|
||||
expect(mockMarkGitSourceApplied).not.toHaveBeenCalled();
|
||||
expect(mockHandoff).not.toHaveBeenCalled();
|
||||
expect(mockAbandon).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -32,6 +32,44 @@ vi.mock('isomorphic-git', () => {
|
||||
|
||||
vi.mock('isomorphic-git/http/node', () => ({ default: {} }));
|
||||
|
||||
|
||||
const {
|
||||
mockCaptureCandidate,
|
||||
mockRecoveryAbandon,
|
||||
mockRecoveryMarkAcquired,
|
||||
mockRecoveryHandoff,
|
||||
mockRecoveryMarkReconciling,
|
||||
mockRecoveryMarkImmediateVerified,
|
||||
mockRecoveryGet,
|
||||
mockRecoveryLinkGateOrRetain,
|
||||
} = vi.hoisted(() => ({
|
||||
mockCaptureCandidate: vi.fn(async () => ({ id: 'rec-test-1' })),
|
||||
mockRecoveryAbandon: vi.fn(async () => true),
|
||||
mockRecoveryMarkAcquired: vi.fn(() => true),
|
||||
mockRecoveryHandoff: vi.fn(() => true),
|
||||
mockRecoveryMarkReconciling: vi.fn(() => true),
|
||||
mockRecoveryMarkImmediateVerified: vi.fn(() => true),
|
||||
mockRecoveryGet: vi.fn(() => ({ id: 'rec-test-1', is_current: 1 })),
|
||||
mockRecoveryLinkGateOrRetain: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../services/StackUpdateRecoveryService', () => ({
|
||||
StackUpdateRecoveryService: {
|
||||
getInstance: () => ({
|
||||
captureCandidate: mockCaptureCandidate,
|
||||
abandon: mockRecoveryAbandon,
|
||||
markAcquired: mockRecoveryMarkAcquired,
|
||||
handoff: mockRecoveryHandoff,
|
||||
markReconciling: mockRecoveryMarkReconciling,
|
||||
markImmediateVerified: mockRecoveryMarkImmediateVerified,
|
||||
get: mockRecoveryGet,
|
||||
linkGateOrRetain: mockRecoveryLinkGateOrRetain,
|
||||
compensateWithCandidate: vi.fn(async () => true),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
|
||||
let tmpDir: string;
|
||||
let GitSourceService: typeof import('../services/GitSourceService').GitSourceService;
|
||||
let GitSourceError: typeof import('../services/GitSourceService').GitSourceError;
|
||||
@@ -50,6 +88,21 @@ afterAll(() => {
|
||||
beforeEach(() => {
|
||||
mockGitClone.mockReset();
|
||||
mockGitLog.mockReset();
|
||||
mockCaptureCandidate.mockReset();
|
||||
mockCaptureCandidate.mockImplementation(async () => ({ id: 'rec-test-1' }));
|
||||
mockRecoveryAbandon.mockReset();
|
||||
mockRecoveryAbandon.mockResolvedValue(true);
|
||||
mockRecoveryMarkAcquired.mockReset();
|
||||
mockRecoveryMarkAcquired.mockReturnValue(true);
|
||||
mockRecoveryHandoff.mockReset();
|
||||
mockRecoveryHandoff.mockReturnValue(true);
|
||||
mockRecoveryMarkReconciling.mockReset();
|
||||
mockRecoveryMarkReconciling.mockReturnValue(true);
|
||||
mockRecoveryMarkImmediateVerified.mockReset();
|
||||
mockRecoveryMarkImmediateVerified.mockReturnValue(true);
|
||||
mockRecoveryGet.mockReset();
|
||||
mockRecoveryLinkGateOrRetain.mockReset();
|
||||
mockRecoveryGet.mockReturnValue({ id: 'rec-test-1', is_current: 1 });
|
||||
|
||||
// Wipe persisted git sources between tests
|
||||
const db = DatabaseService.getInstance();
|
||||
@@ -894,6 +947,47 @@ describe('GitSourceService.handleWebhookPull debounce', () => {
|
||||
expect(result.message).toMatch(/validation failed/i);
|
||||
runSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('routes webhook auto-apply through the shared stack-operation lock', async () => {
|
||||
const sha = 'ffff666ffff666ffff666ffff666ffff666ffff6';
|
||||
mockSuccessfulClone({ sha });
|
||||
const svc = GitSourceService.getInstance();
|
||||
const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true });
|
||||
await svc.upsert({
|
||||
stackName: 'webhook-shared-lock',
|
||||
repoUrl: 'https://github.com/example/repo.git',
|
||||
branch: 'main',
|
||||
composePaths: ['compose.yaml'],
|
||||
contextDir: null,
|
||||
syncEnv: false,
|
||||
envPath: null,
|
||||
authType: 'none',
|
||||
autoApplyOnWebhook: true,
|
||||
autoDeployOnApply: false,
|
||||
});
|
||||
mockGitClone.mockClear();
|
||||
|
||||
const { StackOpLockService } = await import('../services/StackOpLockService');
|
||||
const runExclusive = vi.spyOn(StackOpLockService.getInstance(), 'runExclusive')
|
||||
.mockResolvedValue({
|
||||
ran: false,
|
||||
existing: { action: 'update', actor: 'user:admin', startedAt: Date.now() },
|
||||
} as never);
|
||||
|
||||
const result = await svc.handleWebhookPull('webhook-shared-lock');
|
||||
expect(result.status).toBe('error');
|
||||
expect(result.message).toMatch(/already in progress/i);
|
||||
expect(runExclusive).toHaveBeenCalledWith(
|
||||
expect.any(Number),
|
||||
'webhook-shared-lock',
|
||||
'git_apply',
|
||||
'system:webhook',
|
||||
expect.any(Function),
|
||||
);
|
||||
|
||||
runExclusive.mockRestore();
|
||||
validateSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('GitSourceService per-stack mutex', () => {
|
||||
@@ -1359,7 +1453,7 @@ describe('GitSourceService.apply', () => {
|
||||
const { ComposeService } = await import('../services/ComposeService');
|
||||
const { HealthGateService } = await import('../services/HealthGateService');
|
||||
const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue();
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null });
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-git');
|
||||
const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!;
|
||||
|
||||
@@ -1371,6 +1465,7 @@ describe('GitSourceService.apply', () => {
|
||||
actor: 'system:git-source',
|
||||
});
|
||||
expect(beginSpy).toHaveBeenCalledWith(nodeId, 'apply-deploy-gate', 'deploy', 'system:git-source');
|
||||
expect(mockRecoveryLinkGateOrRetain).toHaveBeenCalledWith('rec-test-1', 'gate-git');
|
||||
} finally {
|
||||
validateSpy.mockRestore();
|
||||
saveSpy.mockRestore();
|
||||
@@ -1470,7 +1565,7 @@ describe('GitSourceService.apply', () => {
|
||||
const TrivyService = (await import('../services/TrivyService')).default;
|
||||
const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue();
|
||||
const listImagesSpy = vi.spyOn(ComposeService.prototype, 'listStackImages').mockResolvedValue(['nginx:bad']);
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue();
|
||||
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null });
|
||||
const trivy = TrivyService.getInstance();
|
||||
const trivyAvailableSpy = vi.spyOn(trivy, 'isTrivyAvailable').mockReturnValue(true);
|
||||
const scanSpy = vi.spyOn(trivy, 'scanImagePreflight').mockResolvedValue({
|
||||
@@ -2178,6 +2273,50 @@ describe('GitSourceService legacy pending apply (migration path)', () => {
|
||||
await cleanupStackDir('legacy-apply');
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects materialize when deleting a stale override fails', async () => {
|
||||
const svc = GitSourceService.getInstance();
|
||||
const { FileSystemService } = await import('../services/FileSystemService');
|
||||
const fsSvc = FileSystemService.getInstance();
|
||||
const stackName = 'legacy-stale-del';
|
||||
await fsSvc.createStack(stackName);
|
||||
await fsSvc.saveStackContent(stackName, 'services:\n web:\n image: nginx\n');
|
||||
await fsSvc.writeStackFile(stackName, 'compose.override.yaml', 'services:\n web:\n environment: [X=1]\n');
|
||||
|
||||
const deleteSpy = vi.spyOn(FileSystemService.prototype, 'deleteStackPath')
|
||||
.mockRejectedValue(Object.assign(new Error('permission denied'), { code: 'EACCES' }));
|
||||
|
||||
const materialize = (svc as unknown as {
|
||||
materialize: (
|
||||
stackName: string,
|
||||
files: Array<{ path: string; content: string }>,
|
||||
contextDir: string | null,
|
||||
syncEnv: boolean,
|
||||
envContent: string | null,
|
||||
prevSpec: { files: string[]; contextDir: string | null } | null,
|
||||
) => Promise<unknown>;
|
||||
}).materialize.bind(svc);
|
||||
|
||||
try {
|
||||
await expect(
|
||||
materialize(
|
||||
stackName,
|
||||
[{ path: 'compose.yaml', content: 'services:\n web:\n image: alpine\n' }],
|
||||
null,
|
||||
false,
|
||||
null,
|
||||
{ files: ['compose.yaml', 'compose.override.yaml'], contextDir: null },
|
||||
),
|
||||
).rejects.toThrow(/permission denied/);
|
||||
expect(deleteSpy).toHaveBeenCalledWith(stackName, 'compose.override.yaml');
|
||||
// Stale override must still be present; apply must not report success over a hybrid.
|
||||
const override = await fsSvc.readStackFile(stackName, 'compose.override.yaml');
|
||||
expect(override.content).toContain('X=1');
|
||||
} finally {
|
||||
deleteSpy.mockRestore();
|
||||
await cleanupStackDir(stackName);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('GitSourceService sync-env stacks with a repo .env (audit C-2)', () => {
|
||||
|
||||
@@ -43,6 +43,7 @@ describe('named stack route permission inventory', () => {
|
||||
['PUT', '/stacks/web/env', 'stack:edit'],
|
||||
['PUT', '/stacks/web/dossier', 'stack:edit'],
|
||||
['PUT', '/stacks/web/labels', 'stack:edit'],
|
||||
['POST', '/stacks/web/fleet-snapshot-apply', 'stack:edit'],
|
||||
['POST', '/stacks/web/deploy', 'stack:deploy'],
|
||||
['POST', '/stacks/web/stop', 'stack:deploy'],
|
||||
['POST', '/stacks/web/services/api/update', 'stack:deploy'],
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* Recovery must execute the generation-captured Compose invocation, not the
|
||||
* live database-derived file / env selection after capture.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import path from 'path';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
|
||||
describe('captured invocation on recovery Compose args', () => {
|
||||
let tmpDir: string;
|
||||
let composeDir: string;
|
||||
let nodeId: number;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
composeDir = process.env.COMPOSE_DIR!;
|
||||
const { DatabaseService } = await import('../services/DatabaseService');
|
||||
const db = DatabaseService.getInstance();
|
||||
const local = db.getDefaultNode();
|
||||
if (!local?.id) throw new Error('missing default node');
|
||||
nodeId = local.id;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (tmpDir) cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
it('uses captured -f order and env-file when live settings diverge', async () => {
|
||||
const stackName = 'inv-capture';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
const fsPromises = await import('fs/promises');
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n a: {}\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.prod.yaml'), 'services:\n b: {}\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'prod.env'), 'A=1\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'dev.env'), 'A=2\n', 'utf8');
|
||||
|
||||
const { DatabaseService } = await import('../services/DatabaseService');
|
||||
const db = DatabaseService.getInstance();
|
||||
db.upsertGitSource({
|
||||
stack_name: stackName,
|
||||
repo_url: 'https://example.com/repo.git',
|
||||
branch: 'main',
|
||||
compose_path: 'compose.yaml',
|
||||
compose_paths: ['compose.yaml', 'compose.prod.yaml'],
|
||||
context_dir: null,
|
||||
sync_env: false,
|
||||
env_path: null,
|
||||
auth_type: 'none',
|
||||
encrypted_token: null,
|
||||
auto_apply_on_webhook: false,
|
||||
auto_deploy_on_apply: false,
|
||||
last_applied_commit_sha: 'abc',
|
||||
last_applied_content_hash: 'h',
|
||||
pending_commit_sha: null,
|
||||
pending_compose_content: null,
|
||||
pending_env_content: null,
|
||||
pending_fetched_at: null,
|
||||
last_debounce_at: null,
|
||||
});
|
||||
db.getDb().prepare(
|
||||
`UPDATE stack_git_sources SET applied_deploy_spec = ? WHERE stack_name = ?`,
|
||||
).run(
|
||||
JSON.stringify({ files: ['compose.yaml'], contextDir: null }),
|
||||
stackName,
|
||||
);
|
||||
db.setStackProjectEnvFiles(nodeId, stackName, ['dev.env']);
|
||||
|
||||
const { ComposeService } = await import('../services/ComposeService');
|
||||
const svc = ComposeService.getInstance(nodeId);
|
||||
const captured = {
|
||||
composeArgsPrefix: [
|
||||
'-f', 'compose.yaml',
|
||||
'-f', 'compose.prod.yaml',
|
||||
'-p', stackName,
|
||||
'--env-file', path.join(stackDir, 'prod.env'),
|
||||
],
|
||||
projectDirectory: null,
|
||||
projectName: stackName,
|
||||
explicitComposeFiles: ['compose.yaml', 'compose.prod.yaml'],
|
||||
};
|
||||
|
||||
const args = await svc.buildComposeArgsWithRecoveryOverride(
|
||||
stackName,
|
||||
['up', '-d'],
|
||||
path.join(stackDir, '.sencho-recovery-aaaaaaaaaaaa.yml'),
|
||||
captured,
|
||||
);
|
||||
|
||||
expect(args).toEqual([
|
||||
'compose',
|
||||
'-f', 'compose.yaml',
|
||||
'-f', 'compose.prod.yaml',
|
||||
'-p', stackName,
|
||||
'--env-file', path.resolve(stackDir, 'prod.env'),
|
||||
'-f', path.join(stackDir, '.sencho-recovery-aaaaaaaaaaaa.yml'),
|
||||
'up', '-d',
|
||||
]);
|
||||
// Live settings would prefer only compose.yaml + dev.env; captured must win.
|
||||
expect(args.join('\0')).not.toContain('dev.env');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* Fifth-audit regressions: structural services_json validation.
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
parseServicesJsonStrict,
|
||||
scrapeRollbackTagsLenient,
|
||||
type StackRecoveryServiceCapture,
|
||||
} from '../services/recoveryServicesJson';
|
||||
|
||||
function validService(over: Partial<StackRecoveryServiceCapture> = {}): StackRecoveryServiceCapture {
|
||||
return {
|
||||
serviceName: 'web',
|
||||
scale: 1,
|
||||
hasBuild: false,
|
||||
declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag',
|
||||
replicas: [{
|
||||
containerId: 'c1',
|
||||
imageId: 'sha256:aaa',
|
||||
repoDigest: null,
|
||||
state: 'running',
|
||||
rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold',
|
||||
}],
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
describe('parseServicesJsonStrict', () => {
|
||||
it('accepts a well-formed services array', () => {
|
||||
const parsed = parseServicesJsonStrict(JSON.stringify([validService()]));
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (parsed.ok) expect(parsed.services[0].serviceName).toBe('web');
|
||||
});
|
||||
|
||||
it('rejects empty object elements like [{}]', () => {
|
||||
expect(parseServicesJsonStrict('[{}]').ok).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects malformed replica arrays', () => {
|
||||
expect(parseServicesJsonStrict(JSON.stringify([{
|
||||
...validService(),
|
||||
replicas: [{ state: 'running' }],
|
||||
}])).ok).toBe(false);
|
||||
|
||||
expect(parseServicesJsonStrict(JSON.stringify([{
|
||||
...validService(),
|
||||
replicas: 'nope',
|
||||
}])).ok).toBe(false);
|
||||
});
|
||||
|
||||
it('scrapeRollbackTagsLenient recovers tags from near-valid JSON', () => {
|
||||
expect(scrapeRollbackTagsLenient(JSON.stringify([{
|
||||
serviceName: 'web',
|
||||
replicas: [{ rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold' }],
|
||||
}]))).toEqual(['sencho-rb/aaaaaaaaaaaa/web:hold']);
|
||||
});
|
||||
|
||||
it('rejects invalid referenceKind or scale', () => {
|
||||
expect(parseServicesJsonStrict(JSON.stringify([{
|
||||
...validService(),
|
||||
referenceKind: 'mystery',
|
||||
}])).ok).toBe(false);
|
||||
|
||||
expect(parseServicesJsonStrict(JSON.stringify([{
|
||||
...validService(),
|
||||
scale: -1,
|
||||
}])).ok).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,103 @@
|
||||
/**
|
||||
* Integration coverage for assessGenerationEligibility: structural services_json
|
||||
* refusal and opaque hold-tag presence checks.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const mockInspectByRef = vi.fn(async (_ref: string) => ({ Id: 'ok' }));
|
||||
|
||||
vi.mock('../services/DockerController', () => ({
|
||||
default: {
|
||||
getInstance: () => ({
|
||||
getDocker: () => ({
|
||||
getImage: (ref: string) => ({
|
||||
inspect: () => mockInspectByRef(ref),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/RollbackGenerationStore', () => ({
|
||||
RollbackGenerationStore: {
|
||||
verifyGenerationContent: vi.fn().mockResolvedValue(true),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/PolicyEnforcement', () => ({
|
||||
enforcePolicyForImageRefs: vi.fn().mockResolvedValue({ ok: true, bypassed: false, violations: [] }),
|
||||
}));
|
||||
|
||||
import { assessGenerationEligibility } from '../services/rollbackEligibility';
|
||||
import type { StackUpdateRecoveryGenerationRow } from '../services/DatabaseService';
|
||||
|
||||
const HOLD_TAG = 'sencho-rb/aaaaaaaaaaaa/web:hold';
|
||||
const IMAGE_ID = 'sha256:aaa';
|
||||
|
||||
function baseRow(over: Partial<StackUpdateRecoveryGenerationRow> = {}): StackUpdateRecoveryGenerationRow {
|
||||
return {
|
||||
id: '11111111-1111-4111-8111-111111111111',
|
||||
node_id: 1,
|
||||
stack_name: 'my-stack',
|
||||
status: 'active',
|
||||
phase: 'reconciling',
|
||||
is_current: 1,
|
||||
backup_slot_id: '11111111-1111-4111-8111-111111111111',
|
||||
content_path: '11111111-1111-4111-8111-111111111111',
|
||||
operation_kind: 'update',
|
||||
override_path: null,
|
||||
services_json: JSON.stringify([{
|
||||
serviceName: 'web',
|
||||
scale: 1,
|
||||
hasBuild: false,
|
||||
declaredImageRef: 'nginx:latest',
|
||||
referenceKind: 'moving_tag',
|
||||
replicas: [{
|
||||
containerId: 'c1',
|
||||
imageId: IMAGE_ID,
|
||||
repoDigest: null,
|
||||
state: 'running',
|
||||
rollbackTag: HOLD_TAG,
|
||||
}],
|
||||
}]),
|
||||
health_gate_id: null,
|
||||
gate_retain_until: null,
|
||||
artifact_expires_at: null,
|
||||
operation_lease_expires_at: null,
|
||||
created_at: Date.now(),
|
||||
updated_at: Date.now(),
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
released_at: null,
|
||||
released_by: null,
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
describe('assessGenerationEligibility', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockInspectByRef.mockResolvedValue({ Id: 'ok' });
|
||||
});
|
||||
|
||||
it('prohibits structurally malformed services_json such as [{}]', async () => {
|
||||
await expect(assessGenerationEligibility(baseRow({ services_json: '[{}]' }))).resolves.toBe('prohibited');
|
||||
});
|
||||
|
||||
it('returns eligible_with_warning when an opaque hold tag is missing', async () => {
|
||||
mockInspectByRef.mockImplementation(async (ref: string) => {
|
||||
if (ref === HOLD_TAG) {
|
||||
throw Object.assign(new Error('No such image'), { statusCode: 404 });
|
||||
}
|
||||
return { Id: 'ok' };
|
||||
});
|
||||
|
||||
await expect(assessGenerationEligibility(baseRow())).resolves.toBe('eligible_with_warning');
|
||||
expect(mockInspectByRef).toHaveBeenCalledWith(IMAGE_ID);
|
||||
expect(mockInspectByRef).toHaveBeenCalledWith(HOLD_TAG);
|
||||
});
|
||||
|
||||
it('returns eligible when image ids and hold tags both resolve', async () => {
|
||||
await expect(assessGenerationEligibility(baseRow())).resolves.toBe('eligible');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* Unit tests for evaluateRollbackEligibility (pure verdict mapping).
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
evaluateRollbackEligibility,
|
||||
type RollbackEligibilityInput,
|
||||
} from '../services/rollbackEligibility';
|
||||
|
||||
const base = (over: Partial<RollbackEligibilityInput> = {}): RollbackEligibilityInput => ({
|
||||
generationIntegrityOk: true,
|
||||
heldImagesPresent: true,
|
||||
securityPostureBlocked: false,
|
||||
...over,
|
||||
});
|
||||
|
||||
describe('evaluateRollbackEligibility', () => {
|
||||
it('returns eligible when every signal is known-good', () => {
|
||||
expect(evaluateRollbackEligibility(base())).toBe('eligible');
|
||||
});
|
||||
|
||||
it('returns prohibited when security posture is blocked', () => {
|
||||
expect(evaluateRollbackEligibility(base({ securityPostureBlocked: true }))).toBe('prohibited');
|
||||
});
|
||||
|
||||
it('returns prohibited when generation integrity is known bad', () => {
|
||||
expect(evaluateRollbackEligibility(base({ generationIntegrityOk: false }))).toBe('prohibited');
|
||||
});
|
||||
|
||||
it('prefers prohibited over other signals when security is blocked', () => {
|
||||
expect(evaluateRollbackEligibility(base({
|
||||
securityPostureBlocked: true,
|
||||
generationIntegrityOk: null,
|
||||
heldImagesPresent: false,
|
||||
}))).toBe('prohibited');
|
||||
});
|
||||
|
||||
it('returns eligible_with_warning when held images are missing', () => {
|
||||
expect(evaluateRollbackEligibility(base({ heldImagesPresent: false }))).toBe('eligible_with_warning');
|
||||
});
|
||||
|
||||
it('returns unknown when any remaining signal is null', () => {
|
||||
expect(evaluateRollbackEligibility(base({ generationIntegrityOk: null }))).toBe('unknown');
|
||||
expect(evaluateRollbackEligibility(base({ heldImagesPresent: null }))).toBe('unknown');
|
||||
expect(evaluateRollbackEligibility(base({ securityPostureBlocked: null }))).toBe('unknown');
|
||||
});
|
||||
|
||||
it('returns unknown when all signals are null', () => {
|
||||
expect(evaluateRollbackEligibility({
|
||||
generationIntegrityOk: null,
|
||||
heldImagesPresent: null,
|
||||
securityPostureBlocked: null,
|
||||
})).toBe('unknown');
|
||||
});
|
||||
});
|
||||
@@ -82,6 +82,8 @@ function makeRow(overrides: Partial<StackUpdateRecoveryGenerationRow> = {}): Sta
|
||||
phase: 'immediate_verified',
|
||||
is_current: 1,
|
||||
backup_slot_id: null,
|
||||
content_path: null,
|
||||
operation_kind: null,
|
||||
override_path: null,
|
||||
services_json: JSON.stringify([{
|
||||
serviceName: 'web',
|
||||
@@ -349,6 +351,22 @@ describe('StackUpdateRecoveryService.releaseGeneration', () => {
|
||||
if (!second.ok) expect(second.reason).toBe('already_released');
|
||||
});
|
||||
|
||||
it('refuses release when services_json is malformed and leaves hold tags intact', async () => {
|
||||
const row = insertRow({ status: 'active', is_current: 1, services_json: '{not-json' });
|
||||
const svc = StackUpdateRecoveryService.getInstance();
|
||||
|
||||
expect(svc.isReleaseEligible(row)).toBe(false);
|
||||
const result = await svc.releaseGeneration(row.id, 'tester');
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) expect(result.reason).toBe('malformed_services');
|
||||
|
||||
const after = DatabaseService.getInstance().getStackUpdateRecoveryGeneration(row.id)!;
|
||||
expect(after.released_at).toBeNull();
|
||||
expect(after.is_current).toBe(1);
|
||||
expect(after.artifacts_retired).toBe(0);
|
||||
expect(mockRemove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('leaves artifacts_retired at 0 (retryable) when Docker tag removal fails', async () => {
|
||||
mockRemove.mockRejectedValueOnce(Object.assign(new Error('docker busy'), { statusCode: 500 }));
|
||||
const row = insertRow({ status: 'active', is_current: 1 });
|
||||
@@ -415,6 +433,18 @@ describe('GET/POST /api/system/rollback/generations', () => {
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.code).toBe('NOT_ELIGIBLE');
|
||||
});
|
||||
|
||||
it('409s releasing a generation with malformed services_json', async () => {
|
||||
const row = insertRow({ status: 'superseded', is_current: 0, services_json: '{not-json' });
|
||||
const res = await request(app)
|
||||
.post(`/api/system/rollback/generations/${row.id}/release`)
|
||||
.set('Authorization', authHeader);
|
||||
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.code).toBe('MALFORMED_SERVICES');
|
||||
const after = DatabaseService.getInstance().getStackUpdateRecoveryGeneration(row.id)!;
|
||||
expect(after.released_at).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('RBAC on the rollback-generation routes', () => {
|
||||
|
||||
@@ -0,0 +1,788 @@
|
||||
/**
|
||||
* Unit tests for RollbackGenerationStore: capture staging, checksum verify,
|
||||
* restore round-trip, mid-capture failure isolation, and managed-set tombstones.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
import { promises as fsPromises } from 'fs';
|
||||
import { randomUUID } from 'crypto';
|
||||
|
||||
const mockState = { composeDir: '', composeDirs: new Map<number, string>() };
|
||||
|
||||
vi.mock('../services/NodeRegistry', () => ({
|
||||
NodeRegistry: {
|
||||
getInstance: () => ({
|
||||
getComposeDir: (nodeId?: number) => mockState.composeDirs.get(nodeId ?? 1) ?? mockState.composeDir,
|
||||
getDefaultNodeId: () => 1,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../utils/debug', () => ({
|
||||
isDebugEnabled: () => false,
|
||||
}));
|
||||
|
||||
import { RollbackGenerationStore } from '../services/RollbackGenerationStore';
|
||||
import type { ResolvedRollbackInventory } from '../types/rollbackGeneration';
|
||||
|
||||
const NODE = 1;
|
||||
|
||||
function inventoryFor(
|
||||
stackName: string,
|
||||
files: Array<{
|
||||
relativePath: string;
|
||||
absolutePath: string | null;
|
||||
kind?: ResolvedRollbackInventory['entries'][number]['dependencyKind'];
|
||||
sensitivity?: ResolvedRollbackInventory['entries'][number]['sensitivity'];
|
||||
}>,
|
||||
): ResolvedRollbackInventory {
|
||||
return {
|
||||
entries: files.map((f) => ({
|
||||
relativePath: f.relativePath,
|
||||
dependencyKind: f.kind ?? 'compose-root',
|
||||
provenance: 'authored' as const,
|
||||
sensitivity: f.sensitivity ?? 'low',
|
||||
absolutePath: f.absolutePath,
|
||||
})),
|
||||
invocation: {
|
||||
composeArgsPrefix: [],
|
||||
projectDirectory: null,
|
||||
projectName: stackName,
|
||||
explicitComposeFiles: files.map((f) => f.relativePath).filter((p) => !p.includes('/')),
|
||||
meshEnabled: false,
|
||||
meshOverrideRelativePath: null,
|
||||
},
|
||||
git: null,
|
||||
appliedDeploySpec: null,
|
||||
lastAppliedContentHash: null,
|
||||
manifestState: null,
|
||||
manifestGeneration: null,
|
||||
exactCoverage: true,
|
||||
coverageRefusal: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe('RollbackGenerationStore', () => {
|
||||
let composeDir: string;
|
||||
let dataDir: string;
|
||||
let originalDataDir: string | undefined;
|
||||
|
||||
beforeEach(async () => {
|
||||
composeDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-compose-'));
|
||||
dataDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-data-'));
|
||||
mockState.composeDir = composeDir;
|
||||
mockState.composeDirs = new Map([[1, composeDir]]);
|
||||
originalDataDir = process.env.DATA_DIR;
|
||||
process.env.DATA_DIR = dataDir;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (originalDataDir === undefined) delete process.env.DATA_DIR;
|
||||
else process.env.DATA_DIR = originalDataDir;
|
||||
await fsPromises.rm(composeDir, { recursive: true, force: true });
|
||||
await fsPromises.rm(dataDir, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('captures and restores a nested path round-trip', async () => {
|
||||
const stackName = 'web';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
const nestedRel = 'includes/base.yaml';
|
||||
await fsPromises.mkdir(path.join(stackDir, 'includes'), { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n app:\n image: a\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, nestedRel), 'services: {}\n', 'utf8');
|
||||
|
||||
const generationId = randomUUID();
|
||||
const inv = inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
{ relativePath: nestedRel, absolutePath: path.join(stackDir, nestedRel), kind: 'include' },
|
||||
]);
|
||||
|
||||
const manifest = await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inv,
|
||||
operationKind: 'manual_backup',
|
||||
});
|
||||
|
||||
expect(manifest.managedRelativePaths).toEqual(['compose.yaml', nestedRel].sort((a, b) => a.localeCompare(b)));
|
||||
const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId);
|
||||
await expect(fsPromises.access(path.join(genDir, 'generation.json'))).resolves.toBeUndefined();
|
||||
await expect(fsPromises.access(path.join(genDir, 'files', nestedRel))).resolves.toBeUndefined();
|
||||
|
||||
// Mutate live files, then restore
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: { mutated: {} }\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, nestedRel), 'services: { mutated: true }\n', 'utf8');
|
||||
|
||||
await RollbackGenerationStore.restoreGeneration(NODE, stackName, generationId, [
|
||||
'compose.yaml',
|
||||
nestedRel,
|
||||
]);
|
||||
|
||||
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf8')).toBe(
|
||||
'services:\n app:\n image: a\n',
|
||||
);
|
||||
expect(await fsPromises.readFile(path.join(stackDir, nestedRel), 'utf8')).toBe('services: {}\n');
|
||||
});
|
||||
|
||||
it('leaves no final generation dir when capture fails mid-write', async () => {
|
||||
const stackName = 'failcap';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
await fsPromises.mkdir(path.join(stackDir, 'configs'), { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'configs', 'app.conf'), 'ok\n', 'utf8');
|
||||
|
||||
const generationId = randomUUID();
|
||||
const inv = inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
{
|
||||
relativePath: 'configs/app.conf',
|
||||
absolutePath: path.join(stackDir, 'configs', 'app.conf'),
|
||||
kind: 'config',
|
||||
sensitivity: 'high',
|
||||
},
|
||||
]);
|
||||
|
||||
const realWriteFile = fsPromises.writeFile.bind(fsPromises);
|
||||
const writeSpy = vi.spyOn(fsPromises, 'writeFile').mockImplementation(async (file, data, options) => {
|
||||
const normalized = path.normalize(String(file));
|
||||
if (normalized.includes(`${path.sep}configs${path.sep}app.conf`) && normalized.includes('staging-')) {
|
||||
throw new Error('disk full during capture');
|
||||
}
|
||||
return realWriteFile(file, data, options);
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(
|
||||
RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inv,
|
||||
}),
|
||||
).rejects.toThrow(/disk full during capture/);
|
||||
|
||||
const finalDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId);
|
||||
await expect(fsPromises.access(finalDir)).rejects.toMatchObject({ code: 'ENOENT' });
|
||||
|
||||
const gensRoot = RollbackGenerationStore.getGenerationsRoot(NODE, stackName);
|
||||
let leftoverStaging = false;
|
||||
try {
|
||||
const entries = await fsPromises.readdir(gensRoot);
|
||||
leftoverStaging = entries.some((e) => e.startsWith('staging-'));
|
||||
} catch (e) {
|
||||
if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e;
|
||||
}
|
||||
expect(leftoverStaging).toBe(false);
|
||||
} finally {
|
||||
writeSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('refuses restore when a checksum is corrupt before mutating live files', async () => {
|
||||
const stackName = 'corrupt';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
const original = 'services:\n app:\n image: good\n';
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), original, 'utf8');
|
||||
|
||||
const generationId = randomUUID();
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
]),
|
||||
});
|
||||
|
||||
const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId);
|
||||
await fsPromises.writeFile(path.join(genDir, 'files', 'compose.yaml'), 'services: { tampered: true }\n', 'utf8');
|
||||
|
||||
const liveMutated = 'services: { live: true }\n';
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), liveMutated, 'utf8');
|
||||
|
||||
await expect(
|
||||
RollbackGenerationStore.restoreGeneration(NODE, stackName, generationId, ['compose.yaml']),
|
||||
).rejects.toThrow(/Checksum mismatch/);
|
||||
|
||||
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf8')).toBe(liveMutated);
|
||||
});
|
||||
|
||||
it('tombstones post-capture managed additions and leaves unrelated files', async () => {
|
||||
const stackName = 'tomb';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(path.join(stackDir, 'configs'), { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'configs', 'app.conf'), 'v1\n', 'utf8');
|
||||
|
||||
const generationId = randomUUID();
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
{
|
||||
relativePath: 'configs/app.conf',
|
||||
absolutePath: path.join(stackDir, 'configs', 'app.conf'),
|
||||
kind: 'config',
|
||||
sensitivity: 'high',
|
||||
},
|
||||
]),
|
||||
});
|
||||
|
||||
// Post-capture addition inside the managed discovery set
|
||||
await fsPromises.writeFile(path.join(stackDir, 'configs', 'extra.conf'), 'new\n', 'utf8');
|
||||
// Unrelated file outside managed discovery
|
||||
await fsPromises.writeFile(path.join(stackDir, 'README.md'), 'keep me\n', 'utf8');
|
||||
// Mutate a present managed file
|
||||
await fsPromises.writeFile(path.join(stackDir, 'configs', 'app.conf'), 'v2\n', 'utf8');
|
||||
|
||||
await RollbackGenerationStore.restoreGeneration(NODE, stackName, generationId, [
|
||||
'compose.yaml',
|
||||
'configs/app.conf',
|
||||
'configs/extra.conf',
|
||||
]);
|
||||
|
||||
expect(await fsPromises.readFile(path.join(stackDir, 'configs', 'app.conf'), 'utf8')).toBe('v1\n');
|
||||
await expect(fsPromises.access(path.join(stackDir, 'configs', 'extra.conf'))).rejects.toMatchObject({
|
||||
code: 'ENOENT',
|
||||
});
|
||||
expect(await fsPromises.readFile(path.join(stackDir, 'README.md'), 'utf8')).toBe('keep me\n');
|
||||
});
|
||||
|
||||
it('encrypts medium sensitivity entries and refuses symlink escapes', async () => {
|
||||
const stackName = 'secure';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, '.env'), 'SECRET=1\n', 'utf8');
|
||||
|
||||
const outside = path.join(composeDir, 'outside-secret.txt');
|
||||
await fsPromises.writeFile(outside, 'escaped\n', 'utf8');
|
||||
const linkPath = path.join(stackDir, 'escape.link');
|
||||
let symlinkOk = true;
|
||||
try {
|
||||
await fsPromises.symlink(outside, linkPath);
|
||||
} catch {
|
||||
symlinkOk = false;
|
||||
}
|
||||
|
||||
if (symlinkOk) {
|
||||
await expect(
|
||||
RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId: randomUUID(),
|
||||
inventory: inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
{
|
||||
relativePath: 'escape.link',
|
||||
absolutePath: linkPath,
|
||||
kind: 'other',
|
||||
sensitivity: 'low',
|
||||
},
|
||||
]),
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'SYMLINK_ESCAPE' });
|
||||
}
|
||||
|
||||
const generationId = randomUUID();
|
||||
const manifest = await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
{
|
||||
relativePath: '.env',
|
||||
absolutePath: path.join(stackDir, '.env'),
|
||||
kind: 'interpolation-env',
|
||||
sensitivity: 'medium',
|
||||
},
|
||||
]),
|
||||
});
|
||||
const envEntry = manifest.entries.find((e) => e.relativePath === '.env');
|
||||
expect(envEntry?.encrypted).toBe(true);
|
||||
expect(envEntry?.state).toBe('present');
|
||||
});
|
||||
it('reverts an interrupted multi-file restore from pre-restore snapshot', async () => {
|
||||
const stackName = 'tx';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(path.join(stackDir, 'includes'), { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'OLD_BASE\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'includes/base.yaml'), 'OLD_INC\n', 'utf8');
|
||||
|
||||
const generationId = randomUUID();
|
||||
const inv = inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
{ relativePath: 'includes/base.yaml', absolutePath: path.join(stackDir, 'includes/base.yaml'), kind: 'include' },
|
||||
]);
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inv,
|
||||
operationKind: 'manual_backup',
|
||||
});
|
||||
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'NEW_BASE\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'includes/base.yaml'), 'NEW_INC\n', 'utf8');
|
||||
|
||||
const { FileSystemService } = await import('../services/FileSystemService');
|
||||
const originalWrite = FileSystemService.prototype.writeStackFile;
|
||||
let n = 0;
|
||||
vi.spyOn(FileSystemService.prototype, 'writeStackFile').mockImplementation(async function (this: InstanceType<typeof FileSystemService>, stack, rel, content) {
|
||||
n += 1;
|
||||
if (n === 2) throw new Error('injected write failure');
|
||||
return originalWrite.call(this, stack, rel, content);
|
||||
});
|
||||
|
||||
await expect(
|
||||
RollbackGenerationStore.restoreGeneration(NODE, stackName, generationId, ['compose.yaml', 'includes/base.yaml']),
|
||||
).rejects.toThrow(/injected write failure/);
|
||||
|
||||
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf8')).toBe('NEW_BASE\n');
|
||||
expect(await fsPromises.readFile(path.join(stackDir, 'includes/base.yaml'), 'utf8')).toBe('NEW_INC\n');
|
||||
|
||||
const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId);
|
||||
await expect(fsPromises.access(path.join(genDir, 'restore-intent.json'))).rejects.toMatchObject({ code: 'ENOENT' });
|
||||
});
|
||||
|
||||
it('restores a snapshotted Git manifesto on compensate path helper', async () => {
|
||||
const stackName = 'gitman';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
|
||||
const managedDir = path.join(dataDir, 'git-managed', '1', stackName);
|
||||
await fsPromises.mkdir(managedDir, { recursive: true });
|
||||
const priorManifest = {
|
||||
manifestVersion: 3,
|
||||
state: 'active',
|
||||
repo: { url: 'https://example.com/r.git', branch: 'main' },
|
||||
resolvedRevision: { commitSha: 'abc1234' },
|
||||
project: {
|
||||
root: '.',
|
||||
composeFiles: ['compose.yaml'],
|
||||
projectName: stackName,
|
||||
effectiveProjectDir: null,
|
||||
invocation: ['-f', 'compose.yaml'],
|
||||
},
|
||||
inputs: [],
|
||||
buildContexts: [],
|
||||
counts: { total: 0, refused: 0 },
|
||||
refusals: [],
|
||||
generation: { appliedDir: 'generations/prior', previousDir: null },
|
||||
};
|
||||
await fsPromises.writeFile(
|
||||
path.join(managedDir, 'manifest.v1.json'),
|
||||
JSON.stringify(priorManifest),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
const generationId = randomUUID();
|
||||
const inv = inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
]);
|
||||
inv.git = {
|
||||
repoUrl: 'https://example.com/r.git',
|
||||
branch: 'main',
|
||||
commitSha: 'abc1234',
|
||||
manifestVersion: 3,
|
||||
};
|
||||
const { GitProjectManifestService } = await import('../services/GitProjectManifestService');
|
||||
const readSpy = vi.spyOn(GitProjectManifestService.getInstance(), 'readManifest')
|
||||
.mockResolvedValue(priorManifest as never);
|
||||
let genDir = '';
|
||||
try {
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inv,
|
||||
});
|
||||
|
||||
genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId);
|
||||
const snap = await fsPromises.readFile(path.join(genDir, 'git-manifest.v1.json'), 'utf8');
|
||||
expect(JSON.parse(snap).resolvedRevision.commitSha).toBe('abc1234');
|
||||
} finally {
|
||||
readSpy.mockRestore();
|
||||
}
|
||||
const newer = { ...priorManifest, resolvedRevision: { commitSha: 'ffffff' } };
|
||||
await fsPromises.writeFile(path.join(managedDir, 'manifest.v1.json'), JSON.stringify(newer), 'utf8');
|
||||
|
||||
const captured = JSON.parse(await fsPromises.readFile(path.join(genDir, 'generation.json'), 'utf8'));
|
||||
await RollbackGenerationStore.restoreCapturedGitManifest(stackName, genDir, captured);
|
||||
const restored = JSON.parse(await fsPromises.readFile(path.join(managedDir, 'manifest.v1.json'), 'utf8'));
|
||||
expect(restored.resolvedRevision.commitSha).toBe('abc1234');
|
||||
});
|
||||
|
||||
it('does not snapshot a corrupt Git manifesto into the generation', async () => {
|
||||
const stackName = 'nocorrupt';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
|
||||
const generationId = randomUUID();
|
||||
const inv = inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
]);
|
||||
inv.git = {
|
||||
repoUrl: 'https://example.com/r.git',
|
||||
branch: 'main',
|
||||
commitSha: 'abc',
|
||||
manifestVersion: 3,
|
||||
};
|
||||
const { GitProjectManifestService } = await import('../services/GitProjectManifestService');
|
||||
const readSpy = vi.spyOn(GitProjectManifestService.getInstance(), 'readManifest')
|
||||
.mockResolvedValue({ corrupt: 'identity mismatch' });
|
||||
try {
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inv,
|
||||
});
|
||||
const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId);
|
||||
await expect(fsPromises.access(path.join(genDir, 'git-manifest.v1.json'))).rejects.toMatchObject({
|
||||
code: 'ENOENT',
|
||||
});
|
||||
const captured = JSON.parse(await fsPromises.readFile(path.join(genDir, 'generation.json'), 'utf8'));
|
||||
expect(captured.priorRecords.gitManifestCaptured).toBe(false);
|
||||
} finally {
|
||||
readSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('clears post-capture manifesto for first-apply preimage', async () => {
|
||||
const stackName = 'firstapply';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
|
||||
const generationId = randomUUID();
|
||||
const inv = inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
]);
|
||||
inv.git = {
|
||||
repoUrl: 'https://example.com/r.git',
|
||||
branch: 'main',
|
||||
commitSha: '',
|
||||
manifestVersion: null,
|
||||
};
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inv,
|
||||
});
|
||||
|
||||
const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId);
|
||||
const captured = JSON.parse(await fsPromises.readFile(path.join(genDir, 'generation.json'), 'utf8'));
|
||||
expect(captured.priorRecords.gitManifestCaptured).toBe(false);
|
||||
|
||||
const managedDir = path.join(dataDir, 'git-managed', '1', stackName);
|
||||
await fsPromises.mkdir(managedDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(managedDir, 'manifest.v1.json'), '{"manifestVersion":3}\n', 'utf8');
|
||||
|
||||
await RollbackGenerationStore.restoreCapturedGitManifest(stackName, genDir, captured);
|
||||
await expect(fsPromises.access(path.join(managedDir, 'manifest.v1.json'))).rejects.toMatchObject({ code: 'ENOENT' });
|
||||
});
|
||||
|
||||
it('does not clear live manifesto for legacy generations without a snapshot', async () => {
|
||||
const stackName = 'legacygen';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
|
||||
const generationId = randomUUID();
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
]),
|
||||
});
|
||||
|
||||
const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId);
|
||||
const captured = JSON.parse(await fsPromises.readFile(path.join(genDir, 'generation.json'), 'utf8'));
|
||||
delete captured.priorRecords.gitManifestCaptured;
|
||||
captured.git = {
|
||||
repoUrl: 'https://example.com/r.git',
|
||||
branch: 'main',
|
||||
commitSha: 'abc',
|
||||
manifestVersion: 3,
|
||||
};
|
||||
await fsPromises.writeFile(path.join(genDir, 'generation.json'), JSON.stringify(captured), 'utf8');
|
||||
|
||||
const managedDir = path.join(dataDir, 'git-managed', '1', stackName);
|
||||
await fsPromises.mkdir(managedDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(managedDir, 'manifest.v1.json'), '{"keep":true}\n', 'utf8');
|
||||
|
||||
await RollbackGenerationStore.restoreCapturedGitManifest(stackName, genDir, captured);
|
||||
expect(await fsPromises.readFile(path.join(managedDir, 'manifest.v1.json'), 'utf8')).toBe('{"keep":true}\n');
|
||||
});
|
||||
|
||||
it('refuses restore when a managed file path is currently a directory', async () => {
|
||||
const stackName = 'dircollide';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'OLD\n', 'utf8');
|
||||
|
||||
const generationId = randomUUID();
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
]),
|
||||
});
|
||||
|
||||
await fsPromises.rm(path.join(stackDir, 'compose.yaml'));
|
||||
await fsPromises.mkdir(path.join(stackDir, 'compose.yaml'), { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml', 'sentinel.txt'), 'keep-me\n', 'utf8');
|
||||
|
||||
await expect(
|
||||
RollbackGenerationStore.restoreGeneration(NODE, stackName, generationId, ['compose.yaml']),
|
||||
).rejects.toMatchObject({ code: 'DIRECTORY_COLLISION' });
|
||||
|
||||
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml', 'sentinel.txt'), 'utf8')).toBe('keep-me\n');
|
||||
});
|
||||
|
||||
it('restores a legitimate file whose name ends with .sencho-tombstone (index-based snapshot)', async () => {
|
||||
const stackName = 'tombname';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
const special = 'config.sencho-tombstone';
|
||||
await fsPromises.writeFile(path.join(stackDir, special), 'KEEP_ME\n', 'utf8');
|
||||
|
||||
const generationId = randomUUID();
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
{ relativePath: special, absolutePath: path.join(stackDir, special), kind: 'other' },
|
||||
]),
|
||||
});
|
||||
|
||||
await fsPromises.writeFile(path.join(stackDir, special), 'CHANGED\n', 'utf8');
|
||||
await RollbackGenerationStore.restoreGeneration(
|
||||
NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
['compose.yaml', special],
|
||||
);
|
||||
|
||||
expect(await fsPromises.readFile(path.join(stackDir, special), 'utf8')).toBe('KEEP_ME\n');
|
||||
});
|
||||
|
||||
it('captures POSIX mode and restores it on generation restore', async () => {
|
||||
if (process.platform === 'win32') return;
|
||||
|
||||
const stackName = 'modes';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
const scriptPath = path.join(stackDir, 'entrypoint.sh');
|
||||
await fsPromises.writeFile(scriptPath, '#!/bin/sh\necho hi\n', 'utf8');
|
||||
await fsPromises.chmod(scriptPath, 0o755);
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
|
||||
const generationId = randomUUID();
|
||||
const manifest = await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
{ relativePath: 'entrypoint.sh', absolutePath: scriptPath, kind: 'other' },
|
||||
]),
|
||||
});
|
||||
const entry = manifest.entries.find((e) => e.relativePath === 'entrypoint.sh');
|
||||
expect(entry?.mode).toBe(0o755);
|
||||
|
||||
await fsPromises.chmod(scriptPath, 0o644);
|
||||
await RollbackGenerationStore.restoreGeneration(
|
||||
NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
['compose.yaml', 'entrypoint.sh'],
|
||||
);
|
||||
const st = await fsPromises.stat(scriptPath);
|
||||
expect(st.mode & 0o777).toBe(0o755);
|
||||
});
|
||||
|
||||
it('reverts an interrupted restore that deleted a post-capture file via index absent entries', async () => {
|
||||
const stackName = 'absenttx';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'OLD\n', 'utf8');
|
||||
|
||||
const generationId = randomUUID();
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
]),
|
||||
});
|
||||
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'NEW\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'extra.yml'), 'EXTRA\n', 'utf8');
|
||||
|
||||
const { FileSystemService } = await import('../services/FileSystemService');
|
||||
const originalWrite = FileSystemService.prototype.writeStackFile;
|
||||
vi.spyOn(FileSystemService.prototype, 'writeStackFile').mockImplementation(async function (
|
||||
this: InstanceType<typeof FileSystemService>,
|
||||
stack,
|
||||
rel,
|
||||
content,
|
||||
) {
|
||||
if (rel === 'compose.yaml') throw new Error('injected write failure');
|
||||
return originalWrite.call(this, stack, rel, content);
|
||||
});
|
||||
|
||||
await expect(
|
||||
RollbackGenerationStore.restoreGeneration(NODE, stackName, generationId, ['compose.yaml', 'extra.yml']),
|
||||
).rejects.toThrow(/injected write failure/);
|
||||
|
||||
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf8')).toBe('NEW\n');
|
||||
expect(await fsPromises.readFile(path.join(stackDir, 'extra.yml'), 'utf8')).toBe('EXTRA\n');
|
||||
});
|
||||
|
||||
it('encrypts sensitive pre-restore snapshots and restores them from ciphertext', async () => {
|
||||
const stackName = 'presensitive';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, '.env'), 'CAPTURED=1\n', 'utf8');
|
||||
|
||||
const generationId = randomUUID();
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
{
|
||||
relativePath: '.env',
|
||||
absolutePath: path.join(stackDir, '.env'),
|
||||
kind: 'interpolation-env',
|
||||
sensitivity: 'medium',
|
||||
},
|
||||
]),
|
||||
});
|
||||
|
||||
await fsPromises.writeFile(path.join(stackDir, '.env'), 'SUPERSECRET=live\n', 'utf8');
|
||||
await RollbackGenerationStore.restoreGeneration(
|
||||
NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
['compose.yaml', '.env'],
|
||||
);
|
||||
|
||||
const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId);
|
||||
const blobsDir = path.join(genDir, 'pre-restore', 'blobs');
|
||||
const blobNames = await fsPromises.readdir(blobsDir);
|
||||
expect(blobNames.length).toBeGreaterThan(0);
|
||||
for (const name of blobNames) {
|
||||
const blob = await fsPromises.readFile(path.join(blobsDir, name));
|
||||
expect(blob.toString('utf8')).not.toContain('SUPERSECRET');
|
||||
expect(blob.toString('utf8')).not.toContain('CAPTURED=1');
|
||||
}
|
||||
const index = JSON.parse(
|
||||
await fsPromises.readFile(path.join(genDir, 'pre-restore', 'index.json'), 'utf8'),
|
||||
) as { entries: Array<{ relativePath: string; encrypted?: boolean; blobId?: string }> };
|
||||
const envEntry = index.entries.find((e) => e.relativePath === '.env');
|
||||
expect(envEntry?.encrypted).toBe(true);
|
||||
expect(envEntry?.blobId).toBeTruthy();
|
||||
const envBlob = await fsPromises.readFile(path.join(blobsDir, envEntry!.blobId!), 'utf8');
|
||||
expect(envBlob.startsWith('enc:')).toBe(true);
|
||||
expect(envBlob).not.toContain('SUPERSECRET');
|
||||
|
||||
await RollbackGenerationStore.reconcileInterruptedRestore(NODE, stackName, generationId);
|
||||
expect(await fsPromises.readFile(path.join(stackDir, '.env'), 'utf8')).toBe('SUPERSECRET=live\n');
|
||||
});
|
||||
|
||||
it('persists runtime image identity on an existing generation', async () => {
|
||||
const stackName = 'imgid';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
const generationId = randomUUID();
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
]),
|
||||
});
|
||||
|
||||
await RollbackGenerationStore.attachImages(NODE, stackName, generationId, [{
|
||||
serviceName: 'web',
|
||||
imageId: 'sha256:abc',
|
||||
repoDigest: 'nginx@sha256:digest',
|
||||
platform: 'linux/amd64',
|
||||
declaredImageRef: 'nginx:latest',
|
||||
}]);
|
||||
|
||||
const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId);
|
||||
const manifest = JSON.parse(
|
||||
await fsPromises.readFile(path.join(genDir, 'generation.json'), 'utf8'),
|
||||
) as { images: Array<{ serviceName: string; imageId: string; repoDigest: string; platform: string }> };
|
||||
expect(manifest.images).toEqual([{
|
||||
serviceName: 'web',
|
||||
imageId: 'sha256:abc',
|
||||
repoDigest: 'nginx@sha256:digest',
|
||||
platform: 'linux/amd64',
|
||||
declaredImageRef: 'nginx:latest',
|
||||
}]);
|
||||
});
|
||||
|
||||
it('does not recursively delete a directory created after an absent-file snapshot', async () => {
|
||||
const stackName = 'dirrace';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'OLD\n', 'utf8');
|
||||
|
||||
const generationId = randomUUID();
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
inventory: inventoryFor(stackName, [
|
||||
{ relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') },
|
||||
]),
|
||||
});
|
||||
|
||||
await RollbackGenerationStore.restoreGeneration(
|
||||
NODE,
|
||||
stackName,
|
||||
generationId,
|
||||
['compose.yaml', 'scratch'],
|
||||
);
|
||||
|
||||
const scratchDir = path.join(stackDir, 'scratch');
|
||||
await fsPromises.mkdir(scratchDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(scratchDir, 'keep.txt'), 'unrelated\n', 'utf8');
|
||||
|
||||
await expect(
|
||||
RollbackGenerationStore.reconcileInterruptedRestore(NODE, stackName, generationId),
|
||||
).rejects.toMatchObject({ code: 'DIRECTORY_COLLISION' });
|
||||
|
||||
expect(await fsPromises.readFile(path.join(scratchDir, 'keep.txt'), 'utf8')).toBe('unrelated\n');
|
||||
const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId);
|
||||
await expect(fsPromises.access(path.join(genDir, 'restore-intent.json'))).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,311 @@
|
||||
/**
|
||||
* Git manifesto inventory: first-apply merge vs established fail-closed.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
import { promises as fsPromises } from 'fs';
|
||||
|
||||
const mockGetGitSource = vi.fn();
|
||||
const mockReadManifest = vi.fn();
|
||||
const mockGetOverrideFilename = vi.fn().mockResolvedValue(null);
|
||||
const mockGetStackProjectEnvFiles = vi.fn().mockReturnValue([]);
|
||||
const mockBuildAuthoredComposeArgs = vi.fn().mockResolvedValue(['compose', '-f', 'compose.yaml', 'config', '--quiet']);
|
||||
const mockAuthoredComposeFileArgs = vi.fn().mockReturnValue(['-f', 'compose.yaml']);
|
||||
const mockAuthoredComposeEnvFileArgs = vi.fn().mockResolvedValue([]);
|
||||
|
||||
vi.mock('../services/DatabaseService', () => ({
|
||||
DatabaseService: {
|
||||
getInstance: () => ({
|
||||
getGitSource: mockGetGitSource,
|
||||
getStackProjectEnvFiles: mockGetStackProjectEnvFiles,
|
||||
isMeshStackEnabled: () => false,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/NodeRegistry', () => ({
|
||||
NodeRegistry: {
|
||||
getInstance: () => ({
|
||||
getDefaultNodeId: () => 1,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/GitProjectManifestService', () => ({
|
||||
GitProjectManifestService: {
|
||||
getInstance: () => ({
|
||||
readManifest: mockReadManifest,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
const mockState = { composeDir: '' };
|
||||
|
||||
vi.mock('../services/FileSystemService', () => ({
|
||||
FileSystemService: {
|
||||
getInstance: () => ({
|
||||
getBaseDir: () => mockState.composeDir,
|
||||
getOverrideFilename: mockGetOverrideFilename,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/ComposeService', () => ({
|
||||
ComposeService: {
|
||||
getInstance: () => ({
|
||||
buildAuthoredComposeArgs: mockBuildAuthoredComposeArgs,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../utils/authoredComposeArgs', () => ({
|
||||
authoredComposeFileArgs: (...args: unknown[]) => mockAuthoredComposeFileArgs(...args),
|
||||
authoredComposeEnvFileArgs: (...args: unknown[]) => mockAuthoredComposeEnvFileArgs(...args),
|
||||
}));
|
||||
|
||||
import { resolveRollbackInventory } from '../services/rollbackInventory';
|
||||
|
||||
describe('resolveRollbackInventory', () => {
|
||||
let composeDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
composeDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-inv-'));
|
||||
mockState.composeDir = composeDir;
|
||||
mockGetOverrideFilename.mockResolvedValue(null);
|
||||
mockGetStackProjectEnvFiles.mockReturnValue([]);
|
||||
mockBuildAuthoredComposeArgs.mockResolvedValue(['compose', '-f', 'compose.yaml', 'config', '--quiet']);
|
||||
mockAuthoredComposeFileArgs.mockReturnValue(['-f', 'compose.yaml']);
|
||||
mockAuthoredComposeEnvFileArgs.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
it('merges authored files with Git identity on first apply when manifesto is missing', async () => {
|
||||
const stackName = 'gitapp';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(
|
||||
path.join(stackDir, 'compose.yaml'),
|
||||
'services:\n web:\n image: nginx\n',
|
||||
'utf8',
|
||||
);
|
||||
|
||||
mockGetGitSource.mockReturnValue({
|
||||
stack_name: stackName,
|
||||
repo_url: 'https://example.com/repo.git',
|
||||
branch: 'main',
|
||||
last_applied_commit_sha: null,
|
||||
last_applied_content_hash: null,
|
||||
applied_deploy_spec: null,
|
||||
sync_env: false,
|
||||
manifest_version: null,
|
||||
manifest_state: null,
|
||||
manifest_generation: null,
|
||||
});
|
||||
mockReadManifest.mockResolvedValue(null);
|
||||
|
||||
const inventory = await resolveRollbackInventory(1, stackName);
|
||||
|
||||
expect(inventory.exactCoverage).toBe(true);
|
||||
expect(inventory.coverageRefusal).toBeNull();
|
||||
expect(inventory.entries.map((e) => e.relativePath)).toContain('compose.yaml');
|
||||
expect(inventory.git).toEqual({
|
||||
repoUrl: 'https://example.com/repo.git',
|
||||
branch: 'main',
|
||||
commitSha: '',
|
||||
manifestVersion: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('fails closed when an established Git stack is missing its manifesto and cannot rebuild from applied spec', async () => {
|
||||
const stackName = 'established';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
|
||||
mockGetGitSource.mockReturnValue({
|
||||
stack_name: stackName,
|
||||
repo_url: 'https://example.com/repo.git',
|
||||
branch: 'main',
|
||||
last_applied_commit_sha: 'abc1234',
|
||||
last_applied_content_hash: 'hash',
|
||||
applied_deploy_spec: null,
|
||||
sync_env: false,
|
||||
manifest_version: 3,
|
||||
manifest_state: 'active',
|
||||
manifest_generation: 'generations/prior',
|
||||
});
|
||||
mockReadManifest.mockResolvedValue(null);
|
||||
|
||||
const inventory = await resolveRollbackInventory(1, stackName);
|
||||
|
||||
expect(inventory.exactCoverage).toBe(false);
|
||||
expect(inventory.coverageRefusal).toMatch(/missing/i);
|
||||
expect(inventory.git?.commitSha).toBe('abc1234');
|
||||
});
|
||||
|
||||
it('fails closed for established missing manifesto instead of applied_deploy_spec exact coverage', async () => {
|
||||
const stackName = 'rebuild';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n a: {}\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.prod.yaml'), 'services:\n b: {}\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'app.env'), 'FOO=1\n', 'utf8');
|
||||
|
||||
mockGetGitSource.mockReturnValue({
|
||||
stack_name: stackName,
|
||||
repo_url: 'https://example.com/repo.git',
|
||||
branch: 'main',
|
||||
last_applied_commit_sha: 'deadbeef',
|
||||
last_applied_content_hash: 'h1',
|
||||
applied_deploy_spec: { files: ['compose.yaml', 'compose.prod.yaml'], contextDir: null },
|
||||
sync_env: false,
|
||||
manifest_version: 3,
|
||||
manifest_state: 'active',
|
||||
manifest_generation: 'generations/x',
|
||||
});
|
||||
mockReadManifest.mockResolvedValue(null);
|
||||
|
||||
const inventory = await resolveRollbackInventory(1, stackName);
|
||||
|
||||
expect(inventory.exactCoverage).toBe(false);
|
||||
expect(inventory.coverageRefusal).toMatch(/missing/i);
|
||||
expect(inventory.git?.commitSha).toBe('deadbeef');
|
||||
});
|
||||
|
||||
it('fails closed when the manifesto is corrupt even if applied_deploy_spec files exist', async () => {
|
||||
const stackName = 'corruptgit';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'app.env'), 'PASS=x\n', 'utf8');
|
||||
|
||||
mockGetGitSource.mockReturnValue({
|
||||
stack_name: stackName,
|
||||
repo_url: 'https://example.com/repo.git',
|
||||
branch: 'main',
|
||||
last_applied_commit_sha: 'abc',
|
||||
last_applied_content_hash: 'h',
|
||||
applied_deploy_spec: { files: ['compose.yaml'], contextDir: null },
|
||||
sync_env: false,
|
||||
manifest_version: 3,
|
||||
manifest_state: 'active',
|
||||
manifest_generation: 'generations/x',
|
||||
});
|
||||
mockReadManifest.mockResolvedValue({ corrupt: 'identity mismatch' });
|
||||
|
||||
const inventory = await resolveRollbackInventory(1, stackName);
|
||||
|
||||
expect(inventory.exactCoverage).toBe(false);
|
||||
expect(inventory.coverageRefusal).toMatch(/unreadable/i);
|
||||
expect(inventory.git?.commitSha).toBe('abc');
|
||||
});
|
||||
|
||||
it('fails closed when the manifesto is corrupt and applied_deploy_spec cannot be rebuilt', async () => {
|
||||
const stackName = 'corrupt-norebuild';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
|
||||
mockGetGitSource.mockReturnValue({
|
||||
stack_name: stackName,
|
||||
repo_url: 'https://example.com/repo.git',
|
||||
branch: 'main',
|
||||
last_applied_commit_sha: 'abc',
|
||||
last_applied_content_hash: 'h',
|
||||
applied_deploy_spec: { files: ['compose.yaml', 'missing.override.yaml'], contextDir: null },
|
||||
sync_env: false,
|
||||
manifest_version: 3,
|
||||
manifest_state: 'active',
|
||||
manifest_generation: 'generations/x',
|
||||
});
|
||||
mockReadManifest.mockResolvedValue({ corrupt: 'identity mismatch' });
|
||||
|
||||
const inventory = await resolveRollbackInventory(1, stackName);
|
||||
|
||||
expect(inventory.exactCoverage).toBe(false);
|
||||
expect(inventory.coverageRefusal).toMatch(/unreadable/i);
|
||||
});
|
||||
|
||||
it('fails closed for missing manifesto with include/env/config dependency files on disk', async () => {
|
||||
const stackName = 'deps-matrix';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(path.join(stackDir, 'configs'), { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n a:\n env_file: [app.env]\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.prod.yaml'), 'include:\n - path: compose.yaml\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'app.env'), 'A=1\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'configs/app.conf'), 'x=1\n', 'utf8');
|
||||
|
||||
mockGetGitSource.mockReturnValue({
|
||||
stack_name: stackName,
|
||||
repo_url: 'https://example.com/repo.git',
|
||||
branch: 'main',
|
||||
last_applied_commit_sha: 'cafebabe',
|
||||
last_applied_content_hash: 'h2',
|
||||
applied_deploy_spec: { files: ['compose.yaml', 'compose.prod.yaml'], contextDir: null },
|
||||
sync_env: false,
|
||||
manifest_version: 3,
|
||||
manifest_state: 'active',
|
||||
manifest_generation: 'generations/y',
|
||||
});
|
||||
mockReadManifest.mockResolvedValue(null);
|
||||
|
||||
const inventory = await resolveRollbackInventory(1, stackName);
|
||||
|
||||
expect(inventory.exactCoverage).toBe(false);
|
||||
expect(inventory.coverageRefusal).toMatch(/missing/i);
|
||||
});
|
||||
|
||||
it('fails closed when the Git manifesto is corrupt even before first apply', async () => {
|
||||
const stackName = 'corrupt-first';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
|
||||
mockGetGitSource.mockReturnValue({
|
||||
stack_name: stackName,
|
||||
repo_url: 'https://example.com/repo.git',
|
||||
branch: 'main',
|
||||
last_applied_commit_sha: null,
|
||||
last_applied_content_hash: null,
|
||||
applied_deploy_spec: null,
|
||||
sync_env: false,
|
||||
manifest_version: null,
|
||||
manifest_state: null,
|
||||
manifest_generation: null,
|
||||
});
|
||||
mockReadManifest.mockResolvedValue({ corrupt: 'identity mismatch' });
|
||||
|
||||
const inventory = await resolveRollbackInventory(1, stackName);
|
||||
|
||||
expect(inventory.exactCoverage).toBe(false);
|
||||
expect(inventory.coverageRefusal).toMatch(/unreadable/i);
|
||||
});
|
||||
|
||||
it('refuses exact coverage when case-colliding managed paths exist', async () => {
|
||||
const stackName = 'casefold';
|
||||
const stackDir = path.join(composeDir, stackName);
|
||||
await fsPromises.mkdir(stackDir, { recursive: true });
|
||||
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'App.conf'), 'A\n', 'utf8');
|
||||
await fsPromises.writeFile(path.join(stackDir, 'app.conf'), 'B\n', 'utf8');
|
||||
|
||||
mockGetGitSource.mockReturnValue(undefined);
|
||||
|
||||
// Force discovery to see both via include parse would be heavy; instead seed
|
||||
// by making them both compose roots is impossible. Use override + compose:
|
||||
mockGetOverrideFilename.mockResolvedValue('App.conf');
|
||||
// Also plant app.conf as project env so both enter the map.
|
||||
mockGetStackProjectEnvFiles.mockReturnValue(['app.conf']);
|
||||
|
||||
const inventory = await resolveRollbackInventory(1, stackName);
|
||||
// On case-sensitive FS both files exist; inventory should refuse collision.
|
||||
// On Windows case-folding FS the second write may overwrite the first.
|
||||
if (process.platform === 'win32') {
|
||||
expect(inventory).toBeTruthy();
|
||||
return;
|
||||
}
|
||||
expect(inventory.exactCoverage).toBe(false);
|
||||
expect(inventory.coverageRefusal).toMatch(/Case-colliding/i);
|
||||
});
|
||||
});
|
||||
@@ -28,6 +28,7 @@ const {
|
||||
mockRunCommand,
|
||||
mockDeployStack,
|
||||
mockBackupStackFiles,
|
||||
mockCaptureCurrentBackup,
|
||||
mockEnforcePolicyPreDeploy,
|
||||
} = vi.hoisted(() => ({
|
||||
mockGetDueScheduledTasks: vi.fn().mockReturnValue([]),
|
||||
@@ -76,6 +77,7 @@ const {
|
||||
mockRunCommand: vi.fn().mockResolvedValue(undefined),
|
||||
mockDeployStack: vi.fn().mockResolvedValue(undefined),
|
||||
mockBackupStackFiles: vi.fn().mockResolvedValue(undefined),
|
||||
mockCaptureCurrentBackup: vi.fn().mockResolvedValue({ id: 'gen-1' }),
|
||||
mockEnforcePolicyPreDeploy: vi.fn(),
|
||||
}));
|
||||
|
||||
@@ -165,6 +167,14 @@ vi.mock('../services/FileSystemService', () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/StackUpdateRecoveryService', () => ({
|
||||
StackUpdateRecoveryService: {
|
||||
getInstance: () => ({
|
||||
captureCurrentBackup: mockCaptureCurrentBackup,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/ImageUpdateService', () => ({
|
||||
ImageUpdateService: {
|
||||
// Default on so existing executeUpdate tests keep prior behavior.
|
||||
@@ -1907,10 +1917,15 @@ describe('SchedulerService - lifecycle actions', () => {
|
||||
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' }));
|
||||
});
|
||||
|
||||
it('auto_backup calls backupStackFiles', async () => {
|
||||
it('auto_backup captures a current recovery generation', async () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup'));
|
||||
await SchedulerService.getInstance().triggerTask(300);
|
||||
expect(mockBackupStackFiles).toHaveBeenCalledWith('my-stack');
|
||||
expect(mockCaptureCurrentBackup).toHaveBeenCalledWith({
|
||||
nodeId: 1,
|
||||
stackName: 'my-stack',
|
||||
createdBy: 'system:scheduler',
|
||||
});
|
||||
expect(mockBackupStackFiles).not.toHaveBeenCalled();
|
||||
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' }));
|
||||
});
|
||||
|
||||
@@ -1924,6 +1939,7 @@ describe('SchedulerService - lifecycle actions', () => {
|
||||
it('auto_backup records failure when node_id is missing', async () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { node_id: null }));
|
||||
await SchedulerService.getInstance().triggerTask(300);
|
||||
expect(mockCaptureCurrentBackup).not.toHaveBeenCalled();
|
||||
expect(mockBackupStackFiles).not.toHaveBeenCalled();
|
||||
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'failure' }));
|
||||
});
|
||||
@@ -2015,6 +2031,7 @@ describe('SchedulerService - lifecycle remote proxy', () => {
|
||||
'http://remote:1852/api/stacks/my-stack/backup',
|
||||
expect.objectContaining({ method: 'POST' }),
|
||||
);
|
||||
expect(mockCaptureCurrentBackup).not.toHaveBeenCalled();
|
||||
expect(mockBackupStackFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -2246,7 +2263,7 @@ describe('SchedulerService - delete_after_run', () => {
|
||||
});
|
||||
|
||||
it('does not delete task when run fails even if delete_after_run is 1', async () => {
|
||||
mockBackupStackFiles.mockRejectedValueOnce(new Error('disk full'));
|
||||
mockCaptureCurrentBackup.mockRejectedValueOnce(new Error('disk full'));
|
||||
mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { delete_after_run: 1 }));
|
||||
await SchedulerService.getInstance().triggerTask(300);
|
||||
expect(mockDeleteScheduledTask).not.toHaveBeenCalled();
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
/**
|
||||
* Integration tests for POST /api/stacks/:stackName/backup, the on-demand
|
||||
* stack-files backup trigger. Covers auth, role, the success path, the
|
||||
* recovery-generation capture. Covers auth, role, the success path, the
|
||||
* missing-stack 404, name validation, and error propagation. The route exists
|
||||
* so a scheduled auto_backup can run on a remote node through the proxy path,
|
||||
* and so an operator can take a snapshot on demand. The backup is available on
|
||||
* and so an operator can capture a current generation on demand. Available on
|
||||
* every tier (no paid gate).
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
|
||||
@@ -11,9 +11,11 @@ import request from 'supertest';
|
||||
import bcrypt from 'bcrypt';
|
||||
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
|
||||
|
||||
const { mockBackupStackFiles, mockHasComposeFile } = vi.hoisted(() => ({
|
||||
const { mockBackupStackFiles, mockHasComposeFile, mockCaptureCurrentBackup, mockGetCurrent } = vi.hoisted(() => ({
|
||||
mockBackupStackFiles: vi.fn(),
|
||||
mockHasComposeFile: vi.fn(),
|
||||
mockCaptureCurrentBackup: vi.fn(),
|
||||
mockGetCurrent: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../services/FileSystemService', () => ({
|
||||
@@ -27,6 +29,15 @@ vi.mock('../services/FileSystemService', () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../services/StackUpdateRecoveryService', () => ({
|
||||
StackUpdateRecoveryService: {
|
||||
getInstance: () => ({
|
||||
getCurrent: mockGetCurrent,
|
||||
captureCurrentBackup: mockCaptureCurrentBackup,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
@@ -59,6 +70,8 @@ afterAll(() => {
|
||||
beforeEach(() => {
|
||||
mockBackupStackFiles.mockReset().mockResolvedValue(undefined);
|
||||
mockHasComposeFile.mockReset().mockResolvedValue(true);
|
||||
mockCaptureCurrentBackup.mockReset().mockResolvedValue({ id: 'gen-1' });
|
||||
mockGetCurrent.mockReset().mockReturnValue(null);
|
||||
tierSpy.mockReturnValue('paid');
|
||||
});
|
||||
|
||||
@@ -67,18 +80,25 @@ describe('POST /api/stacks/:stackName/backup', () => {
|
||||
const res = await request(app).post('/api/stacks/web/backup').set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.success).toBe(true);
|
||||
expect(mockBackupStackFiles).toHaveBeenCalledWith('web');
|
||||
expect(mockCaptureCurrentBackup).toHaveBeenCalledWith(expect.objectContaining({
|
||||
nodeId: expect.any(Number),
|
||||
stackName: 'web',
|
||||
createdBy: expect.any(String),
|
||||
}));
|
||||
expect(mockBackupStackFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns 401 without an auth cookie', async () => {
|
||||
const res = await request(app).post('/api/stacks/web/backup');
|
||||
expect(res.status).toBe(401);
|
||||
expect(mockCaptureCurrentBackup).not.toHaveBeenCalled();
|
||||
expect(mockBackupStackFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns 403 for a viewer (no deploy permission)', async () => {
|
||||
const res = await request(app).post('/api/stacks/web/backup').set('Cookie', viewerCookie);
|
||||
expect(res.status).toBe(403);
|
||||
expect(mockCaptureCurrentBackup).not.toHaveBeenCalled();
|
||||
expect(mockBackupStackFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -87,32 +107,39 @@ describe('POST /api/stacks/:stackName/backup', () => {
|
||||
const res = await request(app).post('/api/stacks/web/backup').set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.success).toBe(true);
|
||||
expect(mockBackupStackFiles).toHaveBeenCalledWith('web');
|
||||
expect(mockCaptureCurrentBackup).toHaveBeenCalledWith(expect.objectContaining({
|
||||
nodeId: expect.any(Number),
|
||||
stackName: 'web',
|
||||
createdBy: expect.any(String),
|
||||
}));
|
||||
expect(mockBackupStackFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns 404 when the stack does not exist', async () => {
|
||||
mockHasComposeFile.mockResolvedValue(false);
|
||||
const res = await request(app).post('/api/stacks/ghost/backup').set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(404);
|
||||
expect(mockCaptureCurrentBackup).not.toHaveBeenCalled();
|
||||
expect(mockBackupStackFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns 400 for an invalid stack name', async () => {
|
||||
const res = await request(app).post('/api/stacks/..bad../backup').set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(400);
|
||||
expect(mockCaptureCurrentBackup).not.toHaveBeenCalled();
|
||||
expect(mockBackupStackFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns 500 when the backup operation fails', async () => {
|
||||
mockBackupStackFiles.mockRejectedValue(new Error('disk full'));
|
||||
mockCaptureCurrentBackup.mockRejectedValue(new Error('disk full'));
|
||||
const res = await request(app).post('/api/stacks/web/backup').set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(500);
|
||||
expect(res.body.error).toContain('disk full');
|
||||
});
|
||||
|
||||
it('returns 409 when the stack is busy with another operation', async () => {
|
||||
// The backup shares the rollback slot, so it must not run while a deploy
|
||||
// holds the stack-op lock for the same stack.
|
||||
// Handoff mutates the current generation, so backup must not run while a
|
||||
// deploy holds the stack-op lock for the same stack.
|
||||
const { StackOpLockService } = await import('../services/StackOpLockService');
|
||||
const localNodeId = DatabaseService.getInstance().getNodes().find(n => n.type === 'local')!.id;
|
||||
StackOpLockService.getInstance().tryAcquire(localNodeId, 'web', 'deploy', 'someone');
|
||||
@@ -120,6 +147,7 @@ describe('POST /api/stacks/:stackName/backup', () => {
|
||||
const res = await request(app).post('/api/stacks/web/backup').set('Cookie', adminCookie);
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body.code).toBe('stack_op_in_progress');
|
||||
expect(mockCaptureCurrentBackup).not.toHaveBeenCalled();
|
||||
expect(mockBackupStackFiles).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
StackOpLockService.getInstance().release(localNodeId, 'web');
|
||||
|
||||
@@ -98,7 +98,7 @@ afterAll(() => {
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
mockDeployStack.mockReset();
|
||||
mockDeployStack.mockReset().mockResolvedValue({ recoveryId: null });
|
||||
mockRunCommand.mockReset();
|
||||
mockRunDown.mockReset();
|
||||
mockUpdateStack.mockReset();
|
||||
@@ -128,7 +128,7 @@ function deferred<T>(): Deferred<T> {
|
||||
|
||||
describe('Stack lifecycle mutex', () => {
|
||||
it('returns 409 with stack_op_in_progress when a deploy is already running', async () => {
|
||||
const gate = deferred<void>();
|
||||
const gate = deferred<{ recoveryId: string | null }>();
|
||||
mockDeployStack.mockImplementationOnce(() => gate.promise);
|
||||
|
||||
const first = request(app)
|
||||
@@ -152,20 +152,20 @@ describe('Stack lifecycle mutex', () => {
|
||||
expect(second.body.error).toMatch(/already deploying/i);
|
||||
expect(typeof second.body.inProgress.startedAt).toBe('number');
|
||||
|
||||
gate.resolve();
|
||||
gate.resolve({ recoveryId: null });
|
||||
const firstRes = await first;
|
||||
expect(firstRes.status).toBe(200);
|
||||
});
|
||||
|
||||
it('releases the lock after a successful deploy so the next request acquires', async () => {
|
||||
mockDeployStack.mockResolvedValueOnce(undefined);
|
||||
mockDeployStack.mockResolvedValueOnce({ recoveryId: null });
|
||||
const first = await request(app)
|
||||
.post('/api/stacks/web/deploy')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ skip_scan: true });
|
||||
expect(first.status).toBe(200);
|
||||
|
||||
mockDeployStack.mockResolvedValueOnce(undefined);
|
||||
mockDeployStack.mockResolvedValueOnce({ recoveryId: null });
|
||||
const second = await request(app)
|
||||
.post('/api/stacks/web/deploy')
|
||||
.set('Cookie', authCookie)
|
||||
@@ -181,7 +181,7 @@ describe('Stack lifecycle mutex', () => {
|
||||
.send({ skip_scan: true });
|
||||
expect(first.status).toBe(500);
|
||||
|
||||
mockDeployStack.mockResolvedValueOnce(undefined);
|
||||
mockDeployStack.mockResolvedValueOnce({ recoveryId: null });
|
||||
const second = await request(app)
|
||||
.post('/api/stacks/web/deploy')
|
||||
.set('Cookie', authCookie)
|
||||
@@ -190,7 +190,7 @@ describe('Stack lifecycle mutex', () => {
|
||||
});
|
||||
|
||||
it('blocks restart while a deploy is in flight on the same stack', async () => {
|
||||
const gate = deferred<void>();
|
||||
const gate = deferred<{ recoveryId: string | null }>();
|
||||
mockDeployStack.mockImplementationOnce(() => gate.promise);
|
||||
|
||||
const deploy = request(app)
|
||||
@@ -207,12 +207,12 @@ describe('Stack lifecycle mutex', () => {
|
||||
expect(restart.body.code).toBe('stack_op_in_progress');
|
||||
expect(restart.body.inProgress.action).toBe('deploy');
|
||||
|
||||
gate.resolve();
|
||||
gate.resolve({ recoveryId: null });
|
||||
await deploy;
|
||||
});
|
||||
|
||||
it('allows concurrent ops on different stacks', async () => {
|
||||
const gate = deferred<void>();
|
||||
const gate = deferred<{ recoveryId: string | null }>();
|
||||
mockDeployStack.mockImplementation(() => gate.promise);
|
||||
|
||||
const webDeploy = request(app)
|
||||
@@ -229,7 +229,7 @@ describe('Stack lifecycle mutex', () => {
|
||||
.then(r => r);
|
||||
await vi.waitFor(() => expect(mockDeployStack).toHaveBeenCalledTimes(2));
|
||||
|
||||
gate.resolve();
|
||||
gate.resolve({ recoveryId: null });
|
||||
const [webRes, apiRes] = await Promise.all([webDeploy, apiDeploy]);
|
||||
expect(webRes.status).toBe(200);
|
||||
expect(apiRes.status).toBe(200);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -39,6 +39,9 @@ describe('classifyStackApiPath', () => {
|
||||
expect(classifyStackApiPath('DELETE', '/stacks/web/git-source')).toEqual({
|
||||
kind: 'named-stack', stackName: 'web', action: 'stack:edit',
|
||||
});
|
||||
expect(classifyStackApiPath('POST', '/stacks/web/fleet-snapshot-apply')).toEqual({
|
||||
kind: 'named-stack', stackName: 'web', action: 'stack:edit',
|
||||
});
|
||||
});
|
||||
|
||||
it('maps deploy routes and service lifecycle ops to stack:deploy', () => {
|
||||
|
||||
@@ -13,6 +13,7 @@ import jwt from 'jsonwebtoken';
|
||||
import { setupTestDb, cleanupTestDb, loginAsTestAdmin, TEST_JWT_SECRET } from './helpers/setupTestDb';
|
||||
import { ComposeRollbackError } from '../services/ComposeService';
|
||||
import * as policyGate from '../helpers/policyGate';
|
||||
import type { StackUpdateRecoveryGenerationRow } from '../services/DatabaseService';
|
||||
|
||||
// ── Hoisted mocks (must come before importing the app) ──────────────────────
|
||||
|
||||
@@ -141,10 +142,10 @@ afterAll(() => {
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
mockDeployStack.mockReset();
|
||||
mockDeployStack.mockReset().mockResolvedValue({ recoveryId: null });
|
||||
mockRunCommand.mockReset();
|
||||
mockRunDown.mockReset();
|
||||
mockUpdateStack.mockReset();
|
||||
mockUpdateStack.mockReset().mockResolvedValue({ recoveryId: null });
|
||||
mockGetContainersByStack.mockReset();
|
||||
mockRestartContainer.mockReset();
|
||||
mockStopContainer.mockReset();
|
||||
@@ -233,7 +234,7 @@ describe('deploy_failure notification on /deploy error', () => {
|
||||
});
|
||||
|
||||
it('uses trusted proxy tier headers for remote atomic deploys', async () => {
|
||||
mockDeployStack.mockResolvedValue(undefined);
|
||||
mockDeployStack.mockResolvedValue({ recoveryId: null });
|
||||
const token = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' });
|
||||
|
||||
const res = await request(app)
|
||||
@@ -259,7 +260,7 @@ describe('health gate begin call sites', () => {
|
||||
});
|
||||
|
||||
it('begins a gate after a manual deploy and returns its id', async () => {
|
||||
mockDeployStack.mockResolvedValue(undefined);
|
||||
mockDeployStack.mockResolvedValue({ recoveryId: null });
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/myapp/deploy')
|
||||
.set('Cookie', authCookie)
|
||||
@@ -269,6 +270,19 @@ describe('health gate begin call sites', () => {
|
||||
expect(res.body.healthGateId).toBe('gate-123');
|
||||
});
|
||||
|
||||
it('links the deploy recovery generation to the observing gate', async () => {
|
||||
mockDeployStack.mockResolvedValue({ recoveryId: 'rec-deploy' });
|
||||
const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService');
|
||||
const linkSpy = vi.spyOn(StackUpdateRecoveryService.getInstance(), 'linkGateOrRetain');
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/myapp/deploy')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ skip_scan: true });
|
||||
expect(res.status).toBe(200);
|
||||
expect(linkSpy).toHaveBeenCalledWith('rec-deploy', 'gate-123');
|
||||
linkSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('begins a gate after a manual update and returns its id', async () => {
|
||||
mockUpdateStack.mockResolvedValue({ recoveryId: null });
|
||||
const res = await request(app)
|
||||
@@ -304,7 +318,7 @@ describe('health gate begin call sites', () => {
|
||||
});
|
||||
|
||||
it('never begins a gate for the rollback recovery path', async () => {
|
||||
mockDeployStack.mockResolvedValue(undefined);
|
||||
mockDeployStack.mockResolvedValue({ recoveryId: null });
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/myapp/rollback')
|
||||
.set('Cookie', authCookie);
|
||||
@@ -370,13 +384,39 @@ describe('failure classification on deploy/update error responses', () => {
|
||||
.set('Cookie', authCookie);
|
||||
|
||||
expect(res.status).toBe(500);
|
||||
expect(res.body.failure.reason).toBe('unknown');
|
||||
expect(res.body.failure).toMatchObject({ reason: 'unknown' });
|
||||
});
|
||||
|
||||
it('classifies mixed replica image capture refusals', async () => {
|
||||
mockDeployStack.mockRejectedValue(
|
||||
new Error('Service "web" has mixed replica images; refusing recovery capture that cannot restore exact prior identity'),
|
||||
);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/myapp/deploy')
|
||||
.set('Cookie', authCookie);
|
||||
|
||||
expect(res.status).toBe(500);
|
||||
expect(res.body.failure).toMatchObject({ reason: 'mixed_replica_images' });
|
||||
});
|
||||
|
||||
it('classifies rollback coverage refusals', async () => {
|
||||
mockDeployStack.mockRejectedValue(
|
||||
new Error('Host-absolute include path cannot be captured for exact rollback'),
|
||||
);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/myapp/deploy')
|
||||
.set('Cookie', authCookie);
|
||||
|
||||
expect(res.status).toBe(500);
|
||||
expect(res.body.failure).toMatchObject({ reason: 'rollback_coverage_unavailable' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('post-deploy scan opt-out', () => {
|
||||
it('does not trigger a post-deploy scan when skip_scan is true', async () => {
|
||||
mockDeployStack.mockResolvedValue(undefined);
|
||||
mockDeployStack.mockResolvedValue({ recoveryId: null });
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/myapp/deploy')
|
||||
@@ -503,6 +543,76 @@ describe('deploy_failure notification on /update error', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('generation rollback error mapping', () => {
|
||||
function stubCurrentGeneration(id: string): StackUpdateRecoveryGenerationRow {
|
||||
return {
|
||||
id,
|
||||
node_id: 1,
|
||||
stack_name: 'myapp',
|
||||
status: 'active',
|
||||
phase: 'immediate_verified',
|
||||
is_current: 1,
|
||||
backup_slot_id: id,
|
||||
content_path: null,
|
||||
operation_kind: null,
|
||||
override_path: '/tmp/override.yml',
|
||||
services_json: '[]',
|
||||
health_gate_id: null,
|
||||
gate_retain_until: null,
|
||||
artifact_expires_at: null,
|
||||
operation_lease_expires_at: null,
|
||||
created_at: 0,
|
||||
updated_at: 0,
|
||||
created_by: null,
|
||||
artifacts_retired: 0,
|
||||
released_at: null,
|
||||
released_by: null,
|
||||
};
|
||||
}
|
||||
|
||||
async function withGenerationRollback(compensate: () => Promise<boolean>) {
|
||||
const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService');
|
||||
const svc = StackUpdateRecoveryService.getInstance();
|
||||
const getSpy = vi.spyOn(svc, 'getCurrent').mockReturnValue(stubCurrentGeneration('gen-1'));
|
||||
const compensateSpy = vi.spyOn(svc, 'compensateWithCandidate').mockImplementation(compensate);
|
||||
try {
|
||||
return await request(app).post('/api/stacks/myapp/rollback').set('Cookie', authCookie);
|
||||
} finally {
|
||||
getSpy.mockRestore();
|
||||
compensateSpy.mockRestore();
|
||||
}
|
||||
}
|
||||
|
||||
it('returns HELD_IMAGE_MISSING when the hold tag is gone', async () => {
|
||||
const res = await withGenerationRollback(async () => {
|
||||
throw Object.assign(new Error('Held recovery image is missing'), { code: 'HELD_IMAGE_MISSING' });
|
||||
});
|
||||
expect(res.status).toBe(500);
|
||||
expect(res.body).toMatchObject({
|
||||
code: 'HELD_IMAGE_MISSING',
|
||||
error: 'Held recovery image is missing',
|
||||
});
|
||||
});
|
||||
|
||||
it('returns RECOVERY_PROBE_FAILED when restore completed but the probe failed', async () => {
|
||||
const res = await withGenerationRollback(async () => {
|
||||
throw Object.assign(new Error('Recovery health probe failed'), { code: 'RECOVERY_PROBE_FAILED' });
|
||||
});
|
||||
expect(res.status).toBe(500);
|
||||
expect(res.body).toMatchObject({
|
||||
code: 'RECOVERY_PROBE_FAILED',
|
||||
error: 'Rollback restore completed but recovery probe failed.',
|
||||
});
|
||||
});
|
||||
|
||||
it('returns a generic restore failure when compensation returns false', async () => {
|
||||
const res = await withGenerationRollback(async () => false);
|
||||
expect(res.status).toBe(500);
|
||||
expect(res.body).toMatchObject({ error: 'Rollback restore did not complete.' });
|
||||
expect(res.body.code).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('rollback file-revert safety on a policy-blocked rollback', () => {
|
||||
it('does not deploy and alerts the operator when the post-block file revert fails', async () => {
|
||||
// The restored backup is blocked by policy after files were already restored.
|
||||
|
||||
@@ -123,7 +123,7 @@ beforeEach(() => {
|
||||
envWritten: false,
|
||||
warnings: [],
|
||||
});
|
||||
mockDeployStack.mockResolvedValue(undefined);
|
||||
mockDeployStack.mockResolvedValue({ recoveryId: null });
|
||||
mockIsTrivyAvailable.mockReturnValue(true);
|
||||
mockListContainers.mockResolvedValue([{ Image: 'nginx:latest' }]);
|
||||
mockGetImageDigest.mockResolvedValue(null);
|
||||
|
||||
@@ -36,6 +36,9 @@ const baseInputs = (over: Partial<RollbackInputs> = {}): RollbackInputs => ({
|
||||
name: 'app-web-1', state: 'running', health: 'healthy', exitCode: null,
|
||||
hasHealthcheck: true, restartPolicy: 'unless-stopped', mounts: ['volume app_data'],
|
||||
}],
|
||||
recoveryGeneration: { exists: false },
|
||||
policyEligibility: null,
|
||||
managedInputs: { covered: true, detail: 'Exact authored-project coverage includes 1 managed path(s).' },
|
||||
...over,
|
||||
});
|
||||
|
||||
@@ -43,19 +46,20 @@ const itemById = (inputs: RollbackInputs, id: string) =>
|
||||
buildRollbackItems(inputs, NOW).find(i => i.id === id)!;
|
||||
|
||||
describe('buildRollbackItems', () => {
|
||||
it('reports a full set of six items', () => {
|
||||
it('reports the full set of rollback readiness items', () => {
|
||||
const items = buildRollbackItems(baseInputs(), NOW);
|
||||
expect(items.map(i => i.id)).toEqual([
|
||||
'compose_source', 'env_keys', 'previous_images', 'last_deploy', 'healthchecks', 'volume_data',
|
||||
'recovery_generation', 'compose_source', 'env_keys', 'previous_images', 'last_deploy', 'healthchecks',
|
||||
'policy_eligibility', 'managed_inputs', 'volume_data',
|
||||
]);
|
||||
});
|
||||
|
||||
it('marks the volume row not_covered unconditionally and names the mounts', () => {
|
||||
const item = itemById(baseInputs(), 'volume_data');
|
||||
expect(item.state).toBe('not_covered');
|
||||
expect(item.detail).toContain('not included in file backups');
|
||||
expect(item.detail).toContain('not included in recovery generations');
|
||||
expect(item.detail).toContain('volume app_data');
|
||||
|
||||
expect(item.detail).toMatch(/managed authored project/i);
|
||||
const noMounts = itemById(baseInputs({ containers: [] }), 'volume_data');
|
||||
expect(noMounts.state).toBe('not_covered');
|
||||
|
||||
@@ -122,6 +126,28 @@ describe('buildRollbackItems', () => {
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
it('marks policy eligibility blocked and warning states', () => {
|
||||
expect(itemById(baseInputs({
|
||||
recoveryGeneration: { exists: true, shortId: 'abc' },
|
||||
policyEligibility: 'prohibited',
|
||||
}), 'policy_eligibility').state).toBe('blocked');
|
||||
expect(itemById(baseInputs({
|
||||
recoveryGeneration: { exists: true, shortId: 'abc' },
|
||||
policyEligibility: 'eligible_with_warning',
|
||||
}), 'policy_eligibility').state).toBe('warning');
|
||||
});
|
||||
|
||||
it('lets recovery_generation supersede the legacy backup slot for compose_source', () => {
|
||||
const items = buildRollbackItems(baseInputs({
|
||||
backup: { exists: false, timestamp: null },
|
||||
recoveryGeneration: { exists: true, shortId: 'deadbeefcaf0' },
|
||||
policyEligibility: 'eligible',
|
||||
}), NOW);
|
||||
expect(items.find(i => i.id === 'compose_source')!.state).toBe('ready');
|
||||
expect(items.find(i => i.id === 'recovery_generation')!.state).toBe('ready');
|
||||
});
|
||||
|
||||
describe('aggregateRollbackOverall', () => {
|
||||
it('is ready when compose, env, and previous image are all covered', () => {
|
||||
expect(aggregateRollbackOverall(buildRollbackItems(baseInputs(), NOW))).toBe('ready');
|
||||
@@ -157,6 +183,27 @@ describe('aggregateRollbackOverall', () => {
|
||||
const items = buildRollbackItems(baseInputs({ containers: 'error' }), NOW);
|
||||
expect(aggregateRollbackOverall(items)).toBe('ready');
|
||||
});
|
||||
it('is not_ready when policy eligibility is blocked', () => {
|
||||
const items = buildRollbackItems(baseInputs({
|
||||
recoveryGeneration: { exists: true, shortId: 'abc' },
|
||||
policyEligibility: 'prohibited',
|
||||
}), NOW);
|
||||
expect(aggregateRollbackOverall(items)).toBe('not_ready');
|
||||
});
|
||||
|
||||
it('is at best partial when policy eligibility is warning or unknown', () => {
|
||||
const warn = buildRollbackItems(baseInputs({
|
||||
recoveryGeneration: { exists: true, shortId: 'abc' },
|
||||
policyEligibility: 'eligible_with_warning',
|
||||
}), NOW);
|
||||
expect(aggregateRollbackOverall(warn)).toBe('partial');
|
||||
const unk = buildRollbackItems(baseInputs({
|
||||
recoveryGeneration: { exists: true, shortId: 'abc' },
|
||||
policyEligibility: 'unknown',
|
||||
}), NOW);
|
||||
expect(aggregateRollbackOverall(unk)).toBe('partial');
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
describe('FileSystemService.getBackupEnvSummary', () => {
|
||||
|
||||
@@ -74,6 +74,8 @@ vi.mock('../services/DatabaseService', () => ({
|
||||
// Rollback-readiness partial-revert disclosure reads the git source row;
|
||||
// no git-managed stacks in these fixtures by default.
|
||||
getGitSource: mockGetGitSource,
|
||||
// Generation supersede path; no current recovery generation in these fixtures.
|
||||
getCurrentStackUpdateRecovery: () => undefined,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
@@ -461,12 +461,15 @@ describe('WebhookService.execute: health gate begin call sites', () => {
|
||||
const { HealthGateService } = await import('../services/HealthGateService');
|
||||
vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined);
|
||||
vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]);
|
||||
vi.spyOn(compose.ComposeService.prototype, 'deployStack').mockResolvedValue(undefined);
|
||||
vi.spyOn(compose.ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: 'rec-hook' });
|
||||
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook');
|
||||
const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService');
|
||||
const linkSpy = vi.spyOn(StackUpdateRecoveryService.getInstance(), 'linkGateOrRetain');
|
||||
|
||||
const result = await WebhookService.getInstance().execute(webhook, 'deploy', 'test', true);
|
||||
expect(result.success).toBe(true);
|
||||
expect(beginSpy).toHaveBeenCalledWith(nodeId, stack, 'deploy', 'system:webhook');
|
||||
expect(linkSpy).toHaveBeenCalledWith('rec-hook', 'gate-hook');
|
||||
});
|
||||
|
||||
it('begins an update gate after a webhook pull succeeds', async () => {
|
||||
|
||||
@@ -145,6 +145,15 @@ export async function startServer(server: Server): Promise<void> {
|
||||
} catch (err) {
|
||||
console.error('[Startup] Deployed stack deletion reconcile failed:', (err as Error).message);
|
||||
}
|
||||
// Interrupted rollback restores must finish before mutation-capable services
|
||||
// or HTTP accept traffic. Fail closed: rethrow so unresolved intents never
|
||||
// leave mutators or HTTP accepting writes.
|
||||
try {
|
||||
await StackUpdateRecoveryService.getInstance().reconcileInterruptedRestoresAtStartup();
|
||||
} catch (err) {
|
||||
console.error('[Startup] Stack update recovery restore reconcile failed:', (err as Error).message);
|
||||
throw err;
|
||||
}
|
||||
StackUpdateRecoveryService.getInstance().start();
|
||||
|
||||
// Synchronous starts: schedule background timers and continue. None of
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
import path from 'path';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { StackOpLockService } from '../services/StackOpLockService';
|
||||
import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService';
|
||||
import { isValidStackName } from '../utils/validation';
|
||||
import { invalidateNodeCaches } from './cacheInvalidation';
|
||||
import {
|
||||
FLEET_SNAPSHOT_APPLY_FILENAMES,
|
||||
type FleetSnapshotApplyFilename,
|
||||
} from '../utils/snapshot-capture';
|
||||
|
||||
export interface FleetSnapshotApplyFile {
|
||||
filename: FleetSnapshotApplyFilename;
|
||||
content: string;
|
||||
}
|
||||
|
||||
/** Keep compose.yaml and .env; ignore any other snapshot filenames. */
|
||||
export function selectFleetSnapshotApplyFiles(
|
||||
files: Array<{ filename: string; content: string }>,
|
||||
): FleetSnapshotApplyFile[] {
|
||||
return files.filter((file): file is FleetSnapshotApplyFile =>
|
||||
FLEET_SNAPSHOT_APPLY_FILENAMES.some((name) => name === file.filename),
|
||||
);
|
||||
}
|
||||
|
||||
export interface ApplyFleetSnapshotFilesInput {
|
||||
nodeId: number;
|
||||
stackName: string;
|
||||
files: FleetSnapshotApplyFile[];
|
||||
actor: string;
|
||||
}
|
||||
|
||||
export interface ApplyFleetSnapshotFilesResult {
|
||||
capturedGenerationId: string | null;
|
||||
}
|
||||
|
||||
type CodedError = Error & { code: string };
|
||||
|
||||
export function getCodedError(error: unknown): CodedError | undefined {
|
||||
if (!(error instanceof Error) || !('code' in error) || typeof error.code !== 'string') {
|
||||
return undefined;
|
||||
}
|
||||
return error as CodedError;
|
||||
}
|
||||
|
||||
export type FleetSnapshotApplyConflictCode = 'stack_op_in_progress' | 'HEALTH_GATE_OBSERVING';
|
||||
|
||||
export function fleetSnapshotApplyConflictCode(error: unknown): FleetSnapshotApplyConflictCode | undefined {
|
||||
const code = getCodedError(error)?.code;
|
||||
if (code === 'stack_op_in_progress' || code === 'HEALTH_GATE_OBSERVING') return code;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function codedError(message: string, code: string): CodedError {
|
||||
return Object.assign(new Error(message), { code });
|
||||
}
|
||||
|
||||
async function writeApplyFile(
|
||||
fsSvc: FileSystemService,
|
||||
stackName: string,
|
||||
file: FleetSnapshotApplyFile,
|
||||
): Promise<void> {
|
||||
switch (file.filename) {
|
||||
case 'compose.yaml':
|
||||
await fsSvc.saveStackContent(stackName, file.content);
|
||||
return;
|
||||
case '.env':
|
||||
await fsSvc.saveEnvContent(stackName, file.content);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture-then-write used by Fleet snapshot restore on the node that owns the
|
||||
* stack. Takes the stack-op lock, then captures and writes. An existing Compose
|
||||
* project creates the current recovery generation before the first snapshot
|
||||
* file is written. Capture failure aborts with the live files unchanged. A
|
||||
* directory with no compose file is treated as a new stack and has no recovery
|
||||
* generation to roll back to.
|
||||
*/
|
||||
export async function applyFleetSnapshotFiles(
|
||||
input: ApplyFleetSnapshotFilesInput,
|
||||
): Promise<ApplyFleetSnapshotFilesResult> {
|
||||
const { nodeId, stackName, files, actor } = input;
|
||||
if (!isValidStackName(stackName)) {
|
||||
throw codedError('Invalid stack name', 'INVALID_STACK_NAME');
|
||||
}
|
||||
if (files.length === 0) {
|
||||
throw codedError('No restoreable snapshot files', 'INVALID_SNAPSHOT_FILES');
|
||||
}
|
||||
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
nodeId,
|
||||
stackName,
|
||||
'backup',
|
||||
actor,
|
||||
async () => {
|
||||
const fsSvc = FileSystemService.getInstance(nodeId);
|
||||
const exists = await fsSvc.hasComposeFile(path.join(fsSvc.getBaseDir(), stackName));
|
||||
let capturedGenerationId: string | null = null;
|
||||
if (exists) {
|
||||
capturedGenerationId = (await StackUpdateRecoveryService.getInstance().captureCurrentBackup({
|
||||
nodeId,
|
||||
stackName,
|
||||
createdBy: actor,
|
||||
})).id;
|
||||
}
|
||||
for (const file of files) {
|
||||
await writeApplyFile(fsSvc, stackName, file);
|
||||
}
|
||||
invalidateNodeCaches(nodeId);
|
||||
return { capturedGenerationId };
|
||||
},
|
||||
);
|
||||
|
||||
if (!lock.ran) {
|
||||
throw codedError(
|
||||
`Cannot restore "${stackName}": another operation (${lock.existing.action}) is already in progress.`,
|
||||
'stack_op_in_progress',
|
||||
);
|
||||
}
|
||||
return lock.result;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Shared enumeration of stack-relative files owned by a managed-project
|
||||
* manifest. Used by Git promotion and by authored-project rollback capture so
|
||||
* the two cannot drift on which paths belong to a generation.
|
||||
*/
|
||||
import type { GitProjectManifest } from '../types/gitProjectManifest';
|
||||
|
||||
/** Exact file paths owned by one manifest, excluding directory-only inventory entries. */
|
||||
export function collectManifestFilePaths(
|
||||
manifest: Pick<GitProjectManifest, 'inputs' | 'buildContexts'>,
|
||||
): string[] {
|
||||
const paths = new Map<string, string>();
|
||||
for (const entry of manifest.inputs) {
|
||||
if (entry.ownership !== 'managed' || entry.state !== 'present' || entry.materializedPath === null) continue;
|
||||
if (entry.dependencyKind === 'build-context' || entry.dependencyKind === 'build-additional-context') continue;
|
||||
paths.set(entry.materializedPath.toLowerCase(), entry.materializedPath);
|
||||
}
|
||||
for (const context of manifest.buildContexts) {
|
||||
for (const file of context.files) {
|
||||
const rel = context.repoPath ? `${context.repoPath}/${file.path}` : file.path;
|
||||
paths.set(rel.toLowerCase(), rel);
|
||||
}
|
||||
}
|
||||
return [...paths.values()].sort((a, b) => a.localeCompare(b));
|
||||
}
|
||||
@@ -80,6 +80,7 @@ const EXACT_SUFFIX_RULES: readonly SuffixRule[] = [
|
||||
{ method: 'PUT', suffix: '/files/permissions', action: 'stack:edit' },
|
||||
{ method: 'PUT', suffix: '/labels', action: 'stack:edit' },
|
||||
{ method: 'PUT', suffix: '/git-source', action: 'stack:edit' },
|
||||
{ method: 'POST', suffix: '/fleet-snapshot-apply', action: 'stack:edit' },
|
||||
{ method: 'DELETE', suffix: '/git-source', action: 'stack:edit' },
|
||||
{ method: 'POST', suffix: '/git-source/pull', action: 'stack:edit' },
|
||||
{ method: 'POST', suffix: '/git-source/apply', action: 'stack:edit' },
|
||||
|
||||
@@ -187,7 +187,10 @@ app.use(errorHandler);
|
||||
installShutdownHandlers(server);
|
||||
|
||||
if (require.main === module) {
|
||||
void startServer(server);
|
||||
void startServer(server).catch((err) => {
|
||||
console.error('[Startup] Fatal startup failure:', (err as Error).message);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
// Exports used by tests (supertest requires the http.Server instance).
|
||||
|
||||
@@ -2,26 +2,31 @@ import express, { type Request, type Response, type NextFunction, type RequestHa
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import { isProxyExemptPath } from '../helpers/proxyExemptPaths';
|
||||
import { SYNC_BODY_LIMIT, SYNC_ERROR_CODES, SYNC_PATH_PREFIX } from '../services/fleetSyncConstants';
|
||||
import { FLEET_SNAPSHOT_APPLY_BODY_LIMIT } from '../utils/snapshot-capture';
|
||||
|
||||
// JSON body parser that also captures the raw bytes for HMAC verification.
|
||||
// `rawBody` is part of the Express.Request augmentation (see types/express.ts);
|
||||
// the cast is required because body-parser's `verify` signature types `req` as
|
||||
// Node's IncomingMessage, not Express's Request.
|
||||
const jsonParser = express.json({
|
||||
verify: (req, _res, buf) => {
|
||||
(req as unknown as Request).rawBody = buf;
|
||||
},
|
||||
});
|
||||
function jsonWithRawBody(limit?: string | number): RequestHandler {
|
||||
return express.json({
|
||||
...(limit === undefined ? {} : { limit }),
|
||||
verify: (req, _res, buf) => {
|
||||
(req as unknown as Request).rawBody = buf;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const jsonParser = jsonWithRawBody();
|
||||
|
||||
// Larger-limit parser for the fleet sync receive endpoint. A control instance
|
||||
// can push up to MAX_SYNC_ROWS rows in a single payload; the default 100 KB
|
||||
// limit is too tight for that.
|
||||
const fleetSyncJsonParser = express.json({
|
||||
limit: SYNC_BODY_LIMIT,
|
||||
verify: (req, _res, buf) => {
|
||||
(req as unknown as Request).rawBody = buf;
|
||||
},
|
||||
});
|
||||
const fleetSyncJsonParser = jsonWithRawBody(SYNC_BODY_LIMIT);
|
||||
|
||||
// Combined compose.yaml + .env restore payload; sized for two snapshot files.
|
||||
const fleetSnapshotApplyJsonParser = jsonWithRawBody(FLEET_SNAPSHOT_APPLY_BODY_LIMIT);
|
||||
|
||||
const FLEET_SNAPSHOT_APPLY_PATH = /^\/api\/stacks\/[^/]+\/fleet-snapshot-apply$/;
|
||||
|
||||
/**
|
||||
* Parse JSON on local requests but preserve the raw stream for remote proxy
|
||||
@@ -62,5 +67,9 @@ export const conditionalJsonParser: RequestHandler = (req: Request, res: Respons
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (FLEET_SNAPSHOT_APPLY_PATH.test(req.path)) {
|
||||
fleetSnapshotApplyJsonParser(req, res, next);
|
||||
return;
|
||||
}
|
||||
jsonParser(req, res, next);
|
||||
};
|
||||
|
||||
+43
-41
@@ -24,7 +24,7 @@ import { ImageOperationService } from '../services/ImageOperationService';
|
||||
import { classifyImageChannel } from '../helpers/imageChannel';
|
||||
import { runPolicyGate, assertPolicyGateAllows, buildPolicyGateOptions } from '../helpers/policyGate';
|
||||
import { remoteSupportsCrossNodeRbac } from '../helpers/remoteCapabilities';
|
||||
import { captureLocalNodeFiles, captureRemoteNodeFiles, buildSnapshotDocumentation, pickDossierFields, dossierHasContent, type SnapshotNodeData, type SnapshotDocumentation } from '../utils/snapshot-capture';
|
||||
import { captureLocalNodeFiles, captureRemoteNodeFiles, buildSnapshotDocumentation, pickDossierFields, dossierHasContent, FLEET_SNAPSHOT_APPLY_TIMEOUT_MS, type SnapshotNodeData, type SnapshotDocumentation } from '../utils/snapshot-capture';
|
||||
import { getLatestVersion, getLatestRelease } from '../utils/version-check';
|
||||
import { isValidStackName } from '../utils/validation';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
@@ -45,6 +45,11 @@ import { POLICY_SEVERITIES } from '../utils/severity';
|
||||
import { isNoOpBlockingPolicy } from '../utils/policy-risk';
|
||||
import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog';
|
||||
import { formatNoTargetError } from '../utils/remoteTarget';
|
||||
import {
|
||||
applyFleetSnapshotFiles,
|
||||
fleetSnapshotApplyConflictCode,
|
||||
selectFleetSnapshotApplyFiles,
|
||||
} from '../helpers/applyFleetSnapshotFiles';
|
||||
import { CloudBackupService } from '../services/CloudBackupService';
|
||||
import { NotificationService } from '../services/NotificationService';
|
||||
import { invalidateRemoteMetaCache } from '../helpers/cacheInvalidation';
|
||||
@@ -2721,12 +2726,21 @@ interface RemoteProxyContext {
|
||||
// of a generic string. The body is truncated to keep the recorded message bounded.
|
||||
async function remoteStackError(action: string, res: Awaited<ReturnType<typeof fetch>>): Promise<Error> {
|
||||
let detail = '';
|
||||
let code: string | undefined;
|
||||
try {
|
||||
detail = (await res.text()).slice(0, 300).trim();
|
||||
const raw = (await res.text()).trim();
|
||||
detail = raw.slice(0, 300);
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (typeof parsed === 'object' && parsed !== null && 'code' in parsed && typeof parsed.code === 'string') {
|
||||
code = parsed.code;
|
||||
}
|
||||
} catch {
|
||||
// Remote body unavailable; the status code alone still names the failure.
|
||||
// Body missing or not JSON; status (and any truncated text) still name the failure.
|
||||
}
|
||||
return new Error(`${action} on remote node (${res.status})${detail ? `: ${detail}` : ''}`);
|
||||
return Object.assign(
|
||||
new Error(`${action} on remote node (${res.status})${detail ? `: ${detail}` : ''}`),
|
||||
{ code, httpStatus: res.status },
|
||||
);
|
||||
}
|
||||
|
||||
// Builds the base URL + proxy headers for a remote node, or null when the node
|
||||
@@ -2745,54 +2759,37 @@ function buildRemoteProxyContext(node: Node): RemoteProxyContext | null {
|
||||
return { baseUrl: proxyTarget.apiUrl.replace(/\/$/, ''), headers };
|
||||
}
|
||||
|
||||
// Writes a snapshot stack's files back to its node: local nodes write to disk
|
||||
// (backing up any current files first), remote nodes receive them over the
|
||||
// proxy. Throws SnapshotProxyTargetError when a remote node is unreachable, and
|
||||
// an Error carrying the remote node's status and reason on a failed remote write.
|
||||
// Writes a snapshot stack's files back to its node. Existing stacks capture a
|
||||
// recovery generation before the first write; capture failure aborts with no
|
||||
// mutation. A later write failure can leave live files changed; the captured
|
||||
// generation remains. Throws SnapshotProxyTargetError when the remote has no
|
||||
// proxy target. A non-OK apply response becomes an Error with the remote
|
||||
// status and body (capture, lock, or write).
|
||||
async function applySnapshotStackFiles(
|
||||
node: Node,
|
||||
stackName: string,
|
||||
files: Array<{ filename: string; content: string }>,
|
||||
): Promise<void> {
|
||||
const applyFiles = selectFleetSnapshotApplyFiles(files);
|
||||
if (node.type === 'local') {
|
||||
const fsService = FileSystemService.getInstance(node.id);
|
||||
try {
|
||||
await fsService.backupStackFiles(stackName);
|
||||
} catch (e) {
|
||||
// Stack may not exist yet before first restore; that is ok.
|
||||
console.warn(`[Fleet Snapshot] Pre-restore backup failed for stack "${stackName}" (may not exist yet):`, getErrorMessage(e, 'unknown'));
|
||||
}
|
||||
for (const file of files) {
|
||||
if (file.filename === 'compose.yaml') {
|
||||
await fsService.saveStackContent(stackName, file.content);
|
||||
} else if (file.filename === '.env') {
|
||||
await fsService.saveEnvContent(stackName, file.content);
|
||||
}
|
||||
}
|
||||
await applyFleetSnapshotFiles({
|
||||
nodeId: node.id,
|
||||
stackName,
|
||||
files: applyFiles,
|
||||
actor: 'system:fleet-snapshot',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const ctx = buildRemoteProxyContext(node);
|
||||
if (!ctx) throw new SnapshotProxyTargetError(formatNoTargetError(node));
|
||||
for (const file of files) {
|
||||
if (file.filename === 'compose.yaml') {
|
||||
const putRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, {
|
||||
method: 'PUT',
|
||||
headers: ctx.headers,
|
||||
body: JSON.stringify({ content: file.content }),
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (!putRes.ok) throw await remoteStackError('Failed to restore compose file', putRes);
|
||||
} else if (file.filename === '.env') {
|
||||
const putRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, {
|
||||
method: 'PUT',
|
||||
headers: ctx.headers,
|
||||
body: JSON.stringify({ content: file.content }),
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (!putRes.ok) throw await remoteStackError('Failed to restore env file', putRes);
|
||||
}
|
||||
}
|
||||
const applyRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}/fleet-snapshot-apply`, {
|
||||
method: 'POST',
|
||||
headers: ctx.headers,
|
||||
body: JSON.stringify({ files: applyFiles }),
|
||||
signal: AbortSignal.timeout(FLEET_SNAPSHOT_APPLY_TIMEOUT_MS),
|
||||
});
|
||||
if (!applyRes.ok) throw await remoteStackError('Failed to restore stack files', applyRes);
|
||||
}
|
||||
|
||||
// Redeploys a stack after its files are restored. The deploy policy gate stays
|
||||
@@ -2958,6 +2955,11 @@ fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request,
|
||||
res.status(503).json({ error: error.message });
|
||||
return;
|
||||
}
|
||||
const conflict = fleetSnapshotApplyConflictCode(error);
|
||||
if (conflict) {
|
||||
res.status(409).json({ error: getErrorMessage(error, 'Restore conflict'), code: conflict });
|
||||
return;
|
||||
}
|
||||
console.error('[Fleet Snapshot] Restore error:', error);
|
||||
res.status(500).json({ error: 'Failed to restore stack from snapshot' });
|
||||
}
|
||||
|
||||
+137
-11
@@ -66,6 +66,12 @@ import {
|
||||
UPDATE_VERIFICATION_INCOMPLETE_WARNING,
|
||||
} from '../services/ImageUpdateService';
|
||||
import { parseComposeSelection, defaultEnvPath } from '../helpers/gitSourceSelection';
|
||||
import {
|
||||
applyFleetSnapshotFiles,
|
||||
fleetSnapshotApplyConflictCode,
|
||||
getCodedError,
|
||||
selectFleetSnapshotApplyFiles,
|
||||
} from '../helpers/applyFleetSnapshotFiles';
|
||||
import { resolveStackEnvSources, discoverStackLocalEnvFiles } from '../helpers/envFileResolution';
|
||||
import { STACK_STATUSES_CACHE_TTL_MS } from '../helpers/constants';
|
||||
import { getTerminalWs, DEPLOY_SESSION_HEADER } from '../websocket/generic';
|
||||
@@ -118,6 +124,7 @@ const STACK_OP_PRESENT_PARTICIPLE: Record<StackOpAction, string> = {
|
||||
rollback: 'rolling back',
|
||||
backup: 'backing up',
|
||||
delete: 'deleting',
|
||||
git_apply: 'applying Git changes',
|
||||
};
|
||||
|
||||
function linkStackUpdateRecoveryGate(recoveryId: string | null | undefined, healthGateId: string | null): void {
|
||||
@@ -1814,7 +1821,7 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => {
|
||||
const debug = isDebugEnabled();
|
||||
const atomic = true;
|
||||
if (debug) console.debug('[Stacks:debug] Deploy starting', { stackName, atomic, nodeId: req.nodeId });
|
||||
await ComposeService.getInstance(req.nodeId).deployStack(
|
||||
const deployResult = await ComposeService.getInstance(req.nodeId).deployStack(
|
||||
stackName,
|
||||
getTerminalWs(req.get(DEPLOY_SESSION_HEADER)),
|
||||
atomic,
|
||||
@@ -1825,6 +1832,7 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => {
|
||||
if (debug) console.debug(`[Stacks:debug] Deploy finished in ${Date.now() - t0}ms`);
|
||||
ok = true;
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'deploy', req.user?.username ?? null);
|
||||
linkStackUpdateRecoveryGate(deployResult.recoveryId, healthGateId);
|
||||
res.json({ message: 'Deployed successfully', healthGateId });
|
||||
notifyActionSuccess('deploy_success', `${stackName} deployed`, stackName, req.user?.username ?? 'system');
|
||||
if (!skipScan) {
|
||||
@@ -2475,6 +2483,65 @@ stacksRouter.post('/:stackName/rollback', async (req: Request, res: Response) =>
|
||||
if (!tryAcquireStackOpLock(req, res, stackName, 'rollback')) return;
|
||||
let revertRestore: (() => Promise<void>) | null = null;
|
||||
try {
|
||||
const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService');
|
||||
const recoverySvc = StackUpdateRecoveryService.getInstance();
|
||||
const currentGen = recoverySvc.getCurrent(req.nodeId, stackName);
|
||||
// Any current recovery row uses compensateWithCandidate. Policy is evaluated
|
||||
// against the restored target inside compensate, not the live pre-restore project.
|
||||
if (currentGen) {
|
||||
dlog(`[Stacks] Rollback initiated via recovery generation: ${sanitizeForLog(stackName)}`);
|
||||
try {
|
||||
const rolledBack = await recoverySvc.compensateWithCandidate(
|
||||
currentGen.id,
|
||||
async (overridePath, invocation) => {
|
||||
await ComposeService.getInstance(req.nodeId).composeUpWithRecoveryOverride(
|
||||
stackName,
|
||||
overridePath,
|
||||
getTerminalWs(req.get(DEPLOY_SESSION_HEADER)),
|
||||
invocation,
|
||||
);
|
||||
},
|
||||
buildPolicyGateOptions(req, { actor: req.user?.username ?? 'system' }),
|
||||
);
|
||||
if (!rolledBack) {
|
||||
res.status(500).json({ error: 'Rollback restore did not complete.' });
|
||||
notifyActionFailure('rollback', stackName, new Error('rollback restore did not complete'), req.user?.username ?? 'system');
|
||||
return;
|
||||
}
|
||||
} catch (compError: unknown) {
|
||||
const compCode = (compError as { code?: string }).code;
|
||||
if (compCode === 'ROLLBACK_PROHIBITED') {
|
||||
res.status(409).json({
|
||||
error: (compError as Error).message || 'Rollback is prohibited for this generation',
|
||||
code: 'ROLLBACK_PROHIBITED',
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (compCode === 'HELD_IMAGE_MISSING') {
|
||||
res.status(500).json({
|
||||
error: (compError as Error).message || 'Held recovery image is missing.',
|
||||
code: 'HELD_IMAGE_MISSING',
|
||||
});
|
||||
notifyActionFailure('rollback', stackName, compError, req.user?.username ?? 'system');
|
||||
return;
|
||||
}
|
||||
if (compCode === 'RECOVERY_PROBE_FAILED') {
|
||||
res.status(500).json({
|
||||
error: 'Rollback restore completed but recovery probe failed.',
|
||||
code: 'RECOVERY_PROBE_FAILED',
|
||||
});
|
||||
notifyActionFailure('rollback', stackName, new Error('recovery probe failed'), req.user?.username ?? 'system');
|
||||
return;
|
||||
}
|
||||
throw compError;
|
||||
}
|
||||
invalidateNodeCaches(req.nodeId);
|
||||
dlog(`[Stacks] Rollback completed: ${sanitizeForLog(stackName)}`);
|
||||
res.json({ message: 'Stack rolled back from recovery generation.', recoveryId: currentGen.id });
|
||||
notifyActionSuccess('deploy_success', `${stackName} rolled back`, stackName, req.user?.username ?? 'system');
|
||||
return;
|
||||
}
|
||||
|
||||
const fsSvc = FileSystemService.getInstance(req.nodeId);
|
||||
const backupInfo = await fsSvc.getBackupInfo(stackName);
|
||||
if (!backupInfo.exists) {
|
||||
@@ -2548,7 +2615,18 @@ stacksRouter.get('/:stackName/backup', async (req: Request, res: Response) => {
|
||||
const stackName = req.params.stackName as string;
|
||||
const fsSvc = FileSystemService.getInstance(req.nodeId);
|
||||
const info = await fsSvc.getBackupInfo(stackName);
|
||||
res.json(info);
|
||||
const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService');
|
||||
const currentGen = StackUpdateRecoveryService.getInstance().getCurrent(req.nodeId, stackName);
|
||||
const recoveryAvailable = !!currentGen;
|
||||
const exists = info.exists || recoveryAvailable;
|
||||
const timestamp = currentGen?.created_at ?? info.timestamp;
|
||||
res.json({
|
||||
exists,
|
||||
timestamp,
|
||||
recoveryAvailable,
|
||||
recoveryId: currentGen?.id ?? null,
|
||||
hasGenerationContent: !!(currentGen && currentGen.content_path),
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
console.error('Failed to get backup info:', error);
|
||||
const message = getErrorMessage(error, 'Failed to get backup info.');
|
||||
@@ -2557,20 +2635,23 @@ stacksRouter.get('/:stackName/backup', async (req: Request, res: Response) => {
|
||||
});
|
||||
|
||||
stacksRouter.post('/:stackName/backup', async (req: Request, res: Response) => {
|
||||
// Triggers a server-side backup of the stack's managed files: the same
|
||||
// rollback snapshot a deploy takes. Exposed so a scheduled backup can run on
|
||||
// a remote node through the proxy path, and so an operator can capture an
|
||||
// on-demand snapshot.
|
||||
// Captures files, holds, and a recovery override, then hands off that
|
||||
// generation as current without compose or a runtime probe. Exposed so a
|
||||
// scheduled backup can run on a remote node through the proxy path, and so
|
||||
// an operator can capture an on-demand snapshot.
|
||||
const stackName = req.params.stackName as string;
|
||||
if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
|
||||
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
|
||||
// The backup slot is shared with the pre-deploy rollback snapshot, so hold the
|
||||
// stack-op lock to keep a backup from interleaving with a concurrent
|
||||
// deploy/update/rollback on the same stack. All early-returns stay inside the
|
||||
// try so finally always releases.
|
||||
// Handoff mutates the current generation, so hold the stack-op lock against
|
||||
// a concurrent deploy/update/rollback. After a successful acquire, release
|
||||
// in finally.
|
||||
if (!tryAcquireStackOpLock(req, res, stackName, 'backup')) return;
|
||||
try {
|
||||
await FileSystemService.getInstance(req.nodeId).backupStackFiles(stackName);
|
||||
await StackUpdateRecoveryService.getInstance().captureCurrentBackup({
|
||||
nodeId: req.nodeId,
|
||||
stackName,
|
||||
createdBy: req.user?.username ?? null,
|
||||
});
|
||||
dlog(`[Stacks] Backup completed: ${sanitizeForLog(stackName)}`);
|
||||
res.json({ success: true });
|
||||
} catch (error: unknown) {
|
||||
@@ -2581,6 +2662,51 @@ stacksRouter.post('/:stackName/backup', async (req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
function isFleetSnapshotFile(value: unknown): value is { filename: string; content: string } {
|
||||
return typeof value === 'object' && value !== null
|
||||
&& 'filename' in value && 'content' in value
|
||||
&& typeof value.filename === 'string'
|
||||
&& typeof value.content === 'string';
|
||||
}
|
||||
|
||||
stacksRouter.post('/:stackName/fleet-snapshot-apply', async (req: Request, res: Response) => {
|
||||
// Node-local capture-then-write used by hub Fleet snapshot restore.
|
||||
const stackName = req.params.stackName as string;
|
||||
if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return;
|
||||
const rawFiles = req.body?.files;
|
||||
if (!Array.isArray(rawFiles) || !rawFiles.every(isFleetSnapshotFile)) {
|
||||
res.status(400).json({ error: 'files must be an array of { filename, content }' });
|
||||
return;
|
||||
}
|
||||
const files = selectFleetSnapshotApplyFiles(rawFiles);
|
||||
if (files.length === 0) {
|
||||
res.status(400).json({ error: 'files must include compose.yaml or .env' });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const result = await applyFleetSnapshotFiles({
|
||||
nodeId: req.nodeId,
|
||||
stackName,
|
||||
files,
|
||||
actor: req.user?.username ?? 'system',
|
||||
});
|
||||
res.json({ success: true, capturedGenerationId: result.capturedGenerationId });
|
||||
} catch (error: unknown) {
|
||||
const code = getCodedError(error)?.code;
|
||||
if (code === 'INVALID_STACK_NAME' || code === 'INVALID_SNAPSHOT_FILES') {
|
||||
res.status(400).json({ error: getErrorMessage(error, 'Invalid restore request'), code });
|
||||
return;
|
||||
}
|
||||
const conflict = fleetSnapshotApplyConflictCode(error);
|
||||
if (conflict) {
|
||||
res.status(409).json({ error: getErrorMessage(error, 'Restore conflict'), code: conflict });
|
||||
return;
|
||||
}
|
||||
console.error('[Stacks] Fleet snapshot apply failed: %s', sanitizeForLog(stackName), error);
|
||||
res.status(500).json({ error: getErrorMessage(error, 'Failed to restore snapshot files') });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Returns the latest post-deploy scan attempt for this stack, or null if
|
||||
* no scan has been attempted yet. Used by the editor UI to flag stacks
|
||||
|
||||
@@ -513,6 +513,8 @@ systemMaintenanceRouter.get('/rollback/generations', async (req: Request, res: R
|
||||
phase: row.phase,
|
||||
createdAt: row.created_at,
|
||||
artifactExpiresAt: row.artifact_expires_at,
|
||||
createdBy: row.created_by,
|
||||
operationKind: row.operation_kind,
|
||||
releasable: service.isReleaseEligible(row),
|
||||
})));
|
||||
} catch (error) {
|
||||
@@ -545,6 +547,11 @@ systemMaintenanceRouter.post('/rollback/generations/:id/release', async (req: Re
|
||||
error: 'This rollback generation cannot be released right now (it may be observing a health gate, mid-recovery, or already in progress).',
|
||||
code: 'NOT_ELIGIBLE',
|
||||
});
|
||||
case 'malformed_services':
|
||||
return res.status(409).json({
|
||||
error: 'This rollback generation has malformed recovery image state and cannot be released until that record is repaired.',
|
||||
code: 'MALFORMED_SERVICES',
|
||||
});
|
||||
default: {
|
||||
const _exhaustive: never = result.reason;
|
||||
throw new Error(`Unhandled release reason: ${_exhaustive}`);
|
||||
|
||||
+1631
-1412
File diff suppressed because it is too large
Load Diff
@@ -18,8 +18,11 @@ import {
|
||||
import { readSnapshotFileRow, type SnapshotFileReadResult, type SnapshotFileRow } from '../helpers/snapshotFileDecrypt';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import type { GitSourceManifestState } from '../types/gitProjectManifest';
|
||||
import type { RollbackOperationKind } from '../types/rollbackGeneration';
|
||||
import { collectImageIds, parseServicesJsonStrict } from './recoveryServicesJson';
|
||||
|
||||
export type { SnapshotFileReadResult } from '../helpers/snapshotFileDecrypt';
|
||||
export type { RollbackOperationKind } from '../types/rollbackGeneration';
|
||||
|
||||
function isPilotMode(): boolean {
|
||||
return process.env.SENCHO_MODE === 'pilot';
|
||||
@@ -249,6 +252,10 @@ export interface StackUpdateRecoveryGenerationRow {
|
||||
phase: 'captured' | 'acquired' | 'handoff_committed' | 'reconciling' | 'immediate_verified';
|
||||
is_current: number;
|
||||
backup_slot_id: string | null;
|
||||
/** Generation content key (often equal to backup_slot_id / generation id). */
|
||||
content_path: string | null;
|
||||
/** Capture trigger: update | deployment | git_apply | manual_backup | unknown. */
|
||||
operation_kind: RollbackOperationKind | null;
|
||||
override_path: string | null;
|
||||
services_json: string;
|
||||
health_gate_id: string | null;
|
||||
@@ -1920,6 +1927,9 @@ export class DatabaseService {
|
||||
// pattern used for health_gate_runs below.
|
||||
maybeAddCol('stack_update_recovery_generations', 'released_at', 'INTEGER');
|
||||
maybeAddCol('stack_update_recovery_generations', 'released_by', 'TEXT');
|
||||
// Authored-project generation content key + capture trigger kind.
|
||||
maybeAddCol('stack_update_recovery_generations', 'content_path', 'TEXT');
|
||||
maybeAddCol('stack_update_recovery_generations', 'operation_kind', 'TEXT');
|
||||
maybeAddCol('stack_update_cleanup_pending', 'required_blueprint_id', 'INTEGER');
|
||||
|
||||
// Distributed API model columns
|
||||
@@ -4203,13 +4213,15 @@ export class DatabaseService {
|
||||
public insertStackUpdateRecoveryGeneration(row: StackUpdateRecoveryGenerationRow): void {
|
||||
this.db.prepare(
|
||||
`INSERT INTO stack_update_recovery_generations (
|
||||
id, node_id, stack_name, status, phase, is_current, backup_slot_id, override_path,
|
||||
services_json, health_gate_id, gate_retain_until, artifact_expires_at,
|
||||
operation_lease_expires_at, created_at, updated_at, created_by, artifacts_retired
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
id, node_id, stack_name, status, phase, is_current, backup_slot_id, content_path,
|
||||
operation_kind, override_path, services_json, health_gate_id, gate_retain_until,
|
||||
artifact_expires_at, operation_lease_expires_at, created_at, updated_at,
|
||||
created_by, artifacts_retired
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
).run(
|
||||
row.id, row.node_id, row.stack_name, row.status, row.phase, row.is_current,
|
||||
row.backup_slot_id, row.override_path, row.services_json, row.health_gate_id,
|
||||
row.backup_slot_id, row.content_path ?? null, row.operation_kind ?? null,
|
||||
row.override_path, row.services_json, row.health_gate_id,
|
||||
row.gate_retain_until, row.artifact_expires_at, row.operation_lease_expires_at,
|
||||
row.created_at, row.updated_at, row.created_by, row.artifacts_retired ?? 0,
|
||||
);
|
||||
@@ -4241,7 +4253,8 @@ export class DatabaseService {
|
||||
id: string,
|
||||
patch: Partial<Pick<StackUpdateRecoveryGenerationRow,
|
||||
'status' | 'phase' | 'is_current' | 'override_path' | 'health_gate_id' |
|
||||
'gate_retain_until' | 'artifact_expires_at' | 'operation_lease_expires_at' | 'services_json'>>,
|
||||
'gate_retain_until' | 'artifact_expires_at' | 'operation_lease_expires_at' | 'services_json' |
|
||||
'content_path' | 'operation_kind'>>,
|
||||
): void {
|
||||
const keys = Object.keys(patch) as Array<keyof typeof patch>;
|
||||
if (keys.length === 0) return;
|
||||
@@ -4432,26 +4445,13 @@ export class DatabaseService {
|
||||
).all(nodeId, now, now) as Array<{ services_json: string }>;
|
||||
const ids = new Set<string>();
|
||||
for (const row of rows) {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(row.services_json);
|
||||
if (!Array.isArray(parsed)) continue;
|
||||
for (const item of parsed) {
|
||||
if (!item || typeof item !== 'object') continue;
|
||||
const replicas = (item as { replicas?: unknown }).replicas;
|
||||
if (Array.isArray(replicas)) {
|
||||
for (const replica of replicas) {
|
||||
if (replica && typeof replica === 'object'
|
||||
&& typeof (replica as { imageId?: unknown }).imageId === 'string'
|
||||
&& (replica as { imageId: string }).imageId.trim()) {
|
||||
ids.add((replica as { imageId: string }).imageId);
|
||||
}
|
||||
}
|
||||
} else if (typeof (item as { imageId?: unknown }).imageId === 'string') {
|
||||
ids.add((item as { imageId: string }).imageId);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Corrupt JSON: skip.
|
||||
const parsed = parseServicesJsonStrict(row.services_json);
|
||||
if (!parsed.ok) {
|
||||
// Fail closed: corrupt hold metadata must not look like "nothing held".
|
||||
throw new Error('Malformed stack recovery services_json while listing held images');
|
||||
}
|
||||
for (const id of collectImageIds(parsed.services)) {
|
||||
ids.add(id);
|
||||
}
|
||||
}
|
||||
return [...ids];
|
||||
@@ -6323,6 +6323,25 @@ export class DatabaseService {
|
||||
).run(commitSha, contentHash, Date.now(), stackName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the last-applied revision identity without removing the Git source
|
||||
* row. Used when compensating to a capture that had a null commit SHA
|
||||
* (first-apply preimage).
|
||||
*/
|
||||
public clearGitSourceAppliedRevision(stackName: string): void {
|
||||
this.db.prepare(
|
||||
`UPDATE stack_git_sources SET
|
||||
last_applied_commit_sha = NULL,
|
||||
last_applied_content_hash = NULL,
|
||||
pending_commit_sha = NULL,
|
||||
pending_compose_content = NULL,
|
||||
pending_env_content = NULL,
|
||||
pending_fetched_at = NULL,
|
||||
updated_at = ?
|
||||
WHERE stack_name = ?`
|
||||
).run(Date.now(), stackName);
|
||||
}
|
||||
|
||||
public touchGitSourceDebounce(stackName: string): void {
|
||||
this.db.prepare('UPDATE stack_git_sources SET last_debounce_at = ? WHERE stack_name = ?')
|
||||
.run(Date.now(), stackName);
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
BLUEPRINT_MARKER_FILENAME,
|
||||
parseBlueprintMarker,
|
||||
} from '../helpers/blueprintMarker';
|
||||
import { scrapeRollbackTagsLenient } from './recoveryServicesJson';
|
||||
|
||||
/**
|
||||
* Directory that may contain recovery override files for a tombstone sweep.
|
||||
@@ -84,23 +85,7 @@ function collectArtifactsFromGenerations(
|
||||
|
||||
for (const gen of generations) {
|
||||
if (gen.override_path) overridePaths.add(gen.override_path);
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(gen.services_json);
|
||||
if (!Array.isArray(parsed)) continue;
|
||||
for (const svc of parsed) {
|
||||
if (!svc || typeof svc !== 'object') continue;
|
||||
const replicas = (svc as { replicas?: unknown }).replicas;
|
||||
if (!Array.isArray(replicas)) continue;
|
||||
for (const replica of replicas) {
|
||||
const tag = replica && typeof replica === 'object'
|
||||
? (replica as { rollbackTag?: unknown }).rollbackTag
|
||||
: null;
|
||||
if (typeof tag === 'string' && tag.trim()) tags.add(tag);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Corrupt capture JSON: skip tags for this generation.
|
||||
}
|
||||
for (const tag of scrapeRollbackTagsLenient(gen.services_json)) tags.add(tag);
|
||||
}
|
||||
|
||||
return { tags: [...tags], overridePaths: [...overridePaths] };
|
||||
|
||||
@@ -24,6 +24,7 @@ import { StackFileRootsService } from './StackFileRootsService';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import { ComposeInputDiscoveryService, type ContextCopyPlan, type CopyEntry } from './ComposeInputDiscoveryService';
|
||||
import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs';
|
||||
import { collectManifestFilePaths } from '../helpers/manifestFilePaths';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { isPathWithinBase, isValidStackName } from '../utils/validation';
|
||||
import type {
|
||||
@@ -411,6 +412,34 @@ export class GitProjectManifestService {
|
||||
await fs.promises.rename(tmp, target);
|
||||
}
|
||||
|
||||
/** Raw on-disk manifesto JSON, or null when the file is absent. */
|
||||
async readRawManifestText(stackName: string): Promise<string | null> {
|
||||
// Inline barrier at the readFile sink (CodeQL path-injection).
|
||||
const root = path.resolve(this.managedRoot(stackName));
|
||||
const target = path.resolve(root, MANIFEST_FILENAME);
|
||||
if (!target.startsWith(root + path.sep)) {
|
||||
throw Object.assign(new Error('Path escapes managed project directory'), { code: 'INVALID_PATH' });
|
||||
}
|
||||
try {
|
||||
return await fs.promises.readFile(target, 'utf8');
|
||||
} catch (e) {
|
||||
if ((e as NodeJS.ErrnoException).code === 'ENOENT') return null;
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove the managed manifesto file (first-apply preimage restore). */
|
||||
async clearManifestFile(stackName: string): Promise<void> {
|
||||
// Inline barrier at the rm sink (CodeQL path-injection).
|
||||
const root = path.resolve(this.managedRoot(stackName));
|
||||
const target = path.resolve(root, MANIFEST_FILENAME);
|
||||
if (!target.startsWith(root + path.sep)) {
|
||||
throw Object.assign(new Error('Path escapes managed project directory'), { code: 'INVALID_PATH' });
|
||||
}
|
||||
await fs.promises.rm(target, { force: true });
|
||||
StackFileRootsService.invalidate(NodeRegistry.getInstance().getDefaultNodeId(), stackName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Public projection for the manifest read endpoint: hashes, size metadata,
|
||||
* provenance, and deletion authority are internal-only, and for
|
||||
@@ -700,19 +729,7 @@ export class GitProjectManifestService {
|
||||
|
||||
/** Exact file paths owned by one manifest, excluding directory inventory entries. */
|
||||
private manifestFilePaths(manifest: Pick<GitProjectManifest, 'inputs' | 'buildContexts'>): string[] {
|
||||
const paths = new Map<string, string>();
|
||||
for (const entry of manifest.inputs) {
|
||||
if (entry.ownership !== 'managed' || entry.state !== 'present' || entry.materializedPath === null) continue;
|
||||
if (entry.dependencyKind === 'build-context' || entry.dependencyKind === 'build-additional-context') continue;
|
||||
paths.set(entry.materializedPath.toLowerCase(), entry.materializedPath);
|
||||
}
|
||||
for (const context of manifest.buildContexts) {
|
||||
for (const file of context.files) {
|
||||
const rel = context.repoPath ? `${context.repoPath}/${file.path}` : file.path;
|
||||
paths.set(rel.toLowerCase(), rel);
|
||||
}
|
||||
}
|
||||
return [...paths.values()].sort((a, b) => a.localeCompare(b));
|
||||
return collectManifestFilePaths(manifest);
|
||||
}
|
||||
|
||||
/** Hash one snapshot file, preserving the distinction between missing and unreadable. */
|
||||
|
||||
@@ -19,6 +19,7 @@ import { isPathWithinBase, isValidRelativeStackPath } from '../utils/validation'
|
||||
import { gitSourceLocalComposeFiles, PRIMARY_COMPOSE_FILENAME } from '../utils/gitComposeFiles';
|
||||
import { ComposeInputDiscoveryService, type ContextCopyPlan } from './ComposeInputDiscoveryService';
|
||||
import { GitProjectManifestService } from './GitProjectManifestService';
|
||||
import { StackUpdateRecoveryService } from './StackUpdateRecoveryService';
|
||||
import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs';
|
||||
import type { ComposeInputEntry, GitProjectManifest, GitSourceManifestState, InventoryResult, ManifestSummary, RefusalInfo } from '../types/gitProjectManifest';
|
||||
import type { GitHttpRequest, GitHttpResponse, HttpClient } from 'isomorphic-git/http/node';
|
||||
@@ -1618,11 +1619,7 @@ export class GitSourceService {
|
||||
for (const old of prevSpec.files) {
|
||||
if (old === PRIMARY_COMPOSE_FILENAME || keep.has(old)) continue;
|
||||
if (!isValidRelativeStackPath(old) || old === '') continue;
|
||||
try {
|
||||
await fsSvc.deleteStackPath(stackName, old);
|
||||
} catch (e) {
|
||||
console.warn(`[GitSource] stale file cleanup skipped ${sanitizeForLog(old)} for ${sanitizeForLog(stackName)}:`, (e as Error).message);
|
||||
}
|
||||
await fsSvc.deleteStackPath(stackName, old);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1743,16 +1740,44 @@ export class GitSourceService {
|
||||
stackName: string,
|
||||
commitSha: string,
|
||||
opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean } = {},
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string }> {
|
||||
return this.withStackLock(stackName, () => this.applyLocked(stackName, commitSha, opts));
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> {
|
||||
return this.withStackLock(stackName, () => this.applyWithSharedLock(stackName, commitSha, opts));
|
||||
}
|
||||
|
||||
/** Body of apply(); assumes the caller already holds the per-stack lock. */
|
||||
/**
|
||||
* Acquire the shared stack-operation lock then run applyLocked.
|
||||
* Callers that already hold the Git per-stack mutex (public apply, webhook
|
||||
* auto-apply) use this so capture/promote/handoff/deploy cannot race other
|
||||
* lifecycle ops. Do not nest withStackLock here.
|
||||
*/
|
||||
private async applyWithSharedLock(
|
||||
stackName: string,
|
||||
commitSha: string,
|
||||
opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean },
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> {
|
||||
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
nodeId,
|
||||
stackName,
|
||||
'git_apply',
|
||||
opts.actor ?? 'system:git-source',
|
||||
() => this.applyLocked(stackName, commitSha, opts),
|
||||
);
|
||||
if (!lock.ran) {
|
||||
throw new GitSourceError(
|
||||
'GIT_ERROR',
|
||||
`Another operation (${lock.existing.action}) is already in progress for ${stackName}.`,
|
||||
);
|
||||
}
|
||||
return lock.result;
|
||||
}
|
||||
|
||||
/** Body of apply(); assumes the caller already holds Git mutex + shared stack lock. */
|
||||
private async applyLocked(
|
||||
stackName: string,
|
||||
commitSha: string,
|
||||
opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean },
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string }> {
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> {
|
||||
const diag = isDebugEnabled();
|
||||
const db = DatabaseService.getInstance();
|
||||
const src = db.getGitSource(stackName);
|
||||
@@ -1775,6 +1800,9 @@ export class GitSourceService {
|
||||
? this.crypto.decrypt(src.pending_env_content)
|
||||
: null;
|
||||
const manifestSvc = GitProjectManifestService.getInstance();
|
||||
const recoverySvc = StackUpdateRecoveryService.getInstance();
|
||||
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
let recoveryId: string | undefined;
|
||||
|
||||
let appliedSpec: GitSourceAppliedSpec | null;
|
||||
if (pending.candidateRelPath !== null && pending.inventory !== null) {
|
||||
@@ -1797,7 +1825,7 @@ export class GitSourceService {
|
||||
// The staged candidate must still exist and be complete; a deleted
|
||||
// candidate (or a node restart that swept it) invalidates the pull.
|
||||
const dataDir = process.env.DATA_DIR || path.join(process.cwd(), 'data');
|
||||
const candidateAbs = path.join(dataDir, 'git-managed', String(NodeRegistry.getInstance().getDefaultNodeId()), stackName, pending.candidateRelPath);
|
||||
const candidateAbs = path.join(dataDir, 'git-managed', String(nodeId), stackName, pending.candidateRelPath);
|
||||
try {
|
||||
await fsPromises.access(candidateAbs);
|
||||
} catch {
|
||||
@@ -1872,8 +1900,8 @@ export class GitSourceService {
|
||||
: null;
|
||||
const invocation: string[] = [];
|
||||
try {
|
||||
invocation.push(...(await authoredComposeFileArgs(stackName, NodeRegistry.getInstance().getDefaultNodeId())));
|
||||
invocation.push(...(await authoredComposeEnvFileArgs(stackName, NodeRegistry.getInstance().getDefaultNodeId())));
|
||||
invocation.push(...(await authoredComposeFileArgs(stackName, nodeId)));
|
||||
invocation.push(...(await authoredComposeEnvFileArgs(stackName, nodeId)));
|
||||
} catch (e) {
|
||||
console.warn(`[GitSource] invocation build failed for ${stackName}:`, (e as Error).message);
|
||||
}
|
||||
@@ -1903,6 +1931,25 @@ export class GitSourceService {
|
||||
...(src.applied_deploy_spec?.files ?? [PRIMARY_COMPOSE_FILENAME]),
|
||||
...(src.sync_env ? ['.env'] : []),
|
||||
];
|
||||
try {
|
||||
const candidate = await recoverySvc.captureCandidate({
|
||||
nodeId,
|
||||
stackName,
|
||||
createdBy: opts.actor ?? 'git-source',
|
||||
operationKind: 'git_apply',
|
||||
});
|
||||
recoveryId = candidate.id;
|
||||
} catch (captureError) {
|
||||
const detail = captureError instanceof Error ? captureError.message : String(captureError);
|
||||
console.error(
|
||||
`[GitSource] Recovery capture failed before apply of ${sanitizeForLog(stackName)}:`,
|
||||
detail,
|
||||
);
|
||||
throw new GitSourceError(
|
||||
'GIT_ERROR',
|
||||
`Rollback capture failed before apply; refusing to promote without recovery coverage: ${scrubCredentials(detail)}`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
await manifestSvc.promoteGeneration(stackName, {
|
||||
sha: commitSha,
|
||||
@@ -1917,6 +1964,16 @@ export class GitSourceService {
|
||||
// The original error is logged with its stack for diagnosis,
|
||||
// and the message is scrubbed of credentials and of the
|
||||
// incoming manifest's high-sensitivity paths.
|
||||
if (recoveryId) {
|
||||
try {
|
||||
await recoverySvc.abandon(recoveryId);
|
||||
} catch (abandonError) {
|
||||
console.warn(
|
||||
`[GitSource] Failed to abandon recovery after promote failure for ${sanitizeForLog(stackName)}:`,
|
||||
abandonError instanceof Error ? abandonError.message : String(abandonError),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (e instanceof GitSourceError) throw e;
|
||||
const raw = e instanceof Error ? e.message : String(e);
|
||||
console.error(`[GitSource] promotion failed for ${sanitizeForLog(stackName)}:`, e instanceof Error ? e.stack ?? e.message : raw);
|
||||
@@ -1937,9 +1994,51 @@ export class GitSourceService {
|
||||
if (diag) console.log(`[GitSource:diag] apply validation fail stack=${stackName}`);
|
||||
throw new GitSourceError('GIT_ERROR', `Compose validation failed: ${validation.error}`);
|
||||
}
|
||||
// Capture the true pre-apply project BEFORE materialize writes new files.
|
||||
try {
|
||||
const captured = await recoverySvc.captureCandidate({
|
||||
nodeId,
|
||||
stackName,
|
||||
createdBy: opts.actor ?? 'git-source',
|
||||
operationKind: 'git_apply',
|
||||
});
|
||||
recoveryId = captured.id;
|
||||
} catch (captureError) {
|
||||
const detail = captureError instanceof Error ? captureError.message : String(captureError);
|
||||
console.error(
|
||||
`[GitSource] Recovery capture failed before legacy apply of ${sanitizeForLog(stackName)}:`,
|
||||
detail,
|
||||
);
|
||||
throw new GitSourceError(
|
||||
'GIT_ERROR',
|
||||
`Rollback capture failed before apply; refusing to materialize without recovery coverage: ${scrubCredentials(detail)}`,
|
||||
);
|
||||
}
|
||||
appliedSpec = await this.materialize(
|
||||
stackName, pending.files, pending.contextDir, src.sync_env, envContent, src.applied_deploy_spec,
|
||||
);
|
||||
).catch(async (materializeError: unknown) => {
|
||||
if (recoveryId) {
|
||||
const reverted = await recoverySvc.revertToGenerationContent(recoveryId);
|
||||
if (!reverted) {
|
||||
throw new GitSourceError(
|
||||
'GIT_ERROR',
|
||||
`Legacy materialize failed and pre-apply generation restore also failed: ${scrubCredentials(
|
||||
materializeError instanceof Error ? materializeError.message : String(materializeError),
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
await recoverySvc.abandon(recoveryId);
|
||||
} catch (abandonError) {
|
||||
console.warn(
|
||||
`[GitSource] Failed to abandon recovery after legacy materialize failure for ${sanitizeForLog(stackName)}:`,
|
||||
abandonError instanceof Error ? abandonError.message : String(abandonError),
|
||||
);
|
||||
}
|
||||
recoveryId = undefined;
|
||||
}
|
||||
throw materializeError;
|
||||
});
|
||||
// Migration: build the conservative manifest from spec + disk.
|
||||
const migrated = await manifestSvc.buildMigratedManifest(stackName, {
|
||||
repo_url: src.repo_url,
|
||||
@@ -1958,9 +2057,25 @@ export class GitSourceService {
|
||||
const shouldDeploy = opts.deploy ?? src.auto_deploy_on_apply;
|
||||
if (diag) console.log('[GitSource:diag] apply wrote stack=%s sha=%s deploy=%s', sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), sanitizeForLog(shouldDeploy));
|
||||
|
||||
const finalizeRecoveryCurrent = async (id: string, immediateVerified: boolean): Promise<void> => {
|
||||
if (!recoverySvc.markAcquired(id)) {
|
||||
await recoverySvc.abandon(id);
|
||||
throw new Error('Failed to mark recovery generation as acquired');
|
||||
}
|
||||
if (!recoverySvc.handoff(id, nodeId, stackName)) {
|
||||
await recoverySvc.abandon(id);
|
||||
throw new Error('Failed to hand off recovery generation');
|
||||
}
|
||||
if (!recoverySvc.markReconciling(id)) {
|
||||
throw new Error('Failed to mark recovery generation as reconciling');
|
||||
}
|
||||
if (immediateVerified && !recoverySvc.markImmediateVerified(id)) {
|
||||
console.warn(`[GitSource] Could not CAS immediate_verified for recovery ${sanitizeForLog(id)}`);
|
||||
}
|
||||
};
|
||||
|
||||
if (shouldDeploy) {
|
||||
try {
|
||||
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
await assertPolicyGateAllows(
|
||||
stackName,
|
||||
nodeId,
|
||||
@@ -1969,34 +2084,73 @@ export class GitSourceService {
|
||||
auditPath: `/api/stacks/${stackName}/git-source/apply`,
|
||||
}),
|
||||
);
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
nodeId, stackName, 'deploy', 'system',
|
||||
() => ComposeService.getInstance(nodeId).deployStack(
|
||||
stackName,
|
||||
undefined,
|
||||
undefined,
|
||||
{ source: 'git_apply', actor: opts.actor ?? 'system:git-source' },
|
||||
),
|
||||
);
|
||||
if (!lock.ran) {
|
||||
const busy = `Auto-deploy skipped: another operation (${lock.existing.action}) is already in progress for ${stackName}.`;
|
||||
console.warn(`[GitSource] ${busy}`);
|
||||
return { applied: true, deployed: false, deployError: busy };
|
||||
if (recoveryId) {
|
||||
await finalizeRecoveryCurrent(recoveryId, false);
|
||||
}
|
||||
// Shared stack lock already held as git_apply for capture→deploy.
|
||||
await ComposeService.getInstance(nodeId).deployStack(
|
||||
stackName,
|
||||
undefined,
|
||||
undefined,
|
||||
{ source: 'git_apply', actor: opts.actor ?? 'system:git-source' },
|
||||
);
|
||||
if (recoveryId) {
|
||||
if (!recoverySvc.markImmediateVerified(recoveryId)) {
|
||||
console.warn(`[GitSource] Could not CAS immediate_verified for recovery ${sanitizeForLog(recoveryId)}`);
|
||||
}
|
||||
}
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(
|
||||
nodeId,
|
||||
stackName,
|
||||
'deploy',
|
||||
'system:git-source',
|
||||
);
|
||||
if (recoveryId) {
|
||||
recoverySvc.linkGateOrRetain(recoveryId, healthGateId);
|
||||
}
|
||||
HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:git-source');
|
||||
console.log(`[GitSource] Applied and deployed ${stackName} at ${commitSha.slice(0, 7)}`);
|
||||
return { applied: true, deployed: true };
|
||||
return { applied: true, deployed: true, recoveryId };
|
||||
} catch (e) {
|
||||
// File is on disk, DB is marked applied. Returning the
|
||||
// error separately lets the UI flag it as a partial
|
||||
// success rather than rolling back the disk.
|
||||
// R1: do not auto-compensate. Keep applied files and leave the
|
||||
// pre-promote generation is_current for manual rollback.
|
||||
if (recoveryId) {
|
||||
const row = recoverySvc.get(recoveryId);
|
||||
if (row && row.is_current !== 1) {
|
||||
try {
|
||||
await finalizeRecoveryCurrent(recoveryId, false);
|
||||
} catch (handoffError) {
|
||||
console.warn(
|
||||
`[GitSource] Failed to hand off recovery after deploy failure for ${sanitizeForLog(stackName)}:`,
|
||||
handoffError instanceof Error ? handoffError.message : String(handoffError),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
const scrubbed = scrubCredentials((e as Error).message || String(e));
|
||||
console.error(`[GitSource] Auto-deploy failed for ${stackName}: ${scrubbed}`);
|
||||
return { applied: true, deployed: false, deployError: scrubbed };
|
||||
return { applied: true, deployed: false, deployError: scrubbed, recoveryId };
|
||||
}
|
||||
}
|
||||
|
||||
if (recoveryId) {
|
||||
try {
|
||||
await finalizeRecoveryCurrent(recoveryId, true);
|
||||
} catch (finalizeError) {
|
||||
const detail = finalizeError instanceof Error ? finalizeError.message : String(finalizeError);
|
||||
console.error(
|
||||
`[GitSource] Failed to finalize recovery for apply-only ${sanitizeForLog(stackName)}:`,
|
||||
detail,
|
||||
);
|
||||
return {
|
||||
applied: true,
|
||||
deployed: false,
|
||||
deployError: `Recovery finalization failed after apply: ${scrubCredentials(detail)}`,
|
||||
recoveryId,
|
||||
};
|
||||
}
|
||||
}
|
||||
console.log(`[GitSource] Applied ${stackName} at ${commitSha.slice(0, 7)}`);
|
||||
return { applied: true, deployed: false };
|
||||
return { applied: true, deployed: false, recoveryId };
|
||||
}
|
||||
|
||||
public dismissPending(stackName: string): void {
|
||||
@@ -2369,7 +2523,10 @@ export class GitSourceService {
|
||||
return { status: 'success', message: `Pending update ready at ${pullResult.commitSha.slice(0, 7)}.` };
|
||||
}
|
||||
|
||||
const applied = await this.applyLocked(stackName, pullResult.commitSha, { deploy: src.auto_deploy_on_apply });
|
||||
const applied = await this.applyWithSharedLock(stackName, pullResult.commitSha, {
|
||||
deploy: src.auto_deploy_on_apply,
|
||||
actor: 'system:webhook',
|
||||
});
|
||||
if (applied.deployError) {
|
||||
// Apply wrote to disk but deploy failed. Surface it so the
|
||||
// webhook_executions row records a degraded outcome instead
|
||||
|
||||
@@ -20,6 +20,7 @@ import { applySuppressions } from '../utils/suppression-filter';
|
||||
import { validateImageRef } from '../utils/image-ref';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import type { RollbackInvocationRecord } from '../types/rollbackGeneration';
|
||||
import {
|
||||
evaluatePolicyRisk,
|
||||
describePolicyInputs,
|
||||
@@ -58,6 +59,11 @@ export interface PolicyEnforcementOptions {
|
||||
auditMethod?: string;
|
||||
/** Request path of the originating route; used for audit attribution. */
|
||||
auditPath?: string;
|
||||
/**
|
||||
* Rollback compensation: list images via this captured Compose invocation
|
||||
* instead of the live database-derived args.
|
||||
*/
|
||||
composeInvocation?: RollbackInvocationRecord | null;
|
||||
}
|
||||
|
||||
export interface PolicyEnforcementResult {
|
||||
@@ -286,7 +292,10 @@ export async function enforcePolicyPreDeploy(
|
||||
|
||||
let imageRefs: string[] = [];
|
||||
try {
|
||||
imageRefs = await ComposeService.getInstance(nodeId).listStackImages(stackName);
|
||||
imageRefs = await ComposeService.getInstance(nodeId).listStackImages(
|
||||
stackName,
|
||||
opts.composeInvocation ?? null,
|
||||
);
|
||||
} catch (err) {
|
||||
const message = getErrorMessage(err, 'compose parse failed');
|
||||
console.error('[Policy] listStackImages failed for %s:', sanitizeForLog(stackName), sanitizeForLog(message));
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -8,6 +8,7 @@ import { ComposeService } from './ComposeService';
|
||||
import { StackUpdateOrchestrator } from './StackUpdateOrchestrator';
|
||||
import { StackOpLockService, stackOpSkipMessage as skipMessage } from './StackOpLockService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { StackUpdateRecoveryService } from './StackUpdateRecoveryService';
|
||||
import { HealthGateService } from './HealthGateService';
|
||||
import { ServiceUpdateRecoveryService } from './ServiceUpdateRecoveryService';
|
||||
import {
|
||||
@@ -559,17 +560,21 @@ export class SchedulerService {
|
||||
this.assertStackTarget(task, 'Auto-backup');
|
||||
if (this.isRemoteNode(task.node_id)) {
|
||||
await this.postToRemoteStack(task.node_id, `${encodeURIComponent(task.target_id)}/backup`);
|
||||
return `Backed up stack "${task.target_id}" files on remote node`;
|
||||
return `Captured a recovery generation for stack "${task.target_id}" on remote node`;
|
||||
}
|
||||
const localNodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId();
|
||||
const lock = await StackOpLockService.getInstance().runExclusive(
|
||||
localNodeId, task.target_id, 'backup', 'system',
|
||||
() => FileSystemService.getInstance(localNodeId).backupStackFiles(task.target_id),
|
||||
() => StackUpdateRecoveryService.getInstance().captureCurrentBackup({
|
||||
nodeId: localNodeId,
|
||||
stackName: task.target_id,
|
||||
createdBy: 'system:scheduler',
|
||||
}),
|
||||
);
|
||||
// Throw (not return) so the skip records as a failed run instead of a
|
||||
// silent success; the next scheduled tick retries once the lock frees.
|
||||
if (!lock.ran) throw new Error(skipMessage(task.target_id, lock.existing.action));
|
||||
return `Backed up stack "${task.target_id}" files`;
|
||||
return `Captured a recovery generation for stack "${task.target_id}"`;
|
||||
}
|
||||
|
||||
private async executeAutoStop(task: ScheduledTask): Promise<string> {
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
/**
|
||||
* Tracks in-flight stack lifecycle operations (deploy, down, restart, stop,
|
||||
* start, update, rollback, backup) per (nodeId, stackName). A second request to
|
||||
* start, update, rollback, backup, delete, git_apply) per (nodeId, stackName). A second request to
|
||||
* the same stack while the first is still running returns 409 instead of racing
|
||||
* the first. Backup is included because it rewrites the shared rollback slot, so
|
||||
* it must not interleave with a deploy/update/rollback on the same stack.
|
||||
* Git apply holds this lock across capture, promote, handoff, and optional deploy.
|
||||
*
|
||||
* State is intentionally process-local: a Sencho restart clears all locks,
|
||||
* which matches the lifecycle of any in-flight `docker compose` child process.
|
||||
*/
|
||||
|
||||
export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup' | 'delete';
|
||||
export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup' | 'delete' | 'git_apply';
|
||||
|
||||
/**
|
||||
* Note returned by a background path that skipped its operation because a manual
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -241,6 +241,35 @@ export class UpdateGuardService {
|
||||
withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'rollback readiness container probe')),
|
||||
]);
|
||||
|
||||
const { StackUpdateRecoveryService, shortGenerationId } = await import('./StackUpdateRecoveryService');
|
||||
const { assessGenerationEligibility } = await import('./rollbackEligibility');
|
||||
const recoverySvc = StackUpdateRecoveryService.getInstance();
|
||||
const currentGen = recoverySvc.getCurrent(nodeId, stackName);
|
||||
|
||||
const recoveryGeneration = currentGen
|
||||
? { exists: true as const, shortId: shortGenerationId(currentGen.id) }
|
||||
: { exists: false as const };
|
||||
|
||||
const policyEligibility = currentGen
|
||||
? await this.collect('rollback eligibility', stackName, () => assessGenerationEligibility(currentGen))
|
||||
: null;
|
||||
|
||||
const managedInputs = await this.collect('managed inputs', stackName, async () => {
|
||||
const { resolveRollbackInventory } = await import('./rollbackInventory');
|
||||
const inventory = await resolveRollbackInventory(nodeId, stackName);
|
||||
if (inventory.exactCoverage) {
|
||||
return {
|
||||
covered: true,
|
||||
detail: `Exact authored-project coverage includes ${inventory.entries.length} managed path(s).`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
covered: false,
|
||||
detail: inventory.coverageRefusal
|
||||
|| 'Exact authored-project coverage is incomplete for this stack.',
|
||||
};
|
||||
});
|
||||
|
||||
const items = buildRollbackItems({
|
||||
backup,
|
||||
envSummary,
|
||||
@@ -255,15 +284,18 @@ export class UpdateGuardService {
|
||||
},
|
||||
lastDeployAt,
|
||||
containers,
|
||||
recoveryGeneration,
|
||||
policyEligibility: policyEligibility === 'error' ? 'error' : policyEligibility,
|
||||
managedInputs: managedInputs === 'error' ? 'error' : managedInputs,
|
||||
}, now);
|
||||
|
||||
// Partial-revert disclosure for Git-managed stacks: rollback restores only
|
||||
// compose files and .env; the rest of the materialized project is not
|
||||
// reverted by the backup slot. State the scope rather than imply a
|
||||
// complete revert.
|
||||
// Partial-revert disclosure for Git-managed stacks when exact generation
|
||||
// coverage is not available. Prefer generation-backed wording when present.
|
||||
let note: string | undefined;
|
||||
const gitSource = db.getGitSource(stackName);
|
||||
if (gitSource && (gitSource.manifest_state === 'active' || gitSource.manifest_state === 'partial' || gitSource.manifest_state === 'migrated')) {
|
||||
if (currentGen) {
|
||||
note = undefined;
|
||||
} else if (gitSource && (gitSource.manifest_state === 'active' || gitSource.manifest_state === 'partial' || gitSource.manifest_state === 'migrated')) {
|
||||
note = 'This stack is Git-managed. Rollback restores compose files and .env; other materialized inputs are not reverted. Re-apply the previous revision from Git to restore them.';
|
||||
}
|
||||
|
||||
|
||||
@@ -151,20 +151,25 @@ export class WebhookService {
|
||||
nodeId, stackName, lockAction, 'system',
|
||||
async () => {
|
||||
switch (action) {
|
||||
case 'deploy':
|
||||
case 'deploy': {
|
||||
await assertPolicyGateAllows(
|
||||
stackName,
|
||||
nodeId,
|
||||
buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }),
|
||||
);
|
||||
await compose.deployStack(
|
||||
const deployResult = await compose.deployStack(
|
||||
stackName,
|
||||
undefined,
|
||||
atomic,
|
||||
{ source: 'webhook', actor: 'system:webhook' },
|
||||
);
|
||||
HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:webhook');
|
||||
const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:webhook');
|
||||
if (deployResult.recoveryId) {
|
||||
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
||||
StackUpdateRecoveryService.getInstance().linkGateOrRetain(deployResult.recoveryId, healthGateId);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'restart':
|
||||
await compose.runCommand(stackName, 'restart');
|
||||
break;
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
/**
|
||||
* Thin Compose project context for safe full-stack updates.
|
||||
* Shared Compose project context for safe full-stack mutations.
|
||||
*
|
||||
* Wraps the current authored compose argument path and atomic file backup/restore.
|
||||
* When a richer shared Compose project context lands, migrate callers to that type;
|
||||
* this module must not become a competing full-manifest resolver.
|
||||
* Resolves the authored inventory (Git managed-project manifest or live stack
|
||||
* discovery), captures/restores generation content through RollbackGenerationStore,
|
||||
* and builds the exact Compose invocation used for deploy/update/rollback.
|
||||
*/
|
||||
import path from 'path';
|
||||
import { randomUUID } from 'crypto';
|
||||
@@ -11,9 +11,18 @@ import { ComposeService } from './ComposeService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { buildEffectiveServiceModel } from './effectiveServiceModel';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { resolveRollbackInventory } from './rollbackInventory';
|
||||
import { RollbackGenerationStore } from './RollbackGenerationStore';
|
||||
import type {
|
||||
RollbackGenerationManifest,
|
||||
RollbackOperationKind,
|
||||
RollbackRestoreTransactionMeta,
|
||||
} from '../types/rollbackGeneration';
|
||||
|
||||
export type ImageReferenceKind = 'moving_tag' | 'digest_pinned' | 'none';
|
||||
|
||||
export type BackupOperation = RollbackOperationKind;
|
||||
|
||||
const DIGEST_PIN_PATTERN = /@sha256:[a-f0-9]{64}$/i;
|
||||
|
||||
export function classifyReferenceKind(declaredImageRef: string | null): ImageReferenceKind {
|
||||
@@ -34,11 +43,18 @@ export interface ComposeProjectContext {
|
||||
readonly nodeId: number;
|
||||
readonly stackName: string;
|
||||
readonly stackDir: string;
|
||||
/** Generation id used as content-store key and backup_slot_id on the DB row. */
|
||||
backupSlotId: string | null;
|
||||
toComposeArgs(action: string[]): Promise<string[]>;
|
||||
validateForMutation(): Promise<void>;
|
||||
backupFromContext(operation: 'update' | 'deployment'): Promise<string>;
|
||||
restoreFromContext(): Promise<void>;
|
||||
/**
|
||||
* Capture a staged generation. When exactCoverage is required and inventory
|
||||
* refuses it, throws before writing any generation content.
|
||||
*/
|
||||
backupFromContext(operation: BackupOperation): Promise<string>;
|
||||
restoreFromContext(
|
||||
transactionMeta?: RollbackRestoreTransactionMeta,
|
||||
): Promise<RollbackGenerationManifest | void>;
|
||||
resolveServiceImageMap(): Promise<Map<string, string | null>>;
|
||||
}
|
||||
|
||||
@@ -60,15 +76,63 @@ class AuthoredComposeProjectContext implements ComposeProjectContext {
|
||||
await requireRenderableModel(this.nodeId, this.stackName);
|
||||
}
|
||||
|
||||
async backupFromContext(_operation: 'update' | 'deployment'): Promise<string> {
|
||||
await FileSystemService.getInstance(this.nodeId).backupStackFiles(this.stackName);
|
||||
const slotId = randomUUID();
|
||||
this.backupSlotId = slotId;
|
||||
return slotId;
|
||||
async backupFromContext(operation: BackupOperation): Promise<string> {
|
||||
const inventory = await resolveRollbackInventory(this.nodeId, this.stackName);
|
||||
if (!inventory.exactCoverage) {
|
||||
throw Object.assign(
|
||||
new Error(
|
||||
inventory.coverageRefusal
|
||||
|| 'Exact authored-project rollback coverage is unavailable for this stack',
|
||||
),
|
||||
{ code: 'ROLLBACK_COVERAGE_UNAVAILABLE' },
|
||||
);
|
||||
}
|
||||
|
||||
// Do not refresh the legacy single-slot backup. Clearing that reused slot
|
||||
// would destroy a pre-migration recovery point if a later capture step fails.
|
||||
|
||||
const generationId = randomUUID();
|
||||
await RollbackGenerationStore.captureGeneration({
|
||||
nodeId: this.nodeId,
|
||||
stackName: this.stackName,
|
||||
generationId,
|
||||
inventory,
|
||||
operationKind: operation,
|
||||
});
|
||||
this.backupSlotId = generationId;
|
||||
return generationId;
|
||||
}
|
||||
|
||||
async restoreFromContext(): Promise<void> {
|
||||
await FileSystemService.getInstance(this.nodeId).restoreStackFiles(this.stackName);
|
||||
async restoreFromContext(
|
||||
transactionMeta?: RollbackRestoreTransactionMeta,
|
||||
): Promise<RollbackGenerationManifest | void> {
|
||||
const generationId = this.backupSlotId;
|
||||
if (!generationId) {
|
||||
// Legacy pre-migration restore: only when no generation id is bound.
|
||||
await FileSystemService.getInstance(this.nodeId).restoreStackFiles(this.stackName);
|
||||
return;
|
||||
}
|
||||
|
||||
const present = await RollbackGenerationStore.verifyGenerationContent(
|
||||
this.nodeId,
|
||||
this.stackName,
|
||||
generationId,
|
||||
);
|
||||
if (!present) {
|
||||
throw Object.assign(
|
||||
new Error('Recovery generation content is missing or incomplete'),
|
||||
{ code: 'GENERATION_CONTENT_MISSING' },
|
||||
);
|
||||
}
|
||||
|
||||
const inventory = await resolveRollbackInventory(this.nodeId, this.stackName);
|
||||
return RollbackGenerationStore.restoreGeneration(
|
||||
this.nodeId,
|
||||
this.stackName,
|
||||
generationId,
|
||||
inventory.entries.map((e) => e.relativePath),
|
||||
transactionMeta,
|
||||
);
|
||||
}
|
||||
|
||||
async resolveServiceImageMap(): Promise<Map<string, string | null>> {
|
||||
@@ -89,6 +153,17 @@ export async function resolveComposeProjectContext(
|
||||
return new AuthoredComposeProjectContext(nodeId, stackName, stackDir);
|
||||
}
|
||||
|
||||
/** Bind an existing generation id onto a fresh context for restore. */
|
||||
export async function resolveComposeProjectContextForGeneration(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
generationId: string,
|
||||
): Promise<ComposeProjectContext> {
|
||||
const ctx = await resolveComposeProjectContext(nodeId, stackName);
|
||||
ctx.backupSlotId = generationId;
|
||||
return ctx;
|
||||
}
|
||||
|
||||
export function describeContextError(error: unknown): string {
|
||||
return getErrorMessage(error, 'Compose project context failed');
|
||||
}
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
/**
|
||||
* Structural validation for stack-update recovery services_json payloads.
|
||||
* Fail closed on any unexpected shape so eligibility, compensate, and probe
|
||||
* share one authoritative parser.
|
||||
*/
|
||||
import type { ImageReferenceKind } from './composeProjectContext';
|
||||
|
||||
export interface StackRecoveryReplicaCapture {
|
||||
containerId: string | null;
|
||||
imageId: string | null;
|
||||
repoDigest: string | null;
|
||||
state: 'running' | 'stopped' | 'none';
|
||||
rollbackTag: string | null;
|
||||
}
|
||||
|
||||
export interface StackRecoveryServiceCapture {
|
||||
serviceName: string;
|
||||
/** Observed running replica count at capture (supported restore scale). */
|
||||
scale: number;
|
||||
hasBuild: boolean;
|
||||
declaredImageRef: string | null;
|
||||
referenceKind: ImageReferenceKind;
|
||||
replicas: StackRecoveryReplicaCapture[];
|
||||
}
|
||||
|
||||
export type ParsedServicesJson =
|
||||
| { ok: true; services: StackRecoveryServiceCapture[] }
|
||||
| { ok: false };
|
||||
|
||||
const REPLICA_STATES = new Set(['running', 'stopped', 'none']);
|
||||
const REFERENCE_KINDS = new Set<ImageReferenceKind>(['moving_tag', 'digest_pinned', 'none']);
|
||||
|
||||
function isNonEmptyString(v: unknown): v is string {
|
||||
return typeof v === 'string' && v.trim().length > 0;
|
||||
}
|
||||
|
||||
/** Accepts null/undefined/string; rejects any other type. */
|
||||
function isNullableString(v: unknown): v is string | null | undefined {
|
||||
return v === null || v === undefined || typeof v === 'string';
|
||||
}
|
||||
|
||||
function asNullableString(v: unknown): string | null {
|
||||
return typeof v === 'string' ? v : null;
|
||||
}
|
||||
|
||||
function parseReplicaCapture(raw: unknown): StackRecoveryReplicaCapture | null {
|
||||
if (!raw || typeof raw !== 'object') return null;
|
||||
const r = raw as Record<string, unknown>;
|
||||
if (!isNullableString(r.containerId)
|
||||
|| !isNullableString(r.imageId)
|
||||
|| !isNullableString(r.repoDigest)
|
||||
|| !isNullableString(r.rollbackTag)) {
|
||||
return null;
|
||||
}
|
||||
if (typeof r.state !== 'string' || !REPLICA_STATES.has(r.state)) return null;
|
||||
|
||||
const state = r.state as StackRecoveryReplicaCapture['state'];
|
||||
const imageId = asNullableString(r.imageId);
|
||||
// Running/stopped replicas must carry a protectable image identity.
|
||||
if ((state === 'running' || state === 'stopped') && !imageId?.trim()) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
containerId: asNullableString(r.containerId),
|
||||
imageId,
|
||||
repoDigest: asNullableString(r.repoDigest),
|
||||
state,
|
||||
rollbackTag: asNullableString(r.rollbackTag),
|
||||
};
|
||||
}
|
||||
|
||||
function parseServiceCapture(raw: unknown): StackRecoveryServiceCapture | null {
|
||||
if (!raw || typeof raw !== 'object') return null;
|
||||
const s = raw as Record<string, unknown>;
|
||||
if (!isNonEmptyString(s.serviceName)) return null;
|
||||
if (typeof s.scale !== 'number' || !Number.isInteger(s.scale) || s.scale < 0) return null;
|
||||
if (typeof s.hasBuild !== 'boolean') return null;
|
||||
if (!isNullableString(s.declaredImageRef)) return null;
|
||||
if (typeof s.referenceKind !== 'string' || !REFERENCE_KINDS.has(s.referenceKind as ImageReferenceKind)) {
|
||||
return null;
|
||||
}
|
||||
if (!Array.isArray(s.replicas)) return null;
|
||||
|
||||
const replicas: StackRecoveryReplicaCapture[] = [];
|
||||
for (const item of s.replicas) {
|
||||
const replica = parseReplicaCapture(item);
|
||||
if (!replica) return null;
|
||||
replicas.push(replica);
|
||||
}
|
||||
return {
|
||||
serviceName: s.serviceName,
|
||||
scale: s.scale,
|
||||
hasBuild: s.hasBuild,
|
||||
declaredImageRef: asNullableString(s.declaredImageRef),
|
||||
referenceKind: s.referenceKind as ImageReferenceKind,
|
||||
replicas,
|
||||
};
|
||||
}
|
||||
|
||||
/** Strict structural validation for recovery services_json (fail closed). */
|
||||
export function parseServicesJsonStrict(raw: string): ParsedServicesJson {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) return { ok: false };
|
||||
const services: StackRecoveryServiceCapture[] = [];
|
||||
for (const item of parsed) {
|
||||
const svc = parseServiceCapture(item);
|
||||
if (!svc) return { ok: false };
|
||||
services.push(svc);
|
||||
}
|
||||
return { ok: true, services };
|
||||
} catch {
|
||||
return { ok: false };
|
||||
}
|
||||
}
|
||||
|
||||
/** Lenient parse for callers that treat empty as no services (legacy). Prefer strict. */
|
||||
export function parseServicesJson(raw: string): StackRecoveryServiceCapture[] {
|
||||
const parsed = parseServicesJsonStrict(raw);
|
||||
return parsed.ok ? parsed.services : [];
|
||||
}
|
||||
|
||||
export function collectImageIds(services: StackRecoveryServiceCapture[]): string[] {
|
||||
const ids = new Set<string>();
|
||||
for (const svc of services) {
|
||||
for (const replica of svc.replicas) {
|
||||
if (replica.imageId?.trim()) ids.add(replica.imageId);
|
||||
}
|
||||
}
|
||||
return [...ids];
|
||||
}
|
||||
|
||||
export function collectImageIdsFromServicesJson(servicesJson: string): string[] {
|
||||
return collectImageIds(parseServicesJson(servicesJson));
|
||||
}
|
||||
|
||||
export function collectRollbackTags(services: StackRecoveryServiceCapture[]): string[] {
|
||||
const tags = new Set<string>();
|
||||
for (const svc of services) {
|
||||
for (const replica of svc.replicas) {
|
||||
if (replica.rollbackTag?.trim()) tags.add(replica.rollbackTag);
|
||||
}
|
||||
}
|
||||
return [...tags];
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort rollbackTag scrape for cleanup paths. Prefer strict parse; on
|
||||
* structural failure walk nested objects for string rollbackTag fields so
|
||||
* opaque holds are still removed when possible.
|
||||
*/
|
||||
export function scrapeRollbackTagsLenient(raw: string): string[] {
|
||||
const strict = parseServicesJsonStrict(raw);
|
||||
if (strict.ok) return collectRollbackTags(strict.services);
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
const tags = new Set<string>();
|
||||
const walk = (value: unknown): void => {
|
||||
if (!value || typeof value !== 'object') return;
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) walk(item);
|
||||
return;
|
||||
}
|
||||
const obj = value as Record<string, unknown>;
|
||||
if (typeof obj.rollbackTag === 'string' && obj.rollbackTag.trim()) {
|
||||
tags.add(obj.rollbackTag);
|
||||
}
|
||||
for (const nested of Object.values(obj)) walk(nested);
|
||||
};
|
||||
walk(parsed);
|
||||
return [...tags];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
/**
|
||||
* Rollback restore eligibility (fail closed on known-bad evidence).
|
||||
*
|
||||
* Pure evaluateRollbackEligibility maps known signals to a verdict.
|
||||
* assessGenerationEligibility gathers best-effort evidence for a recovery row.
|
||||
*/
|
||||
import type { StackUpdateRecoveryGenerationRow } from './DatabaseService';
|
||||
import DockerController from './DockerController';
|
||||
import { enforcePolicyForImageRefs } from './PolicyEnforcement';
|
||||
import {
|
||||
collectImageIds,
|
||||
collectRollbackTags,
|
||||
parseServicesJsonStrict,
|
||||
} from './recoveryServicesJson';
|
||||
import { RollbackGenerationStore } from './RollbackGenerationStore';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
|
||||
type HeldImagesParse =
|
||||
| { ok: true; ids: string[]; rollbackTags: string[] }
|
||||
| { ok: false };
|
||||
|
||||
function parseHeldImageState(servicesJson: string): HeldImagesParse {
|
||||
const parsed = parseServicesJsonStrict(servicesJson);
|
||||
if (!parsed.ok) return { ok: false };
|
||||
return {
|
||||
ok: true,
|
||||
ids: collectImageIds(parsed.services),
|
||||
rollbackTags: collectRollbackTags(parsed.services),
|
||||
};
|
||||
}
|
||||
|
||||
export type RollbackEligibilityVerdict =
|
||||
| 'eligible'
|
||||
| 'eligible_with_warning'
|
||||
| 'prohibited'
|
||||
| 'unknown';
|
||||
|
||||
export interface RollbackEligibilityInput {
|
||||
/** null = unknown */
|
||||
generationIntegrityOk: boolean | null;
|
||||
heldImagesPresent: boolean | null;
|
||||
/** true when known blocked; null = unknown */
|
||||
securityPostureBlocked: boolean | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rules (fail closed on known bad):
|
||||
* - securityPostureBlocked === true → prohibited
|
||||
* - generationIntegrityOk === false → prohibited
|
||||
* - heldImagesPresent === false → eligible_with_warning
|
||||
* - any remaining null → unknown (unless already prohibited)
|
||||
* - else eligible
|
||||
*/
|
||||
export function evaluateRollbackEligibility(
|
||||
input: RollbackEligibilityInput,
|
||||
): RollbackEligibilityVerdict {
|
||||
if (input.securityPostureBlocked === true || input.generationIntegrityOk === false) {
|
||||
return 'prohibited';
|
||||
}
|
||||
if (input.heldImagesPresent === false) return 'eligible_with_warning';
|
||||
if (
|
||||
input.generationIntegrityOk === null
|
||||
|| input.heldImagesPresent === null
|
||||
|| input.securityPostureBlocked === null
|
||||
) {
|
||||
return 'unknown';
|
||||
}
|
||||
return 'eligible';
|
||||
}
|
||||
|
||||
async function checkGenerationIntegrity(
|
||||
row: StackUpdateRecoveryGenerationRow,
|
||||
): Promise<boolean | null> {
|
||||
// Only explicit content_path generations use the content store.
|
||||
const contentKey = row.content_path;
|
||||
if (!contentKey) return null;
|
||||
try {
|
||||
return await RollbackGenerationStore.verifyGenerationContent(
|
||||
row.node_id,
|
||||
row.stack_name,
|
||||
contentKey,
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[RollbackEligibility] Integrity check failed for %s: %s',
|
||||
sanitizeForLog(row.id),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function inspectImagePresent(
|
||||
docker: ReturnType<ReturnType<typeof DockerController.getInstance>['getDocker']>,
|
||||
ref: string,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
await docker.getImage(ref).inspect();
|
||||
return true;
|
||||
} catch (error) {
|
||||
const status = (error as { statusCode?: number }).statusCode;
|
||||
const message = getErrorMessage(error, '').toLowerCase();
|
||||
if (status === 404 || message.includes('no such image') || message.includes('not found')) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Held recovery launch requires both underlying image ids and opaque rollback
|
||||
* tags used by the recovery override to still resolve locally.
|
||||
*/
|
||||
async function checkHeldImagesPresent(
|
||||
row: StackUpdateRecoveryGenerationRow,
|
||||
held: HeldImagesParse,
|
||||
): Promise<boolean | null> {
|
||||
if (!held.ok) return null;
|
||||
const refs = [...new Set([...held.ids, ...held.rollbackTags])];
|
||||
if (refs.length === 0) return true;
|
||||
try {
|
||||
const docker = DockerController.getInstance(row.node_id).getDocker();
|
||||
for (const ref of refs) {
|
||||
if (!(await inspectImagePresent(docker, ref))) return false;
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[RollbackEligibility] Docker check failed for %s: %s',
|
||||
sanitizeForLog(row.id),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function checkSecurityPostureBlocked(
|
||||
row: StackUpdateRecoveryGenerationRow,
|
||||
held: HeldImagesParse,
|
||||
): Promise<boolean | null> {
|
||||
if (!held.ok) return null;
|
||||
const refs = [...new Set([...held.ids, ...held.rollbackTags])];
|
||||
if (refs.length === 0) return false;
|
||||
try {
|
||||
const gate = await enforcePolicyForImageRefs(row.stack_name, row.node_id, refs, {
|
||||
bypass: false,
|
||||
actor: 'rollback-eligibility',
|
||||
auditMethod: 'GET',
|
||||
auditPath: '/api/stacks/rollback-eligibility',
|
||||
});
|
||||
return !gate.ok;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[RollbackEligibility] Security posture check failed for %s: %s',
|
||||
sanitizeForLog(row.id),
|
||||
sanitizeForLog(getErrorMessage(error, 'unknown')),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Best-effort eligibility for a recovery generation row. */
|
||||
export async function assessGenerationEligibility(
|
||||
row: StackUpdateRecoveryGenerationRow,
|
||||
): Promise<RollbackEligibilityVerdict> {
|
||||
const held = parseHeldImageState(row.services_json);
|
||||
// Malformed recovery state cannot be assessed safely; refuse restore.
|
||||
if (!held.ok) return 'prohibited';
|
||||
const generationIntegrityOk = await checkGenerationIntegrity(row);
|
||||
const heldImagesPresent = await checkHeldImagesPresent(row, held);
|
||||
const securityPostureBlocked = await checkSecurityPostureBlocked(row, held);
|
||||
return evaluateRollbackEligibility({
|
||||
generationIntegrityOk,
|
||||
heldImagesPresent,
|
||||
securityPostureBlocked,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,565 @@
|
||||
/**
|
||||
* Resolve the authored-project file set that an atomic rollback generation
|
||||
* must capture. Git-managed stacks consume the managed-project manifest;
|
||||
* authored stacks rediscover against the live stack directory.
|
||||
*/
|
||||
import { promises as fsPromises, readFileSync } from 'fs';
|
||||
import path from 'path';
|
||||
import { DatabaseService, type StackGitSource } from './DatabaseService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { GitProjectManifestService } from './GitProjectManifestService';
|
||||
import { collectManifestFilePaths } from '../helpers/manifestFilePaths';
|
||||
import { isHostAbsolutePath, parseDeclaredInputs } from '../helpers/composeInputParse';
|
||||
import { isValidRelativeStackPath, isValidStackName } from '../utils/validation';
|
||||
import { authoredComposeEnvFileArgs, authoredComposeFileArgs } from '../utils/authoredComposeArgs';
|
||||
import type {
|
||||
ComposeInputEntry,
|
||||
GitProjectManifest,
|
||||
InputSensitivity,
|
||||
} from '../types/gitProjectManifest';
|
||||
import type {
|
||||
ResolvedRollbackInventory,
|
||||
RollbackEntryKind,
|
||||
RollbackEntryProvenance,
|
||||
RollbackInvocationRecord,
|
||||
} from '../types/rollbackGeneration';
|
||||
|
||||
const ROOT_COMPOSE_FILENAMES = [
|
||||
'compose.yaml',
|
||||
'compose.yml',
|
||||
'docker-compose.yaml',
|
||||
'docker-compose.yml',
|
||||
] as const;
|
||||
|
||||
const DOT_ENV = '.' + 'env';
|
||||
|
||||
const COMPOSE_INVOCATION_KINDS = new Set<RollbackEntryKind>([
|
||||
'compose-root',
|
||||
'implicit-override',
|
||||
'explicit',
|
||||
'include',
|
||||
'extends',
|
||||
]);
|
||||
|
||||
type InventoryAccum = {
|
||||
relativePath: string;
|
||||
dependencyKind: RollbackEntryKind;
|
||||
provenance: RollbackEntryProvenance;
|
||||
sensitivity: InputSensitivity;
|
||||
absolutePath: string | null;
|
||||
};
|
||||
|
||||
function posixRel(rel: string): string {
|
||||
return rel.replace(/\\/g, '/').replace(/^\.\//, '');
|
||||
}
|
||||
|
||||
function foldedKey(rel: string): string {
|
||||
return posixRel(rel).toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Prefer exact POSIX paths as map keys so case-distinct Linux paths are kept.
|
||||
* When two different paths collide under case-folding, record a refusal note
|
||||
* instead of silently merging them.
|
||||
*/
|
||||
function upsertEntry(
|
||||
map: Map<string, InventoryAccum>,
|
||||
foldedOwners: Map<string, string>,
|
||||
entry: InventoryAccum,
|
||||
caseCollisions: string[],
|
||||
): void {
|
||||
const exact = posixRel(entry.relativePath);
|
||||
const folded = foldedKey(exact);
|
||||
const owner = foldedOwners.get(folded);
|
||||
if (owner !== undefined && owner !== exact) {
|
||||
caseCollisions.push(`Case-colliding managed paths "${owner}" and "${exact}"`);
|
||||
return;
|
||||
}
|
||||
const prev = map.get(exact);
|
||||
if (!prev) {
|
||||
map.set(exact, { ...entry, relativePath: exact });
|
||||
foldedOwners.set(folded, exact);
|
||||
return;
|
||||
}
|
||||
if (prev.absolutePath !== null && entry.absolutePath === null) return;
|
||||
map.set(exact, {
|
||||
...entry,
|
||||
relativePath: exact,
|
||||
absolutePath: entry.absolutePath ?? prev.absolutePath,
|
||||
dependencyKind: prev.dependencyKind === 'compose-root' && entry.dependencyKind !== 'compose-root'
|
||||
? entry.dependencyKind
|
||||
: entry.dependencyKind,
|
||||
sensitivity: higherSensitivity(entry.sensitivity, prev.sensitivity),
|
||||
});
|
||||
}
|
||||
|
||||
function resolveStackRoot(fsSvc: FileSystemService, stackName: string): string {
|
||||
if (!isValidStackName(stackName)) {
|
||||
throw Object.assign(new Error('Invalid stack name'), { code: 'INVALID_STACK_NAME' });
|
||||
}
|
||||
// Canonical js/path-injection barrier: resolve + startsWith.
|
||||
// CodeQL does not credit isPathWithinBase helpers at later sinks.
|
||||
const base = path.resolve(fsSvc.getBaseDir());
|
||||
const stackRoot = path.resolve(base, stackName);
|
||||
if (!stackRoot.startsWith(base + path.sep)) {
|
||||
throw Object.assign(new Error('Stack name escapes compose directory'), { code: 'INVALID_PATH' });
|
||||
}
|
||||
return stackRoot;
|
||||
}
|
||||
|
||||
function resolveStackRel(
|
||||
stackRoot: string,
|
||||
relRaw: string,
|
||||
): { relativePath: string; absolutePath: string } | null {
|
||||
const relativePath = posixRel(relRaw);
|
||||
if (!relativePath || !isValidRelativeStackPath(relativePath)) return null;
|
||||
// Join-time containment; callers still re-check at each fs sink.
|
||||
const baseResolved = path.resolve(stackRoot);
|
||||
const absolutePath = path.resolve(baseResolved, relativePath);
|
||||
if (!absolutePath.startsWith(baseResolved + path.sep)) return null;
|
||||
return { relativePath, absolutePath };
|
||||
}
|
||||
|
||||
async function pathExistsAsFile(stackRoot: string, relativePath: string): Promise<boolean> {
|
||||
// Inline barrier at the lstat sink.
|
||||
const baseResolved = path.resolve(stackRoot);
|
||||
const abs = path.resolve(baseResolved, relativePath);
|
||||
if (!abs.startsWith(baseResolved + path.sep)) return false;
|
||||
try {
|
||||
const st = await fsPromises.lstat(abs);
|
||||
return st.isFile() || st.isSymbolicLink();
|
||||
} catch (e) {
|
||||
if ((e as NodeJS.ErrnoException).code === 'ENOENT') return false;
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
function authoredSensitivity(kind: RollbackEntryKind): InputSensitivity {
|
||||
if (kind === 'secret' || kind === 'config' || kind === 'build-secret') return 'high';
|
||||
if (
|
||||
kind === 'env_file'
|
||||
|| kind === 'include-env'
|
||||
|| kind === 'interpolation-env'
|
||||
|| kind === 'sync-env'
|
||||
|| kind === 'project-env'
|
||||
|| kind === 'label_file'
|
||||
) {
|
||||
return 'medium';
|
||||
}
|
||||
return 'low';
|
||||
}
|
||||
|
||||
function higherSensitivity(a: InputSensitivity, b: InputSensitivity): InputSensitivity {
|
||||
if (a === 'high' || b === 'high') return 'high';
|
||||
if (a === 'medium' || b === 'medium') return 'medium';
|
||||
return 'low';
|
||||
}
|
||||
|
||||
function appliedDeploySpecString(
|
||||
spec: { files: string[]; contextDir: string | null } | null | undefined,
|
||||
): string | null {
|
||||
if (!spec) return null;
|
||||
return JSON.stringify(spec);
|
||||
}
|
||||
|
||||
function refusedGitInventory(
|
||||
gitSource: StackGitSource,
|
||||
emptyInvocation: RollbackInvocationRecord,
|
||||
coverageRefusal: string,
|
||||
manifestVersion: number | null = gitSource.manifest_version,
|
||||
): ResolvedRollbackInventory {
|
||||
return {
|
||||
entries: [],
|
||||
invocation: emptyInvocation,
|
||||
git: {
|
||||
repoUrl: gitSource.repo_url,
|
||||
branch: gitSource.branch,
|
||||
commitSha: gitSource.last_applied_commit_sha || '',
|
||||
manifestVersion,
|
||||
},
|
||||
appliedDeploySpec: appliedDeploySpecString(gitSource.applied_deploy_spec),
|
||||
lastAppliedContentHash: gitSource.last_applied_content_hash,
|
||||
manifestState: gitSource.manifest_state,
|
||||
manifestGeneration: gitSource.manifest_generation,
|
||||
exactCoverage: false,
|
||||
coverageRefusal,
|
||||
};
|
||||
}
|
||||
|
||||
function isGitManifest(
|
||||
value: GitProjectManifest | { corrupt: string } | null,
|
||||
): value is GitProjectManifest {
|
||||
return value !== null && !('corrupt' in value);
|
||||
}
|
||||
|
||||
function sensitivityForManifestPath(
|
||||
manifest: GitProjectManifest,
|
||||
rel: string,
|
||||
): { kind: RollbackEntryKind; sensitivity: InputSensitivity; provenance: RollbackEntryProvenance } {
|
||||
const key = foldedKey(rel);
|
||||
const input = manifest.inputs.find(
|
||||
(i: ComposeInputEntry) => i.materializedPath !== null && foldedKey(i.materializedPath) === key,
|
||||
);
|
||||
if (input) {
|
||||
return {
|
||||
kind: input.dependencyKind,
|
||||
sensitivity: input.sensitivity,
|
||||
provenance: input.provenance,
|
||||
};
|
||||
}
|
||||
return { kind: 'other', sensitivity: 'low', provenance: 'fetch' };
|
||||
}
|
||||
|
||||
async function resolveGitInventory(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
stackRoot: string,
|
||||
): Promise<ResolvedRollbackInventory | null> {
|
||||
const gitSource = DatabaseService.getInstance().getGitSource(stackName);
|
||||
if (!gitSource) return null;
|
||||
|
||||
const emptyInvocation: RollbackInvocationRecord = {
|
||||
composeArgsPrefix: [],
|
||||
projectDirectory: null,
|
||||
projectName: stackName,
|
||||
explicitComposeFiles: [],
|
||||
meshOverrideRelativePath: null,
|
||||
meshEnabled: false,
|
||||
};
|
||||
|
||||
const read = await GitProjectManifestService.getInstance().readManifest(
|
||||
stackName,
|
||||
gitSource.repo_url,
|
||||
gitSource.branch,
|
||||
);
|
||||
if (!isGitManifest(read)) {
|
||||
const corrupt = Boolean(read && 'corrupt' in read);
|
||||
const reason = corrupt
|
||||
? `Managed-project manifest is unreadable (${(read as { corrupt: string }).corrupt}). Fix or re-link the Git source before capturing rollback coverage.`
|
||||
: 'Managed-project manifest is missing. Pull or re-link the Git source before capturing rollback coverage.';
|
||||
|
||||
const established = Boolean(
|
||||
gitSource.applied_deploy_spec
|
||||
|| gitSource.last_applied_content_hash
|
||||
|| gitSource.last_applied_commit_sha,
|
||||
);
|
||||
|
||||
// Established missing/corrupt manifesto: fail closed. applied_deploy_spec
|
||||
// alone cannot claim exact managed-input coverage (includes, extends, env,
|
||||
// labels, configs, secrets, and build inputs are omitted).
|
||||
if (established || corrupt) {
|
||||
return refusedGitInventory(gitSource, emptyInvocation, reason);
|
||||
}
|
||||
|
||||
// First apply (no applied revision yet): signal incomplete Git coverage so
|
||||
// resolveRollbackInventory can merge authored disk files with this identity.
|
||||
return refusedGitInventory(gitSource, emptyInvocation, reason, null);
|
||||
}
|
||||
|
||||
const map = new Map<string, InventoryAccum>();
|
||||
const foldedOwners = new Map<string, string>();
|
||||
const caseCollisions: string[] = [];
|
||||
for (const rel of collectManifestFilePaths(read)) {
|
||||
const resolved = resolveStackRel(stackRoot, rel);
|
||||
if (!resolved) continue;
|
||||
const meta = sensitivityForManifestPath(read, resolved.relativePath);
|
||||
const exists = await pathExistsAsFile(stackRoot, resolved.relativePath);
|
||||
upsertEntry(map, foldedOwners, {
|
||||
relativePath: resolved.relativePath,
|
||||
dependencyKind: meta.kind,
|
||||
provenance: meta.provenance,
|
||||
sensitivity: meta.sensitivity,
|
||||
absolutePath: exists ? resolved.absolutePath : null,
|
||||
}, caseCollisions);
|
||||
}
|
||||
|
||||
const refused = read.refusals.length > 0
|
||||
|| read.counts.refused > 0
|
||||
|| read.state === 'unsupported'
|
||||
|| read.state === 'partial'
|
||||
|| caseCollisions.length > 0;
|
||||
const coverageRefusal = refused
|
||||
? (caseCollisions[0]
|
||||
?? read.refusals[0]?.reason
|
||||
?? `Managed-project manifest state "${read.state}" does not claim exact coverage`)
|
||||
: null;
|
||||
|
||||
let meshEnabled = false;
|
||||
let meshReadFailed: string | null = null;
|
||||
try {
|
||||
meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(nodeId, stackName);
|
||||
} catch (e) {
|
||||
meshReadFailed = `Could not read Mesh enablement: ${(e as Error).message}`;
|
||||
}
|
||||
|
||||
const invocation: RollbackInvocationRecord = {
|
||||
composeArgsPrefix: [...read.project.invocation],
|
||||
projectDirectory: read.project.effectiveProjectDir,
|
||||
projectName: read.project.projectName || stackName,
|
||||
explicitComposeFiles: [...read.project.composeFiles],
|
||||
meshOverrideRelativePath: null,
|
||||
meshEnabled,
|
||||
};
|
||||
|
||||
const meshRefused = meshReadFailed !== null;
|
||||
return {
|
||||
entries: [...map.values()].sort((a, b) => a.relativePath.localeCompare(b.relativePath)),
|
||||
invocation,
|
||||
git: {
|
||||
repoUrl: read.repo.url,
|
||||
branch: read.repo.branch,
|
||||
commitSha: read.resolvedRevision.commitSha || gitSource.last_applied_commit_sha || '',
|
||||
manifestVersion: read.manifestVersion,
|
||||
},
|
||||
appliedDeploySpec: appliedDeploySpecString(gitSource.applied_deploy_spec),
|
||||
lastAppliedContentHash: gitSource.last_applied_content_hash,
|
||||
manifestState: gitSource.manifest_state,
|
||||
manifestGeneration: gitSource.manifest_generation,
|
||||
exactCoverage: !refused && !meshRefused,
|
||||
coverageRefusal: coverageRefusal ?? meshReadFailed,
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveAuthoredInventory(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
stackRoot: string,
|
||||
fsSvc: FileSystemService,
|
||||
): Promise<ResolvedRollbackInventory> {
|
||||
const map = new Map<string, InventoryAccum>();
|
||||
const foldedOwners = new Map<string, string>();
|
||||
const coverageNotes: string[] = [];
|
||||
const caseCollisions: string[] = [];
|
||||
|
||||
const composePaths: string[] = [];
|
||||
for (const name of ROOT_COMPOSE_FILENAMES) {
|
||||
const resolved = resolveStackRel(stackRoot, name);
|
||||
if (!resolved) continue;
|
||||
if (!(await pathExistsAsFile(stackRoot, resolved.relativePath))) continue;
|
||||
composePaths.push(resolved.relativePath);
|
||||
upsertEntry(map, foldedOwners, {
|
||||
relativePath: resolved.relativePath,
|
||||
dependencyKind: 'compose-root',
|
||||
provenance: 'authored',
|
||||
sensitivity: 'low',
|
||||
absolutePath: resolved.absolutePath,
|
||||
}, caseCollisions);
|
||||
}
|
||||
|
||||
const overrideName = await fsSvc.getOverrideFilename(stackName);
|
||||
if (overrideName) {
|
||||
const resolved = resolveStackRel(stackRoot, overrideName);
|
||||
if (resolved && await pathExistsAsFile(stackRoot, resolved.relativePath)) {
|
||||
if (!composePaths.includes(resolved.relativePath)) {
|
||||
composePaths.push(resolved.relativePath);
|
||||
}
|
||||
upsertEntry(map, foldedOwners, {
|
||||
relativePath: resolved.relativePath,
|
||||
dependencyKind: 'implicit-override',
|
||||
provenance: 'authored',
|
||||
sensitivity: 'low',
|
||||
absolutePath: resolved.absolutePath,
|
||||
}, caseCollisions);
|
||||
}
|
||||
}
|
||||
|
||||
const envCandidates = new Set<string>([DOT_ENV]);
|
||||
for (const f of DatabaseService.getInstance().getStackProjectEnvFiles(nodeId, stackName)) {
|
||||
envCandidates.add(posixRel(f));
|
||||
}
|
||||
for (const envFile of envCandidates) {
|
||||
const resolved = resolveStackRel(stackRoot, envFile);
|
||||
if (!resolved) continue;
|
||||
if (!(await pathExistsAsFile(stackRoot, resolved.relativePath))) continue;
|
||||
upsertEntry(map, foldedOwners, {
|
||||
relativePath: resolved.relativePath,
|
||||
dependencyKind: envFile === DOT_ENV ? 'interpolation-env' : 'project-env',
|
||||
provenance: 'authored',
|
||||
sensitivity: authoredSensitivity(envFile === DOT_ENV ? 'interpolation-env' : 'project-env'),
|
||||
absolutePath: resolved.absolutePath,
|
||||
}, caseCollisions);
|
||||
}
|
||||
|
||||
const readCallback = (repoPath: string): string | null => {
|
||||
const relativePath = posixRel(repoPath);
|
||||
if (!relativePath || !isValidRelativeStackPath(relativePath)) return null;
|
||||
// Inline barrier at the readFileSync sink.
|
||||
const baseResolved = path.resolve(stackRoot);
|
||||
const abs = path.resolve(baseResolved, relativePath);
|
||||
if (!abs.startsWith(baseResolved + path.sep)) return null;
|
||||
try {
|
||||
return readFileSync(abs, 'utf8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
if (composePaths.length > 0) {
|
||||
const orderedContents: Array<{ path: string; content: string }> = [];
|
||||
for (const rel of composePaths) {
|
||||
if (!isValidRelativeStackPath(rel)) continue;
|
||||
// Inline barrier at the readFile sink (same form as readCallback).
|
||||
const baseResolved = path.resolve(stackRoot);
|
||||
const abs = path.resolve(baseResolved, rel);
|
||||
if (!abs.startsWith(baseResolved + path.sep)) continue;
|
||||
try {
|
||||
const content = await fsPromises.readFile(abs, 'utf8');
|
||||
orderedContents.push({ path: rel, content });
|
||||
} catch (e) {
|
||||
coverageNotes.push(`Could not read compose file ${rel}: ${(e as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (orderedContents.length > 0) {
|
||||
const parsed = parseDeclaredInputs(orderedContents, {
|
||||
projectRoot: null,
|
||||
read: readCallback,
|
||||
});
|
||||
|
||||
if (parsed.parseErrors.length > 0) {
|
||||
coverageNotes.push(...parsed.parseErrors);
|
||||
}
|
||||
if (parsed.dynamic.length > 0) {
|
||||
coverageNotes.push(
|
||||
`${parsed.dynamic.length} dynamic path declaration(s) cannot be captured exactly`,
|
||||
);
|
||||
}
|
||||
|
||||
for (const input of parsed.inputs) {
|
||||
const hostAbs = (input.sourcePath !== null && isHostAbsolutePath(input.sourcePath))
|
||||
|| input.baseDir === 'host'
|
||||
|| input.materializedPath === null;
|
||||
|
||||
if (hostAbs) {
|
||||
if (input.kind === 'include' || input.kind === 'extends') {
|
||||
coverageNotes.push(
|
||||
`Host-absolute ${input.kind} path cannot be captured for exact rollback`,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const candidate = input.materializedPath ?? input.sourcePath;
|
||||
if (!candidate) continue;
|
||||
const resolved = resolveStackRel(stackRoot, candidate);
|
||||
if (!resolved) continue;
|
||||
if (!(await pathExistsAsFile(stackRoot, resolved.relativePath))) continue;
|
||||
|
||||
const kind = input.kind;
|
||||
upsertEntry(map, foldedOwners, {
|
||||
relativePath: resolved.relativePath,
|
||||
dependencyKind: kind,
|
||||
provenance: 'authored',
|
||||
sensitivity: authoredSensitivity(kind),
|
||||
absolutePath: resolved.absolutePath,
|
||||
}, caseCollisions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let composeArgsPrefix: string[] = [];
|
||||
try {
|
||||
composeArgsPrefix = [
|
||||
...authoredComposeFileArgs(stackName, nodeId),
|
||||
...(await authoredComposeEnvFileArgs(stackName, nodeId)),
|
||||
];
|
||||
} catch (e) {
|
||||
coverageNotes.push(`Could not build compose invocation args: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
const explicitComposeFiles = [...map.values()]
|
||||
.filter((e) => COMPOSE_INVOCATION_KINDS.has(e.dependencyKind))
|
||||
.map((e) => e.relativePath)
|
||||
.sort((a, b) => a.localeCompare(b));
|
||||
|
||||
if (caseCollisions.length > 0) {
|
||||
coverageNotes.push(caseCollisions[0]);
|
||||
}
|
||||
|
||||
let meshEnabled = false;
|
||||
try {
|
||||
meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(nodeId, stackName);
|
||||
} catch (e) {
|
||||
coverageNotes.push(`Could not read Mesh enablement: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
const exactCoverage = coverageNotes.length === 0 && composePaths.length > 0;
|
||||
const coverageRefusal = exactCoverage
|
||||
? null
|
||||
: (coverageNotes[0]
|
||||
?? (composePaths.length === 0
|
||||
? 'No compose file found in the stack directory'
|
||||
: 'Exact coverage unavailable'));
|
||||
|
||||
return {
|
||||
entries: [...map.values()].sort((a, b) => a.relativePath.localeCompare(b.relativePath)),
|
||||
invocation: {
|
||||
composeArgsPrefix,
|
||||
projectDirectory: null,
|
||||
projectName: stackName,
|
||||
explicitComposeFiles: explicitComposeFiles.length > 0 ? explicitComposeFiles : composePaths,
|
||||
meshOverrideRelativePath: null,
|
||||
meshEnabled,
|
||||
},
|
||||
git: null,
|
||||
appliedDeploySpec: null,
|
||||
lastAppliedContentHash: null,
|
||||
manifestState: null,
|
||||
manifestGeneration: null,
|
||||
exactCoverage,
|
||||
coverageRefusal,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Prefer an exact Git-managed inventory when available. When the manifesto is
|
||||
* missing on a true first apply (no applied revision yet), merge authored disk
|
||||
* discovery with the Git identity fields so capture preserves nullable Git
|
||||
* state. Established missing/corrupt manifesto cases fail closed via
|
||||
* resolveGitInventory and are not overwritten by authored exactCoverage.
|
||||
*/
|
||||
export async function resolveRollbackInventory(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
): Promise<ResolvedRollbackInventory> {
|
||||
const fsSvc = FileSystemService.getInstance(nodeId);
|
||||
const stackRoot = resolveStackRoot(fsSvc, stackName);
|
||||
|
||||
try {
|
||||
const gitInventory = await resolveGitInventory(nodeId, stackName, stackRoot);
|
||||
if (gitInventory?.exactCoverage) return gitInventory;
|
||||
|
||||
const authored = await resolveAuthoredInventory(nodeId, stackName, stackRoot, fsSvc);
|
||||
|
||||
if (gitInventory && !gitInventory.exactCoverage) {
|
||||
const established = Boolean(
|
||||
gitInventory.appliedDeploySpec
|
||||
|| gitInventory.lastAppliedContentHash
|
||||
|| gitInventory.git?.commitSha,
|
||||
);
|
||||
const firstApplyCorrupt = Boolean(gitInventory.coverageRefusal?.includes('unreadable'));
|
||||
// Established or corrupt first-apply: fail closed. Otherwise merge Git
|
||||
// identity onto authored exact coverage for a true first apply.
|
||||
if (established || firstApplyCorrupt || !authored.exactCoverage) {
|
||||
return gitInventory;
|
||||
}
|
||||
return {
|
||||
...authored,
|
||||
git: gitInventory.git,
|
||||
appliedDeploySpec: gitInventory.appliedDeploySpec,
|
||||
lastAppliedContentHash: gitInventory.lastAppliedContentHash,
|
||||
manifestState: gitInventory.manifestState,
|
||||
manifestGeneration: gitInventory.manifestGeneration,
|
||||
};
|
||||
}
|
||||
|
||||
if (authored.exactCoverage) return authored;
|
||||
return gitInventory ?? authored;
|
||||
} catch (e) {
|
||||
console.error(
|
||||
'[rollbackInventory] Failed to resolve inventory:',
|
||||
(e as Error).message,
|
||||
);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
@@ -91,6 +91,18 @@ const RULES: ClassifierRule[] = [
|
||||
suggestion: 'Review the compose file syntax (Compose Doctor can pinpoint the issue), then retry.',
|
||||
pattern: /yaml:|mapping values are not allowed|cannot unmarshal|additional propert|undefined volume|undefined network|invalid compose/i,
|
||||
},
|
||||
{
|
||||
reason: 'mixed_replica_images',
|
||||
label: 'Mixed replica images',
|
||||
suggestion: 'Bring every replica of the service onto the same image, then retry.',
|
||||
pattern: /mixed replica images/i,
|
||||
},
|
||||
{
|
||||
reason: 'rollback_coverage_unavailable',
|
||||
label: 'Exact rollback coverage unavailable',
|
||||
suggestion: 'Remove host-absolute include or extends paths, or capture from a project Sencho can enumerate completely, then retry.',
|
||||
pattern: /cannot be captured for exact rollback|rollback coverage is unavailable/i,
|
||||
},
|
||||
];
|
||||
|
||||
const UNKNOWN_FAILURE: FailureClassification = {
|
||||
|
||||
@@ -331,13 +331,55 @@ export interface RollbackInputs {
|
||||
/** Timestamp of the most recent deploy_success activity event, if any. */
|
||||
lastDeployAt: number | null | Errored;
|
||||
containers: ContainerProbe[] | Errored;
|
||||
/**
|
||||
* Current recovery generation, when one exists. When present, compose_source
|
||||
* readiness is driven by this generation rather than the legacy backup slot.
|
||||
*/
|
||||
recoveryGeneration: { exists: boolean; shortId?: string } | Errored | null;
|
||||
/** Eligibility verdict for the current generation, when assessed. */
|
||||
policyEligibility: 'eligible' | 'eligible_with_warning' | 'prohibited' | 'unknown' | Errored | null;
|
||||
/** Whether managed authored inputs are covered by exact inventory. */
|
||||
managedInputs: { covered: boolean; detail: string } | Errored | null;
|
||||
}
|
||||
|
||||
export function buildRollbackItems(inputs: RollbackInputs, now: number): RollbackReadinessItem[] {
|
||||
const items: RollbackReadinessItem[] = [];
|
||||
|
||||
const recoveryRaw = inputs.recoveryGeneration;
|
||||
const recoveryInfo = recoveryRaw !== null && recoveryRaw !== 'error' ? recoveryRaw : null;
|
||||
const recoveryExists = !!recoveryInfo?.exists;
|
||||
const backupExists = inputs.backup !== 'error' && inputs.backup.exists;
|
||||
if (inputs.backup === 'error') {
|
||||
|
||||
if (recoveryRaw === 'error') {
|
||||
items.push({ id: 'recovery_generation', state: 'unknown', label: 'Recovery generation', detail: 'The recovery generation could not be read.' });
|
||||
} else if (recoveryInfo?.exists) {
|
||||
const short = recoveryInfo.shortId;
|
||||
items.push({
|
||||
id: 'recovery_generation',
|
||||
state: 'ready',
|
||||
label: 'Recovery generation',
|
||||
detail: short
|
||||
? `Current recovery generation ${short} is available for exact restore.`
|
||||
: 'A current recovery generation is available for exact restore.',
|
||||
});
|
||||
} else {
|
||||
items.push({
|
||||
id: 'recovery_generation',
|
||||
state: 'missing',
|
||||
label: 'Recovery generation',
|
||||
detail: 'No recovery generation is current yet. One is created by the next update, atomic deploy, or Git apply.',
|
||||
});
|
||||
}
|
||||
|
||||
// Supersede rule: when a recovery generation exists, compose_source tracks it.
|
||||
if (recoveryExists) {
|
||||
items.push({
|
||||
id: 'compose_source',
|
||||
state: 'ready',
|
||||
label: 'Previous compose file',
|
||||
detail: 'Authored project files are covered by the current recovery generation.',
|
||||
});
|
||||
} else if (inputs.backup === 'error') {
|
||||
items.push({ id: 'compose_source', state: 'unknown', label: 'Previous compose file', detail: 'The backup slot could not be read.' });
|
||||
} else if (backupExists) {
|
||||
const age = inputs.backup.timestamp ? ` from ${formatAge(inputs.backup.timestamp, now)}` : '';
|
||||
@@ -385,6 +427,30 @@ export function buildRollbackItems(inputs: RollbackInputs, now: number): Rollbac
|
||||
items.push({ id: 'healthchecks', state: 'missing', label: 'Healthchecks', detail: 'No service defines a healthcheck; rollback verification relies on run state only.' });
|
||||
}
|
||||
|
||||
if (inputs.policyEligibility === 'error') {
|
||||
items.push({ id: 'policy_eligibility', state: 'unknown', label: 'Rollback eligibility', detail: 'Eligibility could not be assessed.' });
|
||||
} else if (inputs.policyEligibility === null) {
|
||||
items.push({ id: 'policy_eligibility', state: 'ready', label: 'Rollback eligibility', detail: 'No recovery generation to assess yet.' });
|
||||
} else if (inputs.policyEligibility === 'eligible') {
|
||||
items.push({ id: 'policy_eligibility', state: 'ready', label: 'Rollback eligibility', detail: 'Restore is eligible with known-good generation integrity and held images.' });
|
||||
} else if (inputs.policyEligibility === 'eligible_with_warning') {
|
||||
items.push({ id: 'policy_eligibility', state: 'warning', label: 'Rollback eligibility', detail: 'Restore is possible but held images may be missing; moving-tag recoverability is weak.' });
|
||||
} else if (inputs.policyEligibility === 'prohibited') {
|
||||
items.push({ id: 'policy_eligibility', state: 'blocked', label: 'Rollback eligibility', detail: 'Restore is prohibited until generation integrity or security posture is repaired.' });
|
||||
} else {
|
||||
items.push({ id: 'policy_eligibility', state: 'unknown', label: 'Rollback eligibility', detail: 'Eligibility is not fully known yet.' });
|
||||
}
|
||||
|
||||
if (inputs.managedInputs === 'error') {
|
||||
items.push({ id: 'managed_inputs', state: 'unknown', label: 'Managed inputs', detail: 'Managed input coverage could not be read.' });
|
||||
} else if (inputs.managedInputs === null) {
|
||||
items.push({ id: 'managed_inputs', state: 'unknown', label: 'Managed inputs', detail: 'Managed input coverage has not been assessed.' });
|
||||
} else if (inputs.managedInputs.covered) {
|
||||
items.push({ id: 'managed_inputs', state: 'ready', label: 'Managed inputs', detail: inputs.managedInputs.detail });
|
||||
} else {
|
||||
items.push({ id: 'managed_inputs', state: 'warning', label: 'Managed inputs', detail: inputs.managedInputs.detail });
|
||||
}
|
||||
|
||||
const mounts = inputs.containers === 'error'
|
||||
? []
|
||||
: [...new Set(inputs.containers.flatMap(c => c.mounts))];
|
||||
@@ -393,7 +459,7 @@ export function buildRollbackItems(inputs: RollbackInputs, now: number): Rollbac
|
||||
id: 'volume_data',
|
||||
state: 'not_covered',
|
||||
label: 'Application data',
|
||||
detail: `Named volumes and bind-mounted data are not included in file backups. Rolling back restores compose and env files only; application data keeps its current state.${mountDetail}`,
|
||||
detail: `Named volumes and bind-mounted data are not included in recovery generations. Rolling back restores the managed authored project (compose files, overrides, includes/extends, env and related inputs), Git identity when captured, and prior image holds; application data keeps its current state.${mountDetail}`,
|
||||
});
|
||||
|
||||
return items;
|
||||
@@ -406,9 +472,16 @@ export function buildRollbackItems(inputs: RollbackInputs, now: number): Rollbac
|
||||
*/
|
||||
export function aggregateRollbackOverall(items: RollbackReadinessItem[]): RollbackOverall {
|
||||
const byId = new Map(items.map(i => [i.id, i.state]));
|
||||
if (byId.get('policy_eligibility') === 'blocked') {
|
||||
return 'not_ready';
|
||||
}
|
||||
if (byId.get('compose_source') !== 'ready') {
|
||||
return byId.get('compose_source') === 'unknown' ? 'partial' : 'not_ready';
|
||||
}
|
||||
const policy = byId.get('policy_eligibility');
|
||||
if (policy === 'unknown' || policy === 'warning') {
|
||||
return 'partial';
|
||||
}
|
||||
if (byId.get('env_keys') === 'ready' && byId.get('previous_images') === 'ready') {
|
||||
return 'ready';
|
||||
}
|
||||
|
||||
@@ -35,10 +35,10 @@ export interface UpdateReadinessReport {
|
||||
}
|
||||
|
||||
/** State of one rollback readiness item. */
|
||||
export type RollbackItemState = 'ready' | 'missing' | 'unknown' | 'not_covered';
|
||||
export type RollbackItemState = 'ready' | 'missing' | 'unknown' | 'not_covered' | 'blocked' | 'warning';
|
||||
|
||||
export interface RollbackReadinessItem {
|
||||
id: 'compose_source' | 'env_keys' | 'previous_images' | 'last_deploy' | 'healthchecks' | 'volume_data';
|
||||
id: 'compose_source' | 'env_keys' | 'previous_images' | 'last_deploy' | 'healthchecks' | 'volume_data' | 'policy_eligibility' | 'managed_inputs' | 'recovery_generation';
|
||||
state: RollbackItemState;
|
||||
label: string;
|
||||
/** Names only for env coverage; values never appear here. */
|
||||
@@ -117,6 +117,8 @@ export type FailureReason =
|
||||
| 'healthcheck_failed'
|
||||
| 'dependency_unavailable'
|
||||
| 'node_unreachable'
|
||||
| 'mixed_replica_images'
|
||||
| 'rollback_coverage_unavailable'
|
||||
| 'unknown';
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* Authored-project rollback generation content schema.
|
||||
*
|
||||
* The DB row in stack_update_recovery_generations remains the listing /
|
||||
* lifecycle authority. Files under the generation content directory are the
|
||||
* referenced content store (paths, checksums, invocation, tombstones).
|
||||
*/
|
||||
import type { GitSourceManifestState, InputDependencyKind, InputSensitivity, ManifestProvenance } from './gitProjectManifest';
|
||||
|
||||
export const ROLLBACK_GENERATION_SCHEMA_VERSION = 1 as const;
|
||||
|
||||
export type RollbackOperationKind =
|
||||
| 'update'
|
||||
| 'deployment'
|
||||
| 'git_apply'
|
||||
| 'manual_backup'
|
||||
| 'unknown';
|
||||
|
||||
export type RollbackEntryKind =
|
||||
| InputDependencyKind
|
||||
| 'compose-root'
|
||||
| 'project-env'
|
||||
| 'invocation-meta'
|
||||
| 'other';
|
||||
|
||||
export type RollbackEntryProvenance = ManifestProvenance | 'authored' | 'sencho-generated';
|
||||
|
||||
export interface RollbackGenerationEntry {
|
||||
/** Stack-relative POSIX path. */
|
||||
relativePath: string;
|
||||
dependencyKind: RollbackEntryKind;
|
||||
provenance: RollbackEntryProvenance;
|
||||
/** present = file captured; tombstoned = must be absent after restore. */
|
||||
state: 'present' | 'tombstoned';
|
||||
contentSha256: string | null;
|
||||
sizeBytes: number | null;
|
||||
sensitivity: InputSensitivity;
|
||||
/** When true, content is encrypted at rest in the generation store. */
|
||||
encrypted: boolean;
|
||||
/**
|
||||
* POSIX permission bits (mode & 0o777) at capture. Null on legacy generations
|
||||
* or platforms where mode could not be read.
|
||||
*/
|
||||
mode: number | null;
|
||||
}
|
||||
|
||||
export interface RollbackInvocationRecord {
|
||||
/** Ordered compose -f / project-directory / env-file args used for mutation. */
|
||||
composeArgsPrefix: string[];
|
||||
projectDirectory: string | null;
|
||||
projectName: string | null;
|
||||
explicitComposeFiles: string[];
|
||||
/** Stack-relative mesh override path when Mesh was part of the capture invocation. */
|
||||
meshOverrideRelativePath?: string | null;
|
||||
/** True when Mesh was enabled for the stack at capture time. */
|
||||
meshEnabled?: boolean;
|
||||
}
|
||||
|
||||
export interface RollbackGitIdentity {
|
||||
repoUrl: string;
|
||||
branch: string;
|
||||
commitSha: string;
|
||||
manifestVersion: number | null;
|
||||
}
|
||||
|
||||
export interface RollbackImageIdentity {
|
||||
serviceName: string;
|
||||
imageId: string | null;
|
||||
repoDigest: string | null;
|
||||
platform: string | null;
|
||||
declaredImageRef: string | null;
|
||||
}
|
||||
|
||||
/** On-disk generation.json for one recovery content directory. */
|
||||
export interface RollbackGenerationManifest {
|
||||
schemaVersion: typeof ROLLBACK_GENERATION_SCHEMA_VERSION;
|
||||
capabilityVersion: 1;
|
||||
generationId: string;
|
||||
nodeId: number;
|
||||
stackName: string;
|
||||
capturedAt: number;
|
||||
operationKind: RollbackOperationKind;
|
||||
entries: RollbackGenerationEntry[];
|
||||
/**
|
||||
* Full managed relative-path set at capture time. Restore may delete live
|
||||
* files in this set that are not present entries (tombstones / absent-at-
|
||||
* capture). Paths outside this set are never touched by restore unless they
|
||||
* also appear in the caller-supplied liveManagedPaths discovery set.
|
||||
*/
|
||||
managedRelativePaths: string[];
|
||||
invocation: RollbackInvocationRecord;
|
||||
git: RollbackGitIdentity | null;
|
||||
/** Prior applied/deployed/LKG refs when known (opaque strings). */
|
||||
priorRecords: {
|
||||
appliedDeploySpec: string | null;
|
||||
lkgHint: string | null;
|
||||
/** Git DB snapshot at capture (restored with files so Compose args match). */
|
||||
lastAppliedContentHash: string | null;
|
||||
manifestState: string | null;
|
||||
manifestGeneration: string | null;
|
||||
/**
|
||||
* True when git-manifest.v1.json was snapshotted into the generation.
|
||||
* False means the capture-time managed manifesto was absent (first apply
|
||||
* preimage); restore must clear any manifesto written after capture.
|
||||
* Omitted on older generations: inferred from whether the snapshot file exists.
|
||||
*/
|
||||
gitManifestCaptured?: boolean;
|
||||
};
|
||||
images: RollbackImageIdentity[];
|
||||
}
|
||||
|
||||
/** Durable Git database projection stored in restore-intent.json. */
|
||||
export interface RollbackGitDbSnapshot {
|
||||
appliedDeploySpec: { files: string[]; contextDir: string | null } | null;
|
||||
lastAppliedCommitSha: string | null;
|
||||
lastAppliedContentHash: string | null;
|
||||
manifestVersion: number | null;
|
||||
manifestState: GitSourceManifestState | null;
|
||||
manifestGeneration: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Git DB + managed-manifesto preimage recorded before a restore mutates the
|
||||
* live stack. Used as restoreGeneration transactionMeta and as
|
||||
* RollbackRestoreIntent.gitSide.
|
||||
*/
|
||||
export interface RollbackRestoreTransactionMeta {
|
||||
gitDbBefore: RollbackGitDbSnapshot | null;
|
||||
/** Raw manifest.v1.json text before restore, or null when absent. */
|
||||
managedManifestBefore: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Crash-safe restore intent: pre-restore filesystem snapshot plus the Git DB
|
||||
* and managed-manifesto state that must be reinstated if the process dies
|
||||
* before commitRestoreTransaction.
|
||||
*/
|
||||
export interface RollbackRestoreIntent {
|
||||
generationId: string;
|
||||
stackName: string;
|
||||
nodeId: number;
|
||||
paths: string[];
|
||||
at: number;
|
||||
/**
|
||||
* Present when compensate recorded Git side-state. Absent on older intents
|
||||
* (filesystem-only reversion).
|
||||
*/
|
||||
gitSide?: RollbackRestoreTransactionMeta;
|
||||
}
|
||||
|
||||
export interface ResolvedRollbackInventory {
|
||||
entries: Array<{
|
||||
relativePath: string;
|
||||
dependencyKind: RollbackEntryKind;
|
||||
provenance: RollbackEntryProvenance;
|
||||
sensitivity: InputSensitivity;
|
||||
/** Absolute path on the live stack when present; null if absent (tombstone candidate). */
|
||||
absolutePath: string | null;
|
||||
}>;
|
||||
invocation: RollbackInvocationRecord;
|
||||
git: RollbackGitIdentity | null;
|
||||
appliedDeploySpec: string | null;
|
||||
lastAppliedContentHash: string | null;
|
||||
manifestState: string | null;
|
||||
manifestGeneration: string | null;
|
||||
/** True when inventory claims exact atomic coverage. */
|
||||
exactCoverage: boolean;
|
||||
coverageRefusal: string | null;
|
||||
}
|
||||
@@ -104,6 +104,21 @@ export const MAX_SNAPSHOT_FILE_BYTES = 1_000_000;
|
||||
/** The cap rendered in MB for operator-facing warning text. */
|
||||
const MAX_SNAPSHOT_FILE_MB = MAX_SNAPSHOT_FILE_BYTES / 1_000_000;
|
||||
|
||||
/** Snapshot files Fleet restore is allowed to write. */
|
||||
export const FLEET_SNAPSHOT_APPLY_FILENAMES = ['compose.yaml', '.env'] as const;
|
||||
export type FleetSnapshotApplyFilename = typeof FLEET_SNAPSHOT_APPLY_FILENAMES[number];
|
||||
|
||||
/**
|
||||
* JSON body limit for POST /api/stacks/:name/fleet-snapshot-apply. Capture
|
||||
* allows 1 MB per allowed file; the apply POST sends those files in one body,
|
||||
* so the default 100 KB parser would 413 a legal captured pair.
|
||||
*/
|
||||
export const FLEET_SNAPSHOT_APPLY_BODY_LIMIT =
|
||||
FLEET_SNAPSHOT_APPLY_FILENAMES.length * MAX_SNAPSHOT_FILE_BYTES + 256_000;
|
||||
|
||||
/** Hub wait for remote capture-then-write. Capture inspects images before any write. */
|
||||
export const FLEET_SNAPSHOT_APPLY_TIMEOUT_MS = 300_000;
|
||||
|
||||
/**
|
||||
* Minimal node shape accepted by capture functions.
|
||||
* `mode` is required so remote dispatch can emit a tunnel-aware error when
|
||||
|
||||
Reference in New Issue
Block a user