fix(blueprints): fail closed on marker ownership for apply and withdraw (#1694)

* fix(blueprints): fail closed on marker ownership for apply and withdraw

Require a matching .blueprint.json under the stack lock, persist required_blueprint_id on deletion intents, remove the legacy remote apply fallback, and protect the marker in the file explorer.

* fix(blueprints): add CodeQL path barriers on ownership probes

Use the canonical resolve-and-startsWith sanitizer inline at the marker and stack-directory fs sinks so js/path-injection clears.

* fix(blueprints): block delete on failed withdraw and defer marker write

Refuse Blueprint DELETE when pre-delete withdraw does not complete, and write .blueprint.json only after a successful deploy so failed applies cannot orphan stacks or claim an unapplied revision.

* test(blueprints): align lock-order assert with deferred marker write

Update the per-stack lock ordering expectations to compose, cleanup, deploy, then marker after the partial-apply fix.

* fix(deps): bump postcss past GHSA-r28c-9q8g-f849 for npm audit

Raise the Vitest/Vite transitive postcss to 8.5.23 so Backend CI audit --audit-level=high passes.
This commit is contained in:
Anso
2026-07-24 15:57:18 -04:00
committed by GitHub
parent e33eda3c38
commit 17a8dc8a94
19 changed files with 1092 additions and 286 deletions
@@ -89,6 +89,7 @@ describe('Blueprint compose apply (real filesystem)', () => {
const stackDir = path.join(process.env.COMPOSE_DIR!, stackName);
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe(composeContent);
await expectMissing(path.join(stackDir, 'docker-compose.yml'));
expect(await fsPromises.readFile(path.join(stackDir, '.blueprint.json'), 'utf-8')).toBe(markerContent);
const resolved = await FileSystemService.getInstance(nodeId).getComposeFilename(stackName);
expect(resolved).toBe('compose.yaml');
@@ -104,6 +105,10 @@ describe('Blueprint compose apply (real filesystem)', () => {
await fsPromises.writeFile(path.join(stackDir, 'compose.yml'), 'services:\n a:\n image: a\n');
await fsPromises.writeFile(path.join(stackDir, 'docker-compose.yaml'), 'services:\n b:\n image: b\n');
await fsPromises.writeFile(path.join(stackDir, 'docker-compose.yml'), 'services:\n c:\n image: c\n');
await fsPromises.writeFile(
path.join(stackDir, '.blueprint.json'),
JSON.stringify({ blueprintId: 2, revision: 2, lastApplied: 1 }, null, 2),
);
const composeContent = 'services:\n app:\n image: redis:7\n';
const markerContent = JSON.stringify({ blueprintId: 2, revision: 3, lastApplied: Date.now() }, null, 2);
@@ -131,8 +136,83 @@ describe('Blueprint compose apply (real filesystem)', () => {
await expectMissing(path.join(stackDir, 'compose.yml'));
await expectMissing(path.join(stackDir, 'docker-compose.yaml'));
await expectMissing(path.join(stackDir, 'docker-compose.yml'));
expect(await fsPromises.readFile(path.join(stackDir, '.blueprint.json'), 'utf-8')).toBe(markerContent);
expect(deploySpy).toHaveBeenCalledTimes(1);
});
it('does not write a new marker when deploy fails; rolls back a newly created stack', async () => {
const nodeId = seedLocalNode();
const stackName = `bp-partial-${counter}`;
const composeContent = 'services:\n web:\n image: traefik:v3\n';
const markerContent = JSON.stringify({ blueprintId: 7, revision: 1, lastApplied: Date.now() }, null, 2);
const stackDir = path.join(process.env.COMPOSE_DIR!, stackName);
vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue(new Error('docker unavailable'));
await expect(
BlueprintService.getInstance().applyLocalUnderLock(
nodeId,
stackName,
composeContent,
markerContent,
'/api/blueprints/test/apply',
),
).rejects.toThrow(/docker unavailable/);
await expectMissing(path.join(stackDir, '.blueprint.json'));
await expectMissing(stackDir);
});
it('keeps the prior marker when a re-apply deploy fails', async () => {
const nodeId = seedLocalNode();
const stackName = `bp-reapply-fail-${counter}`;
const stackDir = path.join(process.env.COMPOSE_DIR!, stackName);
const priorMarker = JSON.stringify({ blueprintId: 8, revision: 2, lastApplied: 1 }, null, 2);
await fsPromises.mkdir(stackDir, { recursive: true });
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n old:\n image: nginx\n');
await fsPromises.writeFile(path.join(stackDir, '.blueprint.json'), priorMarker);
vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue(new Error('deploy blew up'));
await expect(
BlueprintService.getInstance().applyLocalUnderLock(
nodeId,
stackName,
'services:\n new:\n image: redis:7\n',
JSON.stringify({ blueprintId: 8, revision: 3, lastApplied: Date.now() }, null, 2),
'/api/blueprints/test/apply',
),
).rejects.toThrow(/deploy blew up/);
expect(await fsPromises.readFile(path.join(stackDir, '.blueprint.json'), 'utf-8')).toBe(priorMarker);
expect(await fsPromises.access(stackDir).then(() => true, () => false)).toBe(true);
});
it('refuses to overwrite an existing unmanaged stack directory inside the lock', async () => {
const { BlueprintNameConflictError } = await import('../services/BlueprintService');
const nodeId = seedLocalNode();
const stackName = `bp-hijack-${counter}`;
const stackDir = path.join(process.env.COMPOSE_DIR!, stackName);
await fsPromises.mkdir(stackDir, { recursive: true });
const original = 'services:\n mine:\n image: nginx:alpine\n';
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), original);
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined);
await expect(
BlueprintService.getInstance().applyLocalUnderLock(
nodeId,
stackName,
'services:\n bp:\n image: redis:7\n',
JSON.stringify({ blueprintId: 99, revision: 1, lastApplied: Date.now() }, null, 2),
'/api/blueprints/test/apply',
),
).rejects.toBeInstanceOf(BlueprintNameConflictError);
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe(original);
await expectMissing(path.join(stackDir, '.blueprint.json'));
expect(deploySpy).not.toHaveBeenCalled();
});
});
describe('FileSystemService.removeAlternateRootComposeFiles', () => {
@@ -111,9 +111,9 @@ describe('Blueprint route edge cases', () => {
describe('Blueprint delete guard', () => {
// Rows with no stack of ours on the node must not block delete, AND the route must not run the
// withdraw primitive for them: withdrawFromNode proceeds on a missing marker and would
// down/delete a same-name stack Sencho never owned. A name_conflict is exactly that unmanaged
// stack, so it is excluded even though it carries a last_deployed_at timestamp.
// withdraw primitive for them: there is nothing Sencho owns to remove. A name_conflict is an
// unmanaged same-name stack, so it is excluded even though it may carry a last_deployed_at
// timestamp. When withdraw does run, ownership is enforced under the delete lock.
it.each([
{ label: 'never-deployed pending review', status: 'pending_state_review' as const, last_deployed_at: null },
{ label: 'first-deploy failure', status: 'failed' as const, last_deployed_at: null },
@@ -184,6 +184,34 @@ describe('Blueprint delete guard', () => {
expect(DatabaseService.getInstance().getBlueprint(bp.id)).toBeUndefined();
});
it('refuses to delete a stateless blueprint when pre-delete withdraw fails', async () => {
const node = seedNode();
const bp = seedBlueprint([node.id]);
DatabaseService.getInstance().upsertDeployment({
blueprint_id: bp.id,
node_id: node.id,
status: 'failed',
applied_revision: bp.revision,
last_deployed_at: Date.now(),
last_error: 'Remote node lacks withdraw-local',
});
const withdrawSpy = vi.spyOn(BlueprintService.getInstance(), 'withdrawFromNode').mockResolvedValue({
status: 'failed',
error: 'Remote node does not support atomic blueprint withdraw',
});
const res = await request(app)
.delete(`/api/blueprints/${bp.id}`)
.set('Cookie', adminCookie);
expect(res.status).toBe(409);
expect(res.body.code).toBe('withdraw_failed_blocking_delete');
expect(res.body.nodeId).toBe(node.id);
expect(withdrawSpy).toHaveBeenCalledTimes(1);
expect(DatabaseService.getInstance().getBlueprint(bp.id)).toBeDefined();
expect(DatabaseService.getInstance().listDeployments(bp.id)).toHaveLength(1);
});
it('refuses to delete when a pending review still has a deployed stack (revision drift)', async () => {
const node = seedNode();
const bp = seedBlueprint([node.id], 'stateful');
@@ -253,38 +281,54 @@ describe('BlueprintService marker edge cases', () => {
});
it('refuses to withdraw when the marker belongs to a different blueprint', async () => {
const fs = await import('fs');
const path = await import('path');
const composeDir = process.env.COMPOSE_DIR!;
const localNode = DatabaseService.getInstance().getNodes()[0];
const bp = seedBlueprint([localNode.id]);
DatabaseService.getInstance().getDb()
.prepare('UPDATE nodes SET compose_dir = ? WHERE id = ?')
.run(composeDir, localNode.id);
const refreshed = DatabaseService.getInstance().getNode(localNode.id)!;
const bp = seedBlueprint([refreshed.id]);
const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!;
vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue({
blueprintId: bp.id + 999,
revision: 1,
lastApplied: 0,
DatabaseService.getInstance().upsertDeployment({
blueprint_id: bp.id,
node_id: refreshed.id,
status: 'active',
applied_revision: bpObj.revision,
last_deployed_at: Date.now(),
});
const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, localNode);
const stackDir = path.join(composeDir, bpObj.name);
fs.mkdirSync(stackDir, { recursive: true });
fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n app:\n image: nginx\n');
fs.writeFileSync(
path.join(stackDir, '.blueprint.json'),
JSON.stringify({ blueprintId: bp.id + 999, revision: 1, lastApplied: 0 }),
);
const { DeployedStackDeletionService } = await import('../services/DeployedStackDeletionService');
const deleteSpy = vi.spyOn(DeployedStackDeletionService.getInstance(), 'deleteDeployedStack');
const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, refreshed);
expect(result.status).toBe('name_conflict');
// The deployment row must record the conflict, not silently disappear.
const dep = DatabaseService.getInstance().getDeployment(bp.id, localNode.id);
// deleteDeployedStack is invoked but returns name_conflict without mutating.
expect(deleteSpy).toHaveBeenCalled();
const dep = DatabaseService.getInstance().getDeployment(bp.id, refreshed.id);
expect(dep).toBeDefined();
expect(dep?.status).toBe('name_conflict');
expect(fs.existsSync(path.join(stackDir, 'compose.yaml'))).toBe(true);
});
});
describe('BlueprintService developer-mode diagnostics', () => {
// withdrawFromNode emits its "withdraw inputs" diagnostic line before reading the
// marker, so a cross-blueprint marker stub lets us assert the gate without Docker.
// withdrawFromNode emits its "withdraw inputs" diagnostic before the deletion
// service runs. An absent stack directory is enough to exercise logging without Docker.
function arrangeWithdraw() {
const localNode = DatabaseService.getInstance().getNodes()[0];
const bp = seedBlueprint([localNode.id]);
const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!;
vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue({
blueprintId: bp.id + 999,
revision: 1,
lastApplied: 0,
});
return { bpObj, localNode };
}
@@ -326,6 +326,7 @@ describe('POST /api/blueprints/:id/apply confirm binding', () => {
const conflict = await BlueprintService.getInstance().hasNameConflict(
created.body.name as string,
DatabaseService.getInstance().getNode(node.id)!,
created.body.id as number,
);
expect(conflict).toBe(true);
@@ -342,6 +343,41 @@ describe('POST /api/blueprints/:id/apply confirm binding', () => {
.toBe('services:\n app:\n image: nginx\n');
});
it('hasNameConflict is false for a matching marker and true for a foreign marker', async () => {
const node = seedNode();
counter += 1;
const created = await request(app)
.post('/api/blueprints')
.set('Cookie', adminCookie)
.send(validBlueprintBody(node.id));
expect(created.status).toBe(201);
const composeDir = process.env.COMPOSE_DIR!;
DatabaseService.getInstance().getDb()
.prepare('UPDATE nodes SET compose_dir = ? WHERE id = ?')
.run(composeDir, node.id);
const nodeObj = DatabaseService.getInstance().getNode(node.id)!;
const stackName = created.body.name as string;
const blueprintId = created.body.id as number;
const stackDir = path.join(composeDir, stackName);
fs.mkdirSync(stackDir, { recursive: true });
fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n app:\n image: nginx\n');
fs.writeFileSync(
path.join(stackDir, '.blueprint.json'),
JSON.stringify({ blueprintId, revision: 1, lastApplied: 1 }),
);
const { BlueprintService } = await import('../services/BlueprintService');
const svc = BlueprintService.getInstance();
expect(await svc.hasNameConflict(stackName, nodeObj, blueprintId)).toBe(false);
fs.writeFileSync(
path.join(stackDir, '.blueprint.json'),
JSON.stringify({ blueprintId: blueprintId + 99, revision: 1, lastApplied: 1 }),
);
expect(await svc.hasNameConflict(stackName, nodeObj, blueprintId)).toBe(true);
});
it('blocks rename while a non-withdrawn deployment exists', async () => {
const node = seedNode();
counter += 1;
@@ -101,7 +101,7 @@ describe('BlueprintService remote deploy', () => {
expect(dep?.applied_revision).toBe(bpObj.revision);
});
it('falls back to the legacy create/write/deploy flow when the remote lacks apply-local (404)', async () => {
it('fails closed when the remote lacks apply-local (404); no legacy mutations', async () => {
const node = seedRemoteNode();
const bp = seedBlueprint([node.id]);
const nodeObj = DatabaseService.getInstance().getNode(node.id)!;
@@ -110,25 +110,17 @@ describe('BlueprintService remote deploy', () => {
vi.spyOn(axios, 'get').mockResolvedValue({ status: 200, data: [] });
const putSpy = vi.spyOn(axios, 'put').mockResolvedValue({ status: 200, data: {} });
const postSpy = vi.spyOn(axios, 'post')
.mockResolvedValueOnce({ status: 404, data: {} }) // apply-local missing on older node
.mockResolvedValueOnce({ status: 201, data: {} }) // legacy create stack
.mockResolvedValueOnce({ status: 200, data: {} }); // legacy deploy
.mockResolvedValueOnce({ status: 404, data: {} });
const result = await BlueprintService.getInstance().deployToNode(bpObj, nodeObj);
expect(result.status).toBe('active');
expect(result.status).toBe('failed');
expect(result.error).toMatch(/apply-local|Upgrade/i);
expect(postSpy).toHaveBeenCalledTimes(1);
expect(postSpy.mock.calls[0][0]).toMatch(/\/api\/blueprints\/apply-local$/);
expect(postSpy.mock.calls[1][0]).toMatch(/\/api\/stacks$/);
expect(putSpy).toHaveBeenCalledTimes(2); // compose + marker
const composePutUrl = String(putSpy.mock.calls[0][0]);
const markerPutUrl = String(putSpy.mock.calls[1][0]);
expect(composePutUrl).toMatch(/[?&]path=compose\.yaml(?:&|$)/);
expect(markerPutUrl).toMatch(/[?&]path=\.blueprint\.json(?:&|$)/);
expect(postSpy.mock.calls[2][0]).toMatch(/\/deploy$/);
const [composePutOrder, markerPutOrder] = putSpy.mock.invocationCallOrder;
const deployOrder = postSpy.mock.invocationCallOrder[2];
expect(composePutOrder).toBeLessThan(markerPutOrder);
expect(markerPutOrder).toBeLessThan(deployOrder);
expect(putSpy).not.toHaveBeenCalled();
const dep = DatabaseService.getInstance().getDeployment(bp.id, node.id);
expect(dep?.status).toBe('failed');
});
it('maps a remote apply lock-conflict (409) to status=failed', async () => {
@@ -192,7 +184,7 @@ describe('BlueprintService remote deploy', () => {
expect(dep?.status).toBe('name_conflict');
});
it('withdraws a remote deployment by deleting the stack and removing the row', async () => {
it('withdraws a remote deployment via withdraw-local and removes the row', async () => {
const node = seedRemoteNode();
const bp = seedBlueprint([node.id]);
const nodeObj = DatabaseService.getInstance().getNode(node.id)!;
@@ -204,17 +196,73 @@ describe('BlueprintService remote deploy', () => {
applied_revision: bpObj.revision,
});
vi.spyOn(axios, 'get').mockResolvedValue({ status: 404, data: {} }); // readMarker → null → proceed
vi.spyOn(axios, 'post').mockResolvedValue({ status: 200, data: {} }); // remote down (best-effort)
const delSpy = vi.spyOn(axios, 'delete').mockResolvedValue({ status: 200, data: {} });
const postSpy = vi.spyOn(axios, 'post').mockResolvedValue({
status: 200,
data: { status: 'withdrawn' },
});
const delSpy = vi.spyOn(axios, 'delete');
const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj);
expect(result.status).toBe('withdrawn');
expect(delSpy.mock.calls[0][0]).toMatch(/\/api\/stacks\//);
expect(postSpy.mock.calls[0][0]).toMatch(/\/api\/blueprints\/withdraw-local$/);
expect(delSpy).not.toHaveBeenCalled();
expect(DatabaseService.getInstance().getDeployment(bp.id, node.id)).toBeUndefined();
});
it('maps remote withdraw-local lock conflict to failed without fallback delete', async () => {
const node = seedRemoteNode();
const bp = seedBlueprint([node.id]);
const nodeObj = DatabaseService.getInstance().getNode(node.id)!;
const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!;
DatabaseService.getInstance().upsertDeployment({
blueprint_id: bp.id,
node_id: node.id,
status: 'active',
applied_revision: bpObj.revision,
});
vi.spyOn(axios, 'post').mockResolvedValue({
status: 409,
data: {
error: `${bpObj.name} is busy: another operation (update) is already in progress`,
code: 'stack_op_in_progress',
},
});
const delSpy = vi.spyOn(axios, 'delete');
const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj);
expect(result.status).toBe('failed');
expect(result.error).toMatch(/already in progress/);
expect(delSpy).not.toHaveBeenCalled();
expect(DatabaseService.getInstance().getDeployment(bp.id, node.id)?.status).toBe('failed');
});
it('fails closed when the remote lacks withdraw-local (404); no legacy delete', async () => {
const node = seedRemoteNode();
const bp = seedBlueprint([node.id]);
const nodeObj = DatabaseService.getInstance().getNode(node.id)!;
const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!;
DatabaseService.getInstance().upsertDeployment({
blueprint_id: bp.id,
node_id: node.id,
status: 'active',
applied_revision: bpObj.revision,
});
const postSpy = vi.spyOn(axios, 'post').mockResolvedValue({ status: 404, data: { error: 'Not Found' } });
const delSpy = vi.spyOn(axios, 'delete');
const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj);
expect(result.status).toBe('failed');
expect(result.error).toMatch(/withdraw-local|Upgrade/i);
expect(postSpy.mock.calls[0][0]).toMatch(/\/api\/blueprints\/withdraw-local$/);
expect(delSpy).not.toHaveBeenCalled();
expect(DatabaseService.getInstance().getDeployment(bp.id, node.id)?.status).toBe('failed');
});
it('does not clear hub role assignments when withdrawing a remote deployment', async () => {
const bcrypt = await import('bcrypt');
const db = DatabaseService.getInstance();
@@ -237,15 +285,14 @@ describe('BlueprintService remote deploy', () => {
user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: bpObj.name,
});
vi.spyOn(axios, 'get').mockResolvedValue({ status: 404, data: {} });
vi.spyOn(axios, 'post').mockResolvedValue({ status: 200, data: {} });
const delSpy = vi.spyOn(axios, 'delete').mockResolvedValue({ status: 200, data: {} });
vi.spyOn(axios, 'post').mockResolvedValue({ status: 200, data: { status: 'withdrawn' } });
const delSpy = vi.spyOn(axios, 'delete');
const rbacSpy = vi.spyOn(db, 'deleteRoleAssignmentsByResource');
const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj);
expect(result.status).toBe('withdrawn');
expect(delSpy.mock.calls[0][0]).toMatch(/\/api\/stacks\//);
expect(delSpy).not.toHaveBeenCalled();
expect(rbacSpy).not.toHaveBeenCalled();
expect(db.getAllRoleAssignments(userId)
.some((a) => a.resource_type === 'stack' && a.resource_id === bpObj.name)).toBe(true);
@@ -129,3 +129,71 @@ describe('POST /api/blueprints/apply-local (node-to-node atomic apply)', () => {
expect(StackOpLockService.getInstance().get(1, 'apply-local-busy')?.action).toBe('update');
});
});
describe('POST /api/blueprints/withdraw-local', () => {
let viewerCookie: string;
beforeAll(async () => {
const bcrypt = (await import('bcrypt')).default;
const passwordHash = await bcrypt.hash('bp-wd-viewer-pass', 1);
DatabaseService.getInstance().addUser({
username: 'bp-wd-viewer',
password_hash: passwordHash,
role: 'viewer',
});
const res = await request(app)
.post('/api/auth/login')
.send({ username: 'bp-wd-viewer', password: 'bp-wd-viewer-pass' });
const cookies = res.headers['set-cookie'] as string | string[];
viewerCookie = Array.isArray(cookies) ? cookies[0] : cookies;
});
it('rejects an invalid stack name', async () => {
const res = await request(app)
.post('/api/blueprints/withdraw-local')
.set('Cookie', adminCookie)
.send({ stackName: '../escape', blueprintId: 1 });
expect(res.status).toBe(400);
expect(res.body.error).toContain('Invalid stack name');
});
it('rejects a non-positive blueprintId', async () => {
const res = await request(app)
.post('/api/blueprints/withdraw-local')
.set('Cookie', adminCookie)
.send({ stackName: 'wd-local-stack', blueprintId: 0 });
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/blueprintId/i);
});
it('returns 403 for a viewer without stack:delete', async () => {
const res = await request(app)
.post('/api/blueprints/withdraw-local')
.set('Cookie', viewerCookie)
.send({ stackName: 'wd-local-stack', blueprintId: 1 });
expect(res.status).toBe(403);
});
it('returns 409 self_stack_protected for Sencho own stack', async () => {
const selfStackGuard = await import('../helpers/selfStackGuard');
vi.spyOn(selfStackGuard, 'refuseIfSelfStack').mockImplementation(async (_req, res) => {
res.status(409).json({ error: 'self', code: 'self_stack_protected' });
return true;
});
const res = await request(app)
.post('/api/blueprints/withdraw-local')
.set('Cookie', adminCookie)
.send({ stackName: 'sencho-self', blueprintId: 1 });
expect(res.status).toBe(409);
expect(res.body.code).toBe('self_stack_protected');
});
it('returns already_absent when the stack directory is missing', async () => {
const res = await request(app)
.post('/api/blueprints/withdraw-local')
.set('Cookie', adminCookie)
.send({ stackName: `wd-absent-${Date.now()}`, blueprintId: 42 });
expect(res.status).toBe(200);
expect(res.body.status).toBe('already_absent');
});
});
+21 -9
View File
@@ -399,7 +399,7 @@ describe('BlueprintReconciler developer-mode diagnostics', () => {
});
describe('BlueprintService per-stack lock', () => {
it('deploy under a free lock writes compose then marker, then deploys', async () => {
it('deploy under a free lock writes compose, cleans siblings, deploys, then writes the marker', async () => {
const nodeId = seedNode();
const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] });
const node = DatabaseService.getInstance().getNode(nodeId)!;
@@ -419,7 +419,7 @@ describe('BlueprintService per-stack lock', () => {
source: 'blueprint',
actor: 'system:blueprint',
});
// Compose is written first, then the marker, both before sibling cleanup and deploy.
// Compose is written first; the marker is deferred until after sibling cleanup and deploy.
expect(writeSpy).toHaveBeenCalledTimes(2);
expect(writeSpy.mock.calls[0][1]).toBe('compose.yaml');
expect(writeSpy.mock.calls[0][2]).toBe(bp.compose_content);
@@ -429,9 +429,9 @@ describe('BlueprintService per-stack lock', () => {
const [composeOrder, markerOrder] = writeSpy.mock.invocationCallOrder;
const [cleanupOrder] = cleanupSpy.mock.invocationCallOrder;
const [deployOrder] = deploySpy.mock.invocationCallOrder;
expect(composeOrder).toBeLessThan(markerOrder);
expect(markerOrder).toBeLessThan(cleanupOrder);
expect(composeOrder).toBeLessThan(cleanupOrder);
expect(cleanupOrder).toBeLessThan(deployOrder);
expect(deployOrder).toBeLessThan(markerOrder);
});
it('deploy skips, writes no stack files, and records failed when the stack lock is held', async () => {
@@ -476,7 +476,10 @@ describe('BlueprintService local withdraw clears stack-scoped role assignments',
async function arrangeLocalWithdraw() {
const db = DatabaseService.getInstance();
const composeDir = process.env.COMPOSE_DIR!;
const nodeId = seedNode();
db.getDb().prepare('UPDATE nodes SET compose_dir = ? WHERE id = ?').run(composeDir, nodeId);
const bp = seedBlueprint({ name: `rbac-wd-${counter}`, classification: 'stateless', nodeIds: [nodeId] });
const node = db.getNode(nodeId)!;
db.upsertDeployment({
@@ -499,7 +502,15 @@ describe('BlueprintService local withdraw clears stack-scoped role assignments',
user_id: userId, role: 'deployer', resource_type: 'node', resource_id: String(otherNodeId),
});
vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue(null);
// Matching on-disk marker so ownership passes; FS/down are stubbed.
const fs = await import('fs');
const path = await import('path');
const stackDir = path.join(composeDir, bp.name);
fs.mkdirSync(stackDir, { recursive: true });
fs.writeFileSync(
path.join(stackDir, '.blueprint.json'),
JSON.stringify({ blueprintId: bp.id, revision: bp.revision, lastApplied: Date.now() }),
);
const { ComposeService } = await import('../services/ComposeService');
const { FileSystemService } = await import('../services/FileSystemService');
vi.spyOn(ComposeService.prototype, 'downStack').mockResolvedValue(undefined);
@@ -522,15 +533,16 @@ describe('BlueprintService local withdraw clears stack-scoped role assignments',
db.deleteUser(userId);
});
it('clears the target assignment when deleteStack treats the directory as already absent (ENOENT)', async () => {
// Shared deletion always calls deleteStack; FileSystemService maps ENOENT to success.
it('clears the target assignment when the stack directory is already absent', async () => {
const { bp, node, userId, deleteStackSpy, db } = await arrangeLocalWithdraw();
deleteStackSpy.mockResolvedValue(undefined);
const fs = await import('fs');
const path = await import('path');
fs.rmSync(path.join(process.env.COMPOSE_DIR!, bp.name), { recursive: true, force: true });
const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node);
expect(outcome.status).toBe('withdrawn');
expect(deleteStackSpy).toHaveBeenCalledWith(bp.name);
expect(deleteStackSpy).not.toHaveBeenCalled();
expect(hasAssignment(userId, 'stack', bp.name)).toBe(false);
expect(db.getAllRoleAssignments(userId)).toHaveLength(2);
db.deleteUser(userId);
@@ -0,0 +1,64 @@
/**
* Legacy installations upgrade stack_update_cleanup_pending with nullable
* required_blueprint_id via maybeAddCol.
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
let tmpDir: string;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
});
afterAll(() => cleanupTestDb(tmpDir));
describe('stack_update_cleanup_pending required_blueprint_id migration', () => {
it('upgrades a legacy table without the column and preserves null for old rows', () => {
const db = DatabaseService.getInstance();
const raw = db.getDb();
// Simulate a pre-migration installation: drop the column if present, then
// re-add it the same way initSchema's maybeAddCol does.
try {
raw.exec('ALTER TABLE stack_update_cleanup_pending DROP COLUMN required_blueprint_id');
} catch {
// Column may already be absent in a hand-built fixture.
}
const now = Date.now();
raw.prepare(`
INSERT INTO stack_update_cleanup_pending (
id, node_id, stack_name, status, target_kind, rollback_tags_json,
override_paths_json, prune_volumes_requested, created_at, updated_at
) VALUES (?, ?, ?, 'prepared', 'local_socket', '[]', '[]', 0, ?, ?)
`).run('legacy-1', 1, 'legacy-stack', now, now);
try {
raw.exec('ALTER TABLE stack_update_cleanup_pending ADD COLUMN required_blueprint_id INTEGER');
} catch {
// Idempotent if a parallel path re-added it.
}
const legacy = db.getCleanupPending('legacy-1');
expect(legacy).toBeDefined();
expect(legacy?.required_blueprint_id ?? null).toBeNull();
db.insertCleanupPending({
id: 'owned-1',
node_id: 2,
stack_name: 'bp-stack',
status: 'prepared',
target_kind: 'local_socket',
rollback_tags_json: '[]',
override_paths_json: '[]',
prune_volumes_requested: 0,
required_blueprint_id: 42,
created_at: now,
updated_at: now,
});
expect(db.getCleanupPending('owned-1')?.required_blueprint_id).toBe(42);
});
});
@@ -89,6 +89,7 @@ describe('DeployedStackDeletionService ready transaction', () => {
rollback_tags_json: '[]',
override_paths_json: '[]',
prune_volumes_requested: 0,
required_blueprint_id: null,
created_at: now,
updated_at: now,
};
@@ -111,6 +112,7 @@ describe('DeployedStackDeletionService ready transaction', () => {
rollback_tags_json: '[]',
override_paths_json: '[]',
prune_volumes_requested: 0,
required_blueprint_id: null,
created_at: now,
updated_at: now,
});
@@ -129,6 +131,7 @@ describe('DeployedStackDeletionService ready transaction', () => {
rollback_tags_json: '[]',
override_paths_json: '[]',
prune_volumes_requested: 0,
required_blueprint_id: null,
created_at: now,
updated_at: now,
});
@@ -154,3 +157,39 @@ describe('overrideDeletionContainmentBase', () => {
});
});
describe('DeployedStackDeletionService blueprint ownership probe', () => {
it('returns failed (not name_conflict) when marker read fails with non-ENOENT I/O', async () => {
const { promises: fsPromises } = await import('fs');
const { vi } = await import('vitest');
const composeDir = process.env.COMPOSE_DIR!;
const stackName = `del-probe-${Date.now()}`;
const stackDir = path.join(composeDir, stackName);
await fsPromises.mkdir(stackDir, { recursive: true });
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n');
await fsPromises.writeFile(
path.join(stackDir, '.blueprint.json'),
JSON.stringify({ blueprintId: 7, revision: 1, lastApplied: 0 }),
);
const accessErr = Object.assign(new Error('EACCES'), { code: 'EACCES' });
const readSpy = vi.spyOn(fsPromises, 'readFile').mockRejectedValueOnce(accessErr);
const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({
nodeId: NODE,
stackName,
pruneVolumes: false,
actor: 'test',
requireBlueprintId: 7,
});
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.code).toBe('failed');
expect(result.error).toMatch(/EACCES|Failed to read|permission/i);
}
expect(readSpy).toHaveBeenCalled();
readSpy.mockRestore();
await fsPromises.rm(stackDir, { recursive: true, force: true });
});
});
@@ -95,9 +95,10 @@ describe('FileSystemService stack methods', () => {
expect(fileNames).toEqual(['.env', 'compose.yaml']);
});
it('marks compose.yaml and .env as protected', async () => {
it('marks compose.yaml, .env, and .blueprint.json as protected', async () => {
await fs.writeFile(path.join(stackDir, 'compose.yaml'), '');
await fs.writeFile(path.join(stackDir, '.env'), '');
await fs.writeFile(path.join(stackDir, '.blueprint.json'), '{}');
await fs.writeFile(path.join(stackDir, 'custom.conf'), '');
const service = FileSystemService.getInstance();
@@ -106,6 +107,7 @@ describe('FileSystemService stack methods', () => {
const byName = Object.fromEntries(entries.map(e => [e.name, e]));
expect(byName['compose.yaml'].isProtected).toBe(true);
expect(byName['.env'].isProtected).toBe(true);
expect(byName['.blueprint.json'].isProtected).toBe(true);
expect(byName['custom.conf'].isProtected).toBe(false);
});
@@ -1116,6 +1116,37 @@ describe('PUT /api/stacks/:stackName/files/content', () => {
expect(content).toBe('community-write');
});
it('blocks root trust file writes while a stack op lock is held', async () => {
const { StackOpLockService } = await import('../services/StackOpLockService');
StackOpLockService.getInstance().tryAcquire(1, STACK, 'deploy', 'admin');
try {
const composeRes = await request(app)
.put(`/api/stacks/${STACK}/files/content`)
.query({ path: 'compose.yaml' })
.set('Cookie', adminCookie)
.send({ content: 'services:\n app:\n image: nginx\n' });
expect(composeRes.status).toBe(409);
expect(composeRes.body.code).toBe('stack_op_in_progress');
const markerRes = await request(app)
.put(`/api/stacks/${STACK}/files/content`)
.query({ path: '.blueprint.json' })
.set('Cookie', adminCookie)
.send({ content: '{"blueprintId":1,"revision":1,"lastApplied":0}' });
expect(markerRes.status).toBe(409);
expect(markerRes.body.code).toBe('stack_op_in_progress');
const nestedRes = await request(app)
.put(`/api/stacks/${STACK}/files/content`)
.query({ path: 'config/app.conf' })
.set('Cookie', adminCookie)
.send({ content: 'ok' });
expect(nestedRes.status).toBe(204);
} finally {
StackOpLockService.getInstance().release(1, STACK);
}
});
it('returns 400 when content is not a string', async () => {
const res = await request(app)
.put(`/api/stacks/${STACK}/files/content`)
@@ -1931,6 +1962,16 @@ describe('protected stack files', () => {
expect(res.body.code).toBe('PROTECTED_FILE');
});
it('DELETE /files refuses .blueprint.json with 409 PROTECTED_FILE', async () => {
await fs.writeFile(path.join(stacksDir, STACK, '.blueprint.json'), '{"blueprintId":1,"revision":1}\n');
const res = await request(app)
.delete(`/api/stacks/${STACK}/files`)
.query({ path: '.blueprint.json' })
.set('Cookie', adminCookie);
expect(res.status).toBe(409);
expect(res.body.code).toBe('PROTECTED_FILE');
});
it('PATCH /files/rename refuses compose.yaml as source with 409 PROTECTED_FILE', async () => {
const res = await request(app)
.patch(`/api/stacks/${STACK}/files/rename`)
+28
View File
@@ -0,0 +1,28 @@
/**
* Dependency-neutral blueprint marker parse helpers.
* Used by BlueprintService and DeployedStackDeletionService without a service import cycle.
*/
export const BLUEPRINT_MARKER_FILENAME = '.blueprint.json';
export interface BlueprintMarker {
blueprintId: number;
revision: number;
lastApplied: number;
}
export function parseBlueprintMarker(content: string): BlueprintMarker | null {
try {
const parsed: unknown = JSON.parse(content);
if (!parsed || typeof parsed !== 'object') return null;
const obj = parsed as Record<string, unknown>;
if (typeof obj.blueprintId !== 'number' || typeof obj.revision !== 'number') return null;
return {
blueprintId: obj.blueprintId,
revision: obj.revision,
lastApplied: typeof obj.lastApplied === 'number' ? obj.lastApplied : 0,
};
} catch {
return null;
}
}
+79 -8
View File
@@ -7,7 +7,9 @@ import {
type BlueprintSelector,
type DriftMode,
} from '../services/DatabaseService';
import { BlueprintService } from '../services/BlueprintService';
import { BlueprintService, BlueprintNameConflictError, BlueprintOwnershipProbeError } from '../services/BlueprintService';
import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService';
import { refuseIfSelfStack } from '../helpers/selfStackGuard';
import {
BlueprintReconciler,
messageForConfirmedOutcomes,
@@ -317,12 +319,9 @@ blueprintsRouter.delete('/:id', async (req: Request, res: Response): Promise<voi
return;
}
}
// Best-effort cleanup before delete: withdraw exactly the rows a stateful delete would
// block, i.e. stacks Sencho deployed and still owns (last_deployed_at set, and neither a
// name_conflict nor an already-withdrawn row). Never run the withdraw primitive for a
// never-deployed or unmanaged row: withdrawFromNode proceeds on a missing marker and would
// down/delete a same-name stack Sencho does not own. The blueprint-delete cascade removes
// the rows the loop skips.
// Withdraw owned deployments before delete. Fail closed: if any withdraw does not
// complete as withdrawn, keep the blueprint (and its deployment rows) so the operator
// can retry. Never orphan a live stack by deleting the only control-plane record.
const nodes = DatabaseService.getInstance().getNodes();
const deployments = DatabaseService.getInstance().listDeployments(id);
for (const dep of deployments) {
@@ -330,9 +329,24 @@ blueprintsRouter.delete('/:id', async (req: Request, res: Response): Promise<voi
const node = nodes.find(n => n.id === dep.node_id);
if (!node) continue;
try {
await BlueprintService.getInstance().withdrawFromNode(blueprint, node);
const outcome = await BlueprintService.getInstance().withdrawFromNode(blueprint, node);
if (outcome.status !== 'withdrawn') {
res.status(409).json({
error: `Cannot delete blueprint: withdraw on node "${node.name}" ended as ${outcome.status}. Resolve that deployment, then retry.`,
code: 'withdraw_failed_blocking_delete',
nodeId: node.id,
withdrawStatus: outcome.status,
});
return;
}
} catch (err) {
console.warn(`[Blueprints] Pre-delete withdraw failed for blueprint ${id} on node ${node.id}:`, err);
res.status(409).json({
error: `Cannot delete blueprint: withdraw on node "${node.name}" failed. Resolve that deployment, then retry.`,
code: 'withdraw_failed_blocking_delete',
nodeId: node.id,
});
return;
}
}
DatabaseService.getInstance().deleteBlueprint(id);
@@ -384,11 +398,68 @@ blueprintsRouter.post('/apply-local', async (req: Request, res: Response): Promi
}
res.json({ deployed: true });
} catch (error) {
if (error instanceof BlueprintNameConflictError) {
res.status(409).json({ error: error.message, code: 'name_conflict' });
return;
}
if (error instanceof BlueprintOwnershipProbeError) {
console.error('[Blueprints] apply-local ownership probe failed:', sanitizeForLog(error.message));
res.status(500).json({ error: error.message });
return;
}
console.error('[Blueprints] apply-local error:', sanitizeForLog(getErrorMessage(error, 'apply failed')));
res.status(500).json({ error: getErrorMessage(error, 'Blueprint apply failed') });
}
});
// Node-to-node atomic blueprint withdraw. Ownership is validated under the
// delete lock on this node. Requires stack:delete and refuses Sencho's own stack.
blueprintsRouter.post('/withdraw-local', async (req: Request, res: Response): Promise<void> => {
const body = (req.body ?? {}) as { stackName?: unknown; blueprintId?: unknown };
if (typeof body.stackName !== 'string' || !isValidStackName(body.stackName)) {
res.status(400).json({ error: 'Invalid stack name' });
return;
}
if (typeof body.blueprintId !== 'number' || !Number.isInteger(body.blueprintId) || body.blueprintId <= 0) {
res.status(400).json({ error: 'blueprintId must be a positive integer' });
return;
}
if (!requirePermission(req, res, 'stack:delete', 'stack', body.stackName)) return;
if (await refuseIfSelfStack(req, res, body.stackName)) return;
try {
const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({
nodeId: req.nodeId,
stackName: body.stackName,
pruneVolumes: false,
actor: req.user?.username ?? 'system:blueprint',
requireBlueprintId: body.blueprintId,
});
if (result.ok) {
res.json({
status: result.status === 'already_absent' ? 'already_absent' : 'withdrawn',
});
return;
}
if (result.code === 'name_conflict') {
res.status(409).json({ error: result.error, code: 'name_conflict' });
return;
}
if (result.code === 'lock_conflict') {
res.status(409).json({
error: result.error,
code: 'stack_op_in_progress',
inProgress: { action: result.existingAction },
});
return;
}
res.status(500).json({ error: result.error });
} catch (error) {
console.error('[Blueprints] withdraw-local error:', sanitizeForLog(getErrorMessage(error, 'withdraw failed')));
res.status(500).json({ error: getErrorMessage(error, 'Blueprint withdraw failed') });
}
});
blueprintsRouter.post('/:id/apply', async (req: Request, res: Response): Promise<void> => {
if (!requireAdmin(req, res)) return;
const id = parseIntParam(req, res, 'id');
+39
View File
@@ -135,6 +135,40 @@ function releaseStackOpLock(req: Request, stackName: string): void {
StackOpLockService.getInstance().release(req.nodeId, stackName);
}
/** Root compose + blueprint marker on the stack-source root must not change while a lifecycle op holds the stack lock. */
const STACK_OP_LOCKED_ROOT_TRUST_FILES = new Set([
'compose.yaml',
'compose.yml',
'docker-compose.yaml',
'docker-compose.yml',
'.blueprint.json',
]);
function rejectIfStackOpBlocksRootTrustFileWrite(
req: Request,
res: Response,
stackName: string,
relPath: string,
root: StackFileRoot,
): boolean {
if (root.kind !== 'stack-source') return false;
if (relPath.includes('/')) return false;
const base = relPath.toLowerCase();
if (!STACK_OP_LOCKED_ROOT_TRUST_FILES.has(base)) return false;
const existing = StackOpLockService.getInstance().get(req.nodeId, stackName);
if (!existing) return false;
res.status(409).json({
error: `${stackName} is busy: another operation (${existing.action}) is already in progress`,
code: 'stack_op_in_progress',
inProgress: {
action: existing.action,
startedAt: existing.startedAt,
user: existing.user,
},
});
return true;
}
function stackFileEtag(mtimeMs: number): string {
return `W/"${Math.floor(mtimeMs)}"`;
}
@@ -2884,6 +2918,10 @@ stacksRouter.post(
return res.status(400).json({ error: 'Invalid filename' });
}
const targetRelPath = relPath ? `${relPath}/${originalName}` : originalName;
if (rejectIfStackOpBlocksRootTrustFileWrite(req, res, stackName, targetRelPath, root)) {
await cleanupUploadTemp(req);
return;
}
const overwrite = String(req.query.overwrite) === '1';
// The multer wrapper stashed the route-entry timestamp on the request so
// the success path and the rejection paths share one window. Fall back to
@@ -2987,6 +3025,7 @@ stacksRouter.put('/:stackName/files/content', async (req: Request, res: Response
const expectedVersion = req.header('if-match') || undefined;
const root = await resolveRootForOp(req, res, stackName, 'write');
if (!root) return;
if (rejectIfStackOpBlocksRootTrustFileWrite(req, res, stackName, relPath, root)) return;
const startedAt = Date.now();
logFileDiag('write start', { stackName, relPath, nodeId: req.nodeId, bytes: Buffer.byteLength(content, 'utf-8'), hasIfMatch: expectedVersion !== undefined, rootKind: root.kind });
try {
+244 -181
View File
@@ -19,8 +19,12 @@ import { assertPolicyGateAllows, buildSystemPolicyGateOptions, describePolicyBlo
import { enforcePolicyForImageRefs } from './PolicyEnforcement';
import { BlueprintAnalyzer } from './BlueprintAnalyzer';
import { sanitizeForLog } from '../utils/safeLog';
const MARKER_FILENAME = '.blueprint.json';
import { isPathWithinBase } from '../utils/validation';
import {
BLUEPRINT_MARKER_FILENAME,
parseBlueprintMarker,
type BlueprintMarker,
} from '../helpers/blueprintMarker';
/** On-disk compose name for Blueprint applies. Must match createStack scaffold and Sencho discovery priority. */
const COMPOSE_FILENAME = 'compose.yaml';
const REMOTE_HTTP_TIMEOUT_MS = 30_000;
@@ -41,10 +45,32 @@ function diagnosticLog(message: string, fields: Record<string, string | number |
console.info(`[BlueprintService:diag] ${message}`, safeFields);
}
export interface BlueprintMarker {
blueprintId: number;
revision: number;
lastApplied: number;
export type { BlueprintMarker };
export class BlueprintNameConflictError extends Error {
readonly code = 'name_conflict' as const;
constructor(message: string) {
super(message);
this.name = 'BlueprintNameConflictError';
}
}
/** Thrown when a remote node lacks the atomic apply/withdraw endpoints. */
export class BlueprintRemoteUpgradeRequiredError extends Error {
readonly code = 'remote_upgrade_required' as const;
constructor(message: string) {
super(message);
this.name = 'BlueprintRemoteUpgradeRequiredError';
}
}
/** Thrown when ownership cannot be verified (non-ENOENT I/O or remote probe failure). */
export class BlueprintOwnershipProbeError extends Error {
readonly code = 'ownership_probe_failed' as const;
constructor(message: string) {
super(message);
this.name = 'BlueprintOwnershipProbeError';
}
}
export interface DeployOutcome {
@@ -52,11 +78,17 @@ export interface DeployOutcome {
error?: string;
}
type LocalMarkerRead =
| { kind: 'missing' }
| { kind: 'present'; marker: BlueprintMarker }
| { kind: 'failed'; error: string };
/**
* BlueprintService is the orchestration layer between the reconciler and the
* concrete deploy/withdraw primitives. It owns:
* - per-target marker-file management (writes, reads, validates ownership)
* - name-conflict guard (refuses to touch a stack directory missing the marker)
* - name-conflict guard (refuses apply/withdraw when the directory lacks a matching
* `.blueprint.json` for this blueprint ID)
* - local deploy via ComposeService + FileSystemService
* - remote deploy via direct HTTP calls to the remote Sencho instance
* - per-(blueprint,node) concurrency lock so overlapping ticks don't collide
@@ -128,15 +160,12 @@ export class BlueprintService {
async readMarker(blueprintName: string, node: Node): Promise<BlueprintMarker | null> {
try {
if (node.type === 'local') {
const baseDir = NodeRegistry.getInstance().getComposeDir(node.id);
const markerPath = path.resolve(baseDir, blueprintName, MARKER_FILENAME);
if (!markerPath.startsWith(path.resolve(baseDir))) return null;
const content = await fsPromises.readFile(markerPath, 'utf-8');
return BlueprintService.parseMarker(content);
const markerRead = await this.readLocalMarkerFromDisk(node.id, blueprintName);
return markerRead.kind === 'present' ? markerRead.marker : null;
}
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) return null;
const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/files/content?path=${encodeURIComponent(MARKER_FILENAME)}`;
const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/files/content?path=${encodeURIComponent(BLUEPRINT_MARKER_FILENAME)}`;
const res = await axios.get(url, {
headers: this.remoteHeaders(target.apiToken),
timeout: REMOTE_HTTP_TIMEOUT_MS,
@@ -146,7 +175,7 @@ export class BlueprintService {
const body = res.data;
const content = typeof body === 'string' ? body : (typeof body?.content === 'string' ? body.content : null);
if (content == null) return null;
return BlueprintService.parseMarker(content);
return parseBlueprintMarker(content);
} catch {
return null;
}
@@ -154,47 +183,77 @@ export class BlueprintService {
/**
* Returns true when a stack directory by this name exists on the target
* node but does not carry our marker file. The reconciler must not
* deploy in that case: there is a real user-authored stack with the
* same name and we must not overwrite it.
* node and the on-disk marker is missing, malformed, or references a
* different blueprint ID. Throws BlueprintOwnershipProbeError when the
* directory or marker cannot be probed (non-ENOENT I/O or remote list failure).
*/
async hasNameConflict(blueprintName: string, node: Node): Promise<boolean> {
try {
if (node.type === 'local') {
const baseDir = NodeRegistry.getInstance().getComposeDir(node.id);
const stackDir = path.resolve(baseDir, blueprintName);
if (!stackDir.startsWith(path.resolve(baseDir))) return true;
try {
const stat = await fsPromises.stat(stackDir);
if (!stat.isDirectory()) return false;
} catch {
return false; // directory doesn't exist → no conflict
}
const markerPath = path.join(stackDir, MARKER_FILENAME);
try {
await fsPromises.stat(markerPath);
return false; // marker present → ours
} catch {
return true; // directory exists but no marker → conflict
}
async hasNameConflict(blueprintName: string, node: Node, blueprintId: number): Promise<boolean> {
if (node.type === 'local') {
const baseDir = NodeRegistry.getInstance().getComposeDir(node.id);
const stackDir = path.resolve(baseDir, blueprintName);
if (!isPathWithinBase(stackDir, baseDir)) return true;
try {
const stat = await fsPromises.stat(stackDir);
if (!stat.isDirectory()) return false;
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') return false;
throw BlueprintService.ownershipProbeError(blueprintName, BlueprintService.formatError(err));
}
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) return false;
const baseUrl = target.apiUrl.replace(/\/$/, '');
const listUrl = `${baseUrl}/api/stacks`;
const listRes = await axios.get(listUrl, {
const markerRead = await this.readLocalMarkerFromDisk(node.id, blueprintName);
if (markerRead.kind === 'failed') {
throw BlueprintService.ownershipProbeError(blueprintName, markerRead.error);
}
return markerRead.kind === 'missing' || markerRead.marker.blueprintId !== blueprintId;
}
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) {
throw new BlueprintOwnershipProbeError(
`Cannot verify stack ownership on remote node "${node.name}": no proxy target configured`,
);
}
const baseUrl = target.apiUrl.replace(/\/$/, '');
let listRes;
try {
listRes = await axios.get(`${baseUrl}/api/stacks`, {
headers: this.remoteHeaders(target.apiToken),
timeout: REMOTE_HTTP_TIMEOUT_MS,
validateStatus: () => true,
});
if (listRes.status !== 200) return false;
const stacks = Array.isArray(listRes.data) ? listRes.data as Array<{ name?: string }> : [];
const exists = stacks.some(s => s?.name === blueprintName);
if (!exists) return false;
const marker = await this.readMarker(blueprintName, node);
return marker == null;
} catch {
return false;
} catch (err) {
throw new BlueprintOwnershipProbeError(
`Cannot verify stack ownership on remote node "${node.name}": ${BlueprintService.formatError(err)}`,
);
}
if (listRes.status !== 200) {
throw new BlueprintOwnershipProbeError(
`Cannot verify stack ownership on remote node "${node.name}" (HTTP ${listRes.status})`,
);
}
const stacks = Array.isArray(listRes.data) ? listRes.data as Array<{ name?: string }> : [];
const exists = stacks.some(s => s?.name === blueprintName);
if (!exists) return false;
const marker = await this.readMarker(blueprintName, node);
return marker == null || marker.blueprintId !== blueprintId;
}
/** Read and parse a local on-disk marker without going through the remote HTTP path. */
private async readLocalMarkerFromDisk(nodeId: number, stackName: string): Promise<LocalMarkerRead> {
try {
// Canonical js/path-injection barrier inline with the read sink.
const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId));
const safePath = path.resolve(baseResolved, stackName, BLUEPRINT_MARKER_FILENAME);
if (!safePath.startsWith(baseResolved + path.sep)) {
return { kind: 'failed', error: 'Invalid stack path for blueprint marker' };
}
const content = await fsPromises.readFile(safePath, 'utf-8');
const marker = parseBlueprintMarker(content);
if (!marker) return { kind: 'missing' };
return { kind: 'present', marker };
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') return { kind: 'missing' };
return { kind: 'failed', error: BlueprintService.formatError(err) };
}
}
@@ -222,7 +281,7 @@ export class BlueprintService {
});
try {
this.setStatus(blueprint.id, node.id, 'deploying');
if (await this.hasNameConflict(blueprint.name, node)) {
if (await this.hasNameConflict(blueprint.name, node, blueprint.id)) {
this.setStatus(blueprint.id, node.id, 'name_conflict', {
last_error: `A stack named "${blueprint.name}" already exists on this node and is not managed by Sencho.`,
});
@@ -249,6 +308,12 @@ export class BlueprintService {
sanitizeForLog(blueprint.name), node.id, Date.now() - started);
return { status: 'active' };
} catch (err) {
if (err instanceof BlueprintNameConflictError) {
this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: err.message });
console.warn('[BlueprintService] deploy name conflict blueprint=%s node=%s durationMs=%s',
sanitizeForLog(blueprint.name), node.id, Date.now() - started);
return { status: 'name_conflict', error: 'name_conflict' };
}
const message = BlueprintService.formatError(err);
this.setStatus(blueprint.id, node.id, 'failed', { last_error: message });
console.error('[BlueprintService] deploy failed blueprint=%s node=%s durationMs=%s error=%s',
@@ -280,20 +345,15 @@ export class BlueprintService {
});
try {
this.setStatus(blueprint.id, node.id, 'withdrawing');
// Refuse to withdraw a directory we do not own
const marker = await this.readMarker(blueprint.name, node);
if (marker && marker.blueprintId !== blueprint.id) {
this.setStatus(blueprint.id, node.id, 'name_conflict', {
last_error: `Marker on this node points to a different blueprint (id=${marker.blueprintId}); refusing to withdraw.`,
});
return { status: 'name_conflict' };
}
// Ownership is validated on the node that owns the stack, inside the delete lock.
if (node.type === 'local') {
diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'local' });
await this.withdrawLocal(blueprint, node);
const localOutcome = await this.withdrawLocal(blueprint, node);
if (localOutcome.status !== 'withdrawn') return localOutcome;
} else {
diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'remote' });
await this.withdrawRemote(blueprint, node);
const remoteOutcome = await this.withdrawRemote(blueprint, node);
if (remoteOutcome.status !== 'withdrawn') return remoteOutcome;
}
DatabaseService.getInstance().deleteDeployment(blueprint.id, node.id);
console.info('[BlueprintService] withdraw complete blueprint=%s node=%s durationMs=%s',
@@ -382,15 +442,22 @@ export class BlueprintService {
// ---- local primitives ----
/** Returns whether the stack directory exists. Throws on non-ENOENT I/O. */
private async stackDirExists(nodeId: number, blueprintName: string): Promise<boolean> {
const baseDir = NodeRegistry.getInstance().getComposeDir(nodeId);
const stackDir = path.resolve(baseDir, blueprintName);
if (!stackDir.startsWith(path.resolve(baseDir))) return false;
if (!isPathWithinBase(stackDir, baseDir)) {
throw new BlueprintOwnershipProbeError(`Invalid stack path for "${blueprintName}"`);
}
try {
const stat = await fsPromises.stat(stackDir);
return stat.isDirectory();
} catch {
return false;
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') return false;
throw new BlueprintOwnershipProbeError(
`Cannot access stack directory "${blueprintName}": ${BlueprintService.formatError(err)}`,
);
}
}
@@ -423,14 +490,14 @@ export class BlueprintService {
}
/**
* Create the stack if needed, write the compose and marker files, run the
* deploy policy gate, and deploy, all under the per-stack operation lock so
* none of it can race a manual deploy/update/rollback/backup on the same
* stack and node. Runs on the node that owns the stack: deployLocal calls it
* for the hub's own node, and the /api/blueprints/apply-local route calls it
* on a remote node receiving a blueprint apply from its hub (so the file
* writes hold the remote's lock, not just the deploy). On lock conflict
* nothing is written and { ran: false } is returned.
* Create the stack if needed, write the compose file, run the deploy policy
* gate and deploy, then write the marker, all under the per-stack operation
* lock. The marker is written only after a successful deploy so a failed
* apply cannot claim an applied revision that never ran. Runs on the node
* that owns the stack: deployLocal calls it for the hub's own node, and the
* /api/blueprints/apply-local route calls it on a remote node receiving a
* blueprint apply from its hub. On lock conflict nothing is written and
* { ran: false } is returned.
*/
async applyLocalUnderLock(
nodeId: number,
@@ -439,47 +506,83 @@ export class BlueprintService {
markerContent: string,
auditPath: string,
): Promise<{ ran: true } | { ran: false; existingAction: StackOpAction }> {
const expected = parseBlueprintMarker(markerContent);
if (!expected) {
throw new Error('Invalid blueprint marker');
}
const fs = FileSystemService.getInstance(nodeId);
const lock = await StackOpLockService.getInstance().runExclusive(
nodeId, stackName, 'deploy', 'system',
async () => {
if (!(await this.stackDirExists(nodeId, stackName))) {
let createdStack = false;
if (await this.stackDirExists(nodeId, stackName)) {
const existing = await this.readLocalMarkerFromDisk(nodeId, stackName);
if (existing.kind === 'failed') {
throw new BlueprintOwnershipProbeError(
`Cannot verify ownership of stack "${stackName}": ${existing.error}`,
);
}
if (existing.kind === 'missing' || existing.marker.blueprintId !== expected.blueprintId) {
throw new BlueprintNameConflictError(
`A stack named "${stackName}" already exists on this node and is not managed by this blueprint.`,
);
}
} else {
await fs.createStack(stackName);
createdStack = true;
}
await fs.writeStackFile(stackName, COMPOSE_FILENAME, composeContent);
await fs.writeStackFile(stackName, MARKER_FILENAME, markerContent);
// Clear lower-priority compose siblings so discovery cannot shadow compose.yaml.
// Local + modern apply-local only; legacy remote has no sibling DELETE.
await fs.removeAlternateRootComposeFiles(stackName);
await assertPolicyGateAllows(
stackName,
nodeId,
buildSystemPolicyGateOptions('blueprint', { auditPath }),
);
await ComposeService.getInstance(nodeId).deployStack(
stackName,
undefined,
false,
{ source: 'blueprint', actor: 'system:blueprint' },
);
try {
await assertPolicyGateAllows(
stackName,
nodeId,
buildSystemPolicyGateOptions('blueprint', { auditPath }),
);
await ComposeService.getInstance(nodeId).deployStack(
stackName,
undefined,
false,
{ source: 'blueprint', actor: 'system:blueprint' },
);
await fs.writeStackFile(stackName, BLUEPRINT_MARKER_FILENAME, markerContent);
} catch (err) {
if (createdStack) {
try {
await fs.deleteStack(stackName);
} catch (cleanupErr) {
console.warn(
'[BlueprintService] Failed to roll back newly created stack "%s" after apply error: %s',
sanitizeForLog(stackName),
sanitizeForLog(BlueprintService.formatError(cleanupErr)),
);
}
}
throw err;
}
},
);
return lock.ran ? { ran: true } : { ran: false, existingAction: lock.existing.action };
}
private async withdrawLocal(blueprint: Blueprint, node: Node): Promise<void> {
private async withdrawLocal(blueprint: Blueprint, node: Node): Promise<DeployOutcome> {
const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({
nodeId: node.id,
stackName: blueprint.name,
pruneVolumes: false,
actor: 'system:blueprint',
requireBlueprintId: blueprint.id,
});
if (!result.ok) {
if (result.code === 'lock_conflict') {
throw new Error(result.error);
}
throw new Error(result.error);
if (result.ok) {
return { status: 'withdrawn' };
}
if (result.code === 'name_conflict') {
this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: result.error });
return { status: 'name_conflict' };
}
this.setStatus(blueprint.id, node.id, 'failed', { last_error: result.error });
return { status: 'failed', error: result.error };
}
// ---- remote primitives ----
@@ -500,10 +603,7 @@ export class BlueprintService {
const baseUrl = target.apiUrl.replace(/\/$/, '');
const headers = this.remoteHeaders(target.apiToken);
// Atomic apply: the remote runs create + write compose/marker + deploy
// under its own per-stack lock, so the file writes cannot race a manual
// operation on that node. Older nodes without this route answer 404; we
// fall back to the legacy multi-call flow there (not lock-atomic).
// Atomic apply: the remote validates ownership and writes under its stack lock.
const res = await axios.post(
`${baseUrl}/api/blueprints/apply-local`,
{
@@ -514,11 +614,17 @@ export class BlueprintService {
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (res.status === 404) {
console.warn(`[BlueprintService] remote node ${node.id} lacks /api/blueprints/apply-local; using legacy non-atomic apply`);
await this.deployRemoteLegacy(blueprint, node, marker);
return;
throw new BlueprintRemoteUpgradeRequiredError(
`Remote node "${node.name}" does not support atomic blueprint apply (/api/blueprints/apply-local). Upgrade that Sencho instance, then retry.`,
);
}
if (res.status === 409) {
if (BlueprintService.extractApiCode(res.data) === 'name_conflict') {
throw new BlueprintNameConflictError(
BlueprintService.extractApiError(res.data)
|| `A stack named "${blueprint.name}" already exists on this node and is not managed by this blueprint.`,
);
}
throw new Error(`blueprint apply skipped: ${BlueprintService.extractApiError(res.data) || 'another operation is already in progress'}`);
}
if (res.status >= 400) {
@@ -526,105 +632,62 @@ export class BlueprintService {
}
}
/**
* Legacy remote apply for nodes that predate /api/blueprints/apply-local:
* create, write compose, write marker, deploy as separate calls. The remote
* deploy locks, but the preceding file writes do not, so this is not atomic
* against a concurrent manual operation on that node. Kept only as a
* compatibility fallback.
*/
private async deployRemoteLegacy(blueprint: Blueprint, node: Node, marker: BlueprintMarker): Promise<void> {
private async withdrawRemote(blueprint: Blueprint, node: Node): Promise<DeployOutcome> {
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`);
const baseUrl = target.apiUrl.replace(/\/$/, '');
const headers = this.remoteHeaders(target.apiToken);
// 1. Ensure stack exists. POST returns 409 when already exists; we treat that as success.
const createRes = await axios.post(`${baseUrl}/api/stacks`,
{ stackName: blueprint.name },
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (createRes.status >= 400 && createRes.status !== 409) {
throw new Error(`create stack: HTTP ${createRes.status} ${BlueprintService.extractApiError(createRes.data)}`);
}
// 2. Write the compose file
await this.remotePutFile(baseUrl, headers, blueprint.name, COMPOSE_FILENAME, blueprint.compose_content);
// 3. Write the marker (last so a partial failure leaves us in name_conflict-recoverable state)
await this.remotePutFile(baseUrl, headers, blueprint.name, MARKER_FILENAME, JSON.stringify(marker, null, 2));
// 4. Deploy
const deployRes = await axios.post(
`${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}/deploy`,
{},
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (deployRes.status >= 400) {
throw new Error(`deploy: HTTP ${deployRes.status} ${BlueprintService.extractApiError(deployRes.data)}`);
}
}
private async withdrawRemote(blueprint: Blueprint, node: Node): Promise<void> {
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`);
const baseUrl = target.apiUrl.replace(/\/$/, '');
const headers = this.remoteHeaders(target.apiToken);
// down (best-effort)
let res;
try {
await axios.post(
`${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}/down`,
{},
res = await axios.post(
`${baseUrl}/api/blueprints/withdraw-local`,
{ stackName: blueprint.name, blueprintId: blueprint.id },
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
} catch (err) {
console.warn(`[BlueprintService] remote down failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(err)}`);
const message = BlueprintService.formatError(err);
this.setStatus(blueprint.id, node.id, 'failed', { last_error: message });
return { status: 'failed', error: message };
}
// delete the stack directory entirely
const delRes = await axios.delete(
`${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}`,
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (delRes.status >= 400 && delRes.status !== 404) {
throw new Error(`remote delete: HTTP ${delRes.status} ${BlueprintService.extractApiError(delRes.data)}`);
if (res.status === 404) {
throw new BlueprintRemoteUpgradeRequiredError(
`Remote node "${node.name}" does not support atomic blueprint withdraw (/api/blueprints/withdraw-local). Upgrade that Sencho instance, then retry.`,
);
}
}
private async remotePutFile(
baseUrl: string,
headers: Record<string, string>,
stackName: string,
relPath: string,
content: string,
): Promise<void> {
const url = `${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/files/content?path=${encodeURIComponent(relPath)}`;
const res = await axios.put(url,
{ content },
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (res.status >= 400) {
throw new Error(`PUT ${relPath}: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`);
if (res.status === 200) {
return { status: 'withdrawn' };
}
if (res.status === 409) {
const error = BlueprintService.extractApiError(res.data) || 'withdraw refused';
if (BlueprintService.extractApiCode(res.data) === 'name_conflict') {
this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: error });
return { status: 'name_conflict' };
}
// stack_op_in_progress and any other 409: match local withdraw lock-conflict → failed
this.setStatus(blueprint.id, node.id, 'failed', { last_error: error });
return { status: 'failed', error };
}
const message = `blueprint withdraw: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`;
this.setStatus(blueprint.id, node.id, 'failed', { last_error: message });
return { status: 'failed', error: message };
}
static parseMarker(content: string): BlueprintMarker | null {
try {
const parsed = JSON.parse(content);
if (parsed && typeof parsed === 'object'
&& typeof parsed.blueprintId === 'number'
&& typeof parsed.revision === 'number') {
return {
blueprintId: parsed.blueprintId,
revision: parsed.revision,
lastApplied: typeof parsed.lastApplied === 'number' ? parsed.lastApplied : 0,
};
}
} catch {
// fall through
}
return null;
return parseBlueprintMarker(content);
}
private static ownershipProbeError(blueprintName: string, detail: string): BlueprintOwnershipProbeError {
return new BlueprintOwnershipProbeError(
`Cannot verify stack ownership for "${blueprintName}": ${detail}`,
);
}
static extractApiCode(body: unknown): string {
if (!body || typeof body !== 'object') return '';
const code = (body as Record<string, unknown>).code;
return typeof code === 'string' ? code : '';
}
static formatError(err: unknown): string {
+7 -3
View File
@@ -251,6 +251,8 @@ export interface StackUpdateCleanupPendingRow {
rollback_tags_json: string;
override_paths_json: string;
prune_volumes_requested: number;
/** Blueprint ID that authorized this deletion; null for manual deletes. */
required_blueprint_id: number | null;
created_at: number;
updated_at: number;
}
@@ -1828,6 +1830,7 @@ export class DatabaseService {
`);
maybeAddCol('stack_update_recovery_generations', 'artifacts_retired', 'INTEGER NOT NULL DEFAULT 0');
maybeAddCol('stack_update_cleanup_pending', 'required_blueprint_id', 'INTEGER');
// Distributed API model columns
maybeAddCol('nodes', 'api_url', "TEXT DEFAULT ''");
@@ -3933,12 +3936,12 @@ export class DatabaseService {
this.db.prepare(
`INSERT INTO stack_update_cleanup_pending (
id, node_id, stack_name, status, target_kind, rollback_tags_json,
override_paths_json, prune_volumes_requested, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
override_paths_json, prune_volumes_requested, required_blueprint_id, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
row.id, row.node_id, row.stack_name, row.status, row.target_kind,
row.rollback_tags_json, row.override_paths_json, row.prune_volumes_requested,
row.created_at, row.updated_at,
row.required_blueprint_id, row.created_at, row.updated_at,
);
}
@@ -4409,6 +4412,7 @@ export class DatabaseService {
rollback_tags_json: JSON.stringify(localCleanup.tags),
override_paths_json: JSON.stringify(localCleanup.overridePaths),
prune_volumes_requested: 0,
required_blueprint_id: null,
created_at: now,
updated_at: now,
});
@@ -23,6 +23,10 @@ import { StackOpLockService, stackOpSkipMessage } from './StackOpLockService';
import { getErrorMessage } from '../utils/errors';
import { sanitizeForLog } from '../utils/safeLog';
import { isPathWithinBase, isValidStackName } from '../utils/validation';
import {
BLUEPRINT_MARKER_FILENAME,
parseBlueprintMarker,
} from '../helpers/blueprintMarker';
/**
* Directory that may contain recovery override files for a tombstone sweep.
@@ -46,13 +50,30 @@ export interface DeleteDeployedStackInput {
stackName: string;
pruneVolumes: boolean;
actor: string;
/** When set, deletion requires an on-disk .blueprint.json matching this blueprint ID. */
requireBlueprintId?: number;
/** When true, skip acquiring a new lock (caller already holds delete via continuation). */
continuationIntentId?: string;
}
export type DeleteDeployedStackResult =
| { ok: true }
| { ok: false; code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed'; error: string; existingAction?: string };
| { ok: true; status: 'deleted' | 'already_absent' }
| {
ok: false;
code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed' | 'name_conflict' | 'failed';
error: string;
existingAction?: string;
};
type DirProbe = { kind: 'absent' } | { kind: 'present' } | { kind: 'error'; error: string };
type MarkerProbe =
| { kind: 'match' }
| { kind: 'name_conflict'; error: string }
| { kind: 'failed'; error: string };
function blueprintMarkerMismatchError(stackName: string): string {
return `Stack "${stackName}" exists without a matching blueprint marker; refusing to withdraw.`;
}
function collectArtifactsFromGenerations(
generations: Array<{ override_path: string | null; services_json: string }>,
@@ -100,6 +121,51 @@ function parseJsonStringArray(raw: string): string[] {
}
}
async function probeStackDirectory(nodeId: number, stackName: string): Promise<DirProbe> {
// Canonical js/path-injection barrier inline with the stat sink.
const baseResolved = path.resolve(FileSystemService.getInstance(nodeId).getBaseDir());
const safePath = path.resolve(baseResolved, stackName);
if (!safePath.startsWith(baseResolved + path.sep)) {
return { kind: 'error', error: 'Invalid stack path' };
}
try {
const stat = await fs.stat(safePath);
return stat.isDirectory() ? { kind: 'present' } : { kind: 'absent' };
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code === 'ENOENT') return { kind: 'absent' };
return { kind: 'error', error: getErrorMessage(error, 'Failed to access stack directory') };
}
}
async function probeBlueprintMarkerOwnership(
nodeId: number,
stackName: string,
requireBlueprintId: number,
): Promise<MarkerProbe> {
// Canonical js/path-injection barrier inline with the read sink.
const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId));
const safePath = path.resolve(baseResolved, stackName, BLUEPRINT_MARKER_FILENAME);
if (!safePath.startsWith(baseResolved + path.sep)) {
return { kind: 'failed', error: 'Invalid stack path for blueprint marker' };
}
try {
const content = await fs.readFile(safePath, 'utf-8');
const marker = parseBlueprintMarker(content);
if (!marker || marker.blueprintId !== requireBlueprintId) {
return { kind: 'name_conflict', error: blueprintMarkerMismatchError(stackName) };
}
return { kind: 'match' };
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code === 'ENOENT') {
return { kind: 'name_conflict', error: blueprintMarkerMismatchError(stackName) };
}
return { kind: 'failed', error: getErrorMessage(error, 'Failed to read blueprint marker') };
}
}
export class DeployedStackDeletionService {
private static instance: DeployedStackDeletionService;
@@ -164,6 +230,42 @@ export class DeployedStackDeletionService {
const { nodeId, stackName, pruneVolumes } = input;
const db = DatabaseService.getInstance();
// Continuation loads ownership from the persisted intent; first call uses input.
let requiredBlueprintId: number | null =
typeof input.requireBlueprintId === 'number' ? input.requireBlueprintId : null;
if (existingIntentId) {
const existing = db.getDeletionIntentById(existingIntentId);
if (!existing || existing.status !== 'prepared') {
return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' };
}
if (existing.required_blueprint_id != null) {
requiredBlueprintId = existing.required_blueprint_id;
}
}
let skipPhysical = false;
if (requiredBlueprintId != null) {
const dirProbe = await probeStackDirectory(nodeId, stackName);
if (dirProbe.kind === 'error') {
return { ok: false, code: 'failed', error: dirProbe.error };
}
if (dirProbe.kind === 'absent') {
skipPhysical = true;
} else {
const ownership = await probeBlueprintMarkerOwnership(nodeId, stackName, requiredBlueprintId);
if (ownership.kind === 'failed') {
return { ok: false, code: 'failed', error: ownership.error };
}
if (ownership.kind === 'name_conflict') {
if (existingIntentId) {
db.updateCleanupPendingStatus(existingIntentId, 'cancelled');
}
return { ok: false, code: 'name_conflict', error: ownership.error };
}
}
}
let intentId = existingIntentId;
if (!intentId) {
const { tags, overridePaths } = collectArtifacts(nodeId, stackName);
@@ -177,6 +279,7 @@ export class DeployedStackDeletionService {
rollback_tags_json: JSON.stringify(tags),
override_paths_json: JSON.stringify(overridePaths),
prune_volumes_requested: pruneVolumes ? 1 : 0,
required_blueprint_id: requiredBlueprintId,
created_at: now,
updated_at: now,
};
@@ -197,38 +300,53 @@ export class DeployedStackDeletionService {
return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' };
}
try {
await ComposeService.getInstance(nodeId).downStack(stackName);
} catch (downErr) {
console.warn(
'[DeployedStackDeletion] Compose down failed or no-op for %s:',
sanitizeForLog(stackName),
downErr,
);
}
if (intent.prune_volumes_requested === 1) {
if (!skipPhysical) {
try {
await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]);
} catch (pruneErr) {
await ComposeService.getInstance(nodeId).downStack(stackName);
} catch (downErr) {
console.warn(
'[DeployedStackDeletion] Volume prune failed for %s, continuing delete:',
'[DeployedStackDeletion] Compose down failed or no-op for %s:',
sanitizeForLog(stackName),
pruneErr,
downErr,
);
}
if (intent.prune_volumes_requested === 1) {
try {
await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]);
} catch (pruneErr) {
console.warn(
'[DeployedStackDeletion] Volume prune failed for %s, continuing delete:',
sanitizeForLog(stackName),
pruneErr,
);
}
}
try {
await FileSystemService.getInstance(nodeId).deleteStack(stackName);
} catch (fsErr) {
db.updateCleanupPendingStatus(intentId, 'cancelled');
return {
ok: false,
code: 'fs_failed',
error: getErrorMessage(fsErr, 'Failed to remove stack files'),
};
}
}
try {
await FileSystemService.getInstance(nodeId).deleteStack(stackName);
} catch (fsErr) {
db.updateCleanupPendingStatus(intentId, 'cancelled');
return {
ok: false,
code: 'fs_failed',
error: getErrorMessage(fsErr, 'Failed to remove stack files'),
};
}
const finalized = await this.finalizeLogicalDeletion(input, intentId);
if (!finalized.ok) return finalized;
return { ok: true, status: skipPhysical ? 'already_absent' : 'deleted' };
}
/** Ready transaction, secondary DB/RBAC cleanup, mesh opt-out, sweep, invalidate. */
private async finalizeLogicalDeletion(
input: DeleteDeployedStackInput,
intentId: string,
): Promise<DeleteDeployedStackResult> {
const { nodeId, stackName } = input;
const db = DatabaseService.getInstance();
if (!db.commitStackDeletionReadyTransaction(intentId, nodeId, stackName)) {
return {
@@ -282,7 +400,7 @@ export class DeployedStackDeletionService {
stackName,
ts: Date.now(),
});
return { ok: true };
return { ok: true, status: 'deleted' };
}
/**
@@ -497,6 +615,31 @@ export class DeployedStackDeletionService {
continue;
}
if (intent.required_blueprint_id != null) {
const ownership = await probeBlueprintMarkerOwnership(
nodeId,
stackName,
intent.required_blueprint_id,
);
if (ownership.kind === 'name_conflict') {
db.updateCleanupPendingStatus(intent.id, 'cancelled');
console.warn(
'[DeployedStackDeletion] Startup cancelled blueprint deletion for %s: %s',
sanitizeForLog(stackName),
sanitizeForLog(ownership.error),
);
continue;
}
if (ownership.kind === 'failed') {
console.warn(
'[DeployedStackDeletion] Startup ownership probe failed for %s (leaving prepared): %s',
sanitizeForLog(stackName),
sanitizeForLog(ownership.error),
);
continue;
}
}
const result = await this.deleteDeployedStack({
nodeId,
stackName,
+10 -3
View File
@@ -58,6 +58,13 @@ const PROTECTED_STACK_FILES = new Set([
'.env',
]);
// Explorer-only protection: includes the blueprint ownership marker without
// putting it in PROTECTED_STACK_FILES (backup/rollback orphan removal).
const EXPLORER_PROTECTED_STACK_FILES = new Set([
...PROTECTED_STACK_FILES,
'.blueprint.json',
]);
// Bookkeeping markers Sencho writes into the backup slot. They are never copied
// back into the stack directory on restore: `.timestamp` records when the backup
// was taken; `.checksums` is the integrity manifest verified before a restore.
@@ -169,7 +176,7 @@ function isProtectedRelPath(relPath: string): boolean {
if (normalized.includes('/')) return false;
// Fold case so e.g. a request for COMPOSE.YAML cannot dodge the gate on a
// case-insensitive filesystem where it resolves to the real compose.yaml.
return PROTECTED_STACK_FILES.has(fsCaseKey(normalized));
return EXPLORER_PROTECTED_STACK_FILES.has(fsCaseKey(normalized));
}
function protectedFileError(relPath: string): Error & { code: string } {
@@ -1643,7 +1650,7 @@ export class FileSystemService {
type,
size,
mtime,
isProtected: protectedEnabled && PROTECTED_STACK_FILES.has(dirent.name),
isProtected: protectedEnabled && EXPLORER_PROTECTED_STACK_FILES.has(dirent.name),
};
})
);
@@ -2096,7 +2103,7 @@ export class FileSystemService {
type,
size: stat.isDirectory() ? 0 : stat.size,
mtime: stat.mtimeMs,
isProtected: (scope?.protectedEnabled ?? true) && PROTECTED_STACK_FILES.has(name),
isProtected: (scope?.protectedEnabled ?? true) && EXPLORER_PROTECTED_STACK_FILES.has(name),
};
}
}
@@ -457,15 +457,30 @@ function asApprovedBlueprint(blueprint: Blueprint): Blueprint & BlueprintApprova
}
/** Upgrade create rows to blockers when an unmanaged same-name stack already exists. */
async function applyCreateNameConflictBlockers(blueprintName: string, raw: RawAction[]): Promise<void> {
function blockCreateForOwnership(row: RawAction, detail: string): void {
row.action = 'blocked_name_conflict';
row.severity = 'blocker';
row.detail = detail;
}
async function applyCreateNameConflictBlockers(
blueprintName: string,
blueprintId: number,
raw: RawAction[],
): Promise<void> {
const { BlueprintService } = await import('./BlueprintService');
const svc = BlueprintService.getInstance();
for (const row of raw) {
if (row.action !== 'create') continue;
if (!(await svc.hasNameConflict(blueprintName, row.node))) continue;
row.action = 'blocked_name_conflict';
row.severity = 'blocker';
row.detail = 'Unmanaged stack with this name already exists on this node';
try {
if (!(await svc.hasNameConflict(blueprintName, row.node, blueprintId))) continue;
blockCreateForOwnership(row, 'Unmanaged stack with this name already exists on this node');
} catch (err) {
blockCreateForOwnership(
row,
err instanceof Error ? err.message : 'Cannot verify stack ownership on this node',
);
}
}
}
@@ -477,7 +492,7 @@ export async function buildBlueprintPreview(blueprintId: number): Promise<Bluepr
const deployments = db.listDeployments(blueprintId);
const decision = BlueprintReconciler.getInstance().computeDecisionForPreview(blueprint, allNodes);
const raw = projectActions(blueprint, allNodes, deployments, decision);
await applyCreateNameConflictBlockers(blueprint.name, raw);
await applyCreateNameConflictBlockers(blueprint.name, blueprint.id, raw);
const changes: PreviewChangeRow[] = [];
for (const row of raw) {
+6 -3
View File
@@ -21,7 +21,7 @@ Three moving parts cooperate per blueprint.
1. **The declared spec.** A blueprint is a row in Sencho's database. It carries the compose YAML, the selector (labels or explicit node IDs), the drift policy, and a monotonic revision number. The revision auto-increments every time the compose changes.
2. **The reconciler.** A background loop on the controlling Sencho ticks every 60 seconds (with a 5-second initial delay after startup) and on demand after **Confirm Apply**. Each tick only mutates the fleet when the blueprint's current operational intent is approved. The tick resolves the selector, compares every desired-vs-live node, and runs only the place or remove outcomes the operator already confirmed.
3. **The executor.** Per-node deploy and withdraw run against the local Docker socket on local nodes, and through the standard authenticated proxy to `/api/stacks` on remote nodes. Every blueprint deployment writes a `.blueprint.json` marker into the stack directory; the marker carries the blueprint ID, the revision, and the last-applied timestamp.
3. **The executor.** Per-node deploy and withdraw run against the local Docker socket on local nodes, and through the authenticated proxy to `/api/blueprints/apply-local` and `/api/blueprints/withdraw-local` on remote nodes (older remotes fail closed until upgraded). Every blueprint deployment writes a `.blueprint.json` marker into the stack directory; the marker carries the blueprint ID, the revision, and the last-applied timestamp.
The marker is the trust root. If a directory by the blueprint's name already exists on a node and does not carry a matching marker, the reconciler refuses to touch it and surfaces a **Name conflict** on the deployment row. A Blueprint named `nginx` will never overwrite an existing user-authored `nginx` stack on any node.
@@ -155,7 +155,7 @@ Possible status values:
| **Withdrawing** | A withdraw is in flight. |
| **Withdrawn** | The deployment was successfully removed; the row is about to be cleared. |
| **Evict blocked** | A node left the selector while a stateful deployment was active. The reconciler refuses to evict until you choose an explicit mode. |
| **Name conflict** | A directory by the blueprint's name already exists on the node and does not carry our marker file. The reconciler will not touch it. |
| **Name conflict** | A directory by the blueprint's name already exists on the node and does not carry a matching `.blueprint.json` (missing, malformed, or another blueprint ID). The reconciler will not touch it. |
<Frame caption="A deployment row in Awaiting confirmation. The Confirm deploy action sits at the right; the notes column explains why the reconciler is waiting on the operator.">
<img src="/images/blueprint-model/detail-state-review.png" alt="Blueprint detail sheet with a deployment row in Awaiting confirmation state" />
@@ -309,7 +309,7 @@ A future Volume Migration feature will automate this with app-aware backup tooli
<AccordionGroup>
<Accordion title="A deployment row shows 'Name conflict' and refuses to deploy">
A directory by the blueprint's name already exists on that node and does not carry the `.blueprint.json` marker. The most likely cause is a manually created stack with the same name. Resolution: rename either the existing stack or the blueprint, then click **Apply now** on the detail sheet to retry.
A directory by the blueprint's name already exists on that node without a matching `.blueprint.json` marker (missing, malformed, or owned by another blueprint). The most likely cause is a manually created stack with the same name. Resolution: rename either the existing stack or the blueprint, then click **Apply now** on the detail sheet to retry.
</Accordion>
<Accordion title="A stateful deployment is stuck in 'Awaiting confirmation'">
The reconciler will not auto-deploy a stateful blueprint to a node it has never run on. Click **Confirm deploy** on the row, then choose **Deploy fresh** in the dialog. Sencho will create empty named volumes and start the stack.
@@ -323,6 +323,9 @@ A future Volume Migration feature will automate this with app-aware backup tooli
<Accordion title="A remote node is offline or disconnected during apply">
The row moves to **Failed** with the remote error. Reconnect the node (see [Pilot Agent](/features/pilot-agent) and [Multi-node management](/features/multi-node)), verify whether the stack directory contains `compose.yaml` and `.blueprint.json`, then click **Apply now**. If the directory exists without a matching marker, Sencho treats it as a name conflict until you rename or remove the remote stack manually.
</Accordion>
<Accordion title="Remote apply or withdraw fails asking for an upgrade">
Blueprint apply and withdraw on a remote node require that node's Sencho build to expose the atomic node-local endpoints (`/api/blueprints/apply-local` and `/api/blueprints/withdraw-local`). Older remotes refuse the mutation with a failed deployment row rather than writing files without ownership checks. Upgrade the remote instance to a matching Sencho release, then retry **Apply now** or withdraw.
</Accordion>
<Accordion title="A Docker daemon or registry failure surfaced during apply">
The deployment row moves to **Failed** and records the Docker or registry error. Resolve the daemon, socket, registry credentials, rate limit, disk, or volume-permission issue on the affected node, then click **Apply now**. Watch that node's stack activity and security scan status after retry.
</Accordion>